[RFC V3 PATCH 00/26] Kernel NET policy

@@ -53,6 +53,7 @@
 #include <uapi/linux/if_bonding.h>
 #include <uapi/linux/pkt_cls.h>
 #include <linux/hashtable.h>
+#include <linux/netpolicy.h>
 
 struct netpoll_info;
 struct device;

@@ -1121,6 +1122,9 @@ struct netdev_xdp {
  * int (*ndo_xdp)(struct net_device *dev, struct netdev_xdp *xdp);
  *	This function is used to set or query state related to XDP on the
  *	netdevice. See definition of enum xdp_netdev_command for details.
+ * int(*ndo_netpolicy_init)(struct net_device *dev,
+ *			    struct netpolicy_info *info);
+ *	This function is used to init and get supported policy.
  *
  */
 struct net_device_ops {

@@ -1307,6 +1311,10 @@ struct net_device_ops {
 						       int needed_headroom);
 	int			(*ndo_xdp)(struct net_device *dev,
 					   struct netdev_xdp *xdp);
+#ifdef CONFIG_NETPOLICY
+	int			(*ndo_netpolicy_init)(struct net_device *dev,
+						      struct netpolicy_info *info);
+#endif /* CONFIG_NETPOLICY */
 };
 
 /**

@@ -1619,6 +1627,8 @@ enum netdev_priv_flags {
  *			switch port.
  *
  *	@proc_dev:	device node in proc to configure device net policy
+ *	@netpolicy:	NET policy related information of net device
+ *	@np_lock:	protect the state of NET policy
  *
  *	FIXME: cleanup struct net_device such that network protocol info
  *	moves out.

@@ -1891,6 +1901,8 @@ struct net_device {
 #ifdef CONFIG_PROC_FS
 	struct proc_dir_entry	*proc_dev;
 #endif /* CONFIG_PROC_FS */
+	struct netpolicy_info	*netpolicy;
+	spinlock_t		np_lock;
 #endif /* CONFIG_NETPOLICY */
 };
 #define to_net_dev(d) container_of(d, struct net_device, dev)

@@ -0,0 +1,31 @@
+/*
+ * netpolicy.h: Net policy support
+ * Copyright (c) 2016, Intel Corporation.
+ * Author: Kan Liang (kan.liang@intel.com)
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ */
+#ifndef __LINUX_NETPOLICY_H
+#define __LINUX_NETPOLICY_H
+
+enum netpolicy_name {
+	NET_POLICY_NONE		= 0,
+	NET_POLICY_MAX,
+};
+
+extern const char *policy_name[];
+
+struct netpolicy_info {
+	enum netpolicy_name	cur_policy;
+	unsigned long avail_policy[BITS_TO_LONGS(NET_POLICY_MAX)];
+};
+
+#endif /*__LINUX_NETPOLICY_H*/

@@ -34,14 +34,31 @@
 #include <linux/uaccess.h>
 #include <linux/netdevice.h>
 #include <net/net_namespace.h>
+#include <net/rtnetlink.h>
 
+const char *policy_name[NET_POLICY_MAX] = {
+	"NONE"
+};
 #ifdef CONFIG_PROC_FS
 
 static int net_policy_proc_show(struct seq_file *m, void *v)
 {
 	struct net_device *dev = (struct net_device *)m->private;
-
-	seq_printf(m, "%s doesn't support net policy manager\n", dev->name);
+	int i;
+
+	if (WARN_ON(!dev->netpolicy))
+		return -EINVAL;
+
+	if (dev->netpolicy->cur_policy == NET_POLICY_NONE) {
+		seq_printf(m, "%s: There is no policy applied\n", dev->name);
+		seq_printf(m, "%s: The available policy include:", dev->name);
+		for_each_set_bit(i, dev->netpolicy->avail_policy, NET_POLICY_MAX)
+			seq_printf(m, " %s", policy_name[i]);
+		seq_printf(m, "\n");
+	} else {
+		seq_printf(m, "%s: POLICY %s is running on the system\n",
+			   dev->name, policy_name[dev->netpolicy->cur_policy]);
+	}
 
 	return 0;
 }

@@ -73,33 +90,85 @@ static int netpolicy_proc_dev_init(struct net *net, struct net_device *dev)
 	}
 	return 0;
 }
+#endif /* CONFIG_PROC_FS */
+
+int init_netpolicy(struct net_device *dev)
+{
+	int ret;
+
+	spin_lock(&dev->np_lock);
+	ret = 0;
+
+	if (!dev->netdev_ops->ndo_netpolicy_init) {
+		ret = -ENOTSUPP;
+		goto unlock;
+	}
+
+	if (dev->netpolicy)
+		goto unlock;
+
+	dev->netpolicy = kzalloc(sizeof(*dev->netpolicy), GFP_ATOMIC);
+	if (!dev->netpolicy) {
+		ret = -ENOMEM;
+		goto unlock;
+	}
+
+	ret = dev->netdev_ops->ndo_netpolicy_init(dev, dev->netpolicy);
+	if (ret) {
+		kfree(dev->netpolicy);
+		dev->netpolicy = NULL;
+	}
+
+unlock:
+	spin_unlock(&dev->np_lock);
+	return ret;
+}
+
+void uninit_netpolicy(struct net_device *dev)
+{
+	spin_lock(&dev->np_lock);
+	if (dev->netpolicy) {
+		kfree(dev->netpolicy);
+		dev->netpolicy = NULL;
+	}
+	spin_unlock(&dev->np_lock);
+}
 
 static int __net_init netpolicy_net_init(struct net *net)
 {
 	struct net_device *dev, *aux;
 
+#ifdef CONFIG_PROC_FS
 	net->proc_netpolicy = proc_net_mkdir(net, "netpolicy",
 					     net->proc_net);
 	if (!net->proc_netpolicy)
 		return -ENOMEM;
+#endif /* CONFIG_PROC_FS */
 
+	rtnl_lock();
 	for_each_netdev_safe(net, dev, aux) {
-		netpolicy_proc_dev_init(net, dev);
+		if (!init_netpolicy(dev)) {
+#ifdef CONFIG_PROC_FS
+			if (netpolicy_proc_dev_init(net, dev))
+				uninit_netpolicy(dev);
+			else
+#endif /* CONFIG_PROC_FS */
+			pr_info("NETPOLICY: Init net policy for %s\n", dev->name);
+		}
 	}
+	rtnl_unlock();
 
 	return 0;
 }
 
-#else /* CONFIG_PROC_FS */
-
-static int __net_init netpolicy_net_init(struct net *net)
-{
-	return 0;
-}
-#endif /* CONFIG_PROC_FS */
-
 static void __net_exit netpolicy_net_exit(struct net *net)
 {
+	struct net_device *dev, *aux;
+
+	rtnl_lock();
+	for_each_netdev_safe(net, dev, aux)
+		uninit_netpolicy(dev);
+	rtnl_unlock();
 #ifdef CONFIG_PROC_FS
 	remove_proc_subtree("netpolicy", net->proc_net);
 #endif /* CONFIG_PROC_FS */

@@ -49,6 +49,7 @@ struct netpolicy_sys_info {
 struct netpolicy_info {
 	enum netpolicy_name	cur_policy;
 	unsigned long avail_policy[BITS_TO_LONGS(NET_POLICY_MAX)];
+	bool irq_affinity;
 	/* cpu and queue mapping information */
 	struct netpolicy_sys_info	sys_info;
 };

@@ -162,6 +162,47 @@ static void netpolicy_set_affinity(struct net_device *dev)
 	}
 }
 
+static int netpolicy_disable(struct net_device *dev)
+{
+	if (dev->netpolicy->irq_affinity)
+		netpolicy_clear_affinity(dev);
+	netpolicy_free_sys_map(dev);
+
+	return 0;
+}
+
+static int netpolicy_enable(struct net_device *dev)
+{
+	int ret;
+	struct netpolicy_dev_info d_info;
+	u32 cpu;
+
+	if (WARN_ON(!dev->netpolicy))
+		return -EINVAL;
+
+	/* get driver information */
+	ret = netpolicy_get_dev_info(dev, &d_info);
+	if (ret)
+		return ret;
+
+	/* get cpu information */
+	cpu = netpolicy_get_cpu_information();
+
+	/* create sys map */
+	ret = netpolicy_update_sys_map(dev, &d_info, cpu);
+	if (ret) {
+		netpolicy_free_dev_info(&d_info);
+		return ret;
+	}
+
+	/* set irq affinity */
+	if (dev->netpolicy->irq_affinity)
+		netpolicy_set_affinity(dev);
+
+	netpolicy_free_dev_info(&d_info);
+	return 0;
+}
+
 const char *policy_name[NET_POLICY_MAX] = {
 	"NONE"
 };

@@ -29,6 +29,7 @@
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/init.h>
+#include <linux/irq.h>
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/uaccess.h>

@@ -129,6 +130,38 @@ err:
 	return -ENOMEM;
 }
 
+static void netpolicy_clear_affinity(struct net_device *dev)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+	u32 i;
+
+	for (i = 0; i < s_info->avail_rx_num; i++) {
+		irq_clear_status_flags(s_info->rx[i].irq, IRQ_NO_BALANCING);
+		irq_set_affinity_hint(s_info->rx[i].irq, cpu_online_mask);
+	}
+
+	for (i = 0; i < s_info->avail_tx_num; i++) {
+		irq_clear_status_flags(s_info->tx[i].irq, IRQ_NO_BALANCING);
+		irq_set_affinity_hint(s_info->tx[i].irq, cpu_online_mask);
+	}
+}
+
+static void netpolicy_set_affinity(struct net_device *dev)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+	u32 i;
+
+	for (i = 0; i < s_info->avail_rx_num; i++) {
+		irq_set_status_flags(s_info->rx[i].irq, IRQ_NO_BALANCING);
+		irq_set_affinity_hint(s_info->rx[i].irq, cpumask_of(s_info->rx[i].cpu));
+	}
+
+	for (i = 0; i < s_info->avail_tx_num; i++) {
+		irq_set_status_flags(s_info->tx[i].irq, IRQ_NO_BALANCING);
+		irq_set_affinity_hint(s_info->tx[i].irq, cpumask_of(s_info->tx[i].cpu));
+	}
+}
+
 const char *policy_name[NET_POLICY_MAX] = {
 	"NONE"
 };

@@ -30,9 +30,27 @@ struct netpolicy_dev_info {
 	u32	*tx_irq;
 };
 
+struct netpolicy_sys_map {
+	u32	cpu;
+	u32	queue;
+	u32	irq;
+};
+
+struct netpolicy_sys_info {
+	/*
+	 * Record the cpu and queue 1:1 mapping
+	 */
+	u32				avail_rx_num;
+	struct netpolicy_sys_map	*rx;
+	u32				avail_tx_num;
+	struct netpolicy_sys_map	*tx;
+};
+
 struct netpolicy_info {
 	enum netpolicy_name	cur_policy;
 	unsigned long avail_policy[BITS_TO_LONGS(NET_POLICY_MAX)];
+	/* cpu and queue mapping information */
+	struct netpolicy_sys_info	sys_info;
 };
 
 #endif /*__LINUX_NETPOLICY_H*/

@@ -55,6 +55,80 @@ static u32 netpolicy_get_cpu_information(void)
 	return num_online_cpus();
 }
 
+static void netpolicy_free_sys_map(struct net_device *dev)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+
+	kfree(s_info->rx);
+	s_info->rx = NULL;
+	s_info->avail_rx_num = 0;
+	kfree(s_info->tx);
+	s_info->tx = NULL;
+	s_info->avail_tx_num = 0;
+}
+
+static int netpolicy_update_sys_map(struct net_device *dev,
+				    struct netpolicy_dev_info *d_info,
+				    u32 cpu)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+	u32 num, i, online_cpu;
+	cpumask_var_t cpumask;
+
+	if (!alloc_cpumask_var(&cpumask, GFP_ATOMIC))
+		return -ENOMEM;
+
+	/* update rx cpu map */
+	if (cpu > d_info->rx_num)
+		num = d_info->rx_num;
+	else
+		num = cpu;
+
+	s_info->avail_rx_num = num;
+	s_info->rx = kcalloc(num, sizeof(*s_info->rx), GFP_ATOMIC);
+	if (!s_info->rx)
+		goto err;
+	cpumask_copy(cpumask, cpu_online_mask);
+
+	i = 0;
+	for_each_cpu(online_cpu, cpumask) {
+		if (i == num)
+			break;
+		s_info->rx[i].cpu = online_cpu;
+		s_info->rx[i].queue = i;
+		s_info->rx[i].irq = d_info->rx_irq[i];
+		i++;
+	}
+
+	/* update tx cpu map */
+	if (cpu >= d_info->tx_num)
+		num = d_info->tx_num;
+	else
+		num = cpu;
+
+	s_info->avail_tx_num = num;
+	s_info->tx = kcalloc(num, sizeof(*s_info->tx), GFP_ATOMIC);
+	if (!s_info->tx)
+		goto err;
+
+	i = 0;
+	for_each_cpu(online_cpu, cpumask) {
+		if (i == num)
+			break;
+		s_info->tx[i].cpu = online_cpu;
+		s_info->tx[i].queue = i;
+		s_info->tx[i].irq = d_info->tx_irq[i];
+		i++;
+	}
+
+	free_cpumask_var(cpumask);
+	return 0;
+err:
+	netpolicy_free_sys_map(dev);
+	free_cpumask_var(cpumask);
+	return -ENOMEM;
+}
+
 const char *policy_name[NET_POLICY_MAX] = {
 	"NONE"
 };

@@ -1634,6 +1634,7 @@ enum netdev_priv_flags {
  *	@proc_dev:	device node in proc to configure device net policy
  *	@netpolicy:	NET policy related information of net device
  *	@np_lock:	protect the state of NET policy
+ *	@np_ob_list_lock:	protect the net policy object list
  *
  *	FIXME: cleanup struct net_device such that network protocol info
  *	moves out.

@@ -1908,6 +1909,7 @@ struct net_device {
 #endif /* CONFIG_PROC_FS */
 	struct netpolicy_info	*netpolicy;
 	spinlock_t		np_lock;
+	spinlock_t		np_ob_list_lock;
 #endif /* CONFIG_NETPOLICY */
 };
 #define to_net_dev(d) container_of(d, struct net_device, dev)

@@ -21,6 +21,12 @@ enum netpolicy_name {
 	NET_POLICY_MAX,
 };
 
+enum netpolicy_traffic {
+	NETPOLICY_RX		= 0,
+	NETPOLICY_TX,
+	NETPOLICY_RXTX,
+};
+
 extern const char *policy_name[];
 
 struct netpolicy_dev_info {

@@ -46,12 +52,21 @@ struct netpolicy_sys_info {
 	struct netpolicy_sys_map	*tx;
 };
 
+struct netpolicy_object {
+	struct list_head	list;
+	u32			cpu;
+	u32			queue;
+	atomic_t		refcnt;
+};
+
 struct netpolicy_info {
 	enum netpolicy_name	cur_policy;
 	unsigned long avail_policy[BITS_TO_LONGS(NET_POLICY_MAX)];
 	bool irq_affinity;
 	/* cpu and queue mapping information */
 	struct netpolicy_sys_info	sys_info;
+	/* List of policy objects 0 rx 1 tx */
+	struct list_head	obj_list[NETPOLICY_RXTX][NET_POLICY_MAX];
 };
 
 #endif /*__LINUX_NETPOLICY_H*/

@@ -36,6 +36,7 @@
 #include <linux/netdevice.h>
 #include <net/net_namespace.h>
 #include <net/rtnetlink.h>
+#include <linux/sort.h>
 
 static int netpolicy_get_dev_info(struct net_device *dev,
 				  struct netpolicy_dev_info *d_info)

@@ -162,11 +163,31 @@ static void netpolicy_set_affinity(struct net_device *dev)
 	}
 }
 
+static void netpolicy_free_obj_list(struct net_device *dev)
+{
+	int i, j;
+	struct netpolicy_object *obj, *tmp;
+
+	spin_lock(&dev->np_ob_list_lock);
+	for (i = 0; i < NETPOLICY_RXTX; i++) {
+		for (j = NET_POLICY_NONE; j < NET_POLICY_MAX; j++) {
+			if (list_empty(&dev->netpolicy->obj_list[i][j]))
+				continue;
+			list_for_each_entry_safe(obj, tmp, &dev->netpolicy->obj_list[i][j], list) {
+				list_del(&obj->list);
+				kfree(obj);
+			}
+		}
+	}
+	spin_unlock(&dev->np_ob_list_lock);
+}
+
 static int netpolicy_disable(struct net_device *dev)
 {
 	if (dev->netpolicy->irq_affinity)
 		netpolicy_clear_affinity(dev);
 	netpolicy_free_sys_map(dev);
+	netpolicy_free_obj_list(dev);
 
 	return 0;
 }

@@ -206,6 +227,213 @@ static int netpolicy_enable(struct net_device *dev)
 const char *policy_name[NET_POLICY_MAX] = {
 	"NONE"
 };
+
+static u32 cpu_to_queue(struct net_device *dev,
+			u32 cpu, bool is_rx)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+	int i;
+
+	if (is_rx) {
+		for (i = 0; i < s_info->avail_rx_num; i++) {
+			if (s_info->rx[i].cpu == cpu)
+				return s_info->rx[i].queue;
+		}
+	} else {
+		for (i = 0; i < s_info->avail_tx_num; i++) {
+			if (s_info->tx[i].cpu == cpu)
+				return s_info->tx[i].queue;
+		}
+	}
+
+	return ~0;
+}
+
+static int netpolicy_add_obj(struct net_device *dev,
+			     u32 cpu, bool is_rx,
+			     enum netpolicy_name policy)
+{
+	struct netpolicy_object *obj;
+	int dir = is_rx ? NETPOLICY_RX : NETPOLICY_TX;
+
+	obj = kzalloc(sizeof(*obj), GFP_ATOMIC);
+	if (!obj)
+		return -ENOMEM;
+	obj->cpu = cpu;
+	obj->queue = cpu_to_queue(dev, cpu, is_rx);
+	list_add_tail(&obj->list, &dev->netpolicy->obj_list[dir][policy]);
+
+	return 0;
+}
+
+struct sort_node {
+	int	node;
+	int	distance;
+};
+
+static inline int node_distance_cmp(const void *a, const void *b)
+{
+	const struct sort_node *_a = a;
+	const struct sort_node *_b = b;
+
+	return _a->distance - _b->distance;
+}
+
+static int _netpolicy_gen_obj_list(struct net_device *dev, bool is_rx,
+				   enum netpolicy_name policy,
+				   struct sort_node *nodes, int num_node,
+				   struct cpumask *node_avail_cpumask)
+{
+	cpumask_var_t node_tmp_cpumask, sibling_tmp_cpumask;
+	struct cpumask *node_assigned_cpumask;
+	int i, ret = -ENOMEM;
+	u32 cpu;
+
+	if (!alloc_cpumask_var(&node_tmp_cpumask, GFP_ATOMIC))
+		return ret;
+	if (!alloc_cpumask_var(&sibling_tmp_cpumask, GFP_ATOMIC))
+		goto alloc_fail1;
+
+	node_assigned_cpumask = kcalloc(num_node, sizeof(struct cpumask), GFP_ATOMIC);
+	if (!node_assigned_cpumask)
+		goto alloc_fail2;
+
+	/* Don't share physical core */
+	for (i = 0; i < num_node; i++) {
+		if (cpumask_weight(&node_avail_cpumask[nodes[i].node]) == 0)
+			continue;
+		spin_lock(&dev->np_ob_list_lock);
+		cpumask_copy(node_tmp_cpumask, &node_avail_cpumask[nodes[i].node]);
+		while (cpumask_weight(node_tmp_cpumask)) {
+			cpu = cpumask_first(node_tmp_cpumask);
+
+			/* push to obj list */
+			ret = netpolicy_add_obj(dev, cpu, is_rx, policy);
+			if (ret) {
+				spin_unlock(&dev->np_ob_list_lock);
+				goto err;
+			}
+
+			cpumask_set_cpu(cpu, &node_assigned_cpumask[nodes[i].node]);
+			cpumask_and(sibling_tmp_cpumask, node_tmp_cpumask, topology_sibling_cpumask(cpu));
+			cpumask_xor(node_tmp_cpumask, node_tmp_cpumask, sibling_tmp_cpumask);
+		}
+		spin_unlock(&dev->np_ob_list_lock);
+	}
+
+	for (i = 0; i < num_node; i++) {
+		cpumask_xor(node_tmp_cpumask, &node_avail_cpumask[nodes[i].node], &node_assigned_cpumask[nodes[i].node]);
+		if (cpumask_weight(node_tmp_cpumask) == 0)
+			continue;
+		spin_lock(&dev->np_ob_list_lock);
+		for_each_cpu(cpu, node_tmp_cpumask) {
+			/* push to obj list */
+			ret = netpolicy_add_obj(dev, cpu, is_rx, policy);
+			if (ret) {
+				spin_unlock(&dev->np_ob_list_lock);
+				goto err;
+			}
+			cpumask_set_cpu(cpu, &node_assigned_cpumask[nodes[i].node]);
+		}
+		spin_unlock(&dev->np_ob_list_lock);
+	}
+
+err:
+	kfree(node_assigned_cpumask);
+alloc_fail2:
+	free_cpumask_var(sibling_tmp_cpumask);
+alloc_fail1:
+	free_cpumask_var(node_tmp_cpumask);
+
+	return ret;
+}
+
+static int netpolicy_gen_obj_list(struct net_device *dev,
+				  enum netpolicy_name policy)
+{
+	struct netpolicy_sys_info *s_info = &dev->netpolicy->sys_info;
+	struct cpumask *node_avail_cpumask;
+	struct sort_node *nodes;
+	int i, ret, node = 0;
+	int num_nodes = 1;
+	u32 cpu;
+#ifdef CONFIG_NUMA
+	int dev_node = 0;
+	int val;
+#endif
+	/* The network performance for objects could be different
+	 * because of the queue and cpu topology.
+	 * The objects will be ordered accordingly,
+	 * and put high performance object in the front.
+	 *
+	 * The priority rules as below,
+	 * - The local object. (Local means cpu and queue are in the same node.)
+	 * - The cpu in the object is the only logical core in physical core.
+	 *   The sibiling core's object has not been added in the object list yet.
+	 * - The rest of objects
+	 *
+	 * So the order of object list is as below:
+	 * 1. Local core + the only logical core
+	 * 2. Remote core + the only logical core
+	 * 3. Local core + the core's sibling is already in the object list
+	 * 4. Remote core + the core's sibling is already in the object list
+	 */
+#ifdef CONFIG_NUMA
+	dev_node = dev_to_node(dev->dev.parent);
+	num_nodes = num_online_nodes();
+#endif
+
+	nodes = kcalloc(num_nodes, sizeof(*nodes), GFP_ATOMIC);
+	if (!nodes)
+		return -ENOMEM;
+
+	node_avail_cpumask = kcalloc(num_nodes, sizeof(struct cpumask), GFP_ATOMIC);
+	if (!node_avail_cpumask) {
+		kfree(nodes);
+		return -ENOMEM;
+	}
+
+#ifdef CONFIG_NUMA
+	/* order the node from near to far */
+	for_each_node_mask(i, node_online_map) {
+		val = node_distance(dev_node, i);
+		nodes[node].node = i;
+		nodes[node].distance = val;
+		node++;
+	}
+	sort(nodes, num_nodes, sizeof(*nodes),
+	     node_distance_cmp, NULL);
+#else
+	nodes[0].node = 0;
+#endif
+
+	for (i = 0; i < s_info->avail_rx_num; i++) {
+		cpu = s_info->rx[i].cpu;
+		cpumask_set_cpu(cpu, &node_avail_cpumask[cpu_to_node(cpu)]);
+	}
+	ret = _netpolicy_gen_obj_list(dev, true, policy, nodes,
+				      node, node_avail_cpumask);
+	if (ret)
+		goto err;
+
+	for (i = 0; i < node; i++)
+		cpumask_clear(&node_avail_cpumask[nodes[i].node]);
+
+	for (i = 0; i < s_info->avail_tx_num; i++) {
+		cpu = s_info->tx[i].cpu;
+		cpumask_set_cpu(cpu, &node_avail_cpumask[cpu_to_node(cpu)]);
+	}
+	ret = _netpolicy_gen_obj_list(dev, false, policy, nodes,
+				      node, node_avail_cpumask);
+	if (ret)
+		goto err;
+
+err:
+	kfree(nodes);
+	kfree(node_avail_cpumask);
+	return ret;
+}
+
 #ifdef CONFIG_PROC_FS
 
 static int net_policy_proc_show(struct seq_file *m, void *v)

@@ -261,7 +489,7 @@ static int netpolicy_proc_dev_init(struct net *net, struct net_device *dev)
 
 int init_netpolicy(struct net_device *dev)
 {
-	int ret;
+	int ret, i, j;
 
 	spin_lock(&dev->np_lock);
 	ret = 0;

@@ -284,7 +512,15 @@ int init_netpolicy(struct net_device *dev)
 	if (ret) {
 		kfree(dev->netpolicy);
 		dev->netpolicy = NULL;
+		goto unlock;
+	}
+
+	spin_lock(&dev->np_ob_list_lock);
+	for (i = 0; i < NETPOLICY_RXTX; i++) {
+		for (j = NET_POLICY_NONE; j < NET_POLICY_MAX; j++)
+			INIT_LIST_HEAD(&dev->netpolicy->obj_list[i][j]);
 	}
+	spin_unlock(&dev->np_ob_list_lock);
 
 unlock:
 	spin_unlock(&dev->np_lock);

@@ -97,4 +97,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _UAPI_ASM_SOCKET_H */

@@ -90,4 +90,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _UAPI__ASM_AVR32_SOCKET_H */

@@ -90,5 +90,7 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _ASM_SOCKET_H */
 

@@ -99,4 +99,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _ASM_IA64_SOCKET_H */

@@ -90,4 +90,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _ASM_M32R_SOCKET_H */

@@ -108,4 +108,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _UAPI_ASM_SOCKET_H */

@@ -90,4 +90,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _ASM_SOCKET_H */

@@ -89,4 +89,6 @@
 
 #define SO_CNX_ADVICE		0x402E
 
+#define SO_NETPOLICY		0x402F
+
 #endif /* _UAPI_ASM_SOCKET_H */

@@ -97,4 +97,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif	/* _ASM_POWERPC_SOCKET_H */

@@ -96,4 +96,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* _ASM_SOCKET_H */

@@ -86,6 +86,8 @@
 
 #define SO_CNX_ADVICE		0x0037
 
+#define SO_NETPOLICY		0x0038
+
 /* Security levels - as per NRL IPv6 - don't actually do anything */
 #define SO_SECURITY_AUTHENTICATION		0x5001
 #define SO_SECURITY_ENCRYPTION_TRANSPORT	0x5002

@@ -101,4 +101,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif	/* _XTENSA_SOCKET_H */

@@ -101,7 +101,9 @@ reqsk_alloc(const struct request_sock_ops *ops, struct sock *sk_listener,
 	sk_tx_queue_clear(req_to_sk(req));
 	req->saved_syn = NULL;
 	atomic_set(&req->rsk_refcnt, 0);
-
+#ifdef CONFIG_NETPOLICY
+	memcpy(&req_to_sk(req)->sk_netpolicy, &sk_listener->sk_netpolicy, sizeof(sk_listener->sk_netpolicy));
+#endif
 	return req;
 }
 

@@ -70,6 +70,7 @@
 #include <net/checksum.h>
 #include <net/tcp_states.h>
 #include <linux/net_tstamp.h>
+#include <linux/netpolicy.h>
 
 /*
  * This structure really needs to be cleaned up.

@@ -141,6 +142,7 @@ typedef __u64 __bitwise __addrpair;
  *		%SO_OOBINLINE settings, %SO_TIMESTAMPING settings
  *	@skc_incoming_cpu: record/match cpu processing incoming packets
  *	@skc_refcnt: reference count
+ *	@skc_netpolicy: per socket net policy
  *
  *	This is the minimal network layer representation of sockets, the header
  *	for struct sock and struct inet_timewait_sock.

@@ -200,6 +202,10 @@ struct sock_common {
 		struct sock	*skc_listener; /* request_sock */
 		struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
 	};
+
+#ifdef CONFIG_NETPOLICY
+	struct netpolicy_instance    skc_netpolicy;
+#endif
 	/*
 	 * fields between dontcopy_begin/dontcopy_end
 	 * are not copied in sock_copy()

@@ -339,6 +345,9 @@ struct sock {
 #define sk_incoming_cpu		__sk_common.skc_incoming_cpu
 #define sk_flags		__sk_common.skc_flags
 #define sk_rxhash		__sk_common.skc_rxhash
+#ifdef CONFIG_NETPOLICY
+#define sk_netpolicy		__sk_common.skc_netpolicy
+#endif
 
 	socket_lock_t		sk_lock;
 	struct sk_buff_head	sk_receive_queue;

@@ -92,4 +92,6 @@
 
 #define SO_CNX_ADVICE		53
 
+#define SO_NETPOLICY		54
+
 #endif /* __ASM_GENERIC_SOCKET_H */

@@ -1003,6 +1003,12 @@ set_rcvbuf:
 		if (val == 1)
 			dst_negative_advice(sk);
 		break;
+
+#ifdef CONFIG_NETPOLICY
+	case SO_NETPOLICY:
+		ret = netpolicy_register(&sk->sk_netpolicy, val);
+		break;
+#endif
 	default:
 		ret = -ENOPROTOOPT;
 		break;

@@ -1263,6 +1269,11 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		v.val = sk->sk_incoming_cpu;
 		break;
 
+#ifdef CONFIG_NETPOLICY
+	case SO_NETPOLICY:
+		v.val = sk->sk_netpolicy.policy;
+		break;
+#endif
 	default:
 		/* We implement the SO_SNDLOWAT etc to not be settable
 		 * (1003.1g 7).

@@ -1402,6 +1413,12 @@ struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
 
 		sock_update_classid(&sk->sk_cgrp_data);
 		sock_update_netprioidx(&sk->sk_cgrp_data);
+
+#ifdef CONFIG_NETPOLICY
+		sk->sk_netpolicy.dev = NULL;
+		sk->sk_netpolicy.ptr = (void *)sk;
+		sk->sk_netpolicy.policy = NET_POLICY_INVALID;
+#endif
 	}
 
 	return sk;

@@ -1439,6 +1456,10 @@ static void __sk_destruct(struct rcu_head *head)
 	put_pid(sk->sk_peer_pid);
 	if (likely(sk->sk_net_refcnt))
 		put_net(sock_net(sk));
+#ifdef CONFIG_NETPOLICY
+	if (is_net_policy_valid(sk->sk_netpolicy.policy))
+		netpolicy_unregister(&sk->sk_netpolicy);
+#endif
 	sk_prot_free(sk->sk_prot_creator, sk);
 }
 

@@ -1575,6 +1596,13 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
 		if (sock_needs_netstamp(sk) &&
 		    newsk->sk_flags & SK_FLAGS_TIMESTAMP)
 			net_enable_timestamp();
+
+#ifdef CONFIG_NETPOLICY
+		newsk->sk_netpolicy.ptr = (void *)newsk;
+		if (is_net_policy_valid(newsk->sk_netpolicy.policy))
+			netpolicy_register(&newsk->sk_netpolicy, newsk->sk_netpolicy.policy);
+
+#endif
 	}
 out:
 	return newsk;

@@ -83,6 +83,7 @@ struct netpolicy_info {
 	unsigned long avail_policy[BITS_TO_LONGS(NET_POLICY_MAX)];
 	bool irq_affinity;
 	bool has_mix_policy;
+	bool queue_pair;
 	/* cpu and queue mapping information */
 	struct netpolicy_sys_info	sys_info;
 	/* List of policy objects 0 rx 1 tx */

@@ -538,6 +538,9 @@ int netpolicy_pick_queue(struct netpolicy_instance *instance, bool is_rx)
 	if (!policy_validate(instance))
 		return -EINVAL;
 
+	if (dev->netpolicy->queue_pair)
+		is_rx = true;
+
 	/* fast path */
 	rcu_read_lock();
 	version = rcu_dereference(dev->netpolicy->sys_info.version)->major;

@@ -0,0 +1,157 @@
+What is Linux Net Policy?
+
+It is a big challenge to get good network performance. First, the network
+performance is not good with default system settings. Second, it is too
+difficult to do automatic tuning for all possible workloads, since workloads
+have different requirements. Some workloads may want high throughput. Some may
+need low latency. Last but not least, there are lots of manual configurations.
+Fine grained configuration is too difficult for users.
+
+"NET policy" intends to simplify the network configuration and get a
+good network performance according to the hints(policy) which is applied by
+user. It provides some typical "policies" for user which can be set
+per-socket, per-task or per-device. The kernel automatically figures out
+how to merge different requests to get good network performance.
+
+"Net policy" is designed for multiqueue network devices. This document
+describes the concepts and APIs of "net policy" support.
+
+NET POLICY CONCEPTS
+
+Scope of Net Policies
+
+    Device net policy: this policy applies to the whole device. Once the
+    device net policy is set, it automatically configures the system
+    according to the applied policy. The configuration usually includes IRQ
+    affinity, IRQ balance disable, interrupt moderation, and so on. But the
+    device net policy does not change the packet direction.
+
+    Task net policy: this is a per-task policy. When it is applied to specific
+    task, all packet transmissions of the task will be redirected to the
+    assigned queues accordingly. If a task does not define a task policy,
+    it "falls back" to the system default way to direct the packets. The
+    per-task policy must be compatible with device net policy.
+
+    Socket net policy: this is a per-socket policy. When it is applied to
+    specific socket, all packet transmissions of the socket will be redirected
+    to the assigned queues accordingly. If a socket does not define a socket
+    policy, it "falls back" to the system default way to direct the packets.
+    The per-socket policy must be compatible with both device net policy and
+    per-task policy.
+
+Components of Net Policies
+
+    Net policy object: it is a combination of CPU and queue. The queue IRQ has
+    to set affinity with the CPU. It can be shared between sockets and tasks.
+    A reference counter is used to track the sharing number.
+
+    Net policy object list: each device policy has an object list. Once the
+    device policy is determined, the net policy object will be inserted into
+    the net policy object list. The net policy object list does not change
+    unless the CPU/queue number is changed, the netpolicy is disabled or
+    the device policy is changed.
+    The network performance for objects could be different because of the
+    CPU/queue topology and dev location. The objects which can bring high
+    performance are in the front of the list.
+
+    RCU hash table: an RCU hash table to maintain the relationship between
+    the task/socket and the assigned object. The task/socket can get the
+    assigned object by searching the table.
+    If it is the first time, there is no assigned object in the table. It will
+    go through the object list to find the available object based on position
+    and reference number.
+    If the net policy object list changes, all the assigned objects will become
+    invalid.
+
+NET POLICY APIs
+
+Interfaces between net policy and device driver
+
+    int (*ndo_netpolicy_init)(struct net_device *dev,
+                              struct netpolicy_info *info);
+
+    The device driver who has NET policy support must implement this interface.
+    In this interface, the device driver does necessory initialization, and fill
+    the info for net policy module. The information could include supported
+    policy, MIX policy support, queue pair support and so on.
+
+    int (*ndo_get_irq_info)(struct net_device *dev,
+                            struct netpolicy_dev_info *info);
+
+    This interface is used to get more accurate device IRQ information.
+
+    int (*ndo_set_net_policy)(struct net_device *dev,
+                              enum netpolicy_name name);
+
+    This interface is used to set device net policy by name. It is device driver's
+    responsibility to set driver specific configuration for the given policy.
+
+Interfaces between net policy and kernel
+
+    int netpolicy_register(struct netpolicy_instance *instance);
+    void netpolicy_unregister(struct netpolicy_instance *instance);
+
+    This interface is used to register per task/socket net policy.
+    The socket/task can only be benefited when it register itself with specific
+    policy. After registeration, a record will be created and inserted into RCU
+    hash table, which include all the NET policy related information for the
+    socket/task, such as pointor, policy, object and so on.
+
+    int netpolicy_pick_queue(struct netpolicy_instance *instance, bool is_rx);
+
+    This interface is used to find the proper queue(object) for packet
+    receiving and transmitting. The proper queue is picked from object list
+    according to policy, reference, location and so on.
+
+
+    int netpolicy_set_rules(struct netpolicy_instance *instance);
+
+    This interface is used to add device specific rules. Once the rule is
+    applied, the packet from specific IP and port will be redirected to the
+    given queue. This interface is usually used on receive side.
+
+NET POLICY INTERFACE
+
+Device net policy setting
+
+    /proc/net/netpolicy/$DEV/policy
+
+    Concatenating(cat) the "policy" file can show the available device
+    policies, if there is no device policy applied. Otherwise, the device
+    policy name will be printed out. If it is MIX policy, the policy for each
+    queue will also be printed out.
+    User can set device net policy by writing policy name.
+
+Task policy setting
+
+    /proc/$PID/net_policy
+
+    Concatenating(cat) the "net_policy" file can show the applied per task
+    policy.
+    User can set per task net policy by writing policy name.
+
+    OR
+
+    prctl(PR_SET_NETPOLICY, POLICY_NAME, NULL, NULL, NULL)
+
+    "prctl" is an alternative way to set/get per task policy.
+
+Socket policy setting
+
+    setsockopt(sockfd,SOL_SOCKET,SO_NETPOLICY,&policy,sizeof(int))
+
+    The socket net policy can be set by option SO_NETPOLICY of setsockopt.
+
+AVAILABLE NET POLICIES
+
+    The available net policies are defined as below:
+    - CPU: intends to get higher throughput and lower CPU% (power saving).
+           This policy can be applied as either device net policy or
+           task/socket net policy.
+    - BULK: intends to get highest throughput. This policy can be applied as
+            either device net policy or task/socket net policy.
+    - LATENCY: intends to get lowest latency. This policy can be applied as
+               either device net policy or task/socket net policy.
+    - MIX: combination of other policies, which allows each queue to have a
+           different policy. This policy can only be set as device net policy.
+

@@ -40,6 +40,7 @@ enum netpolicy_traffic {
 #define NETPOLICY_INVALID_QUEUE	-1
 #define NETPOLICY_INVALID_LOC	NETPOLICY_INVALID_QUEUE
 #define POLICY_NAME_LEN_MAX	64
+#define NETPOLICY_MAX_RECORD_NUM	7000
 extern const char *policy_name[];
 
 struct netpolicy_dev_info {