[PATCH 00/13] Add support for secondary queue set in nicvf thunderx driver

@@ -141,7 +141,7 @@ nicvf_base_init(struct nicvf *nic)
 		return NICVF_ERR_BASE_INIT;
 
 	if (nicvf_hw_version(nic) == PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF)
-		nic->hwcap |= NICVF_CAP_TUNNEL_PARSING;
+		nic->hwcap |= NICVF_CAP_TUNNEL_PARSING | NICVF_CAP_CQE_RX2;
 
 	if (nicvf_hw_version(nic) == PCI_SUB_DEVICE_ID_CN81XX_NICVF)
 		nic->hwcap |= NICVF_CAP_TUNNEL_PARSING | NICVF_CAP_CQE_RX2;

@@ -497,9 +497,9 @@ nicvf_qsize_rbdr_roundup(uint32_t val)
 }
 
 int
-nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx,
-			  rbdr_pool_get_handler handler,
-			  void *opaque, uint32_t max_buffs)
+nicvf_qset_rbdr_precharge(void *dev, struct nicvf *nic,
+			  uint16_t ridx, rbdr_pool_get_handler handler,
+			  uint32_t max_buffs)
 {
 	struct rbdr_entry_t *desc, *desc0;
 	struct nicvf_rbdr *rbdr = nic->rbdr;

@@ -514,7 +514,7 @@ nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx,
 		if (count >= max_buffs)
 			break;
 		desc0 = desc + count;
-		phy = handler(opaque);
+		phy = handler(dev, nic);
 		if (phy) {
 			desc0->full_addr = phy;
 			count++;

@@ -85,7 +85,7 @@ enum nicvf_err_e {
 	NICVF_ERR_RSS_GET_SZ,    /* -8171 */
 };
 
-typedef nicvf_phys_addr_t (*rbdr_pool_get_handler)(void *opaque);
+typedef nicvf_phys_addr_t (*rbdr_pool_get_handler)(void *dev, void *opaque);
 
 struct nicvf_hw_rx_qstats {
 	uint64_t q_rx_bytes;

@@ -194,8 +194,8 @@ int nicvf_qset_reclaim(struct nicvf *nic);
 
 int nicvf_qset_rbdr_config(struct nicvf *nic, uint16_t qidx);
 int nicvf_qset_rbdr_reclaim(struct nicvf *nic, uint16_t qidx);
-int nicvf_qset_rbdr_precharge(struct nicvf *nic, uint16_t ridx,
-			      rbdr_pool_get_handler handler, void *opaque,
+int nicvf_qset_rbdr_precharge(void *dev, struct nicvf *nic,
+			      uint16_t ridx, rbdr_pool_get_handler handler,
 			      uint32_t max_buffs);
 int nicvf_qset_rbdr_active(struct nicvf *nic, uint16_t qidx);
 

@@ -691,7 +691,7 @@ nicvf_configure_cpi(struct rte_eth_dev *dev)
 	int ret;
 
 	/* Count started rx queues */
-	for (qidx = qcnt = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++)
+	for (qidx = qcnt = 0; qidx < dev->data->nb_rx_queues; qidx++)
 		if (dev->data->rx_queue_state[qidx] ==
 		    RTE_ETH_QUEUE_STATE_STARTED)
 			qcnt++;

@@ -1023,12 +1023,9 @@ nicvf_stop_rx_queue(struct rte_eth_dev *dev, uint16_t qidx)
 static void
 nicvf_dev_rx_queue_release(void *rx_queue)
 {
-	struct nicvf_rxq *rxq = rx_queue;
-
 	PMD_INIT_FUNC_TRACE();
 
-	if (rxq)
-		rte_free(rxq);
+	rte_free(rx_queue);
 }
 
 static int

@@ -1087,9 +1084,9 @@ nicvf_dev_rx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 		PMD_DRV_LOG(WARNING, "socket_id expected %d, configured %d",
 		socket_id, nic->node);
 
-	/* Mempool memory should be contiguous */
-	if (mp->nb_mem_chunks != 1) {
-		PMD_INIT_LOG(ERR, "Non contiguous mempool, check huge page sz");
+	/* Mempool memory must be contiguous */
+	if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG) {
+		PMD_INIT_LOG(ERR, "Mempool memory must be contiguous");
 		return -EINVAL;
 	}
 

@@ -1212,15 +1209,16 @@ nicvf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 }
 
 static nicvf_phys_addr_t
-rbdr_rte_mempool_get(void *opaque)
+rbdr_rte_mempool_get(void *dev, void *opaque)
 {
 	uint16_t qidx;
 	uintptr_t mbuf;
 	struct nicvf_rxq *rxq;
-	struct nicvf *nic = nicvf_pmd_priv((struct rte_eth_dev *)opaque);
+	struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)dev;
+	struct nicvf *nic __rte_unused = (struct nicvf *)opaque;
 
-	for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
-		rxq = nic->eth_dev->data->rx_queues[qidx];
+	for (qidx = 0; qidx < eth_dev->data->nb_rx_queues; qidx++) {
+		rxq = eth_dev->data->rx_queues[qidx];
 		/* Maintain equal buffer count across all pools */
 		if (rxq->precharge_cnt >= rxq->qlen_mask)
 			continue;

@@ -1354,8 +1352,8 @@ nicvf_dev_start(struct rte_eth_dev *dev)
 	}
 
 	/* Fill rte_mempool buffers in RBDR pool and precharge it */
-	ret = nicvf_qset_rbdr_precharge(nic, 0, rbdr_rte_mempool_get,
-					dev, total_rxq_desc);
+	ret = nicvf_qset_rbdr_precharge(dev, nic, 0, rbdr_rte_mempool_get,
+					total_rxq_desc);
 	if (ret) {
 		PMD_INIT_LOG(ERR, "Failed to fill rbdr %d", ret);
 		goto qset_rbdr_reclaim;

@@ -1721,12 +1719,6 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 		goto malloc_fail;
 	}
 
-	ret = nicvf_mbox_get_rss_size(nic);
-	if (ret) {
-		PMD_INIT_LOG(ERR, "Failed to get rss table size");
-		goto malloc_fail;
-	}
-
 	PMD_INIT_LOG(INFO, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
 		eth_dev->data->port_id, nic->vendor_id, nic->device_id,
 		nic->mac_addr[0], nic->mac_addr[1], nic->mac_addr[2],

@@ -70,19 +70,20 @@ fill_sq_desc_header(union sq_entry_t *entry, struct rte_mbuf *pkt)
 	ol_flags = pkt->ol_flags & NICVF_TX_OFFLOAD_MASK;
 	if (unlikely(ol_flags)) {
 		/* L4 cksum */
-		if (ol_flags & PKT_TX_TCP_CKSUM)
+		uint64_t l4_flags = ol_flags & PKT_TX_L4_MASK;
+		if (l4_flags == PKT_TX_TCP_CKSUM)
 			sqe.hdr.csum_l4 = SEND_L4_CSUM_TCP;
-		else if (ol_flags & PKT_TX_UDP_CKSUM)
+		else if (l4_flags == PKT_TX_UDP_CKSUM)
 			sqe.hdr.csum_l4 = SEND_L4_CSUM_UDP;
 		else
 			sqe.hdr.csum_l4 = SEND_L4_CSUM_DISABLE;
+
+		sqe.hdr.l3_offset = pkt->l2_len;
 		sqe.hdr.l4_offset = pkt->l3_len + pkt->l2_len;
 
 		/* L3 cksum */
-		if (ol_flags & PKT_TX_IP_CKSUM) {
+		if (ol_flags & PKT_TX_IP_CKSUM)
 			sqe.hdr.csum_l3 = 1;
-			sqe.hdr.l3_offset = pkt->l2_len;
-		}
 	}
 
 	entry->buff[0] = sqe.buff[0];

@@ -57,6 +57,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_rxtx.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_hw.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_mbox.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_ethdev.c
+SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_bsvf.c
 
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
 CFLAGS_nicvf_rxtx.o += -fno-prefetch-loop-arrays

@@ -0,0 +1,72 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stddef.h>
+#include <err.h>
+
+#include "nicvf_bsvf.h"
+#include "nicvf_plat.h"
+
+static SIMPLEQ_HEAD(, svf_entry) head = SIMPLEQ_HEAD_INITIALIZER(head);
+
+void
+nicvf_bsvf_push(struct svf_entry *entry)
+{
+	assert(entry != NULL);
+	assert(entry->vf != NULL);
+
+	SIMPLEQ_INSERT_TAIL(&head, entry, next);
+}
+
+struct svf_entry *
+nicvf_bsvf_pop(void)
+{
+	struct svf_entry *entry;
+
+	assert(!SIMPLEQ_EMPTY(&head));
+
+	entry = SIMPLEQ_FIRST(&head);
+
+	assert(entry != NULL);
+	assert(entry->vf != NULL);
+
+	SIMPLEQ_REMOVE_HEAD(&head, next);
+
+	return entry;
+}
+
+int
+nicvf_bsvf_empty(void)
+{
+	return SIMPLEQ_EMPTY(&head);
+}

@@ -0,0 +1,76 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __THUNDERX_NICVF_BSVF_H__
+#define __THUNDERX_NICVF_BSVF_H__
+
+#include <sys/queue.h>
+
+struct nicvf;
+
+/**
+ * The base queue structure to hold secondary qsets.
+ */
+struct svf_entry {
+	SIMPLEQ_ENTRY(svf_entry) next; /**< Next element's pointer */
+	struct nicvf *vf;              /**< Holder of a secondary qset */
+};
+
+/**
+ * Enqueue new entry to secondary qsets.
+ *
+ * @param entry
+ *   Entry to be enqueued.
+ */
+void
+nicvf_bsvf_push(struct svf_entry *entry);
+
+/**
+ * Dequeue an entry from secondary qsets.
+ *
+ * @return
+ *   Dequeued entry.
+ */
+struct svf_entry *
+nicvf_bsvf_pop(void);
+
+/**
+ * Check if the queue of secondary qsets is empty.
+ *
+ * @return
+ *   0 on non-empty
+ *   otherwise empty
+ */
+int
+nicvf_bsvf_empty(void);
+
+#endif /* __THUNDERX_NICVF_BSVF_H__  */

@@ -207,6 +207,7 @@
 #define NICVF_CQE_RX2_RBPTR_WORD        (7)
 
 #define NICVF_STATIC_ASSERT(s) _Static_assert(s, #s)
+#define assert_if_secondary(nic) assert((nic)->sqs_mode == 0)
 
 typedef uint64_t nicvf_phys_addr_t;
 

@@ -62,6 +62,9 @@ static const char *mbox_message[NIC_MBOX_MSG_MAX] =  {
 	[NIC_MBOX_MSG_RESET_STAT_COUNTER] = "NIC_MBOX_MSG_RESET_STAT_COUNTER",
 	[NIC_MBOX_MSG_CFG_DONE]           = "NIC_MBOX_MSG_CFG_DONE",
 	[NIC_MBOX_MSG_SHUTDOWN]           = "NIC_MBOX_MSG_SHUTDOWN",
+	[NIC_MBOX_MSG_RES_BIT]            = "NIC_MBOX_MSG_RES_BIT",
+	[NIC_MBOX_MSG_RSS_SIZE_RES_BIT]   = "NIC_MBOX_MSG_RSS_SIZE",
+	[NIC_MBOX_MSG_ALLOC_SQS_RES_BIT]  = "NIC_MBOX_MSG_ALLOC_SQS",
 };
 
 static inline const char * __attribute__((unused))

@@ -173,7 +176,7 @@ nicvf_handle_mbx_intr(struct nicvf *nic)
 	case NIC_MBOX_MSG_NACK:
 		nic->pf_nacked = true;
 		break;
-	case NIC_MBOX_MSG_RSS_SIZE:
+	case NIC_MBOX_MSG_RSS_SIZE_RES_BIT:
 		nic->rss_info.rss_size = mbx.rss_size.ind_tbl_size;
 		nic->pf_acked = true;
 		break;

@@ -183,6 +186,13 @@ nicvf_handle_mbx_intr(struct nicvf *nic)
 		nic->speed = mbx.link_status.speed;
 		nic->pf_acked = true;
 		break;
+	case NIC_MBOX_MSG_ALLOC_SQS_RES_BIT:
+		assert_if_secondary(nic);
+		assert(mbx.sqs_alloc.qs_count == nic->sqs_count);
+		for (i = 0; i < mbx.sqs_alloc.qs_count; i++)
+			assert(mbx.sqs_alloc.svf[i] == nic->snicvf[i]->vf_id);
+		nic->pf_acked = true;
+		break;
 	default:
 		nicvf_log_error("Invalid message from PF, msg_id=0x%hhx %s",
 				mbx.msg.msg, nicvf_mbox_msg_str(mbx.msg.msg));

@@ -314,11 +324,33 @@ nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg)
 	/* Send a mailbox msg to PF to config Qset */
 	mbx.msg.msg = NIC_MBOX_MSG_QS_CFG;
 	mbx.qs.num = nic->vf_id;
+	mbx.qs.sqs_count = nic->sqs_count;
 	mbx.qs.cfg = qs_cfg->value;
 	return nicvf_mbox_send_msg_to_pf(nic, &mbx);
 }
 
 int
+nicvf_mbox_request_sqs(struct nicvf *nic)
+{
+	struct nic_mbx mbx = { .msg = { 0 } };
+	size_t i;
+
+	assert_if_secondary(nic);
+	assert(nic->sqs_count > 0);
+	assert(nic->sqs_count <= MAX_SQS_PER_VF);
+
+	mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS;
+	mbx.sqs_alloc.spec = 1;
+	mbx.sqs_alloc.qs_count = nic->sqs_count;
+
+	/* Set no of Rx/Tx queues in each of the SQsets */
+	for (i = 0; i < nic->sqs_count; i++)
+		mbx.sqs_alloc.svf[i] = nic->snicvf[i]->vf_id;
+
+	return nicvf_mbox_send_msg_to_pf(nic, &mbx);
+}
+
+int
 nicvf_mbox_rq_drop_config(struct nicvf *nic, uint16_t qidx, bool enable)
 {
 	struct nic_mbx mbx = { .msg = { 0 } };

@@ -36,6 +36,7 @@
 #include <stdint.h>
 
 #include "nicvf_plat.h"
+#include "../nicvf_struct.h"
 
 /* PF <--> VF Mailbox communication
  * Two 64bit registers are shared between PF and VF for each VF

@@ -67,10 +68,16 @@
 #define	NIC_MBOX_MSG_ALLOC_SQS		0x12	/* Allocate secondary Qset */
 #define	NIC_MBOX_MSG_LOOPBACK		0x16	/* Set interface in loopback */
 #define	NIC_MBOX_MSG_RESET_STAT_COUNTER 0x17	/* Reset statistics counters */
-#define	NIC_MBOX_MSG_CFG_DONE		0xF0	/* VF configuration done */
-#define	NIC_MBOX_MSG_SHUTDOWN		0xF1	/* VF is being shutdown */
+#define	NIC_MBOX_MSG_CFG_DONE		0x7E	/* VF configuration done */
+#define	NIC_MBOX_MSG_SHUTDOWN		0x7F	/* VF is being shutdown */
+#define	NIC_MBOX_MSG_RES_BIT		0x80	/* Reset bit from PF */
 #define	NIC_MBOX_MSG_MAX		0x100	/* Maximum number of messages */
 
+#define NIC_MBOX_MSG_RSS_SIZE_RES_BIT \
+	(NIC_MBOX_MSG_RSS_SIZE | NIC_MBOX_MSG_RES_BIT)
+#define NIC_MBOX_MSG_ALLOC_SQS_RES_BIT \
+	(NIC_MBOX_MSG_ALLOC_SQS | NIC_MBOX_MSG_RES_BIT)
+
 /* Get vNIC VF configuration */
 struct nic_cfg_msg {
 	uint8_t    msg;

@@ -155,6 +162,14 @@ struct bgx_link_status {
 	uint32_t   speed;
 };
 
+/* Allocate additional SQS to VF */
+struct sqs_alloc {
+	uint8_t    msg;
+	uint8_t    spec;
+	uint8_t    qs_count;
+	uint8_t    svf[MAX_SQS_PER_VF];
+};
+
 /* Set interface in loopback mode */
 struct set_loopback {
 	uint8_t    msg;

@@ -201,6 +216,7 @@ union {
 	struct rss_sz_msg	rss_size;
 	struct rss_cfg_msg	rss_cfg;
 	struct bgx_link_status  link_status;
+	struct sqs_alloc	sqs_alloc;
 	struct set_loopback	lbk;
 	struct reset_stat_cfg	reset_stat;
 };

@@ -211,6 +227,7 @@ NICVF_STATIC_ASSERT(sizeof(struct nic_mbx) <= 16);
 int nicvf_handle_mbx_intr(struct nicvf *nic);
 int nicvf_mbox_check_pf_ready(struct nicvf *nic);
 int nicvf_mbox_qset_config(struct nicvf *nic, struct pf_qs_cfg *qs_cfg);
+int nicvf_mbox_request_sqs(struct nicvf *nic);
 int nicvf_mbox_rq_config(struct nicvf *nic, uint16_t qidx,
 			 struct pf_rq_cfg *pf_rq_cfg);
 int nicvf_mbox_sq_config(struct nicvf *nic, uint16_t qidx);

@@ -119,6 +119,11 @@ struct nicvf {
 	uint16_t mtu;
 	bool vlan_filter_en;
 	uint8_t mac_addr[ETHER_ADDR_LEN];
+	/* secondary queue set support */
+	uint8_t sqs_id;
+	uint8_t sqs_count;
+#define MAX_SQS_PER_VF 11
+	struct nicvf *snicvf[MAX_SQS_PER_VF];
 } __rte_cache_aligned;
 
 #endif /* _THUNDERX_NICVF_STRUCT_H */

@@ -58,6 +58,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_hw.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_mbox.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_ethdev.c
 SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_bsvf.c
+SRCS-$(CONFIG_RTE_LIBRTE_THUNDERX_NICVF_PMD) += nicvf_svf.c
 
 ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
 CFLAGS_nicvf_rxtx.o += -fno-prefetch-loop-arrays

@@ -0,0 +1,78 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <stddef.h>
+
+#include <rte_debug.h>
+#include <rte_malloc.h>
+
+#include "base/nicvf_bsvf.h"
+
+#include "nicvf_svf.h"
+
+void
+nicvf_svf_push(struct nicvf *vf)
+{
+	struct svf_entry *entry = NULL;
+
+	assert(vf != NULL);
+
+	entry = rte_zmalloc("nicvf", sizeof(*entry), RTE_CACHE_LINE_SIZE);
+	if (entry == NULL)
+		rte_panic("Cannoc allocate memory for svf_entry\n");
+
+	entry->vf = vf;
+
+	nicvf_bsvf_push(entry);
+}
+
+struct nicvf *
+nicvf_svf_pop(void)
+{
+	struct nicvf *vf;
+	struct svf_entry *entry;
+
+	entry = nicvf_bsvf_pop();
+
+	vf = entry->vf;
+
+	rte_free(entry);
+
+	return vf;
+}
+
+int
+nicvf_svf_empty(void)
+{
+	return nicvf_bsvf_empty();
+}

@@ -0,0 +1,66 @@
+/*
+ *   BSD LICENSE
+ *
+ *   Copyright (C) Cavium networks Ltd. 2016.
+ *
+ *   Redistribution and use in source and binary forms, with or without
+ *   modification, are permitted provided that the following conditions
+ *   are met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in
+ *       the documentation and/or other materials provided with the
+ *       distribution.
+ *     * Neither the name of Cavium networks nor the names of its
+ *       contributors may be used to endorse or promote products derived
+ *       from this software without specific prior written permission.
+ *
+ *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __THUNDERX_NICVF_SVF_H__
+#define __THUNDERX_NICVF_SVF_H__
+
+struct nicvf;
+
+/**
+ * Enqueue new VF to secondary qsets.
+ *
+ * @param entry
+ *   Entry to be enqueued.
+ */
+void
+nicvf_svf_push(struct nicvf *vf);
+
+/**
+ * Dequeue a VF from secondary qsets.
+ *
+ * @return
+ *   Dequeued entry.
+ */
+struct nicvf *
+nicvf_svf_pop(void);
+
+/**
+ * Check if the queue of secondary qsets is empty.
+ *
+ * @return
+ *   0 on non-empty
+ *   otherwise empty
+ */
+int
+nicvf_svf_empty(void);
+
+#endif /* __THUNDERX_NICVF_SVF_H__  */

@@ -101,31 +101,40 @@ nicvf_set_eth_link_status(struct nicvf *nic, struct rte_eth_link *link)
 static void
 nicvf_interrupt(void *arg)
 {
-	struct nicvf *nic = arg;
+	struct rte_eth_dev *dev = arg;
+	struct nicvf *nic = nicvf_pmd_priv(dev);
 
 	if (nicvf_reg_poll_interrupts(nic) == NIC_MBOX_MSG_BGX_LINK_CHANGE) {
-		if (nic->eth_dev->data->dev_conf.intr_conf.lsc)
-			nicvf_set_eth_link_status(nic,
-					&nic->eth_dev->data->dev_link);
-		_rte_eth_dev_callback_process(nic->eth_dev,
-				RTE_ETH_EVENT_INTR_LSC);
+		if (dev->data->dev_conf.intr_conf.lsc)
+			nicvf_set_eth_link_status(nic, &dev->data->dev_link);
+		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
 	}
 
 	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
-				nicvf_interrupt, nic);
+				nicvf_interrupt, dev);
+}
+
+static void __rte_unused
+nicvf_vf_interrupt(void *arg)
+{
+	struct nicvf *nic = arg;
+
+	nicvf_reg_poll_interrupts(nic);
+
+	rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
+				nicvf_vf_interrupt, nic);
 }
 
 static int
-nicvf_periodic_alarm_start(struct nicvf *nic)
+nicvf_periodic_alarm_start(void (fn)(void *), void *arg)
 {
-	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000,
-					nicvf_interrupt, nic);
+	return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS * 1000, fn, arg);
 }
 
 static int
-nicvf_periodic_alarm_stop(struct nicvf *nic)
+nicvf_periodic_alarm_stop(void (fn)(void *), void *arg)
 {
-	return rte_eal_alarm_cancel(nicvf_interrupt, nic);
+	return rte_eal_alarm_cancel(fn, arg);
 }
 
 /*

@@ -1512,12 +1521,10 @@ nicvf_dev_stop(struct rte_eth_dev *dev)
 static void
 nicvf_dev_close(struct rte_eth_dev *dev)
 {
-	struct nicvf *nic = nicvf_pmd_priv(dev);
-
 	PMD_INIT_FUNC_TRACE();
 
 	nicvf_dev_stop(dev);
-	nicvf_periodic_alarm_stop(nic);
+	nicvf_periodic_alarm_stop(nicvf_interrupt, dev);
 }
 
 static int

@@ -1668,7 +1675,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 
 	nicvf_disable_all_interrupts(nic);
 
-	ret = nicvf_periodic_alarm_start(nic);
+	ret = nicvf_periodic_alarm_start(nicvf_interrupt, eth_dev);
 	if (ret) {
 		PMD_INIT_LOG(ERR, "Failed to start period alarm");
 		goto fail;

@@ -1729,7 +1736,7 @@ nicvf_eth_dev_init(struct rte_eth_dev *eth_dev)
 malloc_fail:
 	rte_free(eth_dev->data->mac_addrs);
 alarm_fail:
-	nicvf_periodic_alarm_stop(nic);
+	nicvf_periodic_alarm_stop(nicvf_interrupt, eth_dev);
 fail:
 	return ret;
 }

@@ -497,14 +497,14 @@ nicvf_dev_rss_hash_update(struct rte_eth_dev *dev,
 }
 
 static int
-nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx,
-		    uint32_t desc_cnt)
+nicvf_qset_cq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
+		    struct nicvf_rxq *rxq, uint16_t qidx, uint32_t desc_cnt)
 {
 	const struct rte_memzone *rz;
 	uint32_t ring_size = CMP_QUEUE_SZ_MAX * sizeof(union cq_entry_t);
 
-	rz = rte_eth_dma_zone_reserve(nic->eth_dev, "cq_ring", qidx, ring_size,
-					NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
+	rz = rte_eth_dma_zone_reserve(dev, "cq_ring", qidx, ring_size,
+				      NICVF_CQ_BASE_ALIGN_BYTES, nic->node);
 	if (rz == NULL) {
 		PMD_INIT_LOG(ERR, "Failed to allocate mem for cq hw ring");
 		return -ENOMEM;

@@ -520,13 +520,13 @@ nicvf_qset_cq_alloc(struct nicvf *nic, struct nicvf_rxq *rxq, uint16_t qidx,
 }
 
 static int
-nicvf_qset_sq_alloc(struct nicvf *nic,  struct nicvf_txq *sq, uint16_t qidx,
-		    uint32_t desc_cnt)
+nicvf_qset_sq_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
+		    struct nicvf_txq *sq, uint16_t qidx, uint32_t desc_cnt)
 {
 	const struct rte_memzone *rz;
 	uint32_t ring_size = SND_QUEUE_SZ_MAX * sizeof(union sq_entry_t);
 
-	rz = rte_eth_dma_zone_reserve(nic->eth_dev, "sq", qidx, ring_size,
+	rz = rte_eth_dma_zone_reserve(dev, "sq", qidx, ring_size,
 				NICVF_SQ_BASE_ALIGN_BYTES, nic->node);
 	if (rz == NULL) {
 		PMD_INIT_LOG(ERR, "Failed allocate mem for sq hw ring");

@@ -543,7 +543,8 @@ nicvf_qset_sq_alloc(struct nicvf *nic,  struct nicvf_txq *sq, uint16_t qidx,
 }
 
 static int
-nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz)
+nicvf_qset_rbdr_alloc(struct rte_eth_dev *dev, struct nicvf *nic,
+		      uint32_t desc_cnt, uint32_t buffsz)
 {
 	struct nicvf_rbdr *rbdr;
 	const struct rte_memzone *rz;

@@ -558,7 +559,7 @@ nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz)
 	}
 
 	ring_size = sizeof(struct rbdr_entry_t) * RBDR_QUEUE_SZ_MAX;
-	rz = rte_eth_dma_zone_reserve(nic->eth_dev, "rbdr", 0, ring_size,
+	rz = rte_eth_dma_zone_reserve(dev, "rbdr", 0, ring_size,
 				   NICVF_RBDR_BASE_ALIGN_BYTES, nic->node);
 	if (rz == NULL) {
 		PMD_INIT_LOG(ERR, "Failed to allocate mem for rbdr desc ring");

@@ -583,14 +584,15 @@ nicvf_qset_rbdr_alloc(struct nicvf *nic, uint32_t desc_cnt, uint32_t buffsz)
 }
 
 static void
-nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy)
+nicvf_rbdr_release_mbuf(struct rte_eth_dev *dev, struct nicvf *nic __rte_unused,
+			nicvf_phys_addr_t phy)
 {
 	uint16_t qidx;
 	void *obj;
 	struct nicvf_rxq *rxq;
 
-	for (qidx = 0; qidx < nic->eth_dev->data->nb_rx_queues; qidx++) {
-		rxq = nic->eth_dev->data->rx_queues[qidx];
+	for (qidx = 0; qidx < dev->data->nb_rx_queues; qidx++) {
+		rxq = dev->data->rx_queues[qidx];
 		if (rxq->precharge_cnt) {
 			obj = (void *)nicvf_mbuff_phy2virt(phy,
 							   rxq->mbuf_phys_off);

@@ -602,7 +604,7 @@ nicvf_rbdr_release_mbuf(struct nicvf *nic, nicvf_phys_addr_t phy)
 }
 
 static inline void
-nicvf_rbdr_release_mbufs(struct nicvf *nic)
+nicvf_rbdr_release_mbufs(struct rte_eth_dev *dev, struct nicvf *nic)
 {
 	uint32_t qlen_mask, head;
 	struct rbdr_entry_t *entry;

@@ -612,7 +614,7 @@ nicvf_rbdr_release_mbufs(struct nicvf *nic)
 	head = rbdr->head;
 	while (head != rbdr->tail) {
 		entry = rbdr->desc + head;
-		nicvf_rbdr_release_mbuf(nic, entry->full_addr);
+		nicvf_rbdr_release_mbuf(dev, nic, entry->full_addr);
 		head++;
 		head = head & qlen_mask;
 	}

@@ -724,14 +726,13 @@ nicvf_configure_rss(struct rte_eth_dev *dev)
 			dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
 	PMD_DRV_LOG(INFO, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64,
 		    dev->data->dev_conf.rxmode.mq_mode,
-		    nic->eth_dev->data->nb_rx_queues,
-		    nic->eth_dev->data->dev_conf.lpbk_mode, rsshf);
+		    dev->data->nb_rx_queues,
+		    dev->data->dev_conf.lpbk_mode, rsshf);
 
 	if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_NONE)
 		ret = nicvf_rss_term(nic);
 	else if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS)
-		ret = nicvf_rss_config(nic,
-				       nic->eth_dev->data->nb_rx_queues, rsshf);
+		ret = nicvf_rss_config(nic, dev->data->nb_rx_queues, rsshf);
 	if (ret)
 		PMD_INIT_LOG(ERR, "Failed to configure RSS %d", ret);
 

@@ -915,7 +916,7 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 		return -ENOMEM;
 	}
 
-	if (nicvf_qset_sq_alloc(nic, txq, qidx, nb_desc)) {
+	if (nicvf_qset_sq_alloc(dev, nic, txq, qidx, nb_desc)) {
 		PMD_INIT_LOG(ERR, "Failed to allocate mem for sq %d", qidx);
 		nicvf_dev_tx_queue_release(txq);
 		return -ENOMEM;

@@ -932,12 +933,11 @@ nicvf_dev_tx_queue_setup(struct rte_eth_dev *dev, uint16_t qidx,
 }
 
 static inline void
-nicvf_rx_queue_release_mbufs(struct nicvf_rxq *rxq)
+nicvf_rx_queue_release_mbufs(struct rte_eth_dev *dev, struct nicvf_rxq *rxq)
 {
 	uint32_t rxq_cnt;
 	uint32_t nb_pkts, released_pkts = 0;
 	uint32_t refill_cnt = 0;
-	struct rte_eth_dev *dev = rxq->nic->eth_dev;
 	struct rte_mbuf *rx_pkts[NICVF_MAX_RX_FREE_THRESH];
 
 	if (dev->rx_pkt_burst == NULL)