[PATCH v1 2/2] virtio, qtest: Add functionality to handle interrupt

@@ -1043,7 +1043,6 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
 	struct virtio_net_config *config;
 	struct virtio_net_config local_config;
 	struct rte_pci_device *pci_dev;
-	uint32_t dev_flags = RTE_ETH_DEV_DETACHABLE;
 	int ret;
 
 	RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf));

@@ -1067,8 +1066,9 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
 
 	pci_dev = eth_dev->pci_dev;
 
+	eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 	if (pci_dev) {
-		ret = vtpci_init(pci_dev, hw, &dev_flags);
+		ret = vtpci_init(pci_dev, hw, &eth_dev->data->dev_flags);
 		if (ret)
 			return ret;
 	}

@@ -1086,10 +1086,9 @@ eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
 
 	/* If host does not support status then disable LSC */
 	if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS))
-		dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
+		eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
 
 	rte_eth_copy_pci_info(eth_dev, pci_dev);
-	eth_dev->data->dev_flags = dev_flags;
 
 	rx_func_get(eth_dev);
 

@@ -35,7 +35,7 @@
 #define _VIRTIO_QTEST_H_
 
 #define QTEST_DRV_NAME		        "eth_virtio_qtest"
-#define QTEST_DEVICE_NUM                2
+#define QTEST_DEVICE_NUM                3
 
 #include <rte_pci.h>
 #include <linux/pci_regs.h>

@@ -43,6 +43,7 @@
 /* Device information */
 #define VIRTIO_NET_DEVICE_ID            0x1000
 #define VIRTIO_NET_VENDOR_ID            0x1af4
+#define VIRTIO_NET_IRQ_NUM              10
 #define IVSHMEM_DEVICE_ID               0x1110
 #define IVSHMEM_VENDOR_ID               0x1af4
 #define PIIX3_DEVICE_ID                 0x7000

@@ -36,6 +36,7 @@
 #include <sys/un.h>
 #include <pthread.h>
 #include <fcntl.h>
+#include <sys/eventfd.h>
 
 #include <rte_malloc.h>
 

@@ -43,6 +44,12 @@
 #include "../virtio_ethdev.h"
 #include "qtest_utils.h"
 
+/* PIIX3 configuration registers */
+#define PIIX3_REG_ADDR_PIRQA            0x60
+#define PIIX3_REG_ADDR_PIRQB            0x61
+#define PIIX3_REG_ADDR_PIRQC            0x62
+#define PIIX3_REG_ADDR_PIRQD            0x63
+
 /* ivshmem configuration */
 #define IVSHMEM_PROTOCOL_VERSION        0
 

@@ -74,6 +81,14 @@ struct qtest_session {
 	size_t evq_total_len;
 
 	union qtest_pipefds msgfds;
+
+	int irqno;
+	pthread_t intr_th;
+	int intr_th_started;
+	int eventfd;
+	rte_atomic16_t enable_intr;
+	rte_intr_callback_fn cb;
+	void *cb_arg;
 };
 
 static int

@@ -230,6 +245,29 @@ qtest_pci_inb(struct qtest_session *s, uint8_t bus, uint8_t device,
 	return (tmp >> ((offset & 0x3) * 8)) & 0xff;
 }
 
+static void
+qtest_pci_outb(struct qtest_session *s, uint8_t bus, uint8_t device,
+		uint8_t function, uint8_t offset, uint8_t value)
+{
+	uint32_t addr, tmp, pos;
+
+	addr = PCI_CONFIG_ADDR(bus, device, function, offset);
+	pos = (offset % 4) * 8;
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	qtest_raw_out(s, 0xcf8, addr, 'l');
+	tmp = qtest_raw_in(s, 0xcfc, 'l');
+	tmp = (tmp & ~(0xff << pos)) | (value << pos);
+
+	qtest_raw_out(s, 0xcf8, addr, 'l');
+	qtest_raw_out(s, 0xcfc, tmp, 'l');
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot unlock mutex\n");
+}
+
 static uint32_t
 qtest_pci_inl(struct qtest_session *s, uint8_t bus, uint8_t device,
 		uint8_t function, uint8_t offset)

@@ -466,15 +504,112 @@ qtest_get_bar_size(struct qtest_session *s, const char *name,
 	return 0;
 }
 
+int
+qtest_intr_enable(struct qtest_session *s)
+{
+	rte_atomic16_set(&s->enable_intr, 1);
+
+	return 0;
+}
+
+int
+qtest_intr_disable(struct qtest_session *s)
+{
+	rte_atomic16_set(&s->enable_intr, 0);
+
+	return 0;
+}
+
+void
+qtest_intr_callback_register(struct qtest_session *s,
+		rte_intr_callback_fn cb, void *cb_arg)
+{
+	s->cb = cb;
+	s->cb_arg = cb_arg;
+	rte_atomic16_set(&s->enable_intr, 1);
+}
+
+void
+qtest_intr_callback_unregister(struct qtest_session *s,
+		rte_intr_callback_fn cb __rte_unused,
+		void *cb_arg __rte_unused)
+{
+	rte_atomic16_set(&s->enable_intr, 0);
+	s->cb = NULL;
+	s->cb_arg = NULL;
+}
+
+static void *
+qtest_intr_handler(void *data) {
+	struct qtest_session *s = (struct qtest_session *)data;
+	eventfd_t value;
+	int ret;
+
+	for (;;) {
+		ret = eventfd_read(s->eventfd, &value);
+		if (ret < 0)
+			return NULL;
+		s->cb(NULL, s->cb_arg);
+	}
+	return NULL;
+}
+
+static int
+qtest_intr_initialize(struct qtest_session *s)
+{
+	char buf[64];
+	int ret;
+
+	snprintf(buf, sizeof(buf), "irq_intercept_in ioapic\n");
+
+	if (pthread_mutex_lock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	/* To enable interrupt, send "irq_intercept_in" message to QEMU */
+	ret = qtest_raw_send(s->qtest_socket, buf, strlen(buf));
+	if (ret < 0) {
+		pthread_mutex_unlock(&s->qtest_session_lock);
+		return -1;
+	}
+
+	/* just ignore QEMU response */
+	ret = qtest_raw_recv(s->msgfds.readfd, buf, sizeof(buf));
+	if (ret < 0) {
+		pthread_mutex_unlock(&s->qtest_session_lock);
+		return -1;
+	}
+
+	if (pthread_mutex_unlock(&s->qtest_session_lock) < 0)
+		rte_panic("Cannot lock mutex\n");
+
+	return 0;
+}
+
 static void
 qtest_event_send(struct qtest_session *s, char *buf)
 {
+	char interrupt_message[32];
 	int ret;
 
-	/* relay normal message to pipe */
-	ret = qtest_raw_send(s->msgfds.writefd, buf, strlen(buf));
-	if (ret < 0)
-		rte_panic("cannot relay normal message\n");
+	/* This message will come when interrupt occurs */
+	snprintf(interrupt_message, sizeof(interrupt_message),
+			"IRQ raise %d", s->irqno);
+
+	if (strncmp(buf, interrupt_message,
+				strlen(interrupt_message)) == 0) {
+		if (rte_atomic16_read(&s->enable_intr) == 0)
+			return;
+
+		/* relay interrupt to eventfd */
+		ret = eventfd_write(s->eventfd, 1);
+		if (ret < 0)
+			rte_panic("cannot relay interrupt\n");
+	} else {
+		/* relay normal message to pipe */
+		ret = qtest_raw_send(s->msgfds.writefd, buf, strlen(buf));
+		if (ret < 0)
+			rte_panic("cannot relay normal message\n");
+	}
 }
 
 static void

@@ -492,6 +627,7 @@ qtest_close_sockets(struct qtest_session *s)
 	qtest_close_one_socket(&s->qtest_socket);
 	qtest_close_one_socket(&s->msgfds.readfd);
 	qtest_close_one_socket(&s->msgfds.writefd);
+	qtest_close_one_socket(&s->eventfd);
 	qtest_close_one_socket(&s->ivshmem_socket);
 }
 

@@ -595,6 +731,57 @@ qtest_event_handler(void *data) {
 	return NULL;
 }
 
+/* This function should be fixed when multiple target devices are supported */
+int
+qtest_init_piix3_device(struct qtest_session *s, struct qtest_pci_device *dev)
+{
+	uint8_t bus, device, slot = 0;
+	struct qtest_pci_device *tmpdev;
+	uint8_t pcislot2regaddr[] = {	0xff,
+					0xff,
+					0xff,
+					PIIX3_REG_ADDR_PIRQC,
+					PIIX3_REG_ADDR_PIRQD,
+					PIIX3_REG_ADDR_PIRQA,
+					PIIX3_REG_ADDR_PIRQB};
+
+	bus = dev->bus_addr;
+	device = dev->device_addr;
+
+	PMD_DRV_LOG(INFO,
+		"Find %s on virtual PCI bus: %04x:%02x:00.0",
+		dev->name, bus, device);
+
+	/* Get slot id that is connected to target device(virtio-net device) */
+	TAILQ_FOREACH(tmpdev, &s->head, next) {
+		if (strcmp(tmpdev->name, "piix3") != 0 &&
+				strcmp(tmpdev->name, "ivshmem") != 0) {
+			slot = tmpdev->device_addr;
+			break;
+		}
+	}
+
+	if (slot == 0)
+		return -1;
+
+	/*
+	 * Set interrupt routing for target device.
+	 * Here is i440fx/piix3 connection settings
+	 * ---------------------------------------
+	 * PCI Slot3 -> PIRQC
+	 * PCI Slot4 -> PIRQD
+	 * PCI Slot5 -> PIRQA
+	 * PCI Slot6 -> PIRQB
+	 */
+	if (pcislot2regaddr[slot] != 0xff) {
+		qtest_pci_outb(s, bus, device, 0,
+				pcislot2regaddr[slot],
+				s->irqno);
+	}
+
+	return 0;
+}
+
 /*
  * Common initialization of PCI device.
  * To know detail, see pci specification.

@@ -1011,6 +1198,12 @@ qtest_vdev_uninit(struct qtest_session *s)
 		s->event_th_started = 0;
 	}
 
+	if (s->intr_th_started) {
+		pthread_cancel(s->intr_th);
+		pthread_join(s->intr_th, NULL);
+		s->intr_th_started = 0;
+	}
+
 	pthread_mutex_destroy(&s->qtest_session_lock);
 	qtest_remove_target_devices(s);
 	rte_free(s);

@@ -1018,7 +1211,7 @@ qtest_vdev_uninit(struct qtest_session *s)
 
 struct qtest_session *
 qtest_vdev_init(char *qtest_path, char *ivshmem_path,
-		struct qtest_pci_device *devices, int devnum)
+		int irqno, struct qtest_pci_device *devices, int devnum)
 {
 	struct qtest_session *s;
 	int ret;

@@ -1042,12 +1235,21 @@ qtest_vdev_init(char *qtest_path, char *ivshmem_path,
 		goto error;
 	}
 
+	s->eventfd = eventfd(0, 0);
+	if (s->eventfd < 0) {
+		PMD_DRV_LOG(ERR, "Failed to open eventfd");
+		goto error;
+	}
+
 	ret = qtest_register_target_devices(s, devices, devnum);
 	if (ret != 0) {
 		PMD_DRV_LOG(ERR, "Failed to initialize qtest session");
 		goto error;
 	}
 
+	rte_atomic16_set(&s->enable_intr, 0);
+	s->irqno = irqno;
+
 	s->ivshmem_socket = qtest_open_socket(ivshmem_path);
 	if (s->ivshmem_socket < 0) {
 		PMD_DRV_LOG(ERR, "Failed to open %s", ivshmem_path);

@@ -1067,6 +1269,19 @@ qtest_vdev_init(char *qtest_path, char *ivshmem_path,
 	}
 	s->event_th_started = 1;
 
+	ret = pthread_create(&s->intr_th, NULL, qtest_intr_handler, s);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to create interrupt handler");
+		goto error;
+	}
+	s->intr_th_started = 1;
+
+	ret = qtest_intr_initialize(s);
+	if (ret != 0) {
+		PMD_DRV_LOG(ERR, "Failed to initialize interrupt");
+		goto error;
+	}
+
 	ret = qtest_setup_shared_memory(s);
 	if (ret != 0) {
 		PMD_DRV_LOG(ERR, "Failed to setup shared memory");

@@ -134,6 +134,8 @@ struct qtest_pci_device {
  *   Path of qtest socket.
  * @param ivshmem_path
  *   Path of ivshmem socket.
+ * @param irqno
+ *   Interrupt number of the target device(virtio-net device).
  * @param devices
  *   Array of device information. It should contain piix3, ivshmem and target
  *   device(virtio-net device).

@@ -143,7 +145,7 @@ struct qtest_pci_device {
  *   The pointer to qtest session structure.
  */
 struct qtest_session *qtest_vdev_init(char *qtest_path, char *ivshmem_path,
-		struct qtest_pci_device *devices, int devnum);
+		int irqno, struct qtest_pci_device *devices, int devnum);
 
 /**
  * @internal

@@ -156,6 +158,56 @@ void qtest_vdev_uninit(struct qtest_session *s);
 
 /**
  * @internal
+ * Register interrupt callback.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param cb
+ *   The pointer to callback.
+ * @param cb_arg
+ *   The pointer to callback argument.
+ */
+void qtest_intr_callback_register(struct qtest_session *s,
+		rte_intr_callback_fn cb, void *cb_arg);
+
+/**
+ * @internal
+ * Unregister interrupt callback.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param cb
+ *   The pointer to callback.
+ * @param cb_arg
+ *   The pointer to callback argument.
+ */
+void qtest_intr_callback_unregister(struct qtest_session *s,
+		rte_intr_callback_fn cb, void *cb_arg);
+
+/**
+ * @internal
+ * Enable interrupt callback.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @return
+ *   0 on success, negative on error
+ */
+int qtest_intr_enable(struct qtest_session *s);
+
+/**
+ * @internal
+ * Disable interrupt callback.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @return
+ *   0 on success, negative on error
+ */
+int qtest_intr_disable(struct qtest_session *s);
+
+/**
+ * @internal
  * Read a port of QEMU guest.
  *
  * @param s

@@ -274,6 +326,20 @@ int qtest_get_bar_size(struct qtest_session *s, const char *name,
 
 /**
  * @internal
+ * Initialization function of piix3 device.
+ *
+ * @param s
+ *   The pointer to qtest session structure.
+ * @param dev
+ *   The pointer of pci device.
+ * @return
+ *   0 on success, negative on error
+ */
+int qtest_init_piix3_device(struct qtest_session *s,
+			struct qtest_pci_device *dev);
+
+/**
+ * @internal
  * Initialization function of general device.
  *
  * @param s

@@ -51,12 +51,14 @@
 #define ETH_VIRTIO_NET_ARG_IVSHMEM_PATH         "ivshmem"
 #define ETH_VIRTIO_NET_ARG_VIRTIO_NET_ADDR      "virtio-net-addr"
 #define ETH_VIRTIO_NET_ARG_IVSHMEM_ADDR         "ivshmem-addr"
+#define ETH_VIRTIO_NET_ARG_PIIX3_ADDR           "piix3-addr"
 
 static const char *valid_qtest_args[] = {
 	ETH_VIRTIO_NET_ARG_QTEST_PATH,
 	ETH_VIRTIO_NET_ARG_IVSHMEM_PATH,
 	ETH_VIRTIO_NET_ARG_VIRTIO_NET_ADDR,
 	ETH_VIRTIO_NET_ARG_IVSHMEM_ADDR,
+	ETH_VIRTIO_NET_ARG_PIIX3_ADDR,
 	NULL
 };
 

@@ -197,7 +199,7 @@ static int
 virtio_prepare_target_devices(struct qtest_pci_device *devices,
 			struct rte_kvargs *kvlist)
 {
-	struct qtest_pci_device *virtio_net, *ivshmem;
+	struct qtest_pci_device *virtio_net, *ivshmem, *piix3;
 	struct rte_pci_addr default_addr;
 	const struct rte_memseg *ms;
 	int ret;

@@ -224,6 +226,7 @@ virtio_prepare_target_devices(struct qtest_pci_device *devices,
 
 	virtio_net = &devices[0];
 	ivshmem = &devices[1];
+	piix3 = &devices[2];
 
 	virtio_net->name = "virtio-net";
 	virtio_net->device_id = VIRTIO_NET_DEVICE_ID;

@@ -251,6 +254,12 @@ virtio_prepare_target_devices(struct qtest_pci_device *devices,
 	/* In host mode, only one memory segment is vaild */
 	ivshmem->bar[2].region_start = (uint64_t)ms[0].addr;
 
+	/* piix3 is needed to route irqs from virtio-net to ioapic */
+	piix3->name = "piix3";
+	piix3->device_id = PIIX3_DEVICE_ID;
+	piix3->vendor_id = PIIX3_VENDOR_ID;
+	piix3->init = qtest_init_piix3_device;
+
 	/*
 	 * Set pci addresses specified by command line.
 	 * QTest utils will only check specified pci address.

@@ -274,6 +283,13 @@ virtio_prepare_target_devices(struct qtest_pci_device *devices,
 	if (ret < 0)
 		return -1;
 
+	default_addr.devid = 1;
+	ret = virtio_net_eth_pmd_parse_pci_addr(kvlist,
+			ETH_VIRTIO_NET_ARG_PIIX3_ADDR,
+			&piix3->specified_addr, &default_addr);
+	if (ret < 0)
+		return -1;
+
 	return 0;
 }
 /*

@@ -318,7 +334,7 @@ rte_qtest_virtio_pmd_init(const char *name, const char *params)
 
 	hw = eth_dev->data->dev_private;
 	hw->virtio_user_dev = qtest_vdev_init(qtest_path, ivshmem_path,
-			devices, QTEST_DEVICE_NUM);
+			VIRTIO_NET_IRQ_NUM, devices, QTEST_DEVICE_NUM);
 	if (hw->virtio_user_dev == NULL)
 		goto error;
 

@@ -331,8 +347,9 @@ rte_qtest_virtio_pmd_init(const char *name, const char *params)
 	if (ret < 0)
 		goto error;
 
+	TAILQ_INIT(&eth_dev->link_intr_cbs);
+
 	eth_dev->driver = NULL;
-	eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
 	eth_dev->data->kdrv = RTE_KDRV_NONE;
 	eth_dev->data->drv_name = QTEST_DRV_NAME;
 

@@ -58,6 +58,34 @@ check_vq_phys_addr_ok(struct virtqueue *vq)
 	return 1;
 }
 
+static int
+intr_cb_register(struct virtio_hw *hw,
+			rte_intr_callback_fn cb, void *cb_arg)
+{
+	qtest_intr_callback_register(hw->virtio_user_dev, cb, cb_arg);
+	return 0;
+}
+
+static int
+intr_cb_unregister(struct virtio_hw *hw,
+			rte_intr_callback_fn cb, void *cb_arg)
+{
+	qtest_intr_callback_register(hw->virtio_user_dev, cb, cb_arg);
+	return 0;
+}
+
+static int
+intr_enable(struct virtio_hw *hw)
+{
+	return qtest_intr_enable(hw->virtio_user_dev);
+}
+
+static int
+intr_disable(struct virtio_hw *hw)
+{
+	return qtest_intr_disable(hw->virtio_user_dev);
+}
+
 static inline uint8_t
 qtest_read8(struct virtio_hw *hw, uint8_t *addr)
 {

@@ -259,19 +287,23 @@ qtest_notify_queue(struct virtio_hw *hw __rte_unused, struct virtqueue *vq)
 }
 
 const struct virtio_pci_ops modern_qtest_ops = {
-	.read_dev_cfg	= qtest_read_dev_config,
-	.write_dev_cfg	= qtest_write_dev_config,
-	.reset		= qtest_reset,
-	.get_status	= qtest_get_status,
-	.set_status	= qtest_set_status,
-	.get_features	= qtest_get_features,
-	.set_features	= qtest_set_features,
-	.get_isr	= qtest_get_isr,
-	.set_config_irq	= qtest_set_config_irq,
-	.get_queue_num	= qtest_get_queue_num,
-	.setup_queue	= qtest_setup_queue,
-	.del_queue	= qtest_del_queue,
-	.notify_queue	= qtest_notify_queue,
+	.read_dev_cfg		= qtest_read_dev_config,
+	.write_dev_cfg		= qtest_write_dev_config,
+	.reset			= qtest_reset,
+	.get_status		= qtest_get_status,
+	.set_status		= qtest_set_status,
+	.get_features		= qtest_get_features,
+	.set_features		= qtest_set_features,
+	.get_isr		= qtest_get_isr,
+	.set_config_irq		= qtest_set_config_irq,
+	.get_queue_num		= qtest_get_queue_num,
+	.setup_queue		= qtest_setup_queue,
+	.del_queue		= qtest_del_queue,
+	.notify_queue		= qtest_notify_queue,
+	.intr_cb_register	= intr_cb_register,
+	.intr_cb_unregister	= intr_cb_unregister,
+	.intr_enable		= intr_enable,
+	.intr_disable		= intr_disable,
 };
 
 static void *

@@ -396,9 +428,9 @@ qtest_vtpci_init(struct virtio_hw *hw, uint32_t *dev_flags)
 	if (virtio_read_caps(hw) == 0) {
 		PMD_INIT_LOG(INFO, "modern virtio pci detected.");
 		hw->vtpci_ops = &modern_qtest_ops;
-		hw->modern    = 1;
-		/* So far, we don't support LSC interrupt */
-		*dev_flags = 0;
+		hw->use_msix = 0;
+		hw->modern = 1;
+		*dev_flags |= RTE_ETH_DEV_INTR_LSC;
 		return 0;
 	}