Re: [PATCH v1] vdpa/vp_vdpa : add vdpa tool support in vp_vdpa

On Fri, Apr 22, 2022 at 1:57 PM Cindy Lu [off-list ref] wrote:

On Wed, Apr 20, 2022 at 2:27 PM Cindy Lu [off-list ref] wrote:

quoted

On Wed, Apr 20, 2022 at 11:21 AM Jason Wang [off-list ref] wrote:

quoted

在 2022/4/19 09:40, Cindy Lu 写道:

quoted

this patch is to add the support for vdpa tool in vp_vdpa
here is the example steps
modprobe vp_vdpa
modprobe vhost_vdpa

echo 0000:00:06.0>/sys/bus/pci/drivers/virtio-pci/unbind
echo 1af4 1041 > /sys/bus/pci/drivers/vp-vdpa/new_id

vdpa dev add name vdpa1 mgmtdev pci/0000:00:06.0

Signed-off-by: Cindy Lu <redacted>
---
  drivers/vdpa/virtio_pci/vp_vdpa.c | 86 ++++++++++++++++++++++++++++---
  1 file changed, 78 insertions(+), 8 deletions(-)

diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
index cce101e6a940..d8a1d2f8e9bb 100644
--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
+++ b/drivers/vdpa/virtio_pci/vp_vdpa.c

@@ -17,6 +17,8 @@
  #include <linux/virtio_ring.h>
  #include <linux/virtio_pci.h>
  #include <linux/virtio_pci_modern.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/vdpa.h>

  #define VP_VDPA_QUEUE_MAX 256
  #define VP_VDPA_DRIVER_NAME "vp_vdpa"

@@ -41,6 +43,18 @@ struct vp_vdpa {
      int vectors;
  };

+struct vp_vdpa_mgmtdev {
+     struct vdpa_mgmt_dev mgtdev;
+     struct vp_vdpa *vp_vdpa;
+     struct pci_dev *pdev;
+};
+
+#define VP_VDPA_NET_FEATURES                                                   \
+     ((1ULL << VIRTIO_F_ANY_LAYOUT) | (1ULL << VIRTIO_F_VERSION_1) |        \
+      (1ULL << VIRTIO_F_ACCESS_PLATFORM))
+
+#define VP_VDPA_NET_VQ_NUM 2

Let's not go backwards, e.g we've already support any kind of virtio
device with any kind of features. see the comment in vp_vdpa_probe().

got it, I will fix this

quoted

quoted

+
  static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa)
  {
      return container_of(vdpa, struct vp_vdpa, vdpa);

@@ -454,9 +468,14 @@ static void vp_vdpa_free_irq_vectors(void *data)
      pci_free_irq_vectors(data);
  }

-static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
+                        const struct vdpa_dev_set_config *add_config)
  {
+     struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev =
+             container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev);
+
      struct virtio_pci_modern_device *mdev;
+     struct pci_dev *pdev = vp_vdpa_mgtdev->pdev;
      struct device *dev = &pdev->dev;
      struct vp_vdpa *vp_vdpa;
      int ret, i;

@@ -465,8 +484,9 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
      if (ret)
              return ret;

-     vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa,
-                                 dev, &vp_vdpa_ops, NULL, false);
+     vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, dev, &vp_vdpa_ops,
+                                 name, false);

Nit: let's keep the line of vdpa_alloc_device() unchanged to reduce the
changeset.

will fix this

quoted

quoted

+
      if (IS_ERR(vp_vdpa)) {
              dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n");
              return PTR_ERR(vp_vdpa);

@@ -480,9 +500,10 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
              dev_err(&pdev->dev, "Failed to probe modern PCI device\n");
              goto err;
      }
+     vp_vdpa_mgtdev->vp_vdpa = vp_vdpa;

      pci_set_master(pdev);
-     pci_set_drvdata(pdev, vp_vdpa);
+     pci_set_drvdata(pdev, vp_vdpa_mgtdev);

      vp_vdpa->vdpa.dma_dev = &pdev->dev;
      vp_vdpa->queues = vp_modern_get_num_queues(mdev);

@@ -516,7 +537,8 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
      }
      vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR;

-     ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);
+     vp_vdpa->vdpa.mdev = &vp_vdpa_mgtdev->mgtdev;
+     ret = _vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues);
      if (ret) {
              dev_err(&pdev->dev, "Failed to register to vdpa bus\n");
              goto err;

@@ -529,12 +551,60 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
      return ret;
  }

-static void vp_vdpa_remove(struct pci_dev *pdev)
+static void vp_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev,
+                         struct vdpa_device *dev)
  {
-     struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev);
+     struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev =
+             container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev);
+
+     struct vp_vdpa *vp_vdpa = vp_vdpa_mgtdev->vp_vdpa;

      vp_modern_remove(&vp_vdpa->mdev);
-     vdpa_unregister_device(&vp_vdpa->vdpa);
+     _vdpa_unregister_device(&vp_vdpa->vdpa);
+     vp_vdpa_mgtdev->vp_vdpa = NULL;
+}
+
+static const struct vdpa_mgmtdev_ops vp_vdpa_mdev_ops = {
+     .dev_add = vp_vdpa_dev_add,
+     .dev_del = vp_vdpa_dev_del,
+};
+
+static struct virtio_device_id vp_vdpa_id_table[] = {
+     { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
+     { 0 },
+};

Let's not limit the vp_vdpa to net, it can support all other virtio devices.

sure, will fix this

quoted

quoted

+
+static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+     struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev;
+     struct vdpa_mgmt_dev *mgtdev;
+     struct device *dev = &pdev->dev;
+     int err;
+
+     vp_vdpa_mgtdev = kzalloc(sizeof(*vp_vdpa_mgtdev), GFP_KERNEL);
+     if (!vp_vdpa_mgtdev)
+             return -ENOMEM;
+     mgtdev = &vp_vdpa_mgtdev->mgtdev;
+
+     mgtdev->ops = &vp_vdpa_mdev_ops;
+     mgtdev->device = dev;
+     mgtdev->id_table = vp_vdpa_id_table;
+     vp_vdpa_mgtdev->pdev = pdev;
+
+     mgtdev->max_supported_vqs = VP_VDPA_NET_VQ_NUM;

Let's read this from the device instead of using the hard-coded value.

quoted

+     mgtdev->supported_features = VP_VDPA_NET_FEATURES;

Let's read this from the device.

sure will fix this

HI Jason, I have check the code, maybe we can't read these
informations from device here,
because the  virtio_pci_modern_device   was inited in vp_vdpa_dev_add(),
if we want to read these information in vp_vdpa_probe(). maybe we need
to change the struct in vp_vdpa. and separate the
struct virtio_pci_modern_device  from struct vp_vdpa. I'm not sure if
this is ok to do?

Any reason we can't do vdpa_alloc_device() in the vp_vdpa_probe() and
register it in dev_add?

If we can't I'd then decouple virtio_pci_moder_device out of vp_vdpa:

struct vp_vdpa {
    struct vdpa_device vdpa;
    struct virtio_pci_modern_device *mdev;
};

Thanks

struct vp_vdpa {
struct vdpa_device vdpa;
struct virtio_pci_modern_device mdev;
struct vp_vring *vring;
struct vdpa_callback config_cb;
char msix_name[VP_VDPA_NAME_SIZE];
int config_irq;
int queues;
int vectors;
};
Thanks
cindy

quoted

quoted

quoted

+
+     err = vdpa_mgmtdev_register(mgtdev);

Do we need to free the vp_vdpa_mgtdev here?

will add the this

quoted

Thanks

quoted

+
+     return err;
+}
+
+static void vp_vdpa_remove(struct pci_dev *pdev)
+{
+     struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = pci_get_drvdata(pdev);
+
+     vdpa_mgmtdev_unregister(&vp_vdpa_mgtdev->mgtdev);
+     kfree(vp_vdpa_mgtdev);
  }

  static struct pci_driver vp_vdpa_driver = {

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization