vDPA/ifcvf: implement doorbell mapping feature

This series implements doorbell mapping feature for ifcvf.

Please help review

Thanks!
Changes from V2:
assign notify_off_multiplier to notification.size first than
use PAGE_SIZE directly(Jason)

Chagnes from V1:
calculate the doorbell address per vq than per device(Jason)
let upper layer driver decide how to use the non page_size
aligned doorbell(Jason)

Zhu Lingshan (2):
  vDPA/ifcvf: record virtio notify base
  vDPA/ifcvf: implement doorbell mapping for ifcvf

 drivers/vdpa/ifcvf/ifcvf_base.c |  4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h |  2 ++
 drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
 3 files changed, 27 insertions(+)

-- 
2.27.0

This commit records virtio notify base physical addr and
calculate doorbell physical address for vqs.

Signed-off-by: Zhu Lingshan <redacted>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 4 ++++
 drivers/vdpa/ifcvf/ifcvf_base.h | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index 1a661ab45af5..6e197fe0fcf9 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c

@@ -133,6 +133,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 					      &hw->notify_off_multiplier);
 			hw->notify_bar = cap.bar;
 			hw->notify_base = get_cap_addr(hw, &cap);
+			hw->notify_base_pa = pci_resource_start(pdev, cap.bar) +
+					le32_to_cpu(cap.offset);
 			IFCVF_DBG(pdev, "hw->notify_base = %p\n",
 				  hw->notify_base);
 			break;

@@ -161,6 +163,8 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 		notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
 		hw->vring[i].notify_addr = hw->notify_base +
 			notify_off * hw->notify_off_multiplier;
+		hw->vring[i].notify_pa = hw->notify_base_pa +
+			notify_off * hw->notify_off_multiplier;
 	}
 
 	hw->lm_cfg = hw->base[IFCVF_LM_BAR];

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index 0111bfdeb342..447f4ad9c0bf 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h

@@ -73,6 +73,7 @@ struct vring_info {
 	u16 last_avail_idx;
 	bool ready;
 	void __iomem *notify_addr;
+	phys_addr_t notify_pa;
 	u32 irq;
 	struct vdpa_callback cb;
 	char msix_name[256];

@@ -87,6 +88,7 @@ struct ifcvf_hw {
 	u8 notify_bar;
 	/* Notificaiton bar address */
 	void __iomem *notify_base;
+	phys_addr_t notify_base_pa;
 	u32 notify_off_multiplier;
 	u64 req_features;
 	u64 hw_features;

-- 
2.27.0

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <redacted>
---
 drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c

@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
 	return vf->vring[qid].irq;
 }
 
+static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+							       u16 idx)
+{
+	struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+	struct pci_dev *pdev = adapter->pdev;
+	struct vdpa_notification_area area;
+
+	area.addr = vf->vring[idx].notify_pa;
+	if (!vf->notify_off_multiplier)
+		area.size = PAGE_SIZE;
+	else
+		area.size = vf->notify_off_multiplier;
+
+	if (area.addr % PAGE_SIZE)
+		IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);
+
+	return area;
+}
+
 /*
  * IFCVF currently does't have on-chip IOMMU, so not
  * implemented set_map()/dma_map()/dma_unmap()

@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.get_config	= ifcvf_vdpa_get_config,
 	.set_config	= ifcvf_vdpa_set_config,
 	.set_config_cb  = ifcvf_vdpa_set_config_cb,
+	.get_vq_notification = ifcvf_get_vq_notification,
 };
 
 static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

-- 
2.27.0

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

I don't see the reason to keep this, or get_notification is not the proper place to do this kind of warning.

Thanks

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

This commit records virtio notify base physical addr and
calculate doorbell physical address for vqs.

Signed-off-by: Zhu Lingshan <redacted>

Acked-by: Jason Wang <redacted>

7 more quoted lines

On 6/1/2021 4:50 PM, Jason Wang wrote:

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

quoted

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <redacted>
---
  drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
  1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c

@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,

      return vf->vring[qid].irq;
  }
  +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    if (!vf->notify_off_multiplier)
+        area.size = PAGE_SIZE;
+    else
+        area.size = vf->notify_off_multiplier;
+
+    if (area.addr % PAGE_SIZE)
+        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);

I don't see the reason to keep this, or get_notification is not the proper place to do this kind of warning.

Thanks

some customers have ever complained have troubles to enable such features with their IP,
I think this can help them debug.

Thanks

quoted

+
+    return area;
+}
+
  /*
   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {

      .get_config    = ifcvf_vdpa_get_config,
      .set_config    = ifcvf_vdpa_set_config,
      .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
  };
    static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

在 2021/6/1 下午4:56, Zhu, Lingshan 写道:

On 6/1/2021 4:50 PM, Jason Wang wrote:

quoted

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

quoted

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <redacted>
---
  drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
  1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c

@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,

      return vf->vring[qid].irq;
  }
  +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    if (!vf->notify_off_multiplier)
+        area.size = PAGE_SIZE;
+    else
+        area.size = vf->notify_off_multiplier;
+
+    if (area.addr % PAGE_SIZE)
+        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);

I don't see the reason to keep this, or get_notification is not the proper place to do this kind of warning.

4 more quoted lines

If you want to do this, the ifcvf_init_hw() is the proper place.

Note that this function is called by userspace.

Thanks

Thanks

quoted

quoted

+
+    return area;
+}
+
  /*
   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {

      .get_config    = ifcvf_vdpa_get_config,
      .set_config    = ifcvf_vdpa_set_config,
      .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
  };
    static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

On 6/1/2021 4:57 PM, Jason Wang wrote:

在 2021/6/1 下午4:56, Zhu, Lingshan 写道:

quoted

On 6/1/2021 4:50 PM, Jason Wang wrote:

quoted

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

quoted

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <redacted>
---
  drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
  1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c

@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,

      return vf->vring[qid].irq;
  }
  +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    if (!vf->notify_off_multiplier)
+        area.size = PAGE_SIZE;
+    else
+        area.size = vf->notify_off_multiplier;
+
+    if (area.addr % PAGE_SIZE)
+        IFCVF_DBG(pdev, "vq %u doorbell address is not PAGE_SIZE aligned\n", idx);

13 more quoted lines

OK, will move to there.

Thanks!

quoted

Thanks

quoted

quoted

+
+    return area;
+}
+
  /*
   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {

      .get_config    = ifcvf_vdpa_get_config,
      .set_config    = ifcvf_vdpa_set_config,
      .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
  };
    static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)

On 6/1/2021 4:58 PM, Zhu, Lingshan wrote:

On 6/1/2021 4:57 PM, Jason Wang wrote:

quoted

在 2021/6/1 下午4:56, Zhu, Lingshan 写道:

quoted

On 6/1/2021 4:50 PM, Jason Wang wrote:

quoted

在 2021/6/1 下午2:28, Zhu Lingshan 写道:

quoted

This commit implements doorbell mapping feature for ifcvf.
This feature maps the notify page to userspace, to eliminate
vmexit when kick a vq.

Signed-off-by: Zhu Lingshan <redacted>
---
  drivers/vdpa/ifcvf/ifcvf_main.c | 21 +++++++++++++++++++++
  1 file changed, 21 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index ab0ab5cf0f6e..d41db042612c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c

@@ -413,6 +413,26 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,

      return vf->vring[qid].irq;
  }
  +static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+                                   u16 idx)
+{
+    struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);
+    struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+    struct pci_dev *pdev = adapter->pdev;
+    struct vdpa_notification_area area;
+
+    area.addr = vf->vring[idx].notify_pa;
+    if (!vf->notify_off_multiplier)
+        area.size = PAGE_SIZE;
+    else
+        area.size = vf->notify_off_multiplier;
+

19 more quoted lines

oh, I see patch 1 is already has been acked, so I will remove this warning.

Thanks!

quoted

quoted

Thanks

quoted

quoted

+
+    return area;
+}
+
  /*
   * IFCVF currently does't have on-chip IOMMU, so not
   * implemented set_map()/dma_map()/dma_unmap()

@@ -440,6 +460,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {

      .get_config    = ifcvf_vdpa_get_config,
      .set_config    = ifcvf_vdpa_set_config,
      .set_config_cb  = ifcvf_vdpa_set_config_cb,
+    .get_vq_notification = ifcvf_get_vq_notification,
  };
    static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)