--- v2
+++ v1
@@ -7,36 +7,70 @@
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
- drivers/virtio/virtio_ring.c | 29 +++++++++++++++++++----------
- 1 file changed, 19 insertions(+), 10 deletions(-)
+ drivers/virtio/virtio_ring.c | 57 +++++++++++++++++++++++++++++++-----
+ 1 file changed, 49 insertions(+), 8 deletions(-)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
-index 0ebcd4f12d3b..e6d1985a87a8 100644
+index 0b9a8544b0e8..4fd7bd5bcd70 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
-@@ -1089,7 +1089,11 @@ static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
+@@ -1074,10 +1074,45 @@ static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
}
}
-static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
-+#define desc_cache_get_packed(vq, n, gfp) \
-+ _desc_cache_get(vq, n, gfp, struct vring_packed_desc)
++static void desc_cache_chain_free_packed(void *chain)
++{
++ struct vring_packed_desc *desc;
++
++ while (chain) {
++ desc = chain;
++ chain = (void *)desc->addr;
++ kfree(desc);
++ }
++}
++
++static void desc_cache_put_packed(struct vring_virtqueue *vq,
++ struct vring_packed_desc *desc, int n)
++{
++ if (vq->use_desc_cache && n <= VIRT_QUEUE_CACHE_DESC_NUM) {
++ desc->addr = (u64)vq->desc_cache_chain;
++ vq->desc_cache_chain = desc;
++ } else {
++ kfree(desc);
++ }
++}
+
+static struct vring_packed_desc *alloc_indirect_packed(struct vring_virtqueue *vq,
+ unsigned int total_sg,
gfp_t gfp)
{
struct vring_packed_desc *desc;
-@@ -1101,7 +1105,7 @@ static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
++ unsigned int n;
++
++ if (vq->use_desc_cache && total_sg <= VIRT_QUEUE_CACHE_DESC_NUM) {
++ if (vq->desc_cache_chain) {
++ desc = vq->desc_cache_chain;
++ vq->desc_cache_chain = (void *)desc->addr;
++ return desc;
++ }
++ n = VIRT_QUEUE_CACHE_DESC_NUM;
++ } else {
++ n = total_sg;
++ }
+
+ /*
+ * We require lowmem mappings for the descriptors because
+@@ -1086,7 +1121,7 @@ static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
*/
gfp &= ~__GFP_HIGHMEM;
- desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
-+ desc = desc_cache_get_packed(vq, total_sg, gfp);
++ desc = kmalloc_array(n, sizeof(struct vring_packed_desc), gfp);
return desc;
}
-@@ -1121,7 +1125,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
+@@ -1106,7 +1141,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
dma_addr_t addr;
head = vq->packed.next_avail_idx;
@@ -45,16 +79,16 @@
if (unlikely(vq->vq.num_free < 1)) {
pr_debug("Can't add buf len 1 - avail = 0\n");
-@@ -1212,7 +1216,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
+@@ -1197,7 +1232,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
for (i = 0; i < err_idx; i++)
vring_unmap_desc_packed(vq, &desc[i]);
- kfree(desc);
-+ desc_cache_put(vq, desc, total_sg);
++ desc_cache_put_packed(vq, desc, total_sg);
END_USE(vq);
return -ENOMEM;
-@@ -1437,20 +1441,22 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
+@@ -1422,20 +1457,22 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
}
if (vq->indirect) {
@@ -66,11 +100,10 @@
if (!desc)
return;
-+ len = vq->packed.desc_extra[id].len;
+ n = len / sizeof(struct vring_packed_desc);
+
if (vq->use_dma_api) {
-- len = vq->packed.desc_extra[id].len;
+ len = vq->packed.desc_extra[id].len;
- for (i = 0; i < len / sizeof(struct vring_packed_desc);
- i++)
+ for (i = 0; i < n; i++)
@@ -78,30 +111,28 @@
}
- kfree(desc);
+
-+ desc_cache_put(vq, desc, n);
++ desc_cache_put_packed(vq, desc, n);
state->indir_desc = NULL;
} else if (ctx) {
*ctx = state->indir_desc;
-@@ -1768,6 +1774,9 @@ static struct virtqueue *vring_create_virtqueue_packed(
+@@ -1753,6 +1790,8 @@ static struct virtqueue *vring_create_virtqueue_packed(
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
-+ vq->desc_cache_thr = vdev->desc_cache_thr;
-+
-+ INIT_LIST_HEAD(&vq->desc_cache);
++ vq->desc_cache_chain = NULL;
++ vq->use_desc_cache = vdev->desc_cache;
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
-@@ -2389,8 +2398,8 @@ void vring_del_virtqueue(struct virtqueue *_vq)
- if (!vq->packed_ring) {
+@@ -2374,6 +2413,8 @@ void vring_del_virtqueue(struct virtqueue *_vq)
kfree(vq->split.desc_state);
kfree(vq->split.desc_extra);
-- desc_cache_free(&vq->desc_cache);
+ desc_cache_chain_free_split(vq->desc_cache_chain);
++ } else {
++ desc_cache_chain_free_packed(vq->desc_cache_chain);
}
-+ desc_cache_free(&vq->desc_cache);
kfree(vq);
}
- EXPORT_SYMBOL_GPL(vring_del_virtqueue);
--
2.31.0