+ for (i = 0; i < vq_sg->nsgl; ++i) {
+ kfree(vq_sg->sgl[i]);
+ }
+ kfree(vq_sg);
+}
+
+/**
+ * vq_sg_alloc - allocates and returns @nsgl scatter gather lists
+ * @nsgl: amount of scatter gather lists to be allocated
+ * If @nsgl is larger than one then chained lists are used if supported by
+ * architecture.
+ */
+static struct virtqueue_sg *vq_sg_alloc(unsigned int nsgl)
+{
+ struct virtqueue_sg *vq_sg;
+ unsigned int i;
+
+ BUG_ON(!nsgl || nsgl > VIRTQUEUE_SG_NSGL_MAX);
+#ifdef CONFIG_ARCH_NO_SG_CHAIN
+ if (WARN_ON_ONCE(nsgl > 1))
+ return NULL;
+#endif
+
+ vq_sg = kzalloc(sizeof(struct virtqueue_sg) +
+ nsgl * sizeof(struct scatterlist *),
+ GFP_KERNEL);
+
+ if (!vq_sg)
+ return NULL;
+
+ vq_sg->nsgl = nsgl;
+
+ for (i = 0; i < nsgl; ++i) {
+ vq_sg->sgl[i] = kmalloc_array(
+ SG_MAX_SINGLE_ALLOC, sizeof(struct scatterlist),
+ GFP_KERNEL
+ );
+ if (!vq_sg->sgl[i]) {
+ vq_sg_free(vq_sg);
+ return NULL;
+ }
+ sg_init_table(vq_sg->sgl[i], SG_MAX_SINGLE_ALLOC);
+ if (i) {
+ /* chain the lists */
+ sg_chain(vq_sg->sgl[i - 1], SG_MAX_SINGLE_ALLOC,
+ vq_sg->sgl[i]);
+ }
+ }
+ sg_mark_end(&vq_sg->sgl[nsgl - 1][SG_MAX_SINGLE_ALLOC - 1]);
+ return vq_sg;
+}
+
/**
* p9_virtio_close - reclaim resources of a channel
* @client: client instance@@ -158,9 +262,8 @@ static void req_done(struct virtqueue *vq)
/**
* pack_sg_list - pack a scatter gather list from a linear buffer
- * @sg: scatter/gather list to pack into
+ * @vq_sg: scatter/gather lists to pack into
* @start: which segment of the sg_list to start at
- * @limit: maximum segment to pack data to
* @data: data to pack into scatter/gather list
* @count: amount of data to pack into the scatter/gather list
*
@@ -170,11 +273,12 @@ static void req_done(struct virtqueue *vq)
*
*/
-static int pack_sg_list(struct scatterlist *sg, int start,
- int limit, char *data, int count)
+static int pack_sg_list(struct virtqueue_sg *vq_sg, int start,
+ char *data, int count)
{
int s;
int index = start;
+ size_t limit = vq_sg_npages(vq_sg);
while (count) {
s = rest_of_page(data);@@ -182,13 +286,13 @@ static int pack_sg_list(struct scatterlist *sg, int
start, s = count;
BUG_ON(index >= limit);
/* Make sure we don't terminate early. */
- sg_unmark_end(&sg[index]);
- sg_set_buf(&sg[index++], data, s);
+ sg_unmark_end(vq_sg_page(vq_sg, index));
+ sg_set_buf(vq_sg_page(vq_sg, index++), data, s);
count -= s;
data += s;
}
if (index-start)
- sg_mark_end(&sg[index - 1]);
+ sg_mark_end(vq_sg_page(vq_sg, index - 1));
return index-start;
}
@@ -208,21 +312,21 @@ static int p9_virtio_cancelled(struct p9_client
*client, struct p9_req_t *req) /**
* pack_sg_list_p - Just like pack_sg_list. Instead of taking a buffer,
* this takes a list of pages.
- * @sg: scatter/gather list to pack into
+ * @vq_sg: scatter/gather lists to pack into
* @start: which segment of the sg_list to start at
- * @limit: maximum number of pages in sg list.
* @pdata: a list of pages to add into sg.
* @nr_pages: number of pages to pack into the scatter/gather list
* @offs: amount of data in the beginning of first page _not_ to pack
* @count: amount of data to pack into the scatter/gather list
*/
static int
-pack_sg_list_p(struct scatterlist *sg, int start, int limit,
+pack_sg_list_p(struct virtqueue_sg *vq_sg, int start,
struct page **pdata, int nr_pages, size_t offs, int count)
{
int i = 0, s;
int data_off = offs;
int index = start;
+ size_t limit = vq_sg_npages(vq_sg);
BUG_ON(nr_pages > (limit - start));
/*@@ -235,15 +339,16 @@ pack_sg_list_p(struct scatterlist *sg, int start, int
limit, s = count;
BUG_ON(index >= limit);
/* Make sure we don't terminate early. */
- sg_unmark_end(&sg[index]);
- sg_set_page(&sg[index++], pdata[i++], s, data_off);
+ sg_unmark_end(vq_sg_page(vq_sg, index));
+ sg_set_page(vq_sg_page(vq_sg, index++), pdata[i++], s,
+ data_off);
data_off = 0;
count -= s;
nr_pages--;
}
if (index-start)
- sg_mark_end(&sg[index - 1]);
+ sg_mark_end(vq_sg_page(vq_sg, index - 1));
return index - start;
}
@@ -271,15 +376,13 @@ p9_virtio_request(struct p9_client *client, struct
p9_req_t *req)
out_sgs = in_sgs = 0;
/* Handle out VirtIO ring buffers */
- out = pack_sg_list(chan->sg, 0,
- chan->sg_n, req->tc.sdata, req->tc.size);
+ out = pack_sg_list(chan->vq_sg, 0, req->tc.sdata, req->tc.size);
if (out)
- sgs[out_sgs++] = chan->sg;
+ sgs[out_sgs++] = vq_sg_page(chan->vq_sg, 0);
- in = pack_sg_list(chan->sg, out,
- chan->sg_n, req->rc.sdata, req->rc.capacity);
+ in = pack_sg_list(chan->vq_sg, out, req->rc.sdata, req->rc.capacity);
if (in)
- sgs[out_sgs + in_sgs++] = chan->sg + out;
+ sgs[out_sgs + in_sgs++] = vq_sg_page(chan->vq_sg, out);
err = virtqueue_add_sgs(chan->vq, sgs, out_sgs, in_sgs, req,
GFP_ATOMIC);
@@ -448,16 +551,15 @@ p9_virtio_zc_request(struct p9_client *client, struct
p9_req_t *req, out_sgs = in_sgs = 0;
/* out data */
- out = pack_sg_list(chan->sg, 0,
- chan->sg_n, req->tc.sdata, req->tc.size);
+ out = pack_sg_list(chan->vq_sg, 0, req->tc.sdata, req->tc.size);
if (out)
- sgs[out_sgs++] = chan->sg;
+ sgs[out_sgs++] = vq_sg_page(chan->vq_sg, 0);
if (out_pages) {
- sgs[out_sgs++] = chan->sg + out;
- out += pack_sg_list_p(chan->sg, out, chan->sg_n,
- out_pages, out_nr_pages, offs, outlen);
+ sgs[out_sgs++] = vq_sg_page(chan->vq_sg, out);
+ out += pack_sg_list_p(chan->vq_sg, out, out_pages,
+ out_nr_pages, offs, outlen);
}
/*@@ -467,15 +569,14 @@ p9_virtio_zc_request(struct p9_client *client, struct
p9_req_t *req, * Arrange in such a way that server places header in the
* allocated memory and payload onto the user buffer.
*/
- in = pack_sg_list(chan->sg, out,
- chan->sg_n, req->rc.sdata, in_hdr_len);
+ in = pack_sg_list(chan->vq_sg, out, req->rc.sdata, in_hdr_len);
if (in)
- sgs[out_sgs + in_sgs++] = chan->sg + out;
+ sgs[out_sgs + in_sgs++] = vq_sg_page(chan->vq_sg, out);
if (in_pages) {
- sgs[out_sgs + in_sgs++] = chan->sg + out + in;
- in += pack_sg_list_p(chan->sg, out + in, chan->sg_n,
- in_pages, in_nr_pages, offs, inlen);
+ sgs[out_sgs + in_sgs++] = vq_sg_page(chan->vq_sg, out + in);
+ in += pack_sg_list_p(chan->vq_sg, out + in, in_pages,
+ in_nr_pages, offs, inlen);
}
BUG_ON(out_sgs + in_sgs > ARRAY_SIZE(sgs));@@ -576,14 +677,12 @@ static int p9_virtio_probe(struct virtio_device *vdev)
goto fail;
}
- chan->sg = kmalloc_array(VIRTQUEUE_DEFAULT_NUM,
- sizeof(struct scatterlist), GFP_KERNEL);
- if (!chan->sg) {
+ chan->vq_sg = vq_sg_alloc(VIRTQUEUE_SG_NSGL_DEFAULT);
+ if (!chan->vq_sg) {
pr_err("Failed to allocate virtio 9P channel\n");
err = -ENOMEM;
goto out_free_chan_shallow;
}
- chan->sg_n = VIRTQUEUE_DEFAULT_NUM;
chan->vdev = vdev;
@@ -596,8 +695,6 @@ static int p9_virtio_probe(struct virtio_device *vdev)
chan->vq->vdev->priv = chan;
spin_lock_init(&chan->lock);
- sg_init_table(chan->sg, chan->sg_n);
-
chan->inuse = false;
if (virtio_has_feature(vdev, VIRTIO_9P_MOUNT_TAG)) {
virtio_cread(vdev, struct virtio_9p_config, tag_len, &tag_len);@@ -646,7 +743,7 @@ static int p9_virtio_probe(struct virtio_device *vdev)
out_free_vq:
vdev->config->del_vqs(vdev);
out_free_chan:
- kfree(chan->sg);
+ vq_sg_free(chan->vq_sg);
out_free_chan_shallow:
kfree(chan);
fail:
@@ -741,7 +838,7 @@ static void p9_virtio_remove(struct virtio_device *vdev)
kobject_uevent(&(vdev->dev.kobj), KOBJ_CHANGE);
kfree(chan->tag);
kfree(chan->vc_wq);
- kfree(chan->sg);
+ vq_sg_free(chan->vq_sg);
kfree(chan);
}
@@ -780,7 +877,8 @@ static struct p9_trans_module p9_virtio_trans = {
* that are not at page boundary, that can result in an extra
* page in zero copy.
*/
- .maxsize = PAGE_SIZE * (VIRTQUEUE_DEFAULT_NUM - 3),
+ .maxsize = PAGE_SIZE *
+ ((VIRTQUEUE_SG_NSGL_DEFAULT * SG_USER_PAGES_PER_LIST) - 3),
.def = 1,
.owner = THIS_MODULE,
};