The subsequent commit needs to know whether every indirect buffer is
premapped or not. So we need to introduce an extra struct for every
indirect buffer to record this info.
Signed-off-by: Xuan Zhuo <xuanzhuo@xxxxxxxxxxxxxxxxx>
---
drivers/virtio/virtio_ring.c | 67 ++++++++++++++++++++----------------
1 file changed, 37 insertions(+), 30 deletions(-)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 62901bee97c0..7d5fed4ff4f8 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -85,7 +85,11 @@ struct vring_desc_state_split {
struct vring_desc_state_packed {
void *data; /* Data for callback. */
- struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
+
+ /* Indirect extra table and desc table, if any. These two will be
+ * allocated together. So we won't stress more to the memory allocator.
+ */
+ struct vring_desc_extra *indir;
u16 num; /* Descriptor list length. */
u16 last; /* The last desc state in a list. */
};
@@ -1242,27 +1246,13 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
}
}
-static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
- const struct vring_packed_desc *desc)
-{
- u16 flags;
-
- if (!vring_need_unmap_buffer(vq))
- return;
-
- flags = le16_to_cpu(desc->flags);
-
- dma_unmap_page(vring_dma_dev(vq),
- le64_to_cpu(desc->addr),
- le32_to_cpu(desc->len),
- (flags & VRING_DESC_F_WRITE) ?
- DMA_FROM_DEVICE : DMA_TO_DEVICE);
-}
-
static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
+ struct vring_desc_extra **pextra,
gfp_t gfp)
{
+ struct vring_desc_extra *extra;
struct vring_packed_desc *desc;
+ int i;
/*
* We require lowmem mappings for the descriptors because
@@ -1271,7 +1261,16 @@ static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
*/
gfp &= ~__GFP_HIGHMEM;
- desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
+ extra = kmalloc_array(total_sg, sizeof(*desc) + sizeof(*extra), gfp);
+ if (!extra)
+ return NULL;
+
+ desc = (struct vring_packed_desc *)&extra[total_sg];
+
+ for (i = 0; i < total_sg; i++)
+ extra[i].next = i + 1;
+
+ *pextra = extra;
return desc;
}
@@ -1284,6 +1283,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
void *data,
gfp_t gfp)
{
+ struct vring_desc_extra *extra;
struct vring_packed_desc *desc;
struct scatterlist *sg;
unsigned int i, n, err_idx;
@@ -1291,7 +1291,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
dma_addr_t addr;
head = vq->packed.next_avail_idx;
- desc = alloc_indirect_packed(total_sg, gfp);
+ desc = alloc_indirect_packed(total_sg, &extra, gfp);
if (!desc)
return -ENOMEM;
@@ -1316,6 +1316,13 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
0 : VRING_DESC_F_WRITE);
desc[i].addr = cpu_to_le64(addr);
desc[i].len = cpu_to_le32(sg->length);
+
+ if (unlikely(vq->use_dma_api)) {
+ extra[i].addr = addr;
+ extra[i].len = sg->length;
+ extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE;
+ }
+
i++;
}
}
@@ -1371,7 +1378,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
/* Store token and indirect buffer state. */
vq->packed.desc_state[id].num = 1;
vq->packed.desc_state[id].data = data;
- vq->packed.desc_state[id].indir_desc = desc;
+ vq->packed.desc_state[id].indir = extra;
vq->packed.desc_state[id].last = id;
vq->num_added += 1;
@@ -1385,7 +1392,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
err_idx = i;
for (i = 0; i < err_idx; i++)
- vring_unmap_desc_packed(vq, &desc[i]);
+ vring_unmap_extra_packed(vq, &extra[i]);
free_desc:
kfree(desc);
@@ -1508,7 +1515,7 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
/* Store token. */
vq->packed.desc_state[id].num = descs_used;
vq->packed.desc_state[id].data = data;
- vq->packed.desc_state[id].indir_desc = ctx;
+ vq->packed.desc_state[id].indir = ctx;
vq->packed.desc_state[id].last = prev;
/*
@@ -1599,7 +1606,6 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
unsigned int id, void **ctx)
{
struct vring_desc_state_packed *state = NULL;
- struct vring_packed_desc *desc;
unsigned int i, curr;
state = &vq->packed.desc_state[id];
@@ -1621,23 +1627,24 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
}
if (vq->indirect) {
+ struct vring_desc_extra *extra;
u32 len;
/* Free the indirect table, if any, now that it's unmapped. */
- desc = state->indir_desc;
- if (!desc)
+ extra = state->indir;
+ if (!extra)
return;
if (vring_need_unmap_buffer(vq)) {
len = vq->packed.desc_extra[id].len;
for (i = 0; i < len / sizeof(struct vring_packed_desc);
i++)
- vring_unmap_desc_packed(vq, &desc[i]);
+ vring_unmap_extra_packed(vq, &extra[i]);
}
- kfree(desc);
- state->indir_desc = NULL;
+ kfree(extra);
+ state->indir = NULL;
} else if (ctx) {
- *ctx = state->indir_desc;
+ *ctx = state->indir;
}
}
--
2.32.0.3.g01195cf9f
