On Tue, 5 Dec 2023 15:12:03 +0800, Jason Wang <jasowang@xxxxxxxxxx> wrote: > On Mon, Dec 4, 2023 at 3:23 PM Xuan Zhuo <xuanzhuo@xxxxxxxxxxxxxxxxx> wrote: > > > > introduce virtqueue_get_buf_ctx_dma() to collect the dma info when > > get buf from virtio core for premapped mode. > > > > If the virtio queue is premapped mode, the virtio-net send buf may > > have many desc. Every desc dma address need to be unmap. So here we > > introduce a new helper to collect the dma address of the buffer from > > the virtio core. > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@xxxxxxxxxxxxxxxxx> > > --- > > drivers/virtio/virtio_ring.c | 157 +++++++++++++++++++++++------------ > > include/linux/virtio.h | 16 ++++ > > 2 files changed, 121 insertions(+), 52 deletions(-) > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c > > index 51d8f3299c10..103d181f05bb 100644 > > --- a/drivers/virtio/virtio_ring.c > > +++ b/drivers/virtio/virtio_ring.c > > @@ -362,6 +362,26 @@ static struct device *vring_dma_dev(const struct vring_virtqueue *vq) > > return vq->dma_dev; > > } > > > > +static void collect_dma_info(const struct vring_virtqueue *vq, > > Not a native speaker, but it looks to me like "get_dma_info" is better here. get_dma_info? This function collect the dma info to the dma array. Not to the dirver. So do you also think "get_dma_info" is better? > > > + struct virtio_dma_head *dma, > > + dma_addr_t addr, unsigned int length) > > +{ > > + if (WARN_ON_ONCE(!dma)) > > + return; > > So all the callers did: > > if (!vq->do_unmap) > collect_dma_info() > > It's a strong hint that we should move the vq->do_unmap inside > collect_dma_info()? No, vq->do_unmap affects the logic of caller. Such as: if (!vq->do_unmap) { collect_dma_info(vq, dma, le64_to_cpu(desc->addr), le32_to_cpu(desc->len)); return; } So we should move it to this function. > > > + > > + if (unlikely(dma->next >= dma->num)) { > > + dev_warn_once(&vq->vq.vdev->dev, > > + "premapped vq: collect dma miss: %pad %u\n", > > + &addr, length); > > Let's use BAD_RING() for consistency. > > Btw, the message seems ambiguous here. Will fix. > > > + return; > > + } > > + > > + dma->items[dma->next].addr = addr; > > + dma->items[dma->next].length = length; > > + > > + ++dma->next; > > +} > > + > > /* Map one sg entry. */ > > static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg, > > enum dma_data_direction direction, dma_addr_t *addr) > > @@ -441,12 +461,17 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num) > > */ > > > > static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, > > - const struct vring_desc *desc) > > + const struct vring_desc *desc, > > + struct virtio_dma_head *dma) > > { > > u16 flags; > > > > - if (!vq->do_unmap) > > + if (!vq->do_unmap) { > > + collect_dma_info(vq, dma, > > + virtio64_to_cpu(vq->vq.vdev, desc->addr), > > + virtio32_to_cpu(vq->vq.vdev, desc->len)); > > return; > > + } > > > > flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); > > > > @@ -458,7 +483,7 @@ static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, > > } > > > > static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, > > - unsigned int i) > > + unsigned int i, struct virtio_dma_head *dma) > > { > > struct vring_desc_extra *extra = vq->split.desc_extra; > > u16 flags; > > @@ -466,18 +491,15 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, > > flags = extra[i].flags; > > > > if (flags & VRING_DESC_F_INDIRECT) { > > - if (!vq->use_dma_api) > > - goto out; > > - > > - dma_unmap_single(vring_dma_dev(vq), > > - extra[i].addr, > > - extra[i].len, > > - (flags & VRING_DESC_F_WRITE) ? > > - DMA_FROM_DEVICE : DMA_TO_DEVICE); > > + if (vq->use_dma_api) > > + dma_unmap_single(vring_dma_dev(vq), > > + extra[i].addr, > > + extra[i].len, > > + (flags & VRING_DESC_F_WRITE) ? > > + DMA_FROM_DEVICE : DMA_TO_DEVICE); > > + } else if (!vq->do_unmap) { > > + collect_dma_info(vq, dma, extra[i].addr, extra[i].len); > > } else { > > - if (!vq->do_unmap) > > - goto out; > > - > > dma_unmap_page(vring_dma_dev(vq), > > extra[i].addr, > > extra[i].len, > > @@ -485,7 +507,6 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, > > DMA_FROM_DEVICE : DMA_TO_DEVICE); > > } > > > > -out: > > return extra[i].next; > > } > > > > @@ -717,10 +738,10 @@ static inline int virtqueue_add_split(struct virtqueue *_vq, > > if (i == err_idx) > > break; > > if (indirect) { > > - vring_unmap_one_split_indirect(vq, &desc[i]); > > + vring_unmap_one_split_indirect(vq, &desc[i], NULL); > > So this will trigger the WARN_ON_ONCE() in collect_dma_info()? I thought about this. But rethink it, I found I missed one case. Here is not called, when the vq is premapeed mode. But, if the use_dma_api is false, this is called. Will fix it. > > > i = virtio16_to_cpu(_vq->vdev, desc[i].next); > > } else > > - i = vring_unmap_one_split(vq, i); > > + i = vring_unmap_one_split(vq, i, NULL); > > } > > > > free_indirect: > > @@ -763,7 +784,7 @@ static bool virtqueue_kick_prepare_split(struct virtqueue *_vq) > > } > > > > static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, > > - void **ctx) > > + struct virtio_dma_head *dma, void **ctx) > > { > > unsigned int i, j; > > __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); > > @@ -775,12 +796,12 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, > > i = head; > > > > while (vq->split.vring.desc[i].flags & nextflag) { > > - vring_unmap_one_split(vq, i); > > + vring_unmap_one_split(vq, i, dma); > > i = vq->split.desc_extra[i].next; > > vq->vq.num_free++; > > } > > > > - vring_unmap_one_split(vq, i); > > + vring_unmap_one_split(vq, i, dma); > > vq->split.desc_extra[i].next = vq->free_head; > > vq->free_head = head; > > > > @@ -802,10 +823,8 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, > > VRING_DESC_F_INDIRECT)); > > BUG_ON(len == 0 || len % sizeof(struct vring_desc)); > > > > - if (vq->do_unmap) { > > - for (j = 0; j < len / sizeof(struct vring_desc); j++) > > - vring_unmap_one_split_indirect(vq, &indir_desc[j]); > > - } > > + for (j = 0; j < len / sizeof(struct vring_desc); j++) > > + vring_unmap_one_split_indirect(vq, &indir_desc[j], dma); > > > > kfree(indir_desc); > > vq->split.desc_state[head].indir_desc = NULL; > > @@ -822,6 +841,7 @@ static bool more_used_split(const struct vring_virtqueue *vq) > > > > static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, > > unsigned int *len, > > + struct virtio_dma_head *dma, > > void **ctx) > > { > > struct vring_virtqueue *vq = to_vvq(_vq); > > @@ -862,7 +882,7 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, > > > > /* detach_buf_split clears data, so grab it now. */ > > ret = vq->split.desc_state[i].data; > > - detach_buf_split(vq, i, ctx); > > + detach_buf_split(vq, i, dma, ctx); > > vq->last_used_idx++; > > /* If we expect an interrupt for the next entry, tell host > > * by writing event index and flush out the write before > > @@ -984,7 +1004,7 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) > > continue; > > /* detach_buf_split clears data, so grab it now. */ > > buf = vq->split.desc_state[i].data; > > - detach_buf_split(vq, i, NULL); > > + detach_buf_split(vq, i, NULL, NULL); > > vq->split.avail_idx_shadow--; > > vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, > > vq->split.avail_idx_shadow); > > @@ -1221,7 +1241,8 @@ static u16 packed_last_used(u16 last_used_idx) > > } > > > > static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, > > - const struct vring_desc_extra *extra) > > + const struct vring_desc_extra *extra, > > + struct virtio_dma_head *dma) > > { > > u16 flags; > > > > @@ -1235,10 +1256,9 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, > > extra->addr, extra->len, > > (flags & VRING_DESC_F_WRITE) ? > > DMA_FROM_DEVICE : DMA_TO_DEVICE); > > + } else if (!vq->do_unmap) { > > + collect_dma_info(vq, dma, extra->addr, extra->len); > > } else { > > - if (!vq->do_unmap) > > - return; > > - > > dma_unmap_page(vring_dma_dev(vq), > > extra->addr, extra->len, > > (flags & VRING_DESC_F_WRITE) ? > > @@ -1247,12 +1267,15 @@ static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, > > } > > > > static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, > > - const struct vring_packed_desc *desc) > > + const struct vring_packed_desc *desc, > > + struct virtio_dma_head *dma) > > { > > u16 flags; > > > > - if (!vq->do_unmap) > > + if (!vq->do_unmap) { > > + collect_dma_info(vq, dma, le64_to_cpu(desc->addr), le32_to_cpu(desc->len)); > > return; > > + } > > > > flags = le16_to_cpu(desc->flags); > > > > @@ -1389,7 +1412,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, > > err_idx = i; > > > > for (i = 0; i < err_idx; i++) > > - vring_unmap_desc_packed(vq, &desc[i]); > > + vring_unmap_desc_packed(vq, &desc[i], NULL); > > > > free_desc: > > kfree(desc); > > @@ -1539,7 +1562,7 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq, > > for (n = 0; n < total_sg; n++) { > > if (i == err_idx) > > break; > > - vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); > > + vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr], NULL); > > curr = vq->packed.desc_extra[curr].next; > > i++; > > if (i >= vq->packed.vring.num) > > @@ -1600,7 +1623,9 @@ static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) > > } > > > > static void detach_buf_packed(struct vring_virtqueue *vq, > > - unsigned int id, void **ctx) > > + unsigned int id, > > + struct virtio_dma_head *dma, > > + void **ctx) > > { > > struct vring_desc_state_packed *state = NULL; > > struct vring_packed_desc *desc; > > @@ -1615,13 +1640,10 @@ static void detach_buf_packed(struct vring_virtqueue *vq, > > vq->free_head = id; > > vq->vq.num_free += state->num; > > > > - if (unlikely(vq->do_unmap)) { > > - curr = id; > > - for (i = 0; i < state->num; i++) { > > - vring_unmap_extra_packed(vq, > > - &vq->packed.desc_extra[curr]); > > - curr = vq->packed.desc_extra[curr].next; > > - } > > + curr = id; > > + for (i = 0; i < state->num; i++) { > > + vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr], dma); > > + curr = vq->packed.desc_extra[curr].next; > > I wonder if !dma && vq->do_unmap is valid, we need to document it somewhere. Yes, it can be valid. > > And we can keep the unlikely() optimization as is. I don't think there is a high probability for do_unmap. I don't think it's necessary to keep the unlikely(). > > > > } > > > > if (vq->indirect) { > > @@ -1632,12 +1654,9 @@ static void detach_buf_packed(struct vring_virtqueue *vq, > > if (!desc) > > return; > > > > - if (vq->do_unmap) { > > - len = vq->packed.desc_extra[id].len; > > - for (i = 0; i < len / sizeof(struct vring_packed_desc); > > - i++) > > - vring_unmap_desc_packed(vq, &desc[i]); > > - } > > + len = vq->packed.desc_extra[id].len; > > + for (i = 0; i < len / sizeof(struct vring_packed_desc); i++) > > + vring_unmap_desc_packed(vq, &desc[i], dma); > > kfree(desc); > > state->indir_desc = NULL; > > } else if (ctx) { > > @@ -1672,6 +1691,7 @@ static bool more_used_packed(const struct vring_virtqueue *vq) > > > > static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq, > > unsigned int *len, > > + struct virtio_dma_head *dma, > > void **ctx) > > { > > struct vring_virtqueue *vq = to_vvq(_vq); > > @@ -1712,7 +1732,7 @@ static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq, > > > > /* detach_buf_packed clears data, so grab it now. */ > > ret = vq->packed.desc_state[id].data; > > - detach_buf_packed(vq, id, ctx); > > + detach_buf_packed(vq, id, dma, ctx); > > > > last_used += vq->packed.desc_state[id].num; > > if (unlikely(last_used >= vq->packed.vring.num)) { > > @@ -1877,7 +1897,7 @@ static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq) > > continue; > > /* detach_buf clears data, so grab it now. */ > > buf = vq->packed.desc_state[i].data; > > - detach_buf_packed(vq, i, NULL); > > + detach_buf_packed(vq, i, NULL, NULL); > > END_USE(vq); > > return buf; > > } > > @@ -2417,11 +2437,44 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len, > > { > > struct vring_virtqueue *vq = to_vvq(_vq); > > > > - return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) : > > - virtqueue_get_buf_ctx_split(_vq, len, ctx); > > + return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, NULL, ctx) : > > + virtqueue_get_buf_ctx_split(_vq, len, NULL, ctx); > > } > > EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx); > > > > +/** > > + * virtqueue_get_buf_ctx_dma - get the next used buffer with the dma info > > + * @_vq: the struct virtqueue we're talking about. > > + * @len: the length written into the buffer > > + * @dma: the head of the array to store the dma info > > + * @ctx: extra context for the token > > + * > > + * If the device wrote data into the buffer, @len will be set to the > > + * amount written. This means you don't need to clear the buffer > > + * beforehand to ensure there's no data leakage in the case of short > > + * writes. > > + * > > + * Caller must ensure we don't call this with other virtqueue > > + * operations at the same time (except where noted). > > + * > > + * We store the dma info of every descriptor of this buf to the dma->items > > + * array. If the array size is too small, some dma info may be missed, so > > + * the caller must ensure the array is enough. > > large enough? Will fix. > > > The dma->next is the out value > > + * to the caller, > > What do you mean by "out value to the caller" ? The caller know how many items are used by this dma->next. This is out value to the caller. Thanks. > > Thanks > > > > indicates the num of the used items. > > + * > > + * Returns NULL if there are no used buffers, or the "data" token > > + * handed to virtqueue_add_*(). > > + */ > > +void *virtqueue_get_buf_ctx_dma(struct virtqueue *_vq, unsigned int *len, > > + struct virtio_dma_head *dma, void **ctx) > > +{ > > + struct vring_virtqueue *vq = to_vvq(_vq); > > + > > + return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, dma, ctx) : > > + virtqueue_get_buf_ctx_split(_vq, len, dma, ctx); > > +} > > +EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx_dma); > > + > > void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) > > { > > return virtqueue_get_buf_ctx(_vq, len, NULL); > > diff --git a/include/linux/virtio.h b/include/linux/virtio.h > > index 4cc614a38376..572aecec205b 100644 > > --- a/include/linux/virtio.h > > +++ b/include/linux/virtio.h > > @@ -75,6 +75,22 @@ void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len); > > void *virtqueue_get_buf_ctx(struct virtqueue *vq, unsigned int *len, > > void **ctx); > > > > +struct virtio_dma_item { > > + dma_addr_t addr; > > + unsigned int length; > > +}; > > + > > +struct virtio_dma_head { > > + /* total num of items. */ > > + u16 num; > > + /* point to the next item to store dma info. */ > > + u16 next; > > + struct virtio_dma_item items[]; > > +}; > > + > > +void *virtqueue_get_buf_ctx_dma(struct virtqueue *_vq, unsigned int *len, > > + struct virtio_dma_head *dma, void **ctx); > > + > > void virtqueue_disable_cb(struct virtqueue *vq); > > > > bool virtqueue_enable_cb(struct virtqueue *vq); > > -- > > 2.32.0.3.g01195cf9f > > >