vDPA supports the possibility to use user VA in the iotlb messages.
So, let's add support for user VA in vringh to use it in the vDPA
simulators.
Signed-off-by: Stefano Garzarella <sgarzare@xxxxxxxxxx>
---
Notes:
v2:
- replace kmap_atomic() with kmap_local_page() [see previous patch]
- fix cast warnings when build with W=1 C=1
include/linux/vringh.h | 5 +-
drivers/vdpa/mlx5/net/mlx5_vnet.c | 2 +-
drivers/vdpa/vdpa_sim/vdpa_sim.c | 4 +-
drivers/vhost/vringh.c | 247 ++++++++++++++++++++++++------
4 files changed, 205 insertions(+), 53 deletions(-)
diff --git a/include/linux/vringh.h b/include/linux/vringh.h
index 1991a02c6431..d39b9f2dcba0 100644
--- a/include/linux/vringh.h
+++ b/include/linux/vringh.h
@@ -32,6 +32,9 @@ struct vringh {
/* Can we get away with weak barriers? */
bool weak_barriers;
+ /* Use user's VA */
+ bool use_va;
+
/* Last available index we saw (ie. where we're up to). */
u16 last_avail_idx;
@@ -279,7 +282,7 @@ void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb,
spinlock_t *iotlb_lock);
int vringh_init_iotlb(struct vringh *vrh, u64 features,
- unsigned int num, bool weak_barriers,
+ unsigned int num, bool weak_barriers, bool use_va,
struct vring_desc *desc,
struct vring_avail *avail,
struct vring_used *used);
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index 3a0e721aef05..babc8dd171a6 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -2537,7 +2537,7 @@ static int setup_cvq_vring(struct mlx5_vdpa_dev *mvdev)
if (mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))
err = vringh_init_iotlb(&cvq->vring, mvdev->actual_features,
- MLX5_CVQ_MAX_ENT, false,
+ MLX5_CVQ_MAX_ENT, false, false,
(struct vring_desc *)(uintptr_t)cvq->desc_addr,
(struct vring_avail *)(uintptr_t)cvq->driver_addr,
(struct vring_used *)(uintptr_t)cvq->device_addr);
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 6a0a65814626..481eb156658b 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -60,7 +60,7 @@ static void vdpasim_queue_ready(struct vdpasim *vdpasim, unsigned int idx)
struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
uint16_t last_avail_idx = vq->vring.last_avail_idx;
- vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, true,
+ vringh_init_iotlb(&vq->vring, vdpasim->features, vq->num, true, false,
(struct vring_desc *)(uintptr_t)vq->desc_addr,
(struct vring_avail *)
(uintptr_t)vq->driver_addr,
@@ -81,7 +81,7 @@ static void vdpasim_vq_reset(struct vdpasim *vdpasim,
vq->cb = NULL;
vq->private = NULL;
vringh_init_iotlb(&vq->vring, vdpasim->dev_attr.supported_features,
- VDPASIM_QUEUE_MAX, false, NULL, NULL, NULL);
+ VDPASIM_QUEUE_MAX, false, false, NULL, NULL, NULL);
vq->vring.notify = NULL;
}
diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c
index 0ba3ef809e48..61c79cea44ca 100644
--- a/drivers/vhost/vringh.c
+++ b/drivers/vhost/vringh.c
@@ -1094,15 +1094,99 @@ EXPORT_SYMBOL(vringh_need_notify_kern);
#if IS_REACHABLE(CONFIG_VHOST_IOTLB)
-static int iotlb_translate(const struct vringh *vrh,
- u64 addr, u64 len, u64 *translated,
- struct bio_vec iov[],
- int iov_size, u32 perm)
+static int iotlb_translate_va(const struct vringh *vrh,
+ u64 addr, u64 len, u64 *translated,
+ struct iovec iov[],
+ int iov_size, u32 perm)
{
struct vhost_iotlb_map *map;
struct vhost_iotlb *iotlb = vrh->iotlb;
+ u64 s = 0, last = addr + len - 1;
int ret = 0;
+
+ spin_lock(vrh->iotlb_lock);
+
+ while (len > s) {
+ u64 size;
+
+ if (unlikely(ret >= iov_size)) {
+ ret = -ENOBUFS;
+ break;
+ }
+
+ map = vhost_iotlb_itree_first(iotlb, addr, last);
+ if (!map || map->start > addr) {
+ ret = -EINVAL;
+ break;
+ } else if (!(map->perm & perm)) {
+ ret = -EPERM;
+ break;
+ }
+
+ size = map->size - addr + map->start;
+ iov[ret].iov_len = min(len - s, size);
+ iov[ret].iov_base = (void __user *)(unsigned long)
+ (map->addr + addr - map->start);
+ s += size;
+ addr += size;
+ ++ret;
+ }
+
+ spin_unlock(vrh->iotlb_lock);
+
+ if (translated)
+ *translated = min(len, s);
+
+ return ret;
+}
+
+static inline int copy_from_va(const struct vringh *vrh, void *dst, void *src,
+ u64 len, u64 *translated)
+{
+ struct iovec iov[16];
+ struct iov_iter iter;
+ int ret;
+
+ ret = iotlb_translate_va(vrh, (u64)(uintptr_t)src, len, translated, iov,
+ ARRAY_SIZE(iov), VHOST_MAP_RO);
+ if (ret == -ENOBUFS)
+ ret = ARRAY_SIZE(iov);
+ else if (ret < 0)
+ return ret;
+
+ iov_iter_init(&iter, ITER_SOURCE, iov, ret, *translated);
+
+ return copy_from_iter(dst, *translated, &iter);
+}
+
+static inline int copy_to_va(const struct vringh *vrh, void *dst, void *src,
+ u64 len, u64 *translated)
+{
+ struct iovec iov[16];
+ struct iov_iter iter;
+ int ret;
+
+ ret = iotlb_translate_va(vrh, (u64)(uintptr_t)dst, len, translated, iov,
+ ARRAY_SIZE(iov), VHOST_MAP_WO);
+ if (ret == -ENOBUFS)
+ ret = ARRAY_SIZE(iov);
+ else if (ret < 0)
+ return ret;
+
+ iov_iter_init(&iter, ITER_DEST, iov, ret, *translated);
+
+ return copy_to_iter(src, *translated, &iter);
+}
+
+static int iotlb_translate_pa(const struct vringh *vrh,
+ u64 addr, u64 len, u64 *translated,
+ struct bio_vec iov[],
+ int iov_size, u32 perm)
+{
+ struct vhost_iotlb_map *map;
+ struct vhost_iotlb *iotlb = vrh->iotlb;
u64 s = 0, last = addr + len - 1;
+ int ret = 0;
spin_lock(vrh->iotlb_lock);
@@ -1141,28 +1225,61 @@ static int iotlb_translate(const struct vringh *vrh,
return ret;
}
+static inline int copy_from_pa(const struct vringh *vrh, void *dst, void *src,
+ u64 len, u64 *translated)
+{
+ struct bio_vec iov[16];
+ struct iov_iter iter;
+ int ret;
+
+ ret = iotlb_translate_pa(vrh, (u64)(uintptr_t)src, len, translated, iov,
+ ARRAY_SIZE(iov), VHOST_MAP_RO);
+ if (ret == -ENOBUFS)
+ ret = ARRAY_SIZE(iov);
+ else if (ret < 0)
+ return ret;
+
+ iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, *translated);
+
+ return copy_from_iter(dst, *translated, &iter);
+}
+
+static inline int copy_to_pa(const struct vringh *vrh, void *dst, void *src,
+ u64 len, u64 *translated)
+{
+ struct bio_vec iov[16];
+ struct iov_iter iter;
+ int ret;
+
+ ret = iotlb_translate_pa(vrh, (u64)(uintptr_t)dst, len, translated, iov,
+ ARRAY_SIZE(iov), VHOST_MAP_WO);
+ if (ret == -ENOBUFS)
+ ret = ARRAY_SIZE(iov);
+ else if (ret < 0)
+ return ret;
+
+ iov_iter_bvec(&iter, ITER_DEST, iov, ret, *translated);
+
+ return copy_to_iter(src, *translated, &iter);
+}
+
static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
void *src, size_t len)
{
u64 total_translated = 0;
while (total_translated < len) {
- struct bio_vec iov[16];
- struct iov_iter iter;
u64 translated;
int ret;
- ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
- len - total_translated, &translated,
- iov, ARRAY_SIZE(iov), VHOST_MAP_RO);
- if (ret == -ENOBUFS)
- ret = ARRAY_SIZE(iov);
- else if (ret < 0)
- return ret;
-
- iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated);
+ if (vrh->use_va) {
+ ret = copy_from_va(vrh, dst, src,
+ len - total_translated, &translated);
+ } else {
+ ret = copy_from_pa(vrh, dst, src,
+ len - total_translated, &translated);
+ }
- ret = copy_from_iter(dst, translated, &iter);
if (ret < 0)
return ret;
@@ -1180,22 +1297,17 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
u64 total_translated = 0;
while (total_translated < len) {
- struct bio_vec iov[16];
- struct iov_iter iter;
u64 translated;
int ret;
- ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
- len - total_translated, &translated,
- iov, ARRAY_SIZE(iov), VHOST_MAP_WO);
- if (ret == -ENOBUFS)
- ret = ARRAY_SIZE(iov);
- else if (ret < 0)
- return ret;
-
- iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated);
+ if (vrh->use_va) {
+ ret = copy_to_va(vrh, dst, src,
+ len - total_translated, &translated);
+ } else {
+ ret = copy_to_pa(vrh, dst, src,
+ len - total_translated, &translated);
+ }
- ret = copy_to_iter(src, translated, &iter);
if (ret < 0)
return ret;
@@ -1210,20 +1322,37 @@ static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
static inline int getu16_iotlb(const struct vringh *vrh,
u16 *val, const __virtio16 *p)
{
- struct bio_vec iov;
- void *kaddr, *from;
int ret;
/* Atomic read is needed for getu16 */
- ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
- &iov, 1, VHOST_MAP_RO);
- if (ret < 0)
- return ret;
+ if (vrh->use_va) {
+ struct iovec iov;
+ __virtio16 tmp;
+
+ ret = iotlb_translate_va(vrh, (u64)(uintptr_t)p, sizeof(*p),
+ NULL, &iov, 1, VHOST_MAP_RO);
+ if (ret < 0)
+ return ret;
- kaddr = kmap_local_page(iov.bv_page);
- from = kaddr + iov.bv_offset;
- *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
- kunmap_local(kaddr);
+ ret = __get_user(tmp, (__virtio16 __user *)iov.iov_base);
+ if (ret)
+ return ret;
+
+ *val = vringh16_to_cpu(vrh, tmp);
+ } else {
+ struct bio_vec iov;
+ void *kaddr, *from;
+
+ ret = iotlb_translate_pa(vrh, (u64)(uintptr_t)p, sizeof(*p),
+ NULL, &iov, 1, VHOST_MAP_RO);
+ if (ret < 0)
+ return ret;
+
+ kaddr = kmap_local_page(iov.bv_page);
+ from = kaddr + iov.bv_offset;
+ *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
+ kunmap_local(kaddr);
+ }
return 0;
}
@@ -1231,20 +1360,37 @@ static inline int getu16_iotlb(const struct vringh *vrh,
static inline int putu16_iotlb(const struct vringh *vrh,
__virtio16 *p, u16 val)
{
- struct bio_vec iov;
- void *kaddr, *to;
int ret;
/* Atomic write is needed for putu16 */
- ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
- &iov, 1, VHOST_MAP_WO);
- if (ret < 0)
- return ret;
+ if (vrh->use_va) {
+ struct iovec iov;
+ __virtio16 tmp;
- kaddr = kmap_local_page(iov.bv_page);
- to = kaddr + iov.bv_offset;
- WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
- kunmap_local(kaddr);
+ ret = iotlb_translate_va(vrh, (u64)(uintptr_t)p, sizeof(*p),
+ NULL, &iov, 1, VHOST_MAP_RO);
+ if (ret < 0)
+ return ret;
+
+ tmp = cpu_to_vringh16(vrh, val);
+
+ ret = __put_user(tmp, (__virtio16 __user *)iov.iov_base);
+ if (ret)
+ return ret;
+ } else {
+ struct bio_vec iov;
+ void *kaddr, *to;
+
+ ret = iotlb_translate_pa(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL,
+ &iov, 1, VHOST_MAP_WO);
+ if (ret < 0)
+ return ret;
+
+ kaddr = kmap_local_page(iov.bv_page);
+ to = kaddr + iov.bv_offset;
+ WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
+ kunmap_local(kaddr);
+ }
return 0;
}
@@ -1306,6 +1452,7 @@ static inline int putused_iotlb(const struct vringh *vrh,
* @features: the feature bits for this ring.
* @num: the number of elements.
* @weak_barriers: true if we only need memory barriers, not I/O.
+ * @use_va: true if IOTLB contains user VA
* @desc: the userpace descriptor pointer.
* @avail: the userpace avail pointer.
* @used: the userpace used pointer.
@@ -1313,11 +1460,13 @@ static inline int putused_iotlb(const struct vringh *vrh,
* Returns an error if num is invalid.
*/
int vringh_init_iotlb(struct vringh *vrh, u64 features,
- unsigned int num, bool weak_barriers,
+ unsigned int num, bool weak_barriers, bool use_va,
struct vring_desc *desc,
struct vring_avail *avail,
struct vring_used *used)
{
+ vrh->use_va = use_va;
+
return vringh_init_kern(vrh, features, num, weak_barriers,
desc, avail, used);
}
--
2.39.2
