Hi Jason,
On 11/16/22 22:00, Jason Gunthorpe wrote:
> Kernel access is the mode that VFIO "mdevs" use. In this case there is no
> struct device and no IOMMU connection. iommufd acts as a record keeper for
> accesses and returns the actual struct pages back to the caller to use
> however they need. eg with kmap or the DMA API.
>
> Each caller must create a struct iommufd_access with
> iommufd_access_create(), similar to how iommufd_device_bind() works. Using
> this struct the caller can access blocks of IOVA using
> iommufd_access_pin_pages() or iommufd_access_rw().
>
> Callers must provide a callback that immediately unpins any IOVA being
> used within a range. This happens if userspace unmaps the IOVA under the
> pin.
>
> The implementation forwards the access requests directly to the iopt
> infrastructure that manages the iopt_pages_access.
>
> Tested-by: Nicolin Chen <nicolinc@xxxxxxxxxx>
> Tested-by: Yi Liu <yi.l.liu@xxxxxxxxx>
> Tested-by: Lixiao Yang <lixiao.yang@xxxxxxxxx>
> Tested-by: Matthew Rosato <mjrosato@xxxxxxxxxxxxx>
> Reviewed-by: Kevin Tian <kevin.tian@xxxxxxxxx>
> Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
> ---
> drivers/iommu/iommufd/device.c | 314 ++++++++++++++++++++++++
> drivers/iommu/iommufd/io_pagetable.c | 8 +-
> drivers/iommu/iommufd/iommufd_private.h | 10 +
> drivers/iommu/iommufd/main.c | 3 +
> include/linux/iommufd.h | 43 +++-
> 5 files changed, 375 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
> index a71f5740773f84..522469ae7b5770 100644
> --- a/drivers/iommu/iommufd/device.c
> +++ b/drivers/iommu/iommufd/device.c
> @@ -6,6 +6,7 @@
> #include <linux/iommu.h>
> #include <linux/irqdomain.h>
>
> +#include "io_pagetable.h"
> #include "iommufd_private.h"
>
> /*
> @@ -415,3 +416,316 @@ void iommufd_device_detach(struct iommufd_device *idev)
> refcount_dec(&idev->obj.users);
> }
> EXPORT_SYMBOL_NS_GPL(iommufd_device_detach, IOMMUFD);
> +
> +void iommufd_access_destroy_object(struct iommufd_object *obj)
> +{
> + struct iommufd_access *access =
> + container_of(obj, struct iommufd_access, obj);
> +
> + iopt_remove_access(&access->ioas->iopt, access);
> + iommufd_ctx_put(access->ictx);
> + refcount_dec(&access->ioas->obj.users);
> +}
> +
> +/**
> + * iommufd_access_create - Create an iommufd_access
> + * @ictx: iommufd file descriptor
> + * @ioas_id: ID for a IOMMUFD_OBJ_IOAS
> + * @ops: Driver's ops to associate with the access
> + * @data: Opaque data to pass into ops functions
> + *
> + * An iommufd_access allows a driver to read/write to the IOAS without using
> + * DMA. The underlying CPU memory can be accessed using the
> + * iommufd_access_pin_pages() or iommufd_access_rw() functions.
> + *
> + * The provided ops are required to use iommufd_access_pin_pages().
> + */
> +struct iommufd_access *
> +iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
> + const struct iommufd_access_ops *ops, void *data)
> +{
> + struct iommufd_access *access;
> + struct iommufd_object *obj;
> + int rc;
> +
> + /*
> + * There is no uAPI for the access object, but to keep things symmetric
> + * use the object infrastructure anyhow.
> + */
> + access = iommufd_object_alloc(ictx, access, IOMMUFD_OBJ_ACCESS);
> + if (IS_ERR(access))
> + return access;
> +
> + access->data = data;
> + access->ops = ops;
> +
> + obj = iommufd_get_object(ictx, ioas_id, IOMMUFD_OBJ_IOAS);
> + if (IS_ERR(obj)) {
> + rc = PTR_ERR(obj);
> + goto out_abort;
> + }
> + access->ioas = container_of(obj, struct iommufd_ioas, obj);
> + iommufd_ref_to_users(obj);
> +
> + if (ops->needs_pin_pages)
> + access->iova_alignment = PAGE_SIZE;
> + else
> + access->iova_alignment = 1;
> + rc = iopt_add_access(&access->ioas->iopt, access);
> + if (rc)
> + goto out_put_ioas;
> +
> + /* The calling driver is a user until iommufd_access_destroy() */
> + refcount_inc(&access->obj.users);
> + access->ictx = ictx;
> + iommufd_ctx_get(ictx);
> + iommufd_object_finalize(ictx, &access->obj);
> + return access;
> +out_put_ioas:
> + refcount_dec(&access->ioas->obj.users);
> +out_abort:
> + iommufd_object_abort(ictx, &access->obj);
> + return ERR_PTR(rc);
> +}
> +EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD);
> +
> +/**
> + * iommufd_access_destroy - Destroy an iommufd_access
> + * @access: The access to destroy
> + *
> + * The caller must stop using the access before destroying it.
> + */
> +void iommufd_access_destroy(struct iommufd_access *access)
> +{
> + bool was_destroyed;
> +
> + was_destroyed = iommufd_object_destroy_user(access->ictx, &access->obj);
> + WARN_ON(!was_destroyed);
> +}
> +EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD);
> +
> +/**
> + * iommufd_access_notify_unmap - Notify users of an iopt to stop using it
> + * @iopt: iopt to work on
> + * @iova: Starting iova in the iopt
> + * @length: Number of bytes
> + *
> + * After this function returns there should be no users attached to the pages
> + * linked to this iopt that intersect with iova,length. Anyone that has attached
> + * a user through iopt_access_pages() needs to detatch it through
detach
> + * iommufd_access_unpin_pages() before this function returns.
> + *
> + * The unmap callback may not call or wait for a iommufd_access_destroy() to
> + * complete. Once iommufd_access_destroy() returns no ops are running and no
> + * future ops will be called.
I don't understand the above sentence. Is that related to the
+ if (!iommufd_lock_obj(&access->obj))
+ continue;
where is the unmap() called in that case?
> + */
> +void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
> + unsigned long length)
> +{
> + struct iommufd_ioas *ioas =
> + container_of(iopt, struct iommufd_ioas, iopt);
> + struct iommufd_access *access;
> + unsigned long index;
> +
> + xa_lock(&ioas->iopt.access_list);
> + xa_for_each(&ioas->iopt.access_list, index, access) {
> + if (!iommufd_lock_obj(&access->obj))
> + continue;
> + xa_unlock(&ioas->iopt.access_list);
> +
> + access->ops->unmap(access->data, iova, length);
> +
> + iommufd_put_object(&access->obj);
> + xa_lock(&ioas->iopt.access_list);
> + }
> + xa_unlock(&ioas->iopt.access_list);
> +}
> +
> +/**
> + * iommufd_access_unpin_pages() - Undo iommufd_access_pin_pages
> + * @access: IOAS access to act on
> + * @iova: Starting IOVA
> + * @length:- Number of bytes to access
s/-//
> + *
> + * Return the struct page's. The caller must stop accessing them before calling
> + * this. The iova/length must exactly match the one provided to access_pages.
> + */
> +void iommufd_access_unpin_pages(struct iommufd_access *access,
> + unsigned long iova, unsigned long length)
> +{
> + struct io_pagetable *iopt = &access->ioas->iopt;
> + struct iopt_area_contig_iter iter;
> + unsigned long last_iova;
> + struct iopt_area *area;
> +
> + if (WARN_ON(!length) ||
> + WARN_ON(check_add_overflow(iova, length - 1, &last_iova)))
> + return;
> +
> + down_read(&iopt->iova_rwsem);
> + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
> + iopt_area_remove_access(
> + area, iopt_area_iova_to_index(area, iter.cur_iova),
> + iopt_area_iova_to_index(
> + area,
> + min(last_iova, iopt_area_last_iova(area))));
> + up_read(&iopt->iova_rwsem);
> + WARN_ON(!iopt_area_contig_done(&iter));
> +}
> +EXPORT_SYMBOL_NS_GPL(iommufd_access_unpin_pages, IOMMUFD);
> +
> +static bool iopt_area_contig_is_aligned(struct iopt_area_contig_iter *iter)
> +{
> + if (iopt_area_start_byte(iter->area, iter->cur_iova) % PAGE_SIZE)
> + return false;
> +
> + if (!iopt_area_contig_done(iter) &&
> + (iopt_area_start_byte(iter->area, iopt_area_last_iova(iter->area)) %
> + PAGE_SIZE) != (PAGE_SIZE - 1))
> + return false;
> + return true;
> +}
> +
> +static bool check_area_prot(struct iopt_area *area, unsigned int flags)
> +{
> + if (flags & IOMMUFD_ACCESS_RW_WRITE)
> + return area->iommu_prot & IOMMU_WRITE;
> + return area->iommu_prot & IOMMU_READ;
> +}
> +
> +/**
> + * iommufd_access_pin_pages() - Return a list of pages under the iova
> + * @access: IOAS access to act on
> + * @iova: Starting IOVA
> + * @length: Number of bytes to access
> + * @out_pages: Output page list
> + * @flags: IOPMMUFD_ACCESS_RW_* flags
> + *
> + * Reads @length bytes starting at iova and returns the struct page * pointers.
> + * These can be kmap'd by the caller for CPU access.
> + *
> + * The caller must perform iopt_unaccess_pages() when done to balance this.
this function does not exist
> + *
> + * This API always requires a page aligned iova. This happens naturally if the
> + * ioas alignment is >= PAGE_SIZE and the iova is PAGE_SIZE aligned. However
> + * smaller alignments have corner cases where this API can fail on otherwise
> + * aligned iova.
> + */
> +int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
> + unsigned long length, struct page **out_pages,
> + unsigned int flags)
> +{
> + struct io_pagetable *iopt = &access->ioas->iopt;
> + struct iopt_area_contig_iter iter;
> + unsigned long last_iova;
> + struct iopt_area *area;
> + int rc;
> +
> + if (!length)
> + return -EINVAL;
> + if (check_add_overflow(iova, length - 1, &last_iova))
> + return -EOVERFLOW;
> +
> + down_read(&iopt->iova_rwsem);
> + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
> + unsigned long last = min(last_iova, iopt_area_last_iova(area));
> + unsigned long last_index = iopt_area_iova_to_index(area, last);
> + unsigned long index =
> + iopt_area_iova_to_index(area, iter.cur_iova);
> +
> + if (area->prevent_access ||
> + !iopt_area_contig_is_aligned(&iter)) {
> + rc = -EINVAL;
> + goto err_remove;
> + }
> +
> + if (!check_area_prot(area, flags)) {
> + rc = -EPERM;
> + goto err_remove;
> + }
> +
> + rc = iopt_area_add_access(area, index, last_index, out_pages,
> + flags);
> + if (rc)
> + goto err_remove;
> + out_pages += last_index - index + 1;
> + }
> + if (!iopt_area_contig_done(&iter)) {
> + rc = -ENOENT;
> + goto err_remove;
> + }
> +
> + up_read(&iopt->iova_rwsem);
> + return 0;
> +
> +err_remove:
> + if (iova < iter.cur_iova) {
> + last_iova = iter.cur_iova - 1;
> + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova)
> + iopt_area_remove_access(
> + area,
> + iopt_area_iova_to_index(area, iter.cur_iova),
> + iopt_area_iova_to_index(
> + area, min(last_iova,
> + iopt_area_last_iova(area))));
> + }
> + up_read(&iopt->iova_rwsem);
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(iommufd_access_pin_pages, IOMMUFD);
> +
> +/**
> + * iommufd_access_rw - Read or write data under the iova
> + * @access: IOAS access to act on
> + * @iova: Starting IOVA
> + * @data: Kernel buffer to copy to/from
> + * @length: Number of bytes to access
> + * @flags: IOMMUFD_ACCESS_RW_* flags
> + *
> + * Copy kernel to/from data into the range given by IOVA/length. If flags
> + * indicates IOMMUFD_ACCESS_RW_KTHREAD then a large copy can be optimized
> + * by changing it into copy_to/from_user().
> + */
> +int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
> + void *data, size_t length, unsigned int flags)
> +{
> + struct io_pagetable *iopt = &access->ioas->iopt;
> + struct iopt_area_contig_iter iter;
> + struct iopt_area *area;
> + unsigned long last_iova;
> + int rc;
> +
> + if (!length)
> + return -EINVAL;
> + if (check_add_overflow(iova, length - 1, &last_iova))
> + return -EOVERFLOW;
> +
> + down_read(&iopt->iova_rwsem);
> + iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
> + unsigned long last = min(last_iova, iopt_area_last_iova(area));
> + unsigned long bytes = (last - iter.cur_iova) + 1;
> +
> + if (area->prevent_access) {
> + rc = -EINVAL;
> + goto err_out;
> + }
> +
> + if (!check_area_prot(area, flags)) {
> + rc = -EPERM;
> + goto err_out;
> + }
> +
> + rc = iopt_pages_rw_access(
> + area->pages, iopt_area_start_byte(area, iter.cur_iova),
> + data, bytes, flags);
> + if (rc)
> + goto err_out;
> + data += bytes;
> + }
> + if (!iopt_area_contig_done(&iter))
> + rc = -ENOENT;
> +err_out:
> + up_read(&iopt->iova_rwsem);
> + return rc;
> +}
> +EXPORT_SYMBOL_NS_GPL(iommufd_access_rw, IOMMUFD);
> diff --git a/drivers/iommu/iommufd/io_pagetable.c b/drivers/iommu/iommufd/io_pagetable.c
> index 756d347948f0ec..4f4a9d9aac570e 100644
> --- a/drivers/iommu/iommufd/io_pagetable.c
> +++ b/drivers/iommu/iommufd/io_pagetable.c
> @@ -458,6 +458,7 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,
> * is NULL. This prevents domain attach/detatch from running
> * concurrently with cleaning up the area.
> */
> +again:
> down_read(&iopt->domains_rwsem);
> down_write(&iopt->iova_rwsem);
> while ((area = iopt_area_iter_first(iopt, start, last))) {
> @@ -486,8 +487,11 @@ static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,
> area->prevent_access = true;
> up_write(&iopt->iova_rwsem);
> up_read(&iopt->domains_rwsem);
> - /* Later patch calls back to drivers to unmap */
> - return -EBUSY;
> + iommufd_access_notify_unmap(iopt, area_first,
> + iopt_area_length(area));
> + if (WARN_ON(READ_ONCE(area->num_accesses)))
> + return -EDEADLOCK;
> + goto again;
> }
>
> pages = area->pages;
> diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
> index 73345886d969e5..e1653b2276dac9 100644
> --- a/drivers/iommu/iommufd/iommufd_private.h
> +++ b/drivers/iommu/iommufd/iommufd_private.h
> @@ -65,6 +65,8 @@ int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
> unsigned long length, unsigned long *unmapped);
> int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped);
>
> +void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova,
> + unsigned long length);
> int iopt_table_add_domain(struct io_pagetable *iopt,
> struct iommu_domain *domain);
> void iopt_table_remove_domain(struct io_pagetable *iopt,
> @@ -106,6 +108,7 @@ enum iommufd_object_type {
> IOMMUFD_OBJ_DEVICE,
> IOMMUFD_OBJ_HW_PAGETABLE,
> IOMMUFD_OBJ_IOAS,
> + IOMMUFD_OBJ_ACCESS,
> };
>
> /* Base struct for all objects with a userspace ID handle. */
> @@ -246,6 +249,11 @@ void iommufd_hw_pagetable_destroy(struct iommufd_object *obj);
> void iommufd_device_destroy(struct iommufd_object *obj);
>
> struct iommufd_access {
> + struct iommufd_object obj;
> + struct iommufd_ctx *ictx;
> + struct iommufd_ioas *ioas;
> + const struct iommufd_access_ops *ops;
> + void *data;
> unsigned long iova_alignment;
> u32 iopt_access_list_id;
> };
> @@ -253,4 +261,6 @@ struct iommufd_access {
> int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access);
> void iopt_remove_access(struct io_pagetable *iopt,
> struct iommufd_access *access);
> +void iommufd_access_destroy_object(struct iommufd_object *obj);
> +
> #endif
> diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c
> index 8a114ddbdfcde2..c8cc0953dea13a 100644
> --- a/drivers/iommu/iommufd/main.c
> +++ b/drivers/iommu/iommufd/main.c
> @@ -352,6 +352,9 @@ void iommufd_ctx_put(struct iommufd_ctx *ictx)
> EXPORT_SYMBOL_NS_GPL(iommufd_ctx_put, IOMMUFD);
>
> static const struct iommufd_object_ops iommufd_object_ops[] = {
> + [IOMMUFD_OBJ_ACCESS] = {
> + .destroy = iommufd_access_destroy_object,
> + },
> [IOMMUFD_OBJ_DEVICE] = {
> .destroy = iommufd_device_destroy,
> },
> diff --git a/include/linux/iommufd.h b/include/linux/iommufd.h
> index 31efacd8a46cce..fb9a4c275cca86 100644
> --- a/include/linux/iommufd.h
> +++ b/include/linux/iommufd.h
> @@ -9,10 +9,12 @@
> #include <linux/types.h>
> #include <linux/errno.h>
> #include <linux/err.h>
> -#include <linux/device.h>
>
> +struct device;
> struct iommufd_device;
> +struct page;
> struct iommufd_ctx;
> +struct iommufd_access;
> struct file;
>
> struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx,
> @@ -26,6 +28,11 @@ int iommufd_device_attach(struct iommufd_device *idev, u32 *pt_id,
> unsigned int flags);
> void iommufd_device_detach(struct iommufd_device *idev);
>
> +struct iommufd_access_ops {
> + u8 needs_pin_pages : 1;
> + void (*unmap)(void *data, unsigned long iova, unsigned long length);
> +};
> +
> enum {
> IOMMUFD_ACCESS_RW_READ = 0,
> IOMMUFD_ACCESS_RW_WRITE = 1 << 0,
> @@ -33,11 +40,24 @@ enum {
> IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1,
> };
>
> +struct iommufd_access *
> +iommufd_access_create(struct iommufd_ctx *ictx, u32 ioas_id,
> + const struct iommufd_access_ops *ops, void *data);
> +void iommufd_access_destroy(struct iommufd_access *access);
> +
> void iommufd_ctx_get(struct iommufd_ctx *ictx);
>
> #if IS_ENABLED(CONFIG_IOMMUFD)
> struct iommufd_ctx *iommufd_ctx_from_file(struct file *file);
> void iommufd_ctx_put(struct iommufd_ctx *ictx);
> +
> +int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova,
> + unsigned long length, struct page **out_pages,
> + unsigned int flags);
> +void iommufd_access_unpin_pages(struct iommufd_access *access,
> + unsigned long iova, unsigned long length);
> +int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
> + void *data, size_t len, unsigned int flags);
> #else /* !CONFIG_IOMMUFD */
> static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
> {
> @@ -47,5 +67,26 @@ static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file)
> static inline void iommufd_ctx_put(struct iommufd_ctx *ictx)
> {
> }
> +
> +static inline int iommufd_access_pin_pages(struct iommufd_access *access,
> + unsigned long iova,
> + unsigned long length,
> + struct page **out_pages,
> + unsigned int flags)
> +{
> + return -EOPNOTSUPP;
> +}
> +
> +static inline void iommufd_access_unpin_pages(struct iommufd_access *access,
> + unsigned long iova,
> + unsigned long length)
> +{
> +}
> +
> +static inline int iommufd_access_rw(struct iommufd_access *access, unsigned long iova,
> + void *data, size_t len, unsigned int flags)
> +{
> + return -EOPNOTSUPP;
> +}
> #endif /* CONFIG_IOMMUFD */
> #endif
Eric
