[PATCH RFCv2 11/13] iommufd/device: Allow setting IOVAs for MSI(x) vectors

[Nicolin Chen <nicolinc@xxxxxxxxxx>]

Add a resizable array "msi_iovas" to store the user space IOVAs for its
vMSI(x) vectors. And use the corresponding IOVA for set the msi_desc in
the iommufd_sw_msi(). This allows irqchip driver to program the correct
MSI doorbell addresses (in case of requiring IOMMU translations).

Provide a set of helpers to allocate/set/unset the resizable array. The
next patch will call them from the vfio code.

Signed-off-by: Nicolin Chen <nicolinc@xxxxxxxxxx>
---
 drivers/iommu/iommufd/iommufd_private.h |  2 ++
 include/linux/iommufd.h                 |  6 ++++
 drivers/iommu/iommufd/device.c          | 47 ++++++++++++++++++++++++-
 3 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h
index 9f071609f00b..81ff46d630dc 100644
--- a/drivers/iommu/iommufd/iommufd_private.h
+++ b/drivers/iommu/iommufd/iommufd_private.h
@@ -427,6 +427,8 @@ struct iommufd_device {
 	/* protect iopf_enabled counter */
 	struct mutex iopf_lock;
 	unsigned int iopf_enabled;
+	dma_addr_t *msi_iovas;
+	unsigned int num_msi_iovas;
 };
 
 static inline struct iommufd_device *
diff --git a/include/linux/iommufd.h b/include/linux/iommufd.h
index 11110c749200..787dcdcb9b3b 100644
--- a/include/linux/iommufd.h
+++ b/include/linux/iommufd.h
@@ -58,6 +58,12 @@ void iommufd_device_detach(struct iommufd_device *idev);
 
 struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev);
 u32 iommufd_device_to_id(struct iommufd_device *idev);
+int iommufd_device_set_num_msi_iovas(struct iommufd_device *idev,
+				     unsigned int num);
+int iommufd_device_set_msi_iova(struct iommufd_device *idev, unsigned int index,
+				dma_addr_t iova);
+void iommufd_device_unset_msi_iova(struct iommufd_device *idev,
+				   unsigned int index);
 
 struct iommufd_access_ops {
 	u8 needs_pin_pages : 1;
diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c
index 093a3bd798db..3d16967146cc 100644
--- a/drivers/iommu/iommufd/device.c
+++ b/drivers/iommu/iommufd/device.c
@@ -137,6 +137,8 @@ void iommufd_device_destroy(struct iommufd_object *obj)
 	struct iommufd_device *idev =
 		container_of(obj, struct iommufd_device, obj);
 
+	if (idev->num_msi_iovas)
+		kfree(idev->msi_iovas);
 	iommu_device_release_dma_owner(idev->dev);
 	iommufd_put_group(idev->igroup);
 	if (!iommufd_selftest_is_mock_dev(idev->dev))
@@ -294,6 +296,45 @@ u32 iommufd_device_to_id(struct iommufd_device *idev)
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_device_to_id, "IOMMUFD");
 
+int iommufd_device_set_num_msi_iovas(struct iommufd_device *idev,
+				     unsigned int num)
+{
+	dma_addr_t *msi_iovas;
+	int i;
+
+	msi_iovas = krealloc(idev->msi_iovas, sizeof(*idev->msi_iovas) * num,
+			     GFP_KERNEL);
+	if (!msi_iovas)
+		return -ENOMEM;
+
+	for (i = idev->num_msi_iovas; i < num; i++)
+		msi_iovas[i] = PHYS_ADDR_MAX;
+
+	idev->msi_iovas = msi_iovas;
+	idev->num_msi_iovas = num;
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iommufd_device_set_num_msi_iovas, "IOMMUFD");
+
+int iommufd_device_set_msi_iova(struct iommufd_device *idev, unsigned int index,
+				dma_addr_t iova)
+{
+	if (index >= idev->num_msi_iovas)
+		return -EINVAL;
+	idev->msi_iovas[index] = iova;
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(iommufd_device_set_msi_iova, "IOMMUFD");
+
+void iommufd_device_unset_msi_iova(struct iommufd_device *idev,
+				   unsigned int index)
+{
+	if (index >= idev->num_msi_iovas)
+		return;
+	idev->msi_iovas[index] = PHYS_ADDR_MAX;
+}
+EXPORT_SYMBOL_NS_GPL(iommufd_device_unset_msi_iova, "IOMMUFD");
+
 /*
  * Get a iommufd_sw_msi_map for the msi physical address requested by the irq
  * layer. The mapping to IOVA is global to the iommufd file descriptor, every
@@ -403,7 +444,11 @@ int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
 		return rc;
 	set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap);
 
-	iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
+	if (desc->msi_index < handle->idev->num_msi_iovas &&
+	    handle->idev->msi_iovas[desc->msi_index] != PHYS_ADDR_MAX)
+		iova = handle->idev->msi_iovas[desc->msi_index];
+	else
+		iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE;
 	msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT);
 	return 0;
 }
-- 
2.43.0