Add an xarray in iommu_fault_param as place holder for per-{device, pasid}
fault cookie. The iommufd will use it to store the iommufd device pointers.
This allows the iommufd to quickly retrieve the device object ID for a
given {device, pasid} pair in the hot path of I/O page fault delivery.
Otherwise, the iommufd would have to maintain its own data structures to
map {device, pasid} pairs to object IDs, and then look up the object ID on
the critical path. This is not performance friendly.
The iommufd is supposed to set the cookie when a fault capable domain is
attached to the physical device or pasid, and clear the fault cookie when
the domain is removed.
Signed-off-by: Lu Baolu <baolu.lu@xxxxxxxxxxxxxxx>
---
include/linux/iommu.h | 3 +++
drivers/iommu/iommu-priv.h | 15 ++++++++++++
drivers/iommu/io-pgfault.c | 50 ++++++++++++++++++++++++++++++++++++++
3 files changed, 68 insertions(+)
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 2ca3a3eda2e4..615d8a5f9dee 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -608,6 +608,8 @@ struct iommu_device {
* @dev: the device that owns this param
* @queue: IOPF queue
* @queue_list: index into queue->devices
+ * @pasid_cookie: per-pasid fault cookie used by fault message consumers.
+ * This array is self-protected by xa_lock().
* @partial: faults that are part of a Page Request Group for which the last
* request hasn't been submitted yet.
* @faults: holds the pending faults which needs response
@@ -619,6 +621,7 @@ struct iommu_fault_param {
struct device *dev;
struct iopf_queue *queue;
struct list_head queue_list;
+ struct xarray pasid_cookie;
struct list_head partial;
struct list_head faults;
diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
index 2024a2313348..0dc5ad81cbb6 100644
--- a/drivers/iommu/iommu-priv.h
+++ b/drivers/iommu/iommu-priv.h
@@ -27,4 +27,19 @@ void iommu_device_unregister_bus(struct iommu_device *iommu,
struct bus_type *bus,
struct notifier_block *nb);
+#ifdef CONFIG_IOMMU_IOPF
+void *iopf_pasid_cookie_set(struct device *dev, ioasid_t pasid, void *cookie);
+void *iopf_pasid_cookie_get(struct device *dev, ioasid_t pasid);
+#else
+static inline void *iopf_pasid_cookie_set(struct device *dev, ioasid_t pasid, void *cookie)
+{
+ return ERR_PTR(-ENODEV);
+}
+
+static inline void *iopf_pasid_cookie_get(struct device *dev, ioasid_t pasid)
+{
+ return ERR_PTR(-ENODEV);
+}
+#endif /* CONFIG_IOMMU_IOPF */
+
#endif /* __LINUX_IOMMU_PRIV_H */
diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c
index b288c73f2b22..6fa029538deb 100644
--- a/drivers/iommu/io-pgfault.c
+++ b/drivers/iommu/io-pgfault.c
@@ -454,6 +454,7 @@ int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev)
mutex_init(&fault_param->lock);
INIT_LIST_HEAD(&fault_param->faults);
INIT_LIST_HEAD(&fault_param->partial);
+ xa_init(&fault_param->pasid_cookie);
fault_param->dev = dev;
fault_param->users = 1;
list_add(&fault_param->queue_list, &queue->devices);
@@ -575,3 +576,52 @@ void iopf_queue_free(struct iopf_queue *queue)
kfree(queue);
}
EXPORT_SYMBOL_GPL(iopf_queue_free);
+
+/**
+ * iopf_pasid_cookie_set - Set a fault cookie for per-{device, pasid}
+ * @dev: the device to set the cookie
+ * @pasid: the pasid on this device
+ * @cookie: the opaque data
+ *
+ * Return the old cookie on success, or ERR_PTR on failure.
+ */
+void *iopf_pasid_cookie_set(struct device *dev, ioasid_t pasid, void *cookie)
+{
+ struct iommu_fault_param *iopf_param = iopf_get_dev_fault_param(dev);
+ void *curr;
+
+ if (!iopf_param)
+ return ERR_PTR(-ENODEV);
+
+ curr = xa_store(&iopf_param->pasid_cookie, pasid, cookie, GFP_KERNEL);
+ iopf_put_dev_fault_param(iopf_param);
+
+ return xa_is_err(curr) ? ERR_PTR(xa_err(curr)) : curr;
+}
+EXPORT_SYMBOL_NS_GPL(iopf_pasid_cookie_set, IOMMUFD_INTERNAL);
+
+/**
+ * iopf_pasid_cookie_get - Get the fault cookie for {device, pasid}
+ * @dev: the device where the cookie was set
+ * @pasid: the pasid on this device
+ *
+ * Return the cookie on success, or ERR_PTR on failure. Note that NULL is
+ * also a successful return.
+ */
+void *iopf_pasid_cookie_get(struct device *dev, ioasid_t pasid)
+{
+ struct iommu_fault_param *iopf_param = iopf_get_dev_fault_param(dev);
+ void *curr;
+
+ if (!iopf_param)
+ return ERR_PTR(-ENODEV);
+
+ xa_lock(&iopf_param->pasid_cookie);
+ curr = xa_load(&iopf_param->pasid_cookie, pasid);
+ xa_unlock(&iopf_param->pasid_cookie);
+
+ iopf_put_dev_fault_param(iopf_param);
+
+ return curr;
+}
+EXPORT_SYMBOL_NS_GPL(iopf_pasid_cookie_get, IOMMUFD_INTERNAL);
--
2.34.1
