Signed-off-by: Fenghua Yu <fenghua.yu@xxxxxxxxx>
Reviewed-by: Tony Luck <tony.luck@xxxxxxxxx>
---
v2:
- Define a helper free_bind() to simplify error exit code in bind_mm()
(Thomas)
- Fix a ret error code in bind_mm() (Thomas)
- Change pasid's type from "int" to "unsigned int" to have consistent
pasid type in iommu (Thomas)
- Simplify alloc_pasid() a bit.
arch/x86/include/asm/iommu.h | 2 +
arch/x86/include/asm/mmu_context.h | 14 ++++
drivers/iommu/intel/svm.c | 101 +++++++++++++++++++++++++----
3 files changed, 105 insertions(+), 12 deletions(-)
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index bf1ed2ddc74b..ed41259fe7ac 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -26,4 +26,6 @@ arch_rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
return -EINVAL;
}
+void __free_pasid(struct mm_struct *mm);
+
#endif /* _ASM_X86_IOMMU_H */
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 47562147e70b..f8c91ce8c451 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -13,6 +13,7 @@
#include <asm/tlbflush.h>
#include <asm/paravirt.h>
#include <asm/debugreg.h>
+#include <asm/iommu.h>
extern atomic64_t last_mm_ctx_id;
@@ -117,9 +118,22 @@ static inline int init_new_context(struct task_struct *tsk,
init_new_context_ldt(mm);
return 0;
}
+
+static inline void free_pasid(struct mm_struct *mm)
+{
+ if (!IS_ENABLED(CONFIG_INTEL_IOMMU_SVM))
+ return;
+
+ if (!cpu_feature_enabled(X86_FEATURE_ENQCMD))
+ return;
+
+ __free_pasid(mm);
+}
+
static inline void destroy_context(struct mm_struct *mm)
{
destroy_context_ldt(mm);
+ free_pasid(mm);
}
extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index 4e775e12ae52..27dc866b8461 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -425,6 +425,53 @@ int intel_svm_unbind_gpasid(struct device *dev, unsigned int pasid)
return ret;
}
+static void free_bind(struct intel_svm *svm, struct intel_svm_dev *sdev,
+ bool new_pasid)
+{
+ if (new_pasid)
+ ioasid_free(svm->pasid);
+ kfree(svm);
+ kfree(sdev);
+}
+
+/*
+ * If this mm already has a PASID, use it. Otherwise allocate a new one.
+ * Let the caller know if a new PASID is allocated via 'new_pasid'.
+ */
+static int alloc_pasid(struct intel_svm *svm, struct mm_struct *mm,
+ unsigned int pasid_max, bool *new_pasid,
+ unsigned int flags)
+{
+ unsigned int pasid;
+
+ *new_pasid = false;
+
+ /*
+ * Reuse the PASID if the mm already has a PASID and not a private
+ * PASID is requested.
+ */
+ if (mm && mm->pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
+ /*
+ * Once a PASID is allocated for this mm, the PASID
+ * stays with the mm until the mm is dropped. Reuse
+ * the PASID which has been already allocated for the
+ * mm instead of allocating a new one.
+ */
+ ioasid_set_data(mm->pasid, svm);
+
+ return mm->pasid;
+ }
+
+ /* Allocate a new pasid. Do not use PASID 0, reserved for init PASID. */
+ pasid = ioasid_alloc(NULL, PASID_MIN, pasid_max - 1, svm);
+ if (pasid != INVALID_IOASID) {
+ /* A new pasid is allocated. */
+ *new_pasid = true;
+ }
+
+ return pasid;
+}
+
/* Caller must hold pasid_mutex, mm reference */
static int
intel_svm_bind_mm(struct device *dev, unsigned int flags,
@@ -518,6 +565,8 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,
init_rcu_head(&sdev->rcu);
if (!svm) {
+ bool new_pasid;
+
svm = kzalloc(sizeof(*svm), GFP_KERNEL);
if (!svm) {
ret = -ENOMEM;
@@ -529,12 +578,9 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,
if (pasid_max > intel_pasid_max_id)
pasid_max = intel_pasid_max_id;
- /* Do not use PASID 0, reserved for RID to PASID */
- svm->pasid = ioasid_alloc(NULL, PASID_MIN,
- pasid_max - 1, svm);
+ svm->pasid = alloc_pasid(svm, mm, pasid_max, &new_pasid, flags);
if (svm->pasid == INVALID_IOASID) {
- kfree(svm);
- kfree(sdev);
+ free_bind(svm, sdev, new_pasid);
ret = -ENOSPC;
goto out;
}
@@ -547,9 +593,7 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,
if (mm) {
ret = mmu_notifier_register(&svm->notifier, mm);
if (ret) {
- ioasid_free(svm->pasid);
- kfree(svm);
- kfree(sdev);
+ free_bind(svm, sdev, new_pasid);
goto out;
}
}
@@ -565,12 +609,18 @@ intel_svm_bind_mm(struct device *dev, unsigned int flags,
if (ret) {
if (mm)
mmu_notifier_unregister(&svm->notifier, mm);
- ioasid_free(svm->pasid);
- kfree(svm);
- kfree(sdev);
+ free_bind(svm, sdev, new_pasid);
goto out;
}
+ if (mm && new_pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
+ /*
+ * Track the new pasid in the mm. The pasid will be
+ * freed at process exit. Don't track requested
+ * private PASID in the mm.
+ */
+ mm->pasid = svm->pasid;
+ }
list_add_tail(&svm->list, &global_svm_list);
} else {
/*
@@ -640,7 +690,8 @@ static int intel_svm_unbind_mm(struct device *dev, unsigned int pasid)
kfree_rcu(sdev, rcu);
if (list_empty(&svm->devs)) {
- ioasid_free(svm->pasid);
+ /* Clear data in the pasid. */
+ ioasid_set_data(pasid, NULL);
if (svm->mm)
mmu_notifier_unregister(&svm->notifier, svm->mm);
list_del(&svm->list);
@@ -1001,3 +1052,29 @@ unsigned int intel_svm_get_pasid(struct iommu_sva *sva)
return pasid;
}
+
+/*
+ * An invalid pasid is either 0 (init PASID value) or bigger than max PASID
+ * (PASID_MAX - 1).
+ */
+static bool invalid_pasid(unsigned int pasid)
+{
+ return (pasid == INIT_PASID) || (pasid >= PASID_MAX);
+}
+
+/* On process exit free the PASID (if one was allocated). */
+void __free_pasid(struct mm_struct *mm)
+{
+ unsigned int pasid = mm->pasid;
+
+ /* No need to free invalid pasid. */
+ if (invalid_pasid(pasid))
+ return;
+
+ /*
+ * Since the pasid is not bound to any svm by now, there is no race
+ * here with binding/unbinding and no need to protect the free
+ * operation by pasid_mutex.
+ */
+ ioasid_free(pasid);
+}
