Devices behind different IOMMUs can be bound to one sva domain. When a
range of a sva domain address is being invalidated, vt-d driver needs to
issue IOMMU IOTLB and Dev-IOTLB invalidation commands to ask IOMMU
hardware and related devices to invalidate their caches.
The current logic issues both IOTLB invalidation command and device-TLB
command per device, which leads to superfluous IOTLB invalidation (e.g.,
if there are four devices behind a IOMMU are attached to one sva domain.
In the current logic, during handing intel_invalidate_range(), four IOTLB
invalidation commands and four Dev-IOTLB invalidation commands will be
issued. However, only one IOTLB invalidation command and four Dev-IOTLB
invalidation command are necessary.), and therefore impacts run-time
performance.
The patch removes the redundant IOMMU IOTLB invalidations by allowing
issuing IOMMU IOTLB invalidation command per iommu instead of per device.
Signed-off-by: Tina Zhang <tina.zhang@xxxxxxxxx>
---
drivers/iommu/intel/svm.c | 56 +++++++++++++++++++--------------------
1 file changed, 27 insertions(+), 29 deletions(-)
diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index c9a703935908..f684b92a1241 100644
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -135,32 +135,41 @@ void intel_svm_check(struct intel_iommu *iommu)
iommu->flags |= VTD_FLAG_SVM_CAPABLE;
}
-static void __flush_svm_range_dev(struct dmar_domain *domain,
- struct dev_pasid_info *dev_pasid,
+static void __flush_svm_range(struct iommu_domain *domain,
unsigned long address,
unsigned long pages, int ih)
{
- struct device_domain_info *info = dev_iommu_priv_get(dev_pasid->dev);
- struct intel_iommu *iommu = dev_to_intel_iommu(dev_pasid->dev);
- u32 pasid = mm_get_enqcmd_pasid(domain->domain.mm);
+ u32 pasid = mm_get_enqcmd_pasid(domain->mm);
+ struct device_domain_info *dev_info;
+ struct iommu_domain_info *iommu_info;
+ struct dev_pasid_info *dev_pasid;
+ struct intel_iommu *iommu;
+ unsigned long idx;
if (WARN_ON(!pages))
return;
- qi_flush_piotlb(iommu, FLPT_DEFAULT_DID, pasid, address, pages, ih);
- if (info->ats_enabled) {
- qi_flush_dev_iotlb_pasid(iommu, dev_pasid->sid, info->pfsid,
- pasid, dev_pasid->qdep, address,
- order_base_2(pages));
- quirk_extra_dev_tlb_flush(info, address, order_base_2(pages),
- pasid, dev_pasid->qdep);
+ rcu_read_lock();
+ xa_for_each(&to_dmar_domain(domain)->iommu_array, idx, iommu_info)
+ qi_flush_piotlb(iommu_info->iommu, FLPT_DEFAULT_DID,
+ pasid, address, pages, ih);
+
+ list_for_each_entry_rcu(dev_pasid, &to_dmar_domain(domain)->dev_pasids, link_domain) {
+ dev_info = dev_iommu_priv_get(dev_pasid->dev);
+ iommu = dev_to_intel_iommu(dev_pasid->dev);
+ if (dev_info->ats_enabled) {
+ qi_flush_dev_iotlb_pasid(iommu, dev_pasid->sid, dev_info->pfsid,
+ pasid, dev_pasid->qdep, address,
+ order_base_2(pages));
+ quirk_extra_dev_tlb_flush(dev_info, address, order_base_2(pages),
+ pasid, dev_pasid->qdep);
+ }
}
+ rcu_read_unlock();
}
-static void intel_flush_svm_range_dev(struct dmar_domain *domain,
- struct dev_pasid_info *dev_pasid,
- unsigned long address,
- unsigned long pages, int ih)
+static void intel_flush_svm_range(struct iommu_domain *domain, unsigned long address,
+ unsigned long pages, int ih)
{
unsigned long shift = ilog2(__roundup_pow_of_two(pages));
unsigned long align = (1ULL << (VTD_PAGE_SHIFT + shift));
@@ -168,22 +177,11 @@ static void intel_flush_svm_range_dev(struct dmar_domain *domain,
unsigned long end = ALIGN(address + (pages << VTD_PAGE_SHIFT), align);
while (start < end) {
- __flush_svm_range_dev(domain, dev_pasid, start, align >> VTD_PAGE_SHIFT, ih);
+ __flush_svm_range(domain, start, align >> VTD_PAGE_SHIFT, ih);
start += align;
}
}
-static void intel_flush_svm_range(struct dmar_domain *domain, unsigned long address,
- unsigned long pages, int ih)
-{
- struct dev_pasid_info *dev_pasid;
-
- rcu_read_lock();
- list_for_each_entry_rcu(dev_pasid, &domain->dev_pasids, link_domain)
- intel_flush_svm_range_dev(domain, dev_pasid, address, pages, ih);
- rcu_read_unlock();
-}
-
/* Pages have been freed at this point */
static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
struct mm_struct *mm,
@@ -191,7 +189,7 @@ static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
{
struct iommu_domain *domain = container_of(mn, struct iommu_domain, notifier);
- intel_flush_svm_range(to_dmar_domain(domain), start,
+ intel_flush_svm_range(domain, start,
(end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0);
}
--
2.39.3
