When page tables become empty, add them to the freelist so that they
can also be freed.
This is means that a page tables that are outside of the imediat iova
range might be freed as well, therefore, only in the case where such
page tables are going to be freed, we take the writer lock.
Signed-off-by: Pasha Tatashin <pasha.tatashin@xxxxxxxxxx>
---
drivers/iommu/intel/iommu.c | 91 +++++++++++++++++++++++++++++++------
1 file changed, 77 insertions(+), 14 deletions(-)
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 8c7e596728b5..2dedcd4f6060 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1044,7 +1044,7 @@ static void dma_pte_list_pagetables(struct dmar_domain *domain,
static void dma_pte_clear_level(struct dmar_domain *domain, int level,
struct dma_pte *pte, unsigned long pfn,
unsigned long start_pfn, unsigned long last_pfn,
- struct list_head *freelist)
+ struct list_head *freelist, int *freed_level)
{
struct dma_pte *first_pte = NULL, *last_pte = NULL;
@@ -1070,11 +1070,47 @@ static void dma_pte_clear_level(struct dmar_domain *domain, int level,
first_pte = pte;
last_pte = pte;
} else if (level > 1) {
+ struct dma_pte *npte = phys_to_virt(dma_pte_addr(pte));
+ struct page *npage = virt_to_page(npte);
+
/* Recurse down into a level that isn't *entirely* obsolete */
- dma_pte_clear_level(domain, level - 1,
- phys_to_virt(dma_pte_addr(pte)),
+ dma_pte_clear_level(domain, level - 1, npte,
level_pfn, start_pfn, last_pfn,
- freelist);
+ freelist, freed_level);
+
+ /*
+ * Free next level page table if it became empty.
+ *
+ * We only holding the reader lock, and it is possible
+ * that other threads are accessing page table as
+ * readers as well. We can only free page table that
+ * is outside of the request IOVA space only if
+ * we grab the writer lock. Since we need to drop reader
+ * lock, we are incrementing the mapcount in the npage
+ * so it (and the current page table) does not
+ * dissappear due to concurrent unmapping threads.
+ *
+ * Store the size maximum size of the freed page table
+ * into freed_level, so the size of the IOTLB flush
+ * can be determined.
+ */
+ if (freed_level && !atomic_read(&npage->_mapcount)) {
+ atomic_inc(&npage->_mapcount);
+ read_unlock(&domain->pgd_lock);
+ write_lock(&domain->pgd_lock);
+ atomic_dec(&npage->_mapcount);
+ if (!atomic_read(&npage->_mapcount)) {
+ dma_clear_pte(pte);
+ if (!first_pte)
+ first_pte = pte;
+ last_pte = pte;
+ page_mapcount_reset(npage);
+ list_add_tail(&npage->lru, freelist);
+ *freed_level = level;
+ }
+ write_unlock(&domain->pgd_lock);
+ read_lock(&domain->pgd_lock);
+ }
}
next:
pfn = level_pfn + level_size(level);
@@ -1089,7 +1125,8 @@ static void dma_pte_clear_level(struct dmar_domain *domain, int level,
the page tables, and may have cached the intermediate levels. The
pages can only be freed after the IOTLB flush has been done. */
static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
- unsigned long last_pfn, struct list_head *freelist)
+ unsigned long last_pfn, struct list_head *freelist,
+ int *level)
{
if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
WARN_ON(start_pfn > last_pfn))
@@ -1098,7 +1135,8 @@ static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
read_lock(&domain->pgd_lock);
/* we don't need lock here; nobody else touches the iova range */
dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
- domain->pgd, 0, start_pfn, last_pfn, freelist);
+ domain->pgd, 0, start_pfn, last_pfn, freelist,
+ level);
read_unlock(&domain->pgd_lock);
/* free pgd */
@@ -1479,11 +1517,11 @@ static void __iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
struct dmar_domain *domain,
- unsigned long pfn, unsigned int pages,
+ unsigned long pfn, unsigned long pages,
int ih, int map)
{
- unsigned int aligned_pages = __roundup_pow_of_two(pages);
- unsigned int mask = ilog2(aligned_pages);
+ unsigned long aligned_pages = __roundup_pow_of_two(pages);
+ unsigned long mask = ilog2(aligned_pages);
uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
u16 did = domain_id_iommu(domain, iommu);
@@ -1837,7 +1875,8 @@ static void domain_exit(struct dmar_domain *domain)
if (domain->pgd) {
LIST_HEAD(freelist);
- domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
+ domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist,
+ NULL);
iommu_put_pages_list(&freelist);
}
@@ -3419,7 +3458,8 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb,
struct intel_iommu *iommu;
LIST_HEAD(freelist);
- domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
+ domain_unmap(si_domain, start_vpfn, last_vpfn,
+ &freelist, NULL);
rcu_read_lock();
for_each_active_iommu(iommu, drhd)
@@ -4080,6 +4120,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
+ bool queued = iommu_iotlb_gather_queued(gather);
unsigned long start_pfn, last_pfn;
int level = 0;
@@ -4099,7 +4140,16 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
start_pfn = iova >> VTD_PAGE_SHIFT;
last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
- domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
+ /*
+ * pass level only if !queued, which means we will do iotlb
+ * flush callback before freeing pages from freelist.
+ *
+ * When level is passed domain_unamp will attempt to add empty
+ * page tables to freelist, and pass the level number of the highest
+ * page table that was added to the freelist.
+ */
+ domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist,
+ queued ? NULL : &level);
if (dmar_domain->max_addr == iova + size)
dmar_domain->max_addr = iova;
@@ -4108,8 +4158,21 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain,
* We do not use page-selective IOTLB invalidation in flush queue,
* so there is no need to track page and sync iotlb.
*/
- if (!iommu_iotlb_gather_queued(gather))
- iommu_iotlb_gather_add_page(domain, gather, iova, size);
+ if (!queued) {
+ size_t sz = size;
+
+ /*
+ * Increase iova and sz for flushing if level was returned,
+ * as it means we also are freeing some page tables.
+ */
+ if (level) {
+ unsigned long pgsize = level_size(level) << VTD_PAGE_SHIFT;
+
+ iova = ALIGN_DOWN(iova, pgsize);
+ sz = ALIGN(size, pgsize);
+ }
+ iommu_iotlb_gather_add_page(domain, gather, iova, sz);
+ }
return size;
}
--
2.44.0.769.g3c40516874-goog
