From: Leon Romanovsky <leonro@xxxxxxxxxx>
Reuse newly added DMA API to cache IOVA and only link/unlink pages
in fast path.
Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxx>
---
drivers/infiniband/core/umem_odp.c | 61 +++---------------------------
drivers/infiniband/hw/mlx5/odp.c | 7 +++-
include/rdma/ib_umem_odp.h | 8 +---
3 files changed, 12 insertions(+), 64 deletions(-)
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index c628a98c41b7..6e170cb5110c 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -81,20 +81,13 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,
if (!umem_odp->pfn_list)
return -ENOMEM;
- umem_odp->dma_list = kvcalloc(
- ndmas, sizeof(*umem_odp->dma_list), GFP_KERNEL);
- if (!umem_odp->dma_list) {
- ret = -ENOMEM;
- goto out_pfn_list;
- }
umem_odp->iova.dev = dev->dma_device;
umem_odp->iova.size = end - start;
umem_odp->iova.dir = DMA_BIDIRECTIONAL;
ret = dma_alloc_iova(&umem_odp->iova);
if (ret)
- goto out_dma_list;
-
+ goto out_pfn_list;
ret = mmu_interval_notifier_insert(&umem_odp->notifier,
umem_odp->umem.owning_mm,
@@ -107,8 +100,6 @@ static inline int ib_init_umem_odp(struct ib_umem_odp *umem_odp,
out_free_iova:
dma_free_iova(&umem_odp->iova);
-out_dma_list:
- kvfree(umem_odp->dma_list);
out_pfn_list:
kvfree(umem_odp->pfn_list);
return ret;
@@ -286,7 +277,6 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
mutex_unlock(&umem_odp->umem_mutex);
mmu_interval_notifier_remove(&umem_odp->notifier);
dma_free_iova(&umem_odp->iova);
- kvfree(umem_odp->dma_list);
kvfree(umem_odp->pfn_list);
}
put_pid(umem_odp->tgid);
@@ -294,40 +284,10 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
}
EXPORT_SYMBOL(ib_umem_odp_release);
-/*
- * Map for DMA and insert a single page into the on-demand paging page tables.
- *
- * @umem: the umem to insert the page to.
- * @dma_index: index in the umem to add the dma to.
- * @page: the page struct to map and add.
- * @access_mask: access permissions needed for this page.
- *
- * The function returns -EFAULT if the DMA mapping operation fails.
- *
- */
-static int ib_umem_odp_map_dma_single_page(
- struct ib_umem_odp *umem_odp,
- unsigned int dma_index,
- struct page *page)
-{
- struct ib_device *dev = umem_odp->umem.ibdev;
- dma_addr_t *dma_addr = &umem_odp->dma_list[dma_index];
-
- *dma_addr = ib_dma_map_page(dev, page, 0, 1 << umem_odp->page_shift,
- DMA_BIDIRECTIONAL);
- if (ib_dma_mapping_error(dev, *dma_addr)) {
- *dma_addr = 0;
- return -EFAULT;
- }
- umem_odp->npages++;
- return 0;
-}
-
/**
* ib_umem_odp_map_dma_and_lock - DMA map userspace memory in an ODP MR and lock it.
*
* Maps the range passed in the argument to DMA addresses.
- * The DMA addresses of the mapped pages is updated in umem_odp->dma_list.
* Upon success the ODP MR will be locked to let caller complete its device
* page table update.
*
@@ -435,15 +395,6 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt,
__func__, hmm_order, page_shift);
break;
}
-
- ret = ib_umem_odp_map_dma_single_page(
- umem_odp, dma_index, hmm_pfn_to_page(range.hmm_pfns[pfn_index]));
- if (ret < 0) {
- ibdev_dbg(umem_odp->umem.ibdev,
- "ib_umem_odp_map_dma_single_page failed with error %d\n", ret);
- break;
- }
- range.hmm_pfns[pfn_index] |= HMM_PFN_DMA_MAPPED;
}
/* upon success lock should stay on hold for the callee */
if (!ret)
@@ -463,10 +414,8 @@ EXPORT_SYMBOL(ib_umem_odp_map_dma_and_lock);
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
u64 bound)
{
- dma_addr_t dma;
int idx;
u64 addr;
- struct ib_device *dev = umem_odp->umem.ibdev;
lockdep_assert_held(&umem_odp->umem_mutex);
@@ -474,19 +423,19 @@ void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
bound = min_t(u64, bound, ib_umem_end(umem_odp));
for (addr = virt; addr < bound; addr += BIT(umem_odp->page_shift)) {
unsigned long pfn_idx = (addr - ib_umem_start(umem_odp)) >> PAGE_SHIFT;
- struct page *page = hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]);
idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
- dma = umem_odp->dma_list[idx];
if (!(umem_odp->pfn_list[pfn_idx] & HMM_PFN_VALID))
continue;
if (!(umem_odp->pfn_list[pfn_idx] & HMM_PFN_DMA_MAPPED))
continue;
- ib_dma_unmap_page(dev, dma, BIT(umem_odp->page_shift),
- DMA_BIDIRECTIONAL);
+ dma_hmm_unlink_page(&umem_odp->pfn_list[pfn_idx],
+ &umem_odp->iova,
+ idx * (1 << umem_odp->page_shift));
if (umem_odp->pfn_list[pfn_idx] & HMM_PFN_WRITE) {
+ struct page *page = hmm_pfn_to_page(umem_odp->pfn_list[pfn_idx]);
struct page *head_page = compound_head(page);
/*
* set_page_dirty prefers being called with
diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c
index 5713fe25f4de..b2aeaef9d0e1 100644
--- a/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@ -149,6 +149,7 @@ static void populate_mtt(__be64 *pas, size_t idx, size_t nentries,
{
struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
bool downgrade = flags & MLX5_IB_UPD_XLT_DOWNGRADE;
+ struct ib_device *dev = odp->umem.ibdev;
unsigned long pfn;
dma_addr_t pa;
size_t i;
@@ -162,12 +163,16 @@ static void populate_mtt(__be64 *pas, size_t idx, size_t nentries,
/* Initial ODP init */
continue;
- pa = odp->dma_list[idx + i];
+ pa = dma_hmm_link_page(&odp->pfn_list[idx + i], &odp->iova,
+ (idx + i) * (1 << odp->page_shift));
+ WARN_ON_ONCE(ib_dma_mapping_error(dev, pa));
+
pa |= MLX5_IB_MTT_READ;
if ((pfn & HMM_PFN_WRITE) && !downgrade)
pa |= MLX5_IB_MTT_WRITE;
pas[i] = cpu_to_be64(pa);
+ odp->npages++;
}
}
diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h
index a3f4a5c03bf8..653fc076b6ee 100644
--- a/include/rdma/ib_umem_odp.h
+++ b/include/rdma/ib_umem_odp.h
@@ -18,15 +18,9 @@ struct ib_umem_odp {
/* An array of the pfns included in the on-demand paging umem. */
unsigned long *pfn_list;
- /*
- * An array with DMA addresses mapped for pfns in pfn_list.
- * The lower two bits designate access permissions.
- * See ODP_READ_ALLOWED_BIT and ODP_WRITE_ALLOWED_BIT.
- */
- dma_addr_t *dma_list;
struct dma_iova_attrs iova;
/*
- * The umem_mutex protects the page_list and dma_list fields of an ODP
+ * The umem_mutex protects the page_list field of an ODP
* umem, allowing only a single thread to map/unmap pages. The mutex
* also protects access to the mmu notifier counters.
*/
--
2.45.2
