From: Yishai Hadas <yishaih@xxxxxxxxxxxx>
As VMAs for a given range might not be available as part of the
registration phase in ODP, IB_ACCESS_HUGETLB/page_shift must be checked
as part of the page fault flow.
If the application didn't mmap the backed memory with huge pages or
released part of that hugepage area, an error will be set as part of the
page fault flow once be detected.
Fixes: 0008b84ea9af ("IB/umem: Add support to huge ODP")
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
Reviewed-by: Artemy Kovalyov <artemyko@xxxxxxxxxxxx>
Reviewed-by: Aviad Yehezkel <aviadye@xxxxxxxxxxxx>
Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxxxx>
---
drivers/infiniband/core/umem_odp.c | 37 +++++++++++++++---------------
1 file changed, 19 insertions(+), 18 deletions(-)
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
index 2e9ee7adab13..533271897908 100644
--- a/drivers/infiniband/core/umem_odp.c
+++ b/drivers/infiniband/core/umem_odp.c
@@ -241,22 +241,10 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr,
umem_odp->umem.owning_mm = mm = current->mm;
umem_odp->notifier.ops = ops;
- umem_odp->page_shift = PAGE_SHIFT;
- if (access & IB_ACCESS_HUGETLB) {
- struct vm_area_struct *vma;
- struct hstate *h;
-
- down_read(&mm->mmap_sem);
- vma = find_vma(mm, ib_umem_start(umem_odp));
- if (!vma || !is_vm_hugetlb_page(vma)) {
- up_read(&mm->mmap_sem);
- ret = -EINVAL;
- goto err_free;
- }
- h = hstate_vma(vma);
- umem_odp->page_shift = huge_page_shift(h);
- up_read(&mm->mmap_sem);
- }
+ if (access & IB_ACCESS_HUGETLB)
+ umem_odp->page_shift = HPAGE_SHIFT;
+ else
+ umem_odp->page_shift = PAGE_SHIFT;
umem_odp->tgid = get_task_pid(current->group_leader, PIDTYPE_PID);
ret = ib_init_umem_odp(umem_odp, ops);
@@ -266,7 +254,6 @@ struct ib_umem_odp *ib_umem_odp_get(struct ib_udata *udata, unsigned long addr,
err_put_pid:
put_pid(umem_odp->tgid);
-err_free:
kfree(umem_odp);
return ERR_PTR(ret);
}
@@ -403,6 +390,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
int j, k, ret = 0, start_idx, npages = 0;
unsigned int flags = 0, page_shift;
phys_addr_t p = 0;
+ struct vm_area_struct **vmas;
if (access_mask == 0)
return -EINVAL;
@@ -415,6 +403,12 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
if (!local_page_list)
return -ENOMEM;
+ vmas = (struct vm_area_struct **)__get_free_page(GFP_KERNEL);
+ if (!vmas) {
+ ret = -ENOMEM;
+ goto out_free_page_list;
+ }
+
page_shift = umem_odp->page_shift;
page_mask = ~(BIT(page_shift) - 1);
off = user_virt & (~page_mask);
@@ -453,7 +447,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
*/
npages = get_user_pages_remote(owning_process, owning_mm,
user_virt, gup_num_pages,
- flags, local_page_list, NULL, NULL);
+ flags, local_page_list, vmas, NULL);
up_read(&owning_mm->mmap_sem);
if (npages < 0) {
@@ -477,6 +471,11 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
continue;
}
+ if ((1 << page_shift) > vma_kernel_pagesize(vmas[j])) {
+ ret = -EFAULT;
+ break;
+ }
+
ret = ib_umem_odp_map_dma_single_page(
umem_odp, k, local_page_list[j],
access_mask, current_seq);
@@ -517,6 +516,8 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
out_put_task:
if (owning_process)
put_task_struct(owning_process);
+ free_page((unsigned long)vmas);
+out_free_page_list:
free_page((unsigned long)local_page_list);
return ret;
}
--
2.20.1
