From: Ilya Lesokhin <ilyal@xxxxxxxxxxxx>
The mlx5 driver needs to be able to issue invalidation to ODP MRs
even if it cannot allocate memory. To this end it preallocates
emergency pages to use when the situation arises.
This flow should be extremely rare enough, that we don't need
to worry about contention and therefore a single emergency page
is good enough.
Signed-off-by: Ilya Lesokhin <ilyal@xxxxxxxxxxxx>
Signed-off-by: Leon Romanovsky <leon@xxxxxxxxxx>
---
drivers/infiniband/hw/mlx5/main.c | 42 +++++++++++++++++++++++++-----------
drivers/infiniband/hw/mlx5/mlx5_ib.h | 6 +++---
drivers/infiniband/hw/mlx5/mr.c | 11 +++++-----
3 files changed, 37 insertions(+), 22 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index cfe18c43b5e5..5d35c2682a8e 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -92,6 +92,12 @@ static LIST_HEAD(mlx5_ib_dev_list);
*/
static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
+/* We can't use an array for xlt_emergency_page because dma_map_single
+ * doesn't work on kernel modules memory
+ */
+static unsigned long xlt_emergency_page;
+static struct mutex xlt_emergency_page_mutex;
+
struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
{
struct mlx5_ib_dev *dev;
@@ -1706,17 +1712,10 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
#endif
- context->upd_xlt_page = __get_free_page(GFP_KERNEL);
- if (!context->upd_xlt_page) {
- err = -ENOMEM;
- goto out_uars;
- }
- mutex_init(&context->upd_xlt_page_mutex);
-
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
err = mlx5_ib_alloc_transport_domain(dev, &context->tdn);
if (err)
- goto out_page;
+ goto out_uars;
}
INIT_LIST_HEAD(&context->vma_private_list);
@@ -1793,9 +1792,6 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_ib_dealloc_transport_domain(dev, context->tdn);
-out_page:
- free_page(context->upd_xlt_page);
-
out_uars:
deallocate_uars(dev, context);
@@ -1821,7 +1817,6 @@ static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_ib_dealloc_transport_domain(dev, context->tdn);
- free_page(context->upd_xlt_page);
deallocate_uars(dev, context);
kfree(bfregi->sys_pages);
kfree(bfregi->count);
@@ -5293,13 +5288,32 @@ static struct mlx5_interface mlx5_ib_interface = {
.protocol = MLX5_INTERFACE_PROTOCOL_IB,
};
+unsigned long mlx5_ib_get_xlt_emergency_page(void)
+{
+ mutex_lock(&xlt_emergency_page_mutex);
+ return xlt_emergency_page;
+}
+
+void mlx5_ib_put_xlt_emergency_page(void)
+{
+ mutex_unlock(&xlt_emergency_page_mutex);
+}
+
static int __init mlx5_ib_init(void)
{
int err;
+ xlt_emergency_page = __get_free_page(GFP_KERNEL);
+ if (!xlt_emergency_page)
+ return -ENOMEM;
+
+ mutex_init(&xlt_emergency_page_mutex);
+
mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
- if (!mlx5_ib_event_wq)
+ if (!mlx5_ib_event_wq) {
+ free_page(xlt_emergency_page);
return -ENOMEM;
+ }
mlx5_ib_odp_init();
@@ -5312,6 +5326,8 @@ static void __exit mlx5_ib_cleanup(void)
{
mlx5_unregister_interface(&mlx5_ib_interface);
destroy_workqueue(mlx5_ib_event_wq);
+ mutex_destroy(&xlt_emergency_page_mutex);
+ free_page(xlt_emergency_page);
}
module_init(mlx5_ib_init);
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 8285de76278c..c508d1c05bcc 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -130,9 +130,6 @@ struct mlx5_ib_ucontext {
/* protect vma_private_list add/del */
struct mutex vma_private_list_mutex;
- unsigned long upd_xlt_page;
- /* protect ODP/KSM */
- struct mutex upd_xlt_page_mutex;
u64 lib_caps;
};
@@ -1220,4 +1217,7 @@ static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
}
+unsigned long mlx5_ib_get_xlt_emergency_page(void);
+void mlx5_ib_put_xlt_emergency_page(void);
+
#endif /* MLX5_IB_H */
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 8cc52c9370c3..4398969b1f49 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -985,7 +985,6 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
{
struct mlx5_ib_dev *dev = mr->dev;
struct device *ddev = dev->ib_dev.dev.parent;
- struct mlx5_ib_ucontext *uctx = NULL;
int size;
void *xlt;
dma_addr_t dma;
@@ -1001,6 +1000,7 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
size_t pages_to_map = 0;
size_t pages_iter = 0;
gfp_t gfp;
+ bool use_emergency_page = false;
/* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
* so we need to align the offset and length accordingly
@@ -1027,12 +1027,11 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
}
if (!xlt) {
- uctx = to_mucontext(mr->ibmr.pd->uobject->context);
mlx5_ib_warn(dev, "Using XLT emergency buffer\n");
+ xlt = (void *)mlx5_ib_get_xlt_emergency_page();
size = PAGE_SIZE;
- xlt = (void *)uctx->upd_xlt_page;
- mutex_lock(&uctx->upd_xlt_page_mutex);
memset(xlt, 0, size);
+ use_emergency_page = true;
}
pages_iter = size / desc_size;
dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE);
@@ -1096,8 +1095,8 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
free_xlt:
- if (uctx)
- mutex_unlock(&uctx->upd_xlt_page_mutex);
+ if (use_emergency_page)
+ mlx5_ib_put_xlt_emergency_page();
else
free_pages((unsigned long)xlt, get_order(size));
--
2.14.3
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
