[PATCH for-next 9/9] RDMA/rxe: Protect pending send packets

[Bob Pearson <rpearsonhpe@xxxxxxxxx>]

Network interruptions may cause long delays in the processing of
send packets during which time the rxe driver may be unloaded.
This will cause seg faults when the packet is ultimately freed as
it calls the destructor function in the rxe driver. This has been
observed in cable pull fail over fail back testing.

This patch takes a reference on the driver to protect the packet
from this happening.

Signed-off-by: Bob Pearson <rpearsonhpe@xxxxxxxxx>
---
 drivers/infiniband/sw/rxe/rxe.c     | 26 ++++++++++++++++++++++++++
 drivers/infiniband/sw/rxe/rxe.h     |  3 +++
 drivers/infiniband/sw/rxe/rxe_net.c |  7 +++++++
 3 files changed, 36 insertions(+)

diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
index 54c723a6edda..6b55c595f8f8 100644
--- a/drivers/infiniband/sw/rxe/rxe.c
+++ b/drivers/infiniband/sw/rxe/rxe.c
@@ -208,10 +208,33 @@ static struct rdma_link_ops rxe_link_ops = {
 	.newlink = rxe_newlink,
 };
 
+static struct rxe_module {
+	struct kref		ref_cnt;
+	struct completion	complete;
+} rxe_module;
+
+static void rxe_module_release(struct kref *kref)
+{
+	complete(&rxe_module.complete);
+}
+
+int rxe_module_get(void)
+{
+	return kref_get_unless_zero(&rxe_module.ref_cnt);
+}
+
+int rxe_module_put(void)
+{
+	return kref_put(&rxe_module.ref_cnt, rxe_module_release);
+}
+
 static int __init rxe_module_init(void)
 {
 	int err;
 
+	kref_init(&rxe_module.ref_cnt);
+	init_completion(&rxe_module.complete);
+
 	err = rxe_alloc_wq();
 	if (err)
 		return err;
@@ -229,6 +252,9 @@ static int __init rxe_module_init(void)
 
 static void __exit rxe_module_exit(void)
 {
+	rxe_module_put();
+	wait_for_completion(&rxe_module.complete);
+
 	rdma_link_unregister(&rxe_link_ops);
 	ib_unregister_driver(RDMA_DRIVER_RXE);
 	rxe_net_exit();
diff --git a/drivers/infiniband/sw/rxe/rxe.h b/drivers/infiniband/sw/rxe/rxe.h
index d33dd6cf83d3..077e3ad8f39a 100644
--- a/drivers/infiniband/sw/rxe/rxe.h
+++ b/drivers/infiniband/sw/rxe/rxe.h
@@ -158,4 +158,7 @@ void rxe_port_up(struct rxe_dev *rxe);
 void rxe_port_down(struct rxe_dev *rxe);
 void rxe_set_port_state(struct rxe_dev *rxe);
 
+int rxe_module_get(void);
+int rxe_module_put(void);
+
 #endif /* RXE_H */
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index c1b2eaf82334..0e447420a441 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -384,6 +384,7 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
 out_put_ibdev:
 	ib_device_put(&rxe->ib_dev);
 out:
+	rxe_module_put();
 	return;
 }
 
@@ -400,6 +401,12 @@ static int rxe_send(struct sk_buff *skb, struct rxe_pkt_info *pkt)
 	sock_hold(sk);
 	skb->sk = sk;
 
+	/* the module may be potentially removed while this packet
+	 * is waiting on the tx queue causing seg faults. So need
+	 * to protect the module.
+	 */
+	rxe_module_get();
+
 	atomic_inc(&pkt->qp->skb_out);
 
 	sk->sk_user_data = (void *)(long)qp->elem.index;
-- 
2.39.2