[PATCH rdma-core V2 2/4] bnxt_re/lib: Remove db_lock around doorbell ring

[Devesh Sharma <devesh.sharma@xxxxxxxxxxxx>]

Remove the lock around the doorbell. It isn't necessary
and was a bottleneck for multithreaded clients. Performance of
spdk improves by 40% with it removed.

Signed-off-by: Devesh Sharma <devesh.sharma@xxxxxxxxxxxx>
Signed-off-by: JD Zheng <jiandong.zheng@xxxxxxxxxxxx>
Signed-off-by: Jonathan Richardson <jonathan.richardson@xxxxxxxxxxxx>
---
 providers/bnxt_re/db.c     |  9 ++++-----
 providers/bnxt_re/main.c   |  1 -
 providers/bnxt_re/main.h   |  1 -
 providers/bnxt_re/memory.h | 10 ----------
 providers/bnxt_re/verbs.c  |  2 --
 5 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/providers/bnxt_re/db.c b/providers/bnxt_re/db.c
index a79f871..85da182 100644
--- a/providers/bnxt_re/db.c
+++ b/providers/bnxt_re/db.c
@@ -36,7 +36,7 @@
  * Description: Doorbell handling functions.
  */
 
-#include <util/udma_barrier.h>
+#include <util/mmio.h>
 #include "main.h"
 
 static void bnxt_re_ring_db(struct bnxt_re_dpi *dpi,
@@ -44,11 +44,10 @@ static void bnxt_re_ring_db(struct bnxt_re_dpi *dpi,
 {
 	__le64 *dbval;
 
-	pthread_spin_lock(&dpi->db_lock);
 	dbval = (__le64 *)&hdr->indx;
-	udma_to_device_barrier();
-	iowrite64(dpi->dbpage, dbval);
-	pthread_spin_unlock(&dpi->db_lock);
+	mmio_wc_start();
+	mmio_write64_le(dpi->dbpage, *dbval);
+	mmio_flush_writes();
 }
 
 static void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr, uint32_t indx,
diff --git a/providers/bnxt_re/main.c b/providers/bnxt_re/main.c
index 54e3cc3..a1ba06a 100644
--- a/providers/bnxt_re/main.c
+++ b/providers/bnxt_re/main.c
@@ -167,7 +167,6 @@ static void bnxt_re_free_context(struct ibv_context *ibvctx)
 	 * allocated in this context.
 	 */
 	if (cntx->udpi.dbpage && cntx->udpi.dbpage != MAP_FAILED) {
-		pthread_spin_destroy(&cntx->udpi.db_lock);
 		munmap(cntx->udpi.dbpage, dev->pg_size);
 		cntx->udpi.dbpage = NULL;
 	}
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h
index 16f55f9..0b5c749 100644
--- a/providers/bnxt_re/main.h
+++ b/providers/bnxt_re/main.h
@@ -59,7 +59,6 @@
 struct bnxt_re_dpi {
 	__u32 dpindx;
 	__u64 *dbpage;
-	pthread_spinlock_t db_lock;
 };
 
 struct bnxt_re_pd {
diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h
index aac0ff3..75564c4 100644
--- a/providers/bnxt_re/memory.h
+++ b/providers/bnxt_re/memory.h
@@ -81,16 +81,6 @@ static inline unsigned long roundup_pow_of_two(unsigned long val)
 int bnxt_re_alloc_aligned(struct bnxt_re_queue *que, uint32_t pg_size);
 void bnxt_re_free_aligned(struct bnxt_re_queue *que);
 
-static inline void iowrite64(__u64 *dst, __le64 *src)
-{
-	*(volatile __le64 *)dst = *src;
-}
-
-static inline void iowrite32(__u32 *dst, __le32 *src)
-{
-	*(volatile __le32 *)dst = *src;
-}
-
 /* Basic queue operation */
 static inline uint32_t bnxt_re_is_que_full(struct bnxt_re_queue *que)
 {
diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c
index 9ce1454..8e9fd8c 100644
--- a/providers/bnxt_re/verbs.c
+++ b/providers/bnxt_re/verbs.c
@@ -110,8 +110,6 @@ struct ibv_pd *bnxt_re_alloc_pd(struct ibv_context *ibvctx)
 			(void)ibv_cmd_dealloc_pd(&pd->ibvpd);
 			goto out;
 		}
-		pthread_spin_init(&cntx->udpi.db_lock,
-				  PTHREAD_PROCESS_PRIVATE);
         }
 
 	return &pd->ibvpd;
-- 
1.8.3.1