Re: [PATCH] iw_cxgb3: Remove the iw_cxgb3 provider code

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Tuesday, October 10/22/19, 2019 at 23:17:09 +0530, Yuval Shaia wrote:
> Following the removal of iw_cxgb3 module from kerne remove also the
> corresponding rdma-core provider code.
> https://lore.kernel.org/r/20190930074252.20133-1-bharat@xxxxxxxxxxx
> 
> Signed-off-by: Yuval Shaia <yuval.shaia@xxxxxxxxxx>
Hi Yuval,
Thanks for tha patch. There are some more deletions in maintainers file and
spec files which I think needs to be removed. Here is my unsent patch which 
includes them.
Thanks,
Bharat.

> ---
>  CMakeLists.txt                             |   1 -
>  kernel-headers/CMakeLists.txt              |   2 -
>  kernel-headers/rdma/cxgb3-abi.h            |  82 ---
>  kernel-headers/rdma/rdma_user_ioctl_cmds.h |   1 -
>  providers/cxgb3/CMakeLists.txt             |   6 -
>  providers/cxgb3/cq.c                       | 442 ------------
>  providers/cxgb3/cxio_wr.h                  | 758 ---------------------
>  providers/cxgb3/firmware_exports.h         | 148 ----
>  providers/cxgb3/iwch-abi.h                 |  51 --
>  providers/cxgb3/iwch.c                     | 269 --------
>  providers/cxgb3/iwch.h                     | 218 ------
>  providers/cxgb3/qp.c                       | 560 ---------------
>  providers/cxgb3/verbs.c                    | 476 -------------
>  13 files changed, 3014 deletions(-)
>  delete mode 100644 kernel-headers/rdma/cxgb3-abi.h
>  delete mode 100644 providers/cxgb3/CMakeLists.txt
>  delete mode 100644 providers/cxgb3/cq.c
>  delete mode 100644 providers/cxgb3/cxio_wr.h
>  delete mode 100644 providers/cxgb3/firmware_exports.h
>  delete mode 100644 providers/cxgb3/iwch-abi.h
>  delete mode 100644 providers/cxgb3/iwch.c
>  delete mode 100644 providers/cxgb3/iwch.h
>  delete mode 100644 providers/cxgb3/qp.c
>  delete mode 100644 providers/cxgb3/verbs.c
> 
> diff --git a/CMakeLists.txt b/CMakeLists.txt
> index fc17ef36..cdc8d544 100644
> --- a/CMakeLists.txt
> +++ b/CMakeLists.txt
> @@ -616,7 +616,6 @@ add_subdirectory(librdmacm/man)
>  # Providers
>  if (HAVE_COHERENT_DMA)
>  add_subdirectory(providers/bnxt_re)
> -add_subdirectory(providers/cxgb3) # NO SPARSE
>  add_subdirectory(providers/cxgb4) # NO SPARSE
>  add_subdirectory(providers/efa)
>  add_subdirectory(providers/efa/man)
> diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
> index 50bc77e6..b3139b74 100644
> --- a/kernel-headers/CMakeLists.txt
> +++ b/kernel-headers/CMakeLists.txt
> @@ -1,6 +1,5 @@
>  publish_internal_headers(rdma
>    rdma/bnxt_re-abi.h
> -  rdma/cxgb3-abi.h
>    rdma/cxgb4-abi.h
>    rdma/efa-abi.h
>    rdma/hns-abi.h
> @@ -59,7 +58,6 @@ endfunction()
>  # Transform the kernel ABIs used by the providers
>  rdma_kernel_provider_abi(
>    rdma/bnxt_re-abi.h
> -  rdma/cxgb3-abi.h
>    rdma/cxgb4-abi.h
>    rdma/efa-abi.h
>    rdma/hns-abi.h
> diff --git a/kernel-headers/rdma/cxgb3-abi.h b/kernel-headers/rdma/cxgb3-abi.h
> deleted file mode 100644
> index 85aed672..00000000
> --- a/kernel-headers/rdma/cxgb3-abi.h
> +++ /dev/null
> @@ -1,82 +0,0 @@
> -/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
> -/*
> - * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#ifndef CXGB3_ABI_USER_H
> -#define CXGB3_ABI_USER_H
> -
> -#include <linux/types.h>
> -
> -#define IWCH_UVERBS_ABI_VERSION	1
> -
> -/*
> - * Make sure that all structs defined in this file remain laid out so
> - * that they pack the same way on 32-bit and 64-bit architectures (to
> - * avoid incompatibility between 32-bit userspace and 64-bit kernels).
> - * In particular do not use pointer types -- pass pointers in __aligned_u64
> - * instead.
> - */
> -struct iwch_create_cq_req {
> -	__aligned_u64 user_rptr_addr;
> -};
> -
> -struct iwch_create_cq_resp_v0 {
> -	__aligned_u64 key;
> -	__u32 cqid;
> -	__u32 size_log2;
> -};
> -
> -struct iwch_create_cq_resp {
> -	__aligned_u64 key;
> -	__u32 cqid;
> -	__u32 size_log2;
> -	__u32 memsize;
> -	__u32 reserved;
> -};
> -
> -struct iwch_create_qp_resp {
> -	__aligned_u64 key;
> -	__aligned_u64 db_key;
> -	__u32 qpid;
> -	__u32 size_log2;
> -	__u32 sq_size_log2;
> -	__u32 rq_size_log2;
> -};
> -
> -struct iwch_reg_user_mr_resp {
> -	__u32 pbl_addr;
> -};
> -
> -struct iwch_alloc_pd_resp {
> -	__u32 pdid;
> -};
> -
> -#endif /* CXGB3_ABI_USER_H */
> diff --git a/kernel-headers/rdma/rdma_user_ioctl_cmds.h b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
> index b8bb285f..b2680051 100644
> --- a/kernel-headers/rdma/rdma_user_ioctl_cmds.h
> +++ b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
> @@ -88,7 +88,6 @@ enum rdma_driver_id {
>  	RDMA_DRIVER_UNKNOWN,
>  	RDMA_DRIVER_MLX5,
>  	RDMA_DRIVER_MLX4,
> -	RDMA_DRIVER_CXGB3,
>  	RDMA_DRIVER_CXGB4,
>  	RDMA_DRIVER_MTHCA,
>  	RDMA_DRIVER_BNXT_RE,
> diff --git a/providers/cxgb3/CMakeLists.txt b/providers/cxgb3/CMakeLists.txt
> deleted file mode 100644
> index a578105e..00000000
> --- a/providers/cxgb3/CMakeLists.txt
> +++ /dev/null
> @@ -1,6 +0,0 @@
> -rdma_provider(cxgb3
> -  cq.c
> -  iwch.c
> -  qp.c
> -  verbs.c
> -)
> diff --git a/providers/cxgb3/cq.c b/providers/cxgb3/cq.c
> deleted file mode 100644
> index 6cb4fe74..00000000
> --- a/providers/cxgb3/cq.c
> +++ /dev/null
> @@ -1,442 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#include <config.h>
> -
> -#include <stdio.h>
> -#include <pthread.h>
> -#include <sys/errno.h>
> -
> -#include <infiniband/opcode.h>
> -
> -#include "iwch.h"
> -#include "iwch-abi.h"
> -
> -int iwch_arm_cq(struct ibv_cq *ibcq, int solicited)
> -{
> -	int ret;
> -	struct iwch_cq *chp = to_iwch_cq(ibcq);
> -
> -	pthread_spin_lock(&chp->lock);
> -	ret = ibv_cmd_req_notify_cq(ibcq, solicited);
> -	pthread_spin_unlock(&chp->lock);
> -
> -	return ret;
> -}
> -
> -static inline void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
> -{
> -	struct t3_swsq *sqp;
> -	uint32_t ptr = wq->sq_rptr;
> -	int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
> -	
> -	sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
> -	while (count--) {
> -		if (!sqp->signaled) {
> -			ptr++;
> -			sqp = wq->sq + Q_PTR2IDX(ptr,  wq->sq_size_log2);
> -		} else if (sqp->complete) {
> -
> -			/* 
> -			 * Insert this completed cqe into the swcq.
> -			 */
> -			sqp->cqe.header |= htobe32(V_CQE_SWCQE(1));
> -			*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) 
> -				= sqp->cqe;
> -			cq->sw_wptr++;
> -			sqp->signaled = 0;
> -			break;
> -		} else
> -			break;
> -	}
> -}
> -
> -static inline void create_read_req_cqe(struct t3_wq *wq,
> -				       struct t3_cqe *hw_cqe,
> -				       struct t3_cqe *read_cqe)
> -{
> -	CQE_WRID_SQ_WPTR(*read_cqe) = wq->oldest_read->sq_wptr;
> -	read_cqe->len = wq->oldest_read->read_len;
> -	read_cqe->header = htobe32(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
> -				 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
> -				 V_CQE_OPCODE(T3_READ_REQ) |
> -				 V_CQE_TYPE(1));
> -}
> -
> -/*
> - * Return a ptr to the next read wr in the SWSQ or NULL.
> - */
> -static inline void advance_oldest_read(struct t3_wq *wq)
> -{
> -
> -	uint32_t rptr = wq->oldest_read - wq->sq + 1;
> -	uint32_t wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
> -
> -	while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
> -		wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
> -
> -		if (wq->oldest_read->opcode == T3_READ_REQ) {
> -			return;
> -		}
> -		rptr++;
> -	}
> -	wq->oldest_read = NULL;
> -}
> -
> -static inline int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq,
> -		   struct t3_cqe *cqe, uint8_t *cqe_flushed,
> -		   uint64_t *cookie)
> -{
> -	int ret = 0;
> -	struct t3_cqe *hw_cqe, read_cqe;
> -
> -	*cqe_flushed = 0;
> -	hw_cqe = cxio_next_cqe(cq);
> -	udma_from_device_barrier();
> -
> -	/* 
> -	 * Skip cqes not affiliated with a QP.
> -	 */
> -	if (wq == NULL) {
> -		ret = -1;
> -		goto skip_cqe;
> -	}
> -
> -	/*
> -	 * Gotta tweak READ completions:
> -	 * 	1) the cqe doesn't contain the sq_wptr from the wr.
> -	 *	2) opcode not reflected from the wr.
> -	 *	3) read_len not reflected from the wr.
> -	 *	4) cq_type is RQ_TYPE not SQ_TYPE.
> -	 */
> -	if (CQE_OPCODE(*hw_cqe) == T3_READ_RESP) {
> -
> -                /*
> -		 * If this is an unsolicited read response to local stag 1, 
> -		 * then the read was generated by the kernel driver as part 
> -		 * of peer-2-peer connection setup.  So ignore the completion.
> -		 */
> -		if (CQE_WRID_STAG(*hw_cqe) == 1) {
> -			if (CQE_STATUS(*hw_cqe))
> -				wq->error = 1;
> -			ret = -1;
> -			goto skip_cqe;
> -		}
> -		
> -		/* 
> -	 	 * Don't write to the HWCQ, so create a new read req CQE 
> -		 * in local memory.
> -		 */
> -		create_read_req_cqe(wq, hw_cqe, &read_cqe);
> -		hw_cqe = &read_cqe;
> -		advance_oldest_read(wq);
> -	}
> -
> -	/* 
> -	 * Errors.
> -	 */
> -	if (CQE_STATUS(*hw_cqe) || t3_wq_in_error(wq)) {
> -		*cqe_flushed = t3_wq_in_error(wq);
> -		t3_set_wq_in_error(wq);
> -		goto proc_cqe;
> -	}
> -
> -	/*
> -	 * RECV completion.
> -	 */
> -	if (RQ_TYPE(*hw_cqe)) {
> -
> -		/* 
> -		 * HW only validates 4 bits of MSN.  So we must validate that
> -		 * the MSN in the SEND is the next expected MSN.  If its not,
> -		 * then we complete this with TPT_ERR_MSN and mark the wq in 
> -		 * error.
> -		 */
> -		if ((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1))) {
> -			t3_set_wq_in_error(wq);
> -			hw_cqe->header |= htobe32(V_CQE_STATUS(TPT_ERR_MSN));
> -		}
> -		goto proc_cqe;
> -	}
> -
> -	/* 
> - 	 * If we get here its a send completion.
> -	 *
> -	 * Handle out of order completion. These get stuffed
> -	 * in the SW SQ. Then the SW SQ is walked to move any
> -	 * now in-order completions into the SW CQ.  This handles
> -	 * 2 cases:
> -	 * 	1) reaping unsignaled WRs when the first subsequent
> -	 *	   signaled WR is completed.
> -	 *	2) out of order read completions.
> -	 */
> -	if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
> -		struct t3_swsq *sqp;
> -
> -		sqp = wq->sq + 
> -		      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
> -		sqp->cqe = *hw_cqe;
> -		sqp->complete = 1;
> -		ret = -1;
> -		goto flush_wq;
> -	}
> -
> -proc_cqe:
> -	*cqe = *hw_cqe;
> -
> -	/*
> -	 * Reap the associated WR(s) that are freed up with this
> -	 * completion.
> -	 */
> -	if (SQ_TYPE(*hw_cqe)) {
> -		wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
> -		*cookie = (wq->sq + 
> -			   Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
> -		wq->sq_rptr++;
> -	} else {
> -		*cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
> -		wq->rq_rptr++;
> -	}
> -
> -flush_wq:
> -	/*
> -	 * Flush any completed cqes that are now in-order.
> -	 */
> -	flush_completed_wrs(wq, cq);
> -
> -skip_cqe:
> -	if (SW_CQE(*hw_cqe)) {
> -		PDBG("%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x\n", 
> -		     __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
> -		++cq->sw_rptr;
> -	} else {
> -		PDBG("%s cq %p cqid 0x%x skip hw cqe sw_rptr 0x%x\n", 
> -		     __FUNCTION__, cq, cq->cqid, cq->rptr);
> -		++cq->rptr;
> -	}
> -
> -	return ret;
> -}
> -
> -/*
> - * Get one cq entry from cxio and map it to openib.
> - *
> - * Returns:
> - * 	0 			EMPTY;
> - *	1			cqe returned
> - *	-EAGAIN 		caller must try again
> - * 	any other -errno	fatal error
> - */
> -static int iwch_poll_cq_one(struct iwch_device *rhp, struct iwch_cq *chp,
> -		     struct ibv_wc *wc)
> -{
> -	struct iwch_qp *qhp = NULL;
> -	struct t3_cqe cqe, *hw_cqe;
> -	struct t3_wq *wq;
> -	uint8_t cqe_flushed;
> -	uint64_t cookie;
> -	int ret = 1;
> -
> -	hw_cqe = cxio_next_cqe(&chp->cq);
> -	udma_from_device_barrier();
> -
> -	if (!hw_cqe)
> -		return 0;
> -
> -	qhp = rhp->qpid2ptr[CQE_QPID(*hw_cqe)];
> -	if (!qhp)
> -		wq = NULL;
> -	else {
> -		pthread_spin_lock(&qhp->lock);
> -		wq = &(qhp->wq);
> -	}
> -	ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie);
> -	if (ret) {
> -		ret = -EAGAIN;
> -		goto out;
> -	}
> -	ret = 1;
> -
> -	wc->wr_id = cookie;
> -	wc->qp_num = qhp->wq.qpid;
> -	wc->vendor_err = CQE_STATUS(cqe);
> -	wc->wc_flags = 0;
> -
> -	PDBG("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x "
> -	     "lo 0x%x cookie 0x%" PRIx64 "\n", 
> -	     __FUNCTION__, CQE_QPID(cqe), CQE_TYPE(cqe),
> -	     CQE_OPCODE(cqe), CQE_STATUS(cqe), CQE_WRID_HI(cqe),
> -	     CQE_WRID_LOW(cqe), cookie);
> -
> -	if (CQE_TYPE(cqe) == 0) {
> -		if (!CQE_STATUS(cqe))
> -			wc->byte_len = CQE_LEN(cqe);
> -		else
> -			wc->byte_len = 0;
> -		wc->opcode = IBV_WC_RECV;
> -	} else {
> -		switch (CQE_OPCODE(cqe)) {
> -		case T3_RDMA_WRITE:
> -			wc->opcode = IBV_WC_RDMA_WRITE;
> -			break;
> -		case T3_READ_REQ:
> -			wc->opcode = IBV_WC_RDMA_READ;
> -			wc->byte_len = CQE_LEN(cqe);
> -			break;
> -		case T3_SEND:
> -		case T3_SEND_WITH_SE:
> -			wc->opcode = IBV_WC_SEND;
> -			break;
> -		case T3_BIND_MW:
> -			wc->opcode = IBV_WC_BIND_MW;
> -			break;
> -
> -		/* these aren't supported yet */
> -		case T3_SEND_WITH_INV:
> -		case T3_SEND_WITH_SE_INV:
> -		case T3_LOCAL_INV:
> -		case T3_FAST_REGISTER:
> -		default:
> -			PDBG("%s Unexpected opcode %d CQID 0x%x QPID 0x%x\n", 
> -			     __FUNCTION__, CQE_OPCODE(cqe), chp->cq.cqid, 
> -			     CQE_QPID(cqe));
> -			ret = -EINVAL;
> -			goto out;
> -		}
> -	}
> -
> -	if (cqe_flushed) {
> -		wc->status = IBV_WC_WR_FLUSH_ERR;
> -	} else {
> -		
> -		switch (CQE_STATUS(cqe)) {
> -		case TPT_ERR_SUCCESS:
> -			wc->status = IBV_WC_SUCCESS;
> -			break;
> -		case TPT_ERR_STAG:
> -			wc->status = IBV_WC_LOC_ACCESS_ERR;
> -			break;
> -		case TPT_ERR_PDID:
> -			wc->status = IBV_WC_LOC_PROT_ERR;
> -			break;
> -		case TPT_ERR_QPID:
> -		case TPT_ERR_ACCESS:
> -			wc->status = IBV_WC_LOC_ACCESS_ERR;
> -			break;
> -		case TPT_ERR_WRAP:
> -			wc->status = IBV_WC_GENERAL_ERR;
> -			break;
> -		case TPT_ERR_BOUND:
> -			wc->status = IBV_WC_LOC_LEN_ERR;
> -			break;
> -		case TPT_ERR_INVALIDATE_SHARED_MR:
> -		case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
> -			wc->status = IBV_WC_MW_BIND_ERR;
> -			break;
> -		case TPT_ERR_CRC:
> -		case TPT_ERR_MARKER:
> -		case TPT_ERR_PDU_LEN_ERR:
> -		case TPT_ERR_OUT_OF_RQE:
> -		case TPT_ERR_DDP_VERSION:
> -		case TPT_ERR_RDMA_VERSION:
> -		case TPT_ERR_DDP_QUEUE_NUM:
> -		case TPT_ERR_MSN:
> -		case TPT_ERR_TBIT:
> -		case TPT_ERR_MO:
> -		case TPT_ERR_MSN_RANGE:
> -		case TPT_ERR_IRD_OVERFLOW:
> -		case TPT_ERR_OPCODE:
> -			wc->status = IBV_WC_FATAL_ERR;
> -			break;
> -		case TPT_ERR_SWFLUSH:
> -			wc->status = IBV_WC_WR_FLUSH_ERR;
> -			break;
> -		default:
> -			PDBG("%s Unexpected status 0x%x CQID 0x%x QPID 0x%0x\n",
> -			     __FUNCTION__, CQE_STATUS(cqe), chp->cq.cqid, 
> -			     CQE_QPID(cqe));
> -			ret = -EINVAL;
> -		}
> -	}
> -out:
> -	if (wq)
> -		pthread_spin_unlock(&qhp->lock);
> -	return ret;
> -}
> -
> -int t3b_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
> -{
> -	struct iwch_device *rhp;
> -	struct iwch_cq *chp;
> -	int npolled;
> -	int err = 0;
> -
> -	chp = to_iwch_cq(ibcq);
> -	rhp = chp->rhp;
> -
> -	if (rhp->abi_version > 0 && t3_cq_in_error(&chp->cq)) {
> -		t3_reset_cq_in_error(&chp->cq);
> -		iwch_flush_qps(rhp);
> -	}
> -
> -	pthread_spin_lock(&chp->lock);
> -	for (npolled = 0; npolled < num_entries; ++npolled) {
> -
> -		/*
> -	 	 * Because T3 can post CQEs that are out of order,
> -	 	 * we might have to poll again after removing
> -	 	 * one of these.  
> -		 */
> -		do {
> -			err = iwch_poll_cq_one(rhp, chp, wc + npolled);
> -		} while (err == -EAGAIN);
> -		if (err <= 0)
> -			break;
> -	}
> -	pthread_spin_unlock(&chp->lock);
> -
> -	if (err < 0)
> -		return err;
> -	else {
> -		return npolled;
> -	}
> -}
> -
> -int t3a_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
> -{
> -	int ret;
> -	struct iwch_cq *chp = to_iwch_cq(ibcq);
> -	
> -	pthread_spin_lock(&chp->lock);
> -	ret = ibv_cmd_poll_cq(ibcq, num_entries, wc);
> -	pthread_spin_unlock(&chp->lock);
> -	return ret;
> -}
> diff --git a/providers/cxgb3/cxio_wr.h b/providers/cxgb3/cxio_wr.h
> deleted file mode 100644
> index 042bd941..00000000
> --- a/providers/cxgb3/cxio_wr.h
> +++ /dev/null
> @@ -1,758 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#ifndef __CXIO_WR_H__
> -#define __CXIO_WR_H__
> -
> -#include <stddef.h>
> -#include <stdint.h>
> -#include <endian.h>
> -#include <util/udma_barrier.h>
> -#include "firmware_exports.h"
> -
> -#define T3_MAX_NUM_QP (1<<15)
> -#define T3_MAX_NUM_CQ (1<<15)
> -#define T3_MAX_NUM_PD (1<<15)
> -#define T3_MAX_NUM_STAG (1<<15)
> -#define T3_MAX_SGE      4
> -#define T3_MAX_INLINE  64
> -
> -#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
> -#define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
> -				       ((rptr)!=(wptr)) )
> -#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
> -#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
> -#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
> -#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
> -
> -/* FIXME: Move me to a generic PCI mmio accessor */
> -#define cpu_to_pci32(val) htole32(val)
> -
> -#define RING_DOORBELL(doorbell, QPID) { \
> -	*doorbell = cpu_to_pci32(QPID); \
> -}
> -
> -#define SEQ32_GE(x,y) (!( (((uint32_t) (x)) - ((uint32_t) (y))) & 0x80000000 ))
> -
> -enum t3_wr_flags {
> -	T3_COMPLETION_FLAG = 0x01,
> -	T3_NOTIFY_FLAG = 0x02,
> -	T3_SOLICITED_EVENT_FLAG = 0x04,
> -	T3_READ_FENCE_FLAG = 0x08,
> -	T3_LOCAL_FENCE_FLAG = 0x10
> -} __attribute__ ((packed));
> -
> -enum t3_wr_opcode {
> -	T3_WR_BP = FW_WROPCODE_RI_BYPASS,
> -	T3_WR_SEND = FW_WROPCODE_RI_SEND,
> -	T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
> -	T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
> -	T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
> -	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
> -	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
> -	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
> -	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
> -} __attribute__ ((packed));
> -
> -enum t3_rdma_opcode {
> -	T3_RDMA_WRITE,		/* IETF RDMAP v1.0 ... */
> -	T3_READ_REQ,
> -	T3_READ_RESP,
> -	T3_SEND,
> -	T3_SEND_WITH_INV,
> -	T3_SEND_WITH_SE,
> -	T3_SEND_WITH_SE_INV,
> -	T3_TERMINATE,
> -	T3_RDMA_INIT,		/* CHELSIO RI specific ... */
> -	T3_BIND_MW,
> -	T3_FAST_REGISTER,
> -	T3_LOCAL_INV,
> -	T3_QP_MOD,
> -	T3_BYPASS
> -} __attribute__ ((packed));
> -
> -static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
> -{
> -	switch (wrop) {
> -		case T3_WR_BP: return T3_BYPASS;
> -		case T3_WR_SEND: return T3_SEND;
> -		case T3_WR_WRITE: return T3_RDMA_WRITE;
> -		case T3_WR_READ: return T3_READ_REQ;
> -		case T3_WR_INV_STAG: return T3_LOCAL_INV;
> -		case T3_WR_BIND: return T3_BIND_MW;
> -		case T3_WR_INIT: return T3_RDMA_INIT;
> -		case T3_WR_QP_MOD: return T3_QP_MOD;
> -		default: break;
> -	}
> -	return -1;
> -}
> -
> -
> -/* Work request id */
> -union t3_wrid {
> -	struct {
> -		uint32_t hi:32;
> -		uint32_t low:32;
> -	} id0;
> -	uint64_t id1;
> -};
> -
> -#define WRID(wrid)      	(wrid.id1)
> -#define WRID_GEN(wrid)		(wrid.id0.wr_gen)
> -#define WRID_IDX(wrid)		(wrid.id0.wr_idx)
> -#define WRID_LO(wrid)		(wrid.id0.wr_lo)
> -
> -struct fw_riwrh {
> -	uint32_t op_seop_flags;
> -	uint32_t gen_tid_len;
> -};
> -
> -#define S_FW_RIWR_OP		24
> -#define M_FW_RIWR_OP		0xff
> -#define V_FW_RIWR_OP(x)		((x) << S_FW_RIWR_OP)
> -#define G_FW_RIWR_OP(x)   	((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
> -
> -#define S_FW_RIWR_SOPEOP	22
> -#define M_FW_RIWR_SOPEOP	0x3
> -#define V_FW_RIWR_SOPEOP(x)	((x) << S_FW_RIWR_SOPEOP)
> -
> -#define S_FW_RIWR_FLAGS		8
> -#define M_FW_RIWR_FLAGS		0x3fffff
> -#define V_FW_RIWR_FLAGS(x)	((x) << S_FW_RIWR_FLAGS)
> -#define G_FW_RIWR_FLAGS(x)   	((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
> -
> -#define S_FW_RIWR_TID		8
> -#define V_FW_RIWR_TID(x)	((x) << S_FW_RIWR_TID)
> -
> -#define S_FW_RIWR_LEN		0
> -#define V_FW_RIWR_LEN(x)	((x) << S_FW_RIWR_LEN)
> -
> -#define S_FW_RIWR_GEN           31
> -#define V_FW_RIWR_GEN(x)        ((x)  << S_FW_RIWR_GEN)
> -
> -struct t3_sge {
> -	uint32_t stag;
> -	uint32_t len;
> -	uint64_t to;
> -};
> -
> -/* If num_sgle is zero, flit 5+ contains immediate data.*/
> -struct t3_send_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -
> -	enum t3_rdma_opcode rdmaop:8;
> -	uint32_t reserved:24;	/* 2 */
> -	uint32_t rem_stag;	/* 2 */
> -	uint32_t plen;		/* 3 */
> -	uint32_t num_sgle;
> -	struct t3_sge sgl[T3_MAX_SGE];	/* 4+ */
> -};
> -
> -struct t3_local_inv_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	uint32_t stag;		/* 2 */
> -	uint32_t reserved3;
> -};
> -
> -struct t3_rdma_write_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	enum t3_rdma_opcode rdmaop:8;	/* 2 */
> -	uint32_t reserved:24;	/* 2 */
> -	uint32_t stag_sink;
> -	uint64_t to_sink;	/* 3 */
> -	uint32_t plen;		/* 4 */
> -	uint32_t num_sgle;
> -	struct t3_sge sgl[T3_MAX_SGE];	/* 5+ */
> -};
> -
> -struct t3_rdma_read_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	enum t3_rdma_opcode rdmaop:8;	/* 2 */
> -	uint32_t reserved:24;
> -	uint32_t rem_stag;
> -	uint64_t rem_to;	/* 3 */
> -	uint32_t local_stag;	/* 4 */
> -	uint32_t local_len;
> -	uint64_t local_to;	/* 5 */
> -};
> -
> -enum t3_addr_type {
> -	T3_VA_BASED_TO = 0x0,
> -	T3_ZERO_BASED_TO = 0x1
> -} __attribute__ ((packed));
> -
> -enum t3_mem_perms {
> -	T3_MEM_ACCESS_LOCAL_READ = 0x1,
> -	T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
> -	T3_MEM_ACCESS_REM_READ = 0x4,
> -	T3_MEM_ACCESS_REM_WRITE = 0x8
> -} __attribute__ ((packed));
> -
> -struct t3_bind_mw_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	uint32_t reserved:16;
> -	enum t3_addr_type type:8;
> -	enum t3_mem_perms perms:8;	/* 2 */
> -	uint32_t mr_stag;
> -	uint32_t mw_stag;	/* 3 */
> -	uint32_t mw_len;
> -	uint64_t mw_va;		/* 4 */
> -	uint32_t mr_pbl_addr;	/* 5 */
> -	uint32_t reserved2:24;
> -	uint32_t mr_pagesz:8;
> -};
> -
> -struct t3_receive_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	uint8_t pagesz[T3_MAX_SGE];
> -	uint32_t num_sgle;		/* 2 */
> -	struct t3_sge sgl[T3_MAX_SGE];	/* 3+ */
> -	uint32_t pbl_addr[T3_MAX_SGE];
> -};
> -
> -struct t3_bypass_wr {
> -	struct fw_riwrh wrh;
> -	union t3_wrid wrid;	/* 1 */
> -};
> -
> -struct t3_modify_qp_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	uint32_t flags;		/* 2 */
> -	uint32_t quiesce;	/* 2 */
> -	uint32_t max_ird;	/* 3 */
> -	uint32_t max_ord;	/* 3 */
> -	uint64_t sge_cmd;	/* 4 */
> -	uint64_t ctx1;		/* 5 */
> -	uint64_t ctx0;		/* 6 */
> -};
> -
> -enum t3_modify_qp_flags {
> -	MODQP_QUIESCE  = 0x01,
> -	MODQP_MAX_IRD  = 0x02,
> -	MODQP_MAX_ORD  = 0x04,
> -	MODQP_WRITE_EC = 0x08,
> -	MODQP_READ_EC  = 0x10,
> -};
> -	
> -
> -enum t3_mpa_attrs {
> -	uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
> -	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
> -	uP_RI_MPA_CRC_ENABLE = 0x4,
> -	uP_RI_MPA_IETF_ENABLE = 0x8
> -} __attribute__ ((packed));
> -
> -enum t3_qp_caps {
> -	uP_RI_QP_RDMA_READ_ENABLE = 0x01,
> -	uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
> -	uP_RI_QP_BIND_ENABLE = 0x04,
> -	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
> -	uP_RI_QP_STAG0_ENABLE = 0x10
> -} __attribute__ ((packed));
> -
> -struct t3_rdma_init_attr {
> -	uint32_t tid;
> -	uint32_t qpid;
> -	uint32_t pdid;
> -	uint32_t scqid;
> -	uint32_t rcqid;
> -	uint32_t rq_addr;
> -	uint32_t rq_size;
> -	enum t3_mpa_attrs mpaattrs;
> -	enum t3_qp_caps qpcaps;
> -	uint16_t tcp_emss;
> -	uint32_t ord;
> -	uint32_t ird;
> -	uint64_t qp_dma_addr;
> -	uint32_t qp_dma_size;
> -	uint8_t rqes_posted;
> -};
> -
> -struct t3_rdma_init_wr {
> -	struct fw_riwrh wrh;	/* 0 */
> -	union t3_wrid wrid;	/* 1 */
> -	uint32_t qpid;		/* 2 */
> -	uint32_t pdid;
> -	uint32_t scqid;		/* 3 */
> -	uint32_t rcqid;
> -	uint32_t rq_addr;	/* 4 */
> -	uint32_t rq_size;
> -	enum t3_mpa_attrs mpaattrs:8;	/* 5 */
> -	enum t3_qp_caps qpcaps:8;
> -	uint32_t ulpdu_size:16;
> -	uint32_t rqes_posted;	/* bits 31-1 - reservered */
> -				/* bit     0 - set if RECV posted */
> -	uint32_t ord;		/* 6 */
> -	uint32_t ird;
> -	uint64_t qp_dma_addr;	/* 7 */
> -	uint32_t qp_dma_size;	/* 8 */
> -	uint32_t rsvd;
> -};
> -
> -union t3_wr {
> -	struct t3_send_wr send;
> -	struct t3_rdma_write_wr write;
> -	struct t3_rdma_read_wr read;
> -	struct t3_receive_wr recv;
> -	struct t3_local_inv_wr local_inv;
> -	struct t3_bind_mw_wr bind;
> -	struct t3_bypass_wr bypass;
> -	struct t3_rdma_init_wr init;
> -	struct t3_modify_qp_wr qp_mod;
> -	uint64_t flit[16];
> -};
> -
> -#define T3_SQ_CQE_FLIT 	  13
> -#define T3_SQ_COOKIE_FLIT 14
> -
> -#define T3_RQ_COOKIE_FLIT 13
> -#define T3_RQ_CQE_FLIT 	  14
> -
> -static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
> -				  enum t3_wr_flags flags, uint8_t genbit, 
> -				  uint32_t tid, uint8_t len)
> -{
> -	wqe->op_seop_flags = htobe32(V_FW_RIWR_OP(op) |
> -				   V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
> -				   V_FW_RIWR_FLAGS(flags));
> -	udma_to_device_barrier();
> -	wqe->gen_tid_len = htobe32(V_FW_RIWR_GEN(genbit) | V_FW_RIWR_TID(tid) |
> -				 V_FW_RIWR_LEN(len));
> -	/* 2nd gen bit... */
> -        ((union t3_wr *)wqe)->flit[15] = htobe64(genbit);
> -}
> -
> -/*
> - * T3 ULP2_TX commands
> - */
> -enum t3_utx_mem_op {
> -	T3_UTX_MEM_READ = 2,
> -	T3_UTX_MEM_WRITE = 3
> -};
> -
> -/* T3 MC7 RDMA TPT entry format */
> -
> -enum tpt_mem_type {
> -	TPT_NON_SHARED_MR = 0x0,
> -	TPT_SHARED_MR = 0x1,
> -	TPT_MW = 0x2,
> -	TPT_MW_RELAXED_PROTECTION = 0x3
> -};
> -
> -enum tpt_addr_type {
> -	TPT_ZBTO = 0,
> -	TPT_VATO = 1
> -};
> -
> -enum tpt_mem_perm {
> -	TPT_LOCAL_READ = 0x8,
> -	TPT_LOCAL_WRITE = 0x4,
> -	TPT_REMOTE_READ = 0x2,
> -	TPT_REMOTE_WRITE = 0x1
> -};
> -
> -struct tpt_entry {
> -	uint32_t valid_stag_pdid;
> -	uint32_t flags_pagesize_qpid;
> -
> -	uint32_t rsvd_pbl_addr;
> -	uint32_t len;
> -	uint32_t va_hi;
> -	uint32_t va_low_or_fbo;
> -
> -	uint32_t rsvd_bind_cnt_or_pstag;
> -	uint32_t rsvd_pbl_size;
> -};
> -
> -#define S_TPT_VALID		31
> -#define V_TPT_VALID(x)		((x) << S_TPT_VALID)
> -#define F_TPT_VALID		V_TPT_VALID(1U)
> -
> -#define S_TPT_STAG_KEY		23
> -#define M_TPT_STAG_KEY		0xFF
> -#define V_TPT_STAG_KEY(x)	((x) << S_TPT_STAG_KEY)
> -#define G_TPT_STAG_KEY(x)	(((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
> -
> -#define S_TPT_STAG_STATE	22
> -#define V_TPT_STAG_STATE(x)	((x) << S_TPT_STAG_STATE)
> -#define F_TPT_STAG_STATE	V_TPT_STAG_STATE(1U)
> -
> -#define S_TPT_STAG_TYPE		20
> -#define M_TPT_STAG_TYPE		0x3
> -#define V_TPT_STAG_TYPE(x)	((x) << S_TPT_STAG_TYPE)
> -#define G_TPT_STAG_TYPE(x)	(((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
> -
> -#define S_TPT_PDID		0
> -#define M_TPT_PDID		0xFFFFF
> -#define V_TPT_PDID(x)		((x) << S_TPT_PDID)
> -#define G_TPT_PDID(x)		(((x) >> S_TPT_PDID) & M_TPT_PDID)
> -
> -#define S_TPT_PERM		28
> -#define M_TPT_PERM		0xF
> -#define V_TPT_PERM(x)		((x) << S_TPT_PERM)
> -#define G_TPT_PERM(x)		(((x) >> S_TPT_PERM) & M_TPT_PERM)
> -
> -#define S_TPT_REM_INV_DIS	27
> -#define V_TPT_REM_INV_DIS(x)	((x) << S_TPT_REM_INV_DIS)
> -#define F_TPT_REM_INV_DIS	V_TPT_REM_INV_DIS(1U)
> -
> -#define S_TPT_ADDR_TYPE		26
> -#define V_TPT_ADDR_TYPE(x)	((x) << S_TPT_ADDR_TYPE)
> -#define F_TPT_ADDR_TYPE		V_TPT_ADDR_TYPE(1U)
> -
> -#define S_TPT_MW_BIND_ENABLE	25
> -#define V_TPT_MW_BIND_ENABLE(x)	((x) << S_TPT_MW_BIND_ENABLE)
> -#define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)
> -
> -#define S_TPT_PAGE_SIZE		20
> -#define M_TPT_PAGE_SIZE		0x1F
> -#define V_TPT_PAGE_SIZE(x)	((x) << S_TPT_PAGE_SIZE)
> -#define G_TPT_PAGE_SIZE(x)	(((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
> -
> -#define S_TPT_PBL_ADDR		0
> -#define M_TPT_PBL_ADDR		0x1FFFFFFF
> -#define V_TPT_PBL_ADDR(x)	((x) << S_TPT_PBL_ADDR)
> -#define G_TPT_PBL_ADDR(x)       (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
> -
> -#define S_TPT_QPID		0
> -#define M_TPT_QPID		0xFFFFF
> -#define V_TPT_QPID(x)		((x) << S_TPT_QPID)
> -#define G_TPT_QPID(x)		(((x) >> S_TPT_QPID) & M_TPT_QPID)
> -
> -#define S_TPT_PSTAG		0
> -#define M_TPT_PSTAG		0xFFFFFF
> -#define V_TPT_PSTAG(x)		((x) << S_TPT_PSTAG)
> -#define G_TPT_PSTAG(x)		(((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
> -
> -#define S_TPT_PBL_SIZE		0
> -#define M_TPT_PBL_SIZE		0xFFFFF
> -#define V_TPT_PBL_SIZE(x)	((x) << S_TPT_PBL_SIZE)
> -#define G_TPT_PBL_SIZE(x)	(((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
> -
> -/*
> - * CQE defs
> - */
> -struct t3_cqe {
> -	uint32_t header:32;
> -	uint32_t len:32;
> -	uint32_t wrid_hi_stag:32;
> -	uint32_t wrid_low_msn:32;
> -};
> -
> -#define S_CQE_OOO	  31
> -#define M_CQE_OOO	  0x1
> -#define G_CQE_OOO(x)	  ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
> -#define V_CEQ_OOO(x)	  ((x)<<S_CQE_OOO)
> -
> -#define S_CQE_QPID        12
> -#define M_CQE_QPID        0x7FFFF
> -#define G_CQE_QPID(x)     ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
> -#define V_CQE_QPID(x) 	  ((x)<<S_CQE_QPID)
> -
> -#define S_CQE_SWCQE       11
> -#define M_CQE_SWCQE       0x1
> -#define G_CQE_SWCQE(x)    ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
> -#define V_CQE_SWCQE(x) 	  ((x)<<S_CQE_SWCQE)
> -
> -#define S_CQE_GENBIT      10
> -#define M_CQE_GENBIT      0x1
> -#define G_CQE_GENBIT(x)   (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
> -#define V_CQE_GENBIT(x)	  ((x)<<S_CQE_GENBIT)
> -
> -#define S_CQE_STATUS      5
> -#define M_CQE_STATUS      0x1F
> -#define G_CQE_STATUS(x)   ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
> -#define V_CQE_STATUS(x)   ((x)<<S_CQE_STATUS)
> -
> -#define S_CQE_TYPE        4
> -#define M_CQE_TYPE        0x1
> -#define G_CQE_TYPE(x)     ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
> -#define V_CQE_TYPE(x)     ((x)<<S_CQE_TYPE)
> -
> -#define S_CQE_OPCODE      0
> -#define M_CQE_OPCODE      0xF
> -#define G_CQE_OPCODE(x)   ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
> -#define V_CQE_OPCODE(x)   ((x)<<S_CQE_OPCODE)
> -
> -#define SW_CQE(x)         (G_CQE_SWCQE(be32toh((x).header)))
> -#define CQE_OOO(x)        (G_CQE_OOO(be32toh((x).header)))
> -#define CQE_QPID(x)       (G_CQE_QPID(be32toh((x).header)))
> -#define CQE_GENBIT(x)     (G_CQE_GENBIT(be32toh((x).header)))
> -#define CQE_TYPE(x)       (G_CQE_TYPE(be32toh((x).header)))
> -#define SQ_TYPE(x)	  (CQE_TYPE((x)))
> -#define RQ_TYPE(x)	  (!CQE_TYPE((x)))
> -#define CQE_STATUS(x)     (G_CQE_STATUS(be32toh((x).header)))
> -#define CQE_OPCODE(x)     (G_CQE_OPCODE(be32toh((x).header)))
> -
> -#define CQE_LEN(x)        (be32toh((x).len))
> -
> -#define CQE_WRID_HI(x)    (be32toh((x).wrid_hi_stag))
> -#define CQE_WRID_LOW(x)   (be32toh((x).wrid_low_msn))
> -
> -/* used for RQ completion processing */
> -#define CQE_WRID_STAG(x)  (be32toh((x).wrid_hi_stag))
> -#define CQE_WRID_MSN(x)   (be32toh((x).wrid_low_msn))
> -
> -/* used for SQ completion processing */
> -#define CQE_WRID_SQ_WPTR(x)	((x).wrid_hi_stag)
> -#define CQE_WRID_WPTR(x)   	((x).wrid_low_msn)
> -
> -#define TPT_ERR_SUCCESS                     0x0
> -#define TPT_ERR_STAG                        0x1	 /* STAG invalid: either the */
> -						 /* STAG is offlimt, being 0, */
> -						 /* or STAG_key mismatch */
> -#define TPT_ERR_PDID                        0x2	 /* PDID mismatch */
> -#define TPT_ERR_QPID                        0x3	 /* QPID mismatch */
> -#define TPT_ERR_ACCESS                      0x4	 /* Invalid access right */
> -#define TPT_ERR_WRAP                        0x5	 /* Wrap error */
> -#define TPT_ERR_BOUND                       0x6	 /* base and bounds voilation */
> -#define TPT_ERR_INVALIDATE_SHARED_MR        0x7	 /* attempt to invalidate a  */
> -						 /* shared memory region */
> -#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8	 /* attempt to invalidate a  */
> -						 /* shared memory region */
> -#define TPT_ERR_ECC                         0x9	 /* ECC error detected */
> -#define TPT_ERR_ECC_PSTAG                   0xA	 /* ECC error detected when  */
> -						 /* reading PSTAG for a MW  */
> -						 /* Invalidate */
> -#define TPT_ERR_PBL_ADDR_BOUND              0xB	 /* pbl addr out of bounds:  */
> -						 /* software error */
> -#define TPT_ERR_SWFLUSH			    0xC	 /* SW FLUSHED */
> -#define TPT_ERR_CRC                         0x10 /* CRC error */
> -#define TPT_ERR_MARKER                      0x11 /* Marker error */
> -#define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */
> -#define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */
> -#define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */
> -#define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */
> -#define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */
> -#define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */
> -#define TPT_ERR_MSN                         0x18 /* MSN error */
> -#define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */
> -#define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */
> -						 /* or READ_REQ */
> -#define TPT_ERR_MSN_GAP                     0x1B
> -#define TPT_ERR_MSN_RANGE                   0x1C
> -#define TPT_ERR_IRD_OVERFLOW                0x1D
> -#define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */
> -						 /* software error */
> -#define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
> -						 /* mismatch) */
> -
> -struct t3_swsq {
> -	uint64_t 		wr_id;
> -	struct t3_cqe 		cqe;
> -	uint32_t		sq_wptr;
> -	uint32_t		read_len;
> -	int 			opcode;
> -	int			complete;
> -	int			signaled;	
> -};
> -
> -/*
> - * A T3 WQ implements both the SQ and RQ.
> - */
> -struct t3_wq {
> -	union t3_wr *queue;		/* DMA Mapped work queue */
> -	uint32_t error;			/* 1 once we go to ERROR */
> -	uint32_t qpid;
> -	uint32_t wptr;			/* idx to next available WR slot */
> -	uint32_t size_log2;		/* total wq size */
> -	struct t3_swsq *sq;		/* SW SQ */
> -	struct t3_swsq *oldest_read;	/* tracks oldest pending read */
> -	uint32_t sq_wptr;		/* sq_wptr - sq_rptr == count of */
> -	uint32_t sq_rptr;		/* pending wrs */
> -	uint32_t sq_size_log2;		/* sq size */
> -	uint64_t *rq;			/* SW RQ (holds consumer wr_ids) */
> -	uint32_t rq_wptr;		/* rq_wptr - rq_rptr == count of */
> -	uint32_t rq_rptr;		/* pending wrs */
> -	uint32_t rq_size_log2;		/* rq size */
> -	volatile uint32_t *doorbell;	/* mapped adapter doorbell register */
> -	int flushed;
> -};
> -
> -struct t3_cq {
> -	uint32_t cqid;
> -	uint32_t rptr;
> -	uint32_t wptr;
> -	uint32_t size_log2;
> -	struct t3_cqe *queue;
> -	struct t3_cqe *sw_queue;
> -	uint32_t sw_rptr;
> -	uint32_t sw_wptr;
> -	uint32_t memsize;
> -};
> -
> -static inline unsigned t3_wq_depth(struct t3_wq *wq)
> -{
> -	return (1UL<<wq->size_log2);
> -}
> -
> -static inline unsigned t3_sq_depth(struct t3_wq *wq)
> -{
> -	return (1UL<<wq->sq_size_log2);
> -}
> -
> -static inline unsigned t3_rq_depth(struct t3_wq *wq)
> -{
> -	return (1UL<<wq->rq_size_log2);
> -}
> -
> -static inline unsigned t3_cq_depth(struct t3_cq *cq)
> -{
> -	return (1UL<<cq->size_log2);
> -}
> -
> -extern unsigned long iwch_page_size;
> -extern unsigned long iwch_page_shift;
> -extern unsigned long iwch_page_mask;
> -
> -#define PAGE_ALIGN(x) (((x) + iwch_page_mask) & ~iwch_page_mask)
> -
> -static inline unsigned t3_wq_memsize(struct t3_wq *wq)
> -{
> -	return PAGE_ALIGN((1UL<<wq->size_log2) * sizeof (union t3_wr));
> -}
> -
> -static inline unsigned t3_cq_memsize(struct t3_cq *cq)
> -{
> -	return cq->memsize;
> -}
> -
> -static inline unsigned t3_mmid(uint32_t stag)
> -{
> -	return (stag>>8);
> -}
> -
> -struct t3_cq_status_page {
> -	uint32_t cq_err;
> -};
> -
> -static inline int t3_cq_in_error(struct t3_cq *cq)
> -{
> -	return ((struct t3_cq_status_page *)
> -	       &cq->queue[1 << cq->size_log2])->cq_err;
> -}
> -
> -static inline void t3_set_cq_in_error(struct t3_cq *cq)
> -{
> -	((struct t3_cq_status_page *)
> -		&cq->queue[1 << cq->size_log2])->cq_err = 1;
> -}
> -
> -static inline void t3_reset_cq_in_error(struct t3_cq *cq)
> -{
> -	((struct t3_cq_status_page *)
> -		&cq->queue[1 << cq->size_log2])->cq_err = 0;
> -}
> -
> -static inline int t3_wq_in_error(struct t3_wq *wq)
> -{
> -	/*
> -	 * The kernel sets bit 0 in the first WR of the WQ memory
> -	 * when the QP moves out of RTS...
> -	 */
> -        return (wq->queue->flit[13] & 1);
> -}
> -
> -static inline void t3_set_wq_in_error(struct t3_wq *wq)
> -{
> -        wq->queue->flit[13] |= 1;
> -}
> -
> -static inline int t3_wq_db_enabled(struct t3_wq *wq)
> -{
> -	return !(wq->queue->flit[13] & 2);
> -}
> -
> -#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
> -					 CQE_GENBIT(*cqe))
> -
> -static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
> -{
> -	struct t3_cqe *cqe;
> -
> -	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
> -	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
> -		return cqe;
> -	return NULL;
> -}
> -
> -static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
> -{
> -	struct t3_cqe *cqe;
> -
> -	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
> -		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
> -		return cqe;
> -	}
> -	return NULL;
> -}
> -
> -static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
> -{
> -	struct t3_cqe *cqe;
> -
> -	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
> -		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
> -		return cqe;
> -	}
> -	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
> -	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
> -		return cqe;
> -	return NULL;
> -}
> -
> -/*
> - * Return a ptr to the next read wr in the SWSQ or NULL.
> - */
> -static inline struct t3_swsq *next_read_wr(struct t3_wq *wq)
> -{
> -	uint32_t rptr = wq->oldest_read - wq->sq + 1;
> -	int count = Q_COUNT(rptr, wq->sq_wptr);
> -	struct t3_swsq *sqp;
> -
> -	while (count--) {
> -		sqp = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
> -
> -		if (sqp->opcode == T3_READ_REQ)
> -			return sqp;
> -
> -		rptr++;
> -	}
> -	return NULL;
> -}
> -#endif
> diff --git a/providers/cxgb3/firmware_exports.h b/providers/cxgb3/firmware_exports.h
> deleted file mode 100644
> index 831140a4..00000000
> --- a/providers/cxgb3/firmware_exports.h
> +++ /dev/null
> @@ -1,148 +0,0 @@
> -/*
> - * Copyright (c) 2004-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#ifndef _FIRMWARE_EXPORTS_H_
> -#define _FIRMWARE_EXPORTS_H_
> -
> -/* WR OPCODES supported by the firmware.
> - */
> -#define	FW_WROPCODE_FORWARD			0x01
> -#define FW_WROPCODE_BYPASS			0x05
> -
> -#define FW_WROPCODE_TUNNEL_TX_PKT		0x03
> -
> -#define FW_WROPOCDE_ULPTX_DATA_SGL		0x00
> -#define FW_WROPCODE_ULPTX_MEM_READ		0x02
> -#define FW_WROPCODE_ULPTX_PKT			0x04
> -#define FW_WROPCODE_ULPTX_INVALIDATE		0x06
> -
> -#define FW_WROPCODE_TUNNEL_RX_PKT		0x07
> -
> -#define FW_WROPCODE_TOE_GETTCB_RPL		0x08
> -#define FW_WROPCODE_TOE_CLOSE_CON		0x09
> -#define FW_WROPCODE_TOE_TP_ABORT_CON_REQ	0x0A
> -#define FW_WROPCODE_TOE_HOST_ABORT_CON_RPL	0x0F
> -#define FW_WROPCODE_TOE_HOST_ABORT_CON_REQ	0x0B
> -#define FW_WROPCODE_TOE_TP_ABORT_CON_RPL	0x0C
> -#define FW_WROPCODE_TOE_TX_DATA			0x0D
> -#define FW_WROPCODE_TOE_TX_DATA_ACK		0x0E
> -
> -#define FW_WROPCODE_RI_RDMA_INIT		0x10
> -#define FW_WROPCODE_RI_RDMA_WRITE		0x11
> -#define FW_WROPCODE_RI_RDMA_READ_REQ		0x12
> -#define FW_WROPCODE_RI_RDMA_READ_RESP		0x13
> -#define FW_WROPCODE_RI_SEND			0x14
> -#define FW_WROPCODE_RI_TERMINATE		0x15
> -#define FW_WROPCODE_RI_RDMA_READ		0x16
> -#define FW_WROPCODE_RI_RECEIVE			0x17
> -#define FW_WROPCODE_RI_BIND_MW			0x18
> -#define FW_WROPCODE_RI_FASTREGISTER_MR		0x19
> -#define FW_WROPCODE_RI_LOCAL_INV		0x1A
> -#define FW_WROPCODE_RI_MODIFY_QP		0x1B
> -#define FW_WROPCODE_RI_BYPASS			0x1C
> -
> -#define FW_WROPOCDE_RSVD			0x1E
> -
> -#define FW_WROPCODE_SGE_EGRESSCONTEXT_RR	0x1F
> -
> -#define FW_WROPCODE_MNGT			0x1D
> -#define FW_MNGTOPCODE_PKTSCHED_SET		0x00
> -
> -/* Maximum size of a WR sent from the host, limited by the SGE. 
> - *
> - * Note: WR coming from ULP or TP are only limited by CIM. 
> - */
> -#define FW_WR_SIZE			128
> -
> -/* Maximum number of outstanding WRs sent from the host. Value must be
> - * programmed in the CTRL/TUNNEL/QP SGE Egress Context and used by TOM to 
> - * limit the number of WRs per connection.
> - */
> -#ifndef N3
> -# define FW_WR_NUM			16
> -#else
> -# define FW_WR_NUM			7
> -#endif
> -
> -/* FW_TUNNEL_NUM corresponds to the number of supported TUNNEL Queues. These
> - * queues must start at SGE Egress Context FW_TUNNEL_SGEEC_START and must
> - * start at 'TID' (or 'uP Token') FW_TUNNEL_TID_START.
> - *
> - * Ingress Traffic (e.g. DMA completion credit)  for TUNNEL Queue[i] is sent 
> - * to RESP Queue[i].
> - */
> -#define FW_TUNNEL_NUM                   8
> -#define FW_TUNNEL_SGEEC_START           8
> -#define FW_TUNNEL_TID_START             65544
> -
> -
> -/* FW_CTRL_NUM corresponds to the number of supported CTRL Queues. These queues
> - * must start at SGE Egress Context FW_CTRL_SGEEC_START and must start at 'TID'
> - * (or 'uP Token') FW_CTRL_TID_START.
> - *
> - * Ingress Traffic for CTRL Queue[i] is sent to RESP Queue[i].
> - */ 
> -#define FW_CTRL_NUM			8
> -#define FW_CTRL_SGEEC_START		65528
> -#define FW_CTRL_TID_START		65536
> -
> -/* FW_TOE_NUM corresponds to the number of supported TOE Queues. These queues 
> - * must start at SGE Egress Context FW_TOE_SGEEC_START. 
> - * 
> - * Note: the 'uP Token' in the SGE Egress Context fields is irrelevant for 
> - * TOE Queues, as the host is responsible for providing the correct TID in
> - * every WR.
> - *
> - * Ingress Trafffic for TOE Queue[i] is sent to RESP Queue[i].
> - */
> -#define FW_TOE_NUM			8
> -#define FW_TOE_SGEEC_START		0
> -
> -/*
> - * 
> - */
> -#define FW_RI_NUM			1
> -#define FW_RI_SGEEC_START		65527
> -#define FW_RI_TID_START			65552
> -
> -/*
> - * The RX_PKT_TID 
> - */
> -#define FW_RX_PKT_NUM			1
> -#define FW_RX_PKT_TID_START		65553
> -
> -/* FW_WRC_NUM corresponds to the number of Work Request Context that supported
> - * by the firmware.
> - */
> -#define FW_WRC_NUM			(65536 + FW_TUNNEL_NUM + FW_CTRL_NUM +\
> -					 FW_RI_NUM + FW_RX_PKT_NUM)
> -
> -#endif /* _FIRMWARE_EXPORTS_H_ */
> diff --git a/providers/cxgb3/iwch-abi.h b/providers/cxgb3/iwch-abi.h
> deleted file mode 100644
> index 047f84b7..00000000
> --- a/providers/cxgb3/iwch-abi.h
> +++ /dev/null
> @@ -1,51 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#ifndef IWCH_ABI_H
> -#define IWCH_ABI_H
> -
> -#include <stdint.h>
> -#include <infiniband/kern-abi.h>
> -#include <rdma/cxgb3-abi.h>
> -#include <kernel-abi/cxgb3-abi.h>
> -
> -DECLARE_DRV_CMD(uiwch_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
> -		empty, iwch_alloc_pd_resp);
> -DECLARE_DRV_CMD(uiwch_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
> -		iwch_create_cq_req, iwch_create_cq_resp);
> -DECLARE_DRV_CMD(uiwch_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
> -		empty, iwch_create_qp_resp);
> -DECLARE_DRV_CMD(uiwch_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
> -		empty, empty);
> -DECLARE_DRV_CMD(uiwch_reg_mr, IB_USER_VERBS_CMD_REG_MR,
> -		empty, iwch_reg_user_mr_resp);
> -
> -#endif				/* IWCH_ABI_H */
> diff --git a/providers/cxgb3/iwch.c b/providers/cxgb3/iwch.c
> deleted file mode 100644
> index 6f3c8b9f..00000000
> --- a/providers/cxgb3/iwch.c
> +++ /dev/null
> @@ -1,269 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#include <config.h>
> -
> -#include <stdio.h>
> -#include <stdlib.h>
> -#include <unistd.h>
> -#include <errno.h>
> -#include <sys/mman.h>
> -#include <pthread.h>
> -#include <string.h>
> -
> -#include "iwch.h"
> -#include "iwch-abi.h"
> -
> -#define PCI_VENDOR_ID_CHELSIO		0x1425
> -#define PCI_DEVICE_ID_CHELSIO_PE9000_2C	0x0020
> -#define PCI_DEVICE_ID_CHELSIO_T302E	0x0021
> -#define PCI_DEVICE_ID_CHELSIO_T310E	0x0022
> -#define PCI_DEVICE_ID_CHELSIO_T320X	0x0023
> -#define PCI_DEVICE_ID_CHELSIO_T302X	0x0024
> -#define PCI_DEVICE_ID_CHELSIO_T320E	0x0025
> -#define PCI_DEVICE_ID_CHELSIO_T310X	0x0026
> -#define PCI_DEVICE_ID_CHELSIO_T3B10	0x0030
> -#define PCI_DEVICE_ID_CHELSIO_T3B20	0x0031
> -#define PCI_DEVICE_ID_CHELSIO_T3B02	0x0032
> -#define PCI_DEVICE_ID_CHELSIO_T3C20	0x0035
> -#define PCI_DEVICE_ID_CHELSIO_S320E	0x0036
> -
> -#define HCA(v, d, t)                                                           \
> -	VERBS_PCI_MATCH(PCI_VENDOR_ID_##v, PCI_DEVICE_ID_CHELSIO_##d,          \
> -			(void *)(CHELSIO_##t))
> -static const struct verbs_match_ent hca_table[] = {
> -	HCA(CHELSIO, PE9000_2C, T3B),
> -	HCA(CHELSIO, T302E, T3A),
> -	HCA(CHELSIO, T302X, T3A),
> -	HCA(CHELSIO, T310E, T3A),
> -	HCA(CHELSIO, T310X, T3A),
> -	HCA(CHELSIO, T320E, T3A),
> -	HCA(CHELSIO, T320X, T3A),
> -	HCA(CHELSIO, T3B10, T3B),
> -	HCA(CHELSIO, T3B20, T3B),
> -	HCA(CHELSIO, T3B02, T3B),
> -	HCA(CHELSIO, T3C20, T3B),
> -	HCA(CHELSIO, S320E, T3B),
> -	{},
> -};
> -
> -static const struct verbs_context_ops iwch_ctx_common_ops = {
> -	.query_device = iwch_query_device,
> -	.query_port = iwch_query_port,
> -	.alloc_pd = iwch_alloc_pd,
> -	.dealloc_pd = iwch_free_pd,
> -	.reg_mr = iwch_reg_mr,
> -	.dereg_mr = iwch_dereg_mr,
> -	.create_cq = iwch_create_cq,
> -	.resize_cq = iwch_resize_cq,
> -	.destroy_cq = iwch_destroy_cq,
> -	.create_srq = iwch_create_srq,
> -	.modify_srq = iwch_modify_srq,
> -	.destroy_srq = iwch_destroy_srq,
> -	.create_qp = iwch_create_qp,
> -	.modify_qp = iwch_modify_qp,
> -	.destroy_qp = iwch_destroy_qp,
> -	.query_qp = iwch_query_qp,
> -	.create_ah = iwch_create_ah,
> -	.destroy_ah = iwch_destroy_ah,
> -	.attach_mcast = iwch_attach_mcast,
> -	.detach_mcast = iwch_detach_mcast,
> -	.post_srq_recv = iwch_post_srq_recv,
> -	.req_notify_cq = iwch_arm_cq,
> -};
> -
> -static const struct verbs_context_ops iwch_ctx_t3a_ops = {
> -	.poll_cq = t3a_poll_cq,
> -	.post_recv = t3a_post_recv,
> -	.post_send = t3a_post_send,
> -};
> -
> -static const struct verbs_context_ops iwch_ctx_t3b_ops = {
> -	.async_event = t3b_async_event,
> -	.poll_cq = t3b_poll_cq,
> -	.post_recv = t3b_post_recv,
> -	.post_send = t3b_post_send,
> -};
> -
> -unsigned long iwch_page_size;
> -unsigned long iwch_page_shift;
> -unsigned long iwch_page_mask;
> -
> -static struct verbs_context *iwch_alloc_context(struct ibv_device *ibdev,
> -						int cmd_fd,
> -						void *private_data)
> -{
> -	struct iwch_context *context;
> -	struct ibv_get_context cmd;
> -	struct uiwch_alloc_ucontext_resp resp;
> -	struct iwch_device *rhp = to_iwch_dev(ibdev);
> -
> -	context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
> -					       RDMA_DRIVER_CXGB3);
> -	if (!context)
> -		return NULL;
> -
> -	if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
> -				&resp.ibv_resp, sizeof resp))
> -		goto err_free;
> -
> -	verbs_set_ops(&context->ibv_ctx, &iwch_ctx_common_ops);
> -
> -	switch (rhp->hca_type) {
> -	case CHELSIO_T3B:
> -		PDBG("%s T3B device\n", __FUNCTION__);
> -		verbs_set_ops(&context->ibv_ctx, &iwch_ctx_t3b_ops);
> -		break;
> -	case CHELSIO_T3A:
> -		PDBG("%s T3A device\n", __FUNCTION__);
> -		verbs_set_ops(&context->ibv_ctx, &iwch_ctx_t3a_ops);
> -		break;
> -	default:
> -		PDBG("%s unknown hca type %d\n", __FUNCTION__, rhp->hca_type);
> -		goto err_free;
> -		break;
> -	}	
> -
> -	return &context->ibv_ctx;
> -
> -err_free:
> -	verbs_uninit_context(&context->ibv_ctx);
> -	free(context);
> -	return NULL;
> -}
> -
> -static void iwch_free_context(struct ibv_context *ibctx)
> -{
> -	struct iwch_context *context = to_iwch_ctx(ibctx);
> -
> -	verbs_uninit_context(&context->ibv_ctx);
> -	free(context);
> -}
> -
> -static void iwch_uninit_device(struct verbs_device *verbs_device)
> -{
> -	struct iwch_device *dev = to_iwch_dev(&verbs_device->device);
> -
> -	free(dev);
> -}
> -
> -static bool iwch_device_match(struct verbs_sysfs_dev *sysfs_dev)
> -{
> -	char value[32], *cp;
> -	unsigned int fw_maj, fw_min;
> -
> -	/* Rely on the core code to match PCI devices */
> -	if (!sysfs_dev->match)
> -		return false;
> -
> -	/* 
> -	 * Verify that the firmware major number matches.  Major number
> -	 * mismatches are fatal.  Minor number mismatches are tolerated.
> -	 */
> -	if (ibv_get_fw_ver(value, sizeof(value), sysfs_dev))
> -		return false;
> -
> -	cp = strtok(value+1, ".");
> -	sscanf(cp, "%i", &fw_maj);
> -	cp = strtok(NULL, ".");
> -	sscanf(cp, "%i", &fw_min);
> -
> -	if (fw_maj < FW_MAJ) {
> -		fprintf(stderr, "libcxgb3: Fatal firmware version mismatch.  "
> -			"Firmware major number is %u and libcxgb3 needs %u.\n",
> -			fw_maj, FW_MAJ);	
> -		fflush(stderr);
> -		return false;
> -	}
> -
> -	DBGLOG("libcxgb3");
> -
> -	if ((signed int)fw_min < FW_MIN) {
> -		PDBG("libcxgb3: non-fatal firmware version mismatch.  "
> -			"Firmware minor number is %u and libcxgb3 needs %u.\n",
> -			fw_min, FW_MIN);
> -		fflush(stderr);
> -	}
> -
> -	return true;
> -}
> -
> -static struct verbs_device *iwch_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
> -{
> -	struct iwch_device *dev;
> -
> -	dev = calloc(1, sizeof(*dev));
> -	if (!dev)
> -		return NULL;
> -
> -	pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE);
> -	dev->hca_type = (uintptr_t)sysfs_dev->match->driver_data;
> -	dev->abi_version = sysfs_dev->abi_ver;
> -
> -	iwch_page_size = sysconf(_SC_PAGESIZE);
> -	iwch_page_shift = long_log2(iwch_page_size);
> -	iwch_page_mask = iwch_page_size - 1;
> -
> -	dev->mmid2ptr = calloc(T3_MAX_NUM_STAG, sizeof(void *));
> -	if (!dev->mmid2ptr) {
> -		goto err1;
> -	}
> -	dev->qpid2ptr = calloc(T3_MAX_NUM_QP, sizeof(void *)); 
> -	if (!dev->qpid2ptr) {
> -		goto err2;
> -	}
> -	dev->cqid2ptr = calloc(T3_MAX_NUM_CQ, sizeof(void *));
> -	if (!dev->cqid2ptr) 
> -		goto err3;
> -
> -	return &dev->ibv_dev;
> -
> -err3:
> -	free(dev->qpid2ptr);
> -err2:
> -	free(dev->mmid2ptr);
> -err1:
> -	free(dev);
> -	return NULL;
> -}
> -
> -static const struct verbs_device_ops iwch_dev_ops = {
> -	.name = "cxgb3",
> -	.match_min_abi_version = 0,
> -	.match_max_abi_version = ABI_VERS,
> -	.match_table = hca_table,
> -	.match_device = iwch_device_match,
> -	.alloc_device = iwch_device_alloc,
> -	.uninit_device = iwch_uninit_device,
> -	.alloc_context = iwch_alloc_context,
> -	.free_context = iwch_free_context,
> -};
> -PROVIDER_DRIVER(cxgb3, iwch_dev_ops);
> diff --git a/providers/cxgb3/iwch.h b/providers/cxgb3/iwch.h
> deleted file mode 100644
> index c7d85d3a..00000000
> --- a/providers/cxgb3/iwch.h
> +++ /dev/null
> @@ -1,218 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#ifndef IWCH_H
> -#define IWCH_H
> -
> -#include <pthread.h>
> -#include <inttypes.h>
> -#include <stddef.h>
> -
> -#include <infiniband/driver.h>
> -#include <util/udma_barrier.h>
> -
> -#include "cxio_wr.h"
> -
> -enum iwch_hca_type {
> -	CHELSIO_T3A = 0,
> -	CHELSIO_T3B = 1,
> -};
> -
> -struct iwch_mr;
> -
> -#define ABI_VERS 1
> -
> -struct iwch_device {
> -	struct verbs_device ibv_dev;
> -	enum iwch_hca_type hca_type;
> -	struct iwch_mr **mmid2ptr;
> -	struct iwch_qp **qpid2ptr;
> -	struct iwch_cq **cqid2ptr;
> -	pthread_spinlock_t lock;
> -	int abi_version;
> -};
> -
> -static inline int t3b_device(struct iwch_device *dev)
> -{
> -	return (dev->hca_type == CHELSIO_T3B);
> -}
> -
> -static inline int t3a_device(struct iwch_device *dev)
> -{
> -	return (dev->hca_type == CHELSIO_T3A);
> -}
> -
> -struct iwch_context {
> -	struct verbs_context ibv_ctx;
> -};
> -
> -struct iwch_pd {
> -	struct ibv_pd ibv_pd;
> -};
> -
> -struct iwch_mr {
> -	struct verbs_mr vmr;
> -	uint64_t va_fbo;
> -	uint32_t page_size;
> -	uint32_t pbl_addr;
> -	uint32_t len;
> -};
> -
> -struct iwch_cq {
> -	struct ibv_cq ibv_cq;
> -	struct iwch_device *rhp;
> -	struct t3_cq cq;
> -	pthread_spinlock_t lock;
> -};
> -
> -struct iwch_qp {
> -	struct ibv_qp ibv_qp;
> -	struct iwch_device *rhp;
> -	struct t3_wq wq;
> -	pthread_spinlock_t lock;
> -	int sq_sig_all;
> -};
> -
> -#define to_iwch_xxx(xxx, type)                                                 \
> -	container_of(ib##xxx, struct iwch_##type, ibv_##xxx)
> -
> -static inline struct iwch_device *to_iwch_dev(struct ibv_device *ibdev)
> -{
> -	return container_of(ibdev, struct iwch_device, ibv_dev.device);
> -}
> -
> -static inline struct iwch_context *to_iwch_ctx(struct ibv_context *ibctx)
> -{
> -	return container_of(ibctx, struct iwch_context, ibv_ctx.context);
> -}
> -
> -static inline struct iwch_pd *to_iwch_pd(struct ibv_pd *ibpd)
> -{
> -	return to_iwch_xxx(pd, pd);
> -}
> -
> -static inline struct iwch_cq *to_iwch_cq(struct ibv_cq *ibcq)
> -{
> -	return to_iwch_xxx(cq, cq);
> -}
> -
> -static inline struct iwch_qp *to_iwch_qp(struct ibv_qp *ibqp)
> -{
> -	return to_iwch_xxx(qp, qp);
> -}
> -
> -static inline struct iwch_mr *to_iwch_mr(struct verbs_mr *vmr)
> -{
> -	return container_of(vmr, struct iwch_mr, vmr);
> -}
> -
> -static inline unsigned long long_log2(unsigned long x)
> -{
> -        unsigned long r = 0;
> -        for (x >>= 1; x > 0; x >>= 1)
> -                r++;
> -        return r;
> -}
> -
> -extern int iwch_query_device(struct ibv_context *context,
> -			     struct ibv_device_attr *attr);
> -extern int iwch_query_port(struct ibv_context *context, uint8_t port,
> -			   struct ibv_port_attr *attr);
> -
> -extern struct ibv_pd *iwch_alloc_pd(struct ibv_context *context);
> -extern int iwch_free_pd(struct ibv_pd *pd);
> -
> -extern struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> -				  uint64_t hca_va, int access);
> -extern int iwch_dereg_mr(struct verbs_mr *mr);
> -
> -struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
> -			      struct ibv_comp_channel *channel,
> -			      int comp_vector);
> -extern int iwch_resize_cq(struct ibv_cq *cq, int cqe);
> -extern int iwch_destroy_cq(struct ibv_cq *cq);
> -extern int t3a_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
> -extern int t3b_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
> -extern int iwch_arm_cq(struct ibv_cq *cq, int solicited);
> -extern void iwch_cq_event(struct ibv_cq *cq);
> -extern void iwch_init_cq_buf(struct iwch_cq *cq, int nent);
> -
> -extern struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
> -				       struct ibv_srq_init_attr *attr);
> -extern int iwch_modify_srq(struct ibv_srq *srq,
> -			   struct ibv_srq_attr *attr,
> -			   int mask);
> -extern int iwch_destroy_srq(struct ibv_srq *srq);
> -extern int iwch_post_srq_recv(struct ibv_srq *ibsrq,
> -			      struct ibv_recv_wr *wr,
> -			      struct ibv_recv_wr **bad_wr);
> -
> -extern struct ibv_qp *iwch_create_qp(struct ibv_pd *pd,
> -				     struct ibv_qp_init_attr *attr);
> -extern int iwch_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
> -			  int attr_mask);
> -extern int iwch_destroy_qp(struct ibv_qp *qp);
> -extern int iwch_query_qp(struct ibv_qp *qp,
> -			 struct ibv_qp_attr *attr,
> -			 int attr_mask,
> -			 struct ibv_qp_init_attr *init_attr);
> -extern void iwch_flush_qp(struct iwch_qp *qhp);
> -extern void iwch_flush_qps(struct iwch_device *dev);
> -extern int t3a_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
> -			  struct ibv_send_wr **bad_wr);
> -extern int t3b_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
> -			  struct ibv_send_wr **bad_wr);
> -extern int t3a_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
> -			  struct ibv_recv_wr **bad_wr);
> -extern int t3b_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
> -			  struct ibv_recv_wr **bad_wr);
> -extern struct ibv_ah *iwch_create_ah(struct ibv_pd *pd,
> -			     struct ibv_ah_attr *ah_attr);
> -extern int iwch_destroy_ah(struct ibv_ah *ah);
> -extern int iwch_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
> -			     uint16_t lid);
> -extern int iwch_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
> -			     uint16_t lid);
> -extern void t3b_async_event(struct ibv_context *context,
> -			    struct ibv_async_event *event);
> -#ifdef DEBUG
> -#include <syslog.h>
> -#define DBGLOG(s) openlog(s, LOG_NDELAY|LOG_PID, LOG_LOCAL7)
> -#define PDBG(fmt, args...) do {syslog(LOG_DEBUG, fmt, ##args);} while (0)
> -#else
> -#define DBGLOG(s) 
> -#define PDBG(fmt, args...) do {} while (0)
> -#endif
> -
> -#define FW_MAJ 5
> -#define FW_MIN 0
> -
> -#endif				/* IWCH_H */
> diff --git a/providers/cxgb3/qp.c b/providers/cxgb3/qp.c
> deleted file mode 100644
> index 4a1e7397..00000000
> --- a/providers/cxgb3/qp.c
> +++ /dev/null
> @@ -1,560 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#include <config.h>
> -
> -#include <stdlib.h>
> -#include <pthread.h>
> -#include <string.h>
> -
> -#include "iwch.h"
> -#include <stdio.h>
> -
> -#define ROUNDUP8(a) (((a) + 7) & ~7)
> -
> -static inline int iwch_build_rdma_send(union t3_wr *wqe, struct ibv_send_wr *wr,
> -				       uint8_t *flit_cnt)
> -{
> -	int i;
> -
> -	if (wr->num_sge > T3_MAX_SGE)
> -		return -1;
> -	if (wr->send_flags & IBV_SEND_SOLICITED)
> -		wqe->send.rdmaop = T3_SEND_WITH_SE;
> -	else
> -		wqe->send.rdmaop = T3_SEND;
> -	wqe->send.rem_stag = 0;
> -	wqe->send.reserved = 0;
> -	if ((wr->send_flags & IBV_SEND_INLINE) || wr->num_sge == 0) {
> -		uint8_t *datap;
> -
> -		wqe->send.plen = 0;
> -		datap = (uint8_t *)&wqe->send.sgl[0];
> -		wqe->send.num_sgle = 0;	/* indicates in-line data */
> -		for (i = 0; i < wr->num_sge; i++) {
> -			if ((wqe->send.plen + wr->sg_list[i].length) > 
> -			    T3_MAX_INLINE)
> -				return -1;
> -			wqe->send.plen += wr->sg_list[i].length;
> -			memcpy(datap, 
> -			       (void *)(unsigned long)wr->sg_list[i].addr, 
> -			       wr->sg_list[i].length);
> -			datap += wr->sg_list[i].length;
> -		}
> -		*flit_cnt = 4 + (ROUNDUP8(wqe->send.plen) >> 3);
> -		wqe->send.plen = htobe32(wqe->send.plen);
> -	} else {
> -		wqe->send.plen = 0;
> -		for (i = 0; i < wr->num_sge; i++) {
> -			if ((wqe->send.plen + wr->sg_list[i].length) < 
> -			    wqe->send.plen) {
> -				return -1;
> -			}
> -			wqe->send.plen += wr->sg_list[i].length;
> -			wqe->send.sgl[i].stag =
> -			    htobe32(wr->sg_list[i].lkey);
> -			wqe->send.sgl[i].len =
> -			    htobe32(wr->sg_list[i].length);
> -			wqe->send.sgl[i].to = htobe64(wr->sg_list[i].addr);
> -		}
> -		wqe->send.plen = htobe32(wqe->send.plen);
> -		wqe->send.num_sgle = htobe32(wr->num_sge);
> -		*flit_cnt = 4 + ((wr->num_sge) << 1);
> -	}
> -	return 0;
> -}
> -
> -static inline int iwch_build_rdma_write(union t3_wr *wqe, 
> -					struct ibv_send_wr *wr,
> -					uint8_t *flit_cnt)
> -{
> -	int i;
> -
> -	if (wr->num_sge > T3_MAX_SGE)
> -		return -1;
> -	wqe->write.rdmaop = T3_RDMA_WRITE;
> -	wqe->write.reserved = 0;
> -	wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey);
> -	wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr);
> -
> -	wqe->write.num_sgle = wr->num_sge;
> -
> -	if ((wr->send_flags & IBV_SEND_INLINE) || wr->num_sge == 0) {
> -		uint8_t *datap;
> -
> -		wqe->write.plen = 0;
> -		datap = (uint8_t *)&wqe->write.sgl[0];
> -		wqe->write.num_sgle = 0;	/* indicates in-line data */
> -		for (i = 0; i < wr->num_sge; i++) {
> -			if ((wqe->write.plen + wr->sg_list[i].length) >
> -			    T3_MAX_INLINE)
> -				return -1;
> -			wqe->write.plen += wr->sg_list[i].length;
> -			memcpy(datap, 
> -			       (void *)(unsigned long)wr->sg_list[i].addr, 
> -			       wr->sg_list[i].length);
> -			datap += wr->sg_list[i].length;
> -		}
> -		*flit_cnt = 5 + (ROUNDUP8(wqe->write.plen) >> 3);
> -		wqe->write.plen = htobe32(wqe->write.plen);
> -	} else {
> -		wqe->write.plen = 0;
> -		for (i = 0; i < wr->num_sge; i++) {
> -			if ((wqe->write.plen + wr->sg_list[i].length) < 
> -			    wqe->write.plen) {
> -				return -1;
> -			}
> -			wqe->write.plen += wr->sg_list[i].length;
> -			wqe->write.sgl[i].stag =
> -			    htobe32(wr->sg_list[i].lkey);
> -			wqe->write.sgl[i].len =
> -			    htobe32(wr->sg_list[i].length);
> -			wqe->write.sgl[i].to =
> -			    htobe64(wr->sg_list[i].addr);
> -		}
> -		wqe->write.plen = htobe32(wqe->write.plen);
> -		wqe->write.num_sgle = htobe32(wr->num_sge);
> -		*flit_cnt = 5 + ((wr->num_sge) << 1);
> -	}
> -	return 0;
> -}
> -
> -static inline int iwch_build_rdma_read(union t3_wr *wqe, struct ibv_send_wr *wr,
> -				       uint8_t *flit_cnt)
> -{
> -	if (wr->num_sge > 1)
> -		return -1;
> -	wqe->read.rdmaop = T3_READ_REQ;
> -	wqe->read.reserved = 0;
> -	if (wr->num_sge == 1 && wr->sg_list[0].length > 0) {
> -		wqe->read.rem_stag = htobe32(wr->wr.rdma.rkey);
> -		wqe->read.rem_to = htobe64(wr->wr.rdma.remote_addr);
> -		wqe->read.local_stag = htobe32(wr->sg_list[0].lkey);
> -		wqe->read.local_len = htobe32(wr->sg_list[0].length);
> -		wqe->read.local_to = htobe64(wr->sg_list[0].addr);
> -	} else {
> -
> -		/* build passable 0B read request */
> -		wqe->read.rem_stag = 2;
> -		wqe->read.rem_to = 2;
> -		wqe->read.local_stag = 2;
> -		wqe->read.local_len = 0;
> -		wqe->read.local_to = 2;
> -	}
> -	*flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3;
> -	return 0;
> -}
> -
> -int t3b_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 
> -		  struct ibv_send_wr **bad_wr)
> -{
> -	int err = 0;
> -	uint8_t t3_wr_flit_cnt;
> -	enum t3_wr_opcode t3_wr_opcode = 0;
> -	enum t3_wr_flags t3_wr_flags;
> -	struct iwch_qp *qhp;
> -	uint32_t idx;
> -	union t3_wr *wqe;
> -	uint32_t num_wrs;
> -	struct t3_swsq *sqp;
> -
> -	qhp = to_iwch_qp(ibqp);
> -	pthread_spin_lock(&qhp->lock);
> -	if (t3_wq_in_error(&qhp->wq)) {
> -		iwch_flush_qp(qhp);
> -		pthread_spin_unlock(&qhp->lock);
> -		return -1;
> -	}
> -	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 
> -		  qhp->wq.sq_size_log2);
> -	if (num_wrs <= 0) {
> -		pthread_spin_unlock(&qhp->lock);
> -		return -1;
> -	}
> -	while (wr) {
> -		if (num_wrs == 0) {
> -			err = -1;
> -			*bad_wr = wr;
> -			break;
> -		}
> -		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
> -		wqe = (union t3_wr *) (qhp->wq.queue + idx);
> -		t3_wr_flags = 0;
> -		if (wr->send_flags & IBV_SEND_SOLICITED)
> -			t3_wr_flags |= T3_SOLICITED_EVENT_FLAG;
> -		if (wr->send_flags & IBV_SEND_FENCE)
> -			t3_wr_flags |= T3_READ_FENCE_FLAG;
> -		if ((wr->send_flags & IBV_SEND_SIGNALED) || qhp->sq_sig_all)
> -			t3_wr_flags |= T3_COMPLETION_FLAG;
> -		sqp = qhp->wq.sq + 
> -		      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
> -		switch (wr->opcode) {
> -		case IBV_WR_SEND:
> -			t3_wr_opcode = T3_WR_SEND;
> -			err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt);
> -			break;
> -		case IBV_WR_RDMA_WRITE:
> -			t3_wr_opcode = T3_WR_WRITE;
> -			err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt);
> -			break;
> -		case IBV_WR_RDMA_READ:
> -			t3_wr_opcode = T3_WR_READ;
> -			t3_wr_flags = 0;
> -			err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt);
> -			if (err)
> -				break;
> -			sqp->read_len = wqe->read.local_len;
> -			if (!qhp->wq.oldest_read)
> -				qhp->wq.oldest_read = sqp;
> -			break;
> -		default:
> -			PDBG("%s post of type=%d TBD!\n", __FUNCTION__, 
> -			     wr->opcode);
> -			err = -1;
> -		}
> -		if (err) {
> -			*bad_wr = wr;
> -			break;
> -		}
> -		wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
> -		sqp->wr_id = wr->wr_id;
> -		sqp->opcode = wr2opcode(t3_wr_opcode);
> -		sqp->sq_wptr = qhp->wq.sq_wptr;
> -		sqp->complete = 0;
> -		sqp->signaled = (wr->send_flags & IBV_SEND_SIGNALED);
> -
> -		build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags,
> -			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
> -			       0, t3_wr_flit_cnt);
> -		PDBG("%s cookie 0x%" PRIx64 
> -		     " wq idx 0x%x swsq idx %ld opcode %d\n", 
> -		     __FUNCTION__, wr->wr_id, idx, 
> -		     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),
> -		     sqp->opcode);
> -		wr = wr->next;
> -		num_wrs--;
> -		++(qhp->wq.wptr);
> -		++(qhp->wq.sq_wptr);
> -	}
> -	pthread_spin_unlock(&qhp->lock);
> -	if (t3_wq_db_enabled(&qhp->wq))
> -		RING_DOORBELL(qhp->wq.doorbell, qhp->wq.qpid);
> -	return err;
> -}
> -
> -int t3a_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 
> -		   struct ibv_send_wr **bad_wr)
> -{
> -	int ret;
> -	struct iwch_qp *qhp = to_iwch_qp(ibqp);
> -
> -	pthread_spin_lock(&qhp->lock);
> -	ret = ibv_cmd_post_send(ibqp, wr, bad_wr);
> -	pthread_spin_unlock(&qhp->lock);
> -	return ret;
> -}
> -
> -static inline int iwch_build_rdma_recv(struct iwch_device *rhp,
> -				       union t3_wr *wqe, 
> -				       struct ibv_recv_wr *wr)
> -{
> -	int i;
> -	if (wr->num_sge > T3_MAX_SGE)
> -		return -1;
> -
> -	wqe->recv.num_sgle = htobe32(wr->num_sge);
> -	for (i = 0; i < wr->num_sge; i++) {
> -		wqe->recv.sgl[i].stag = htobe32(wr->sg_list[i].lkey);
> -		wqe->recv.sgl[i].len = htobe32(wr->sg_list[i].length);
> -		wqe->recv.sgl[i].to = htobe64(wr->sg_list[i].addr);
> -	}
> -	for (; i < T3_MAX_SGE; i++) {
> -		wqe->recv.sgl[i].stag = 0;
> -		wqe->recv.sgl[i].len = 0;
> -		wqe->recv.sgl[i].to = 0;
> -	}
> -	return 0;
> -}
> -
> -static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
> -{
> -	struct t3_cqe cqe;
> -
> -	PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __FUNCTION__, 
> -	     wq, cq, cq->sw_rptr, cq->sw_wptr);
> -	memset(&cqe, 0, sizeof(cqe));
> -	cqe.header = V_CQE_STATUS(TPT_ERR_SWFLUSH) | 
> -		     V_CQE_OPCODE(T3_SEND) | 
> -		     V_CQE_TYPE(0) |
> -		     V_CQE_SWCQE(1) |
> -		     V_CQE_QPID(wq->qpid) | 
> -		     V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, cq->size_log2));
> -	cqe.header = htobe32(cqe.header);
> -	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
> -	cq->sw_wptr++;
> -}
> -
> -static void flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
> -{
> -	uint32_t ptr;
> -
> -	/* flush RQ */
> -	PDBG("%s rq_rptr 0x%x rq_wptr 0x%x skip count %u\n", __FUNCTION__, 
> -	     wq->rq_rptr, wq->rq_wptr, count);
> -	ptr = wq->rq_rptr + count;
> -	while (ptr++ != wq->rq_wptr) {
> -		insert_recv_cqe(wq, cq);
> -	}
> -}
> -
> -static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq, 
> -		          struct t3_swsq *sqp)
> -{
> -	struct t3_cqe cqe;
> -
> -	PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __FUNCTION__, 
> -	     wq, cq, cq->sw_rptr, cq->sw_wptr);
> -	memset(&cqe, 0, sizeof(cqe));
> -	cqe.header = V_CQE_STATUS(TPT_ERR_SWFLUSH) | 
> -		     V_CQE_OPCODE(sqp->opcode) |
> -		     V_CQE_TYPE(1) |
> -		     V_CQE_SWCQE(1) |
> -		     V_CQE_QPID(wq->qpid) | 
> -		     V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, cq->size_log2));
> -	cqe.header = htobe32(cqe.header);
> -	CQE_WRID_SQ_WPTR(cqe) = sqp->sq_wptr;
> -
> -	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
> -	cq->sw_wptr++;
> -}
> -
> -static void flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
> -{
> -	uint32_t ptr;
> -	struct t3_swsq *sqp;
> -
> -	ptr = wq->sq_rptr + count;
> -	sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
> -	while (ptr != wq->sq_wptr) {
> -		insert_sq_cqe(wq, cq, sqp);
> -		ptr++;
> -		sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
> -	}
> -}
> -
> -/* 
> - * Move all CQEs from the HWCQ into the SWCQ.
> - */
> -static void flush_hw_cq(struct t3_cq *cq)
> -{
> -	struct t3_cqe *cqe, *swcqe;
> -
> -	PDBG("%s cq %p cqid 0x%x\n", __FUNCTION__, cq, cq->cqid);
> -	cqe = cxio_next_hw_cqe(cq);
> -	while (cqe) {
> -		PDBG("%s flushing hwcq rptr 0x%x to swcq wptr 0x%x\n", 
> -		     __FUNCTION__, cq->rptr, cq->sw_wptr);
> -		swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
> -		*swcqe = *cqe;
> -		swcqe->header |= htobe32(V_CQE_SWCQE(1));
> -		cq->sw_wptr++;
> -		cq->rptr++;
> -		cqe = cxio_next_hw_cqe(cq);
> -	}
> -}
> -
> -static void count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
> -{
> -	struct t3_cqe *cqe;
> -	uint32_t ptr;
> -
> -	*count = 0;
> -	ptr = cq->sw_rptr;
> -	while (!Q_EMPTY(ptr, cq->sw_wptr)) {
> -		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
> -		if ((SQ_TYPE(*cqe) || 
> -		     (CQE_OPCODE(*cqe) == T3_READ_RESP && CQE_WRID_STAG(*cqe) != 1)) &&
> -		    (CQE_QPID(*cqe) == wq->qpid))
> -			(*count)++;
> -		ptr++;
> -	}	
> -	PDBG("%s cq %p count %d\n", __FUNCTION__, cq, *count);
> -}
> -
> -static void count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
> -{
> -	struct t3_cqe *cqe;
> -	uint32_t ptr;
> -
> -	*count = 0;
> -	ptr = cq->sw_rptr;
> -	while (!Q_EMPTY(ptr, cq->sw_wptr)) {
> -		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
> -		if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) && 
> -		    (CQE_QPID(*cqe) == wq->qpid))
> -			(*count)++;
> -		ptr++;
> -	}	
> -	PDBG("%s cq %p count %d\n", __FUNCTION__, cq, *count);
> -}
> -
> -/*
> - * Assumes qhp lock is held.
> - */
> -void iwch_flush_qp(struct iwch_qp *qhp)
> -{
> -	struct iwch_cq *rchp, *schp;
> -	int count;
> -
> -	if (qhp->wq.flushed)
> -		return;
> -
> -	rchp = qhp->rhp->cqid2ptr[to_iwch_cq(qhp->ibv_qp.recv_cq)->cq.cqid];
> -	schp = qhp->rhp->cqid2ptr[to_iwch_cq(qhp->ibv_qp.send_cq)->cq.cqid];
> -	
> -	PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp);
> -	qhp->wq.flushed = 1;
> -
> -#ifdef notyet
> -	/* take a ref on the qhp since we must release the lock */
> -	atomic_inc(&qhp->refcnt);
> -#endif
> -	pthread_spin_unlock(&qhp->lock);
> -
> -	/* locking heirarchy: cq lock first, then qp lock. */
> -	pthread_spin_lock(&rchp->lock);
> -	pthread_spin_lock(&qhp->lock);
> -	flush_hw_cq(&rchp->cq);
> -	count_rcqes(&rchp->cq, &qhp->wq, &count);
> -	flush_rq(&qhp->wq, &rchp->cq, count);
> -	pthread_spin_unlock(&qhp->lock);
> -	pthread_spin_unlock(&rchp->lock);
> -
> -	/* locking heirarchy: cq lock first, then qp lock. */
> -	pthread_spin_lock(&schp->lock);
> -	pthread_spin_lock(&qhp->lock);
> -	flush_hw_cq(&schp->cq);
> -	count_scqes(&schp->cq, &qhp->wq, &count);
> -	flush_sq(&qhp->wq, &schp->cq, count);
> -	pthread_spin_unlock(&qhp->lock);
> -	pthread_spin_unlock(&schp->lock);
> -
> -#ifdef notyet
> -	/* deref */
> -	if (atomic_dec_and_test(&qhp->refcnt))
> -                wake_up(&qhp->wait);
> -#endif
> -	pthread_spin_lock(&qhp->lock);
> -}
> -
> -void iwch_flush_qps(struct iwch_device *dev)
> -{
> -	int i;
> -
> -	pthread_spin_lock(&dev->lock);
> -	for (i=0; i < T3_MAX_NUM_QP; i++) {
> -		struct iwch_qp *qhp = dev->qpid2ptr[i];
> -		if (qhp) {
> -			if (!qhp->wq.flushed && t3_wq_in_error(&qhp->wq)) {
> -				pthread_spin_lock(&qhp->lock);
> -				iwch_flush_qp(qhp);
> -				pthread_spin_unlock(&qhp->lock);
> -			}
> -		}
> -	}
> -	pthread_spin_unlock(&dev->lock);
> -
> -}
> -
> -int t3b_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
> -		   struct ibv_recv_wr **bad_wr)
> -{
> -	int err = 0;
> -	struct iwch_qp *qhp;
> -	uint32_t idx;
> -	union t3_wr *wqe;
> -	uint32_t num_wrs;
> -
> -	qhp = to_iwch_qp(ibqp);
> -	pthread_spin_lock(&qhp->lock);
> -	if (t3_wq_in_error(&qhp->wq)) {
> -		iwch_flush_qp(qhp);
> -		pthread_spin_unlock(&qhp->lock);
> -		return -1;
> -	}
> -	num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 
> -			    qhp->wq.rq_size_log2) - 1;
> -	if (!wr) {
> -		pthread_spin_unlock(&qhp->lock);
> -		return -1;
> -	}
> -	while (wr) {
> -		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
> -		wqe = (union t3_wr *) (qhp->wq.queue + idx);
> -		if (num_wrs)
> -			err = iwch_build_rdma_recv(qhp->rhp, wqe, wr);
> -		else
> -			err = -1;
> -		if (err) {
> -			*bad_wr = wr;
> -			break;
> -		}
> -		qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = 
> -			wr->wr_id;
> -		build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG,
> -			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
> -			       0, sizeof(struct t3_receive_wr) >> 3);
> -		PDBG("%s cookie 0x%" PRIx64 
> -		     " idx 0x%x rq_wptr 0x%x rw_rptr 0x%x "
> -		     "wqe %p \n", __FUNCTION__, wr->wr_id, idx, 
> -		     qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);
> -		++(qhp->wq.rq_wptr);
> -		++(qhp->wq.wptr);
> -		wr = wr->next;
> -		num_wrs--;
> -	}
> -	pthread_spin_unlock(&qhp->lock);
> -	if (t3_wq_db_enabled(&qhp->wq))
> -		RING_DOORBELL(qhp->wq.doorbell, qhp->wq.qpid);
> -	return err;
> -}
> -
> -int t3a_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
> -		   struct ibv_recv_wr **bad_wr)
> -{
> -	int ret;
> -	struct iwch_qp *qhp = to_iwch_qp(ibqp);
> -
> -	pthread_spin_lock(&qhp->lock);
> -	ret = ibv_cmd_post_recv(ibqp, wr, bad_wr);
> -	pthread_spin_unlock(&qhp->lock);
> -	return ret;
> -}
> diff --git a/providers/cxgb3/verbs.c b/providers/cxgb3/verbs.c
> deleted file mode 100644
> index 39a44192..00000000
> --- a/providers/cxgb3/verbs.c
> +++ /dev/null
> @@ -1,476 +0,0 @@
> -/*
> - * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
> - *
> - * This software is available to you under a choice of one of two
> - * licenses.  You may choose to be licensed under the terms of the GNU
> - * General Public License (GPL) Version 2, available from the file
> - * COPYING in the main directory of this source tree, or the
> - * OpenIB.org BSD license below:
> - *
> - *     Redistribution and use in source and binary forms, with or
> - *     without modification, are permitted provided that the following
> - *     conditions are met:
> - *
> - *      - Redistributions of source code must retain the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer.
> - *
> - *      - Redistributions in binary form must reproduce the above
> - *        copyright notice, this list of conditions and the following
> - *        disclaimer in the documentation and/or other materials
> - *        provided with the distribution.
> - *
> - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> - * SOFTWARE.
> - */
> -#include <config.h>
> -
> -#include <stdlib.h>
> -#include <stdio.h>
> -#include <string.h>
> -#include <errno.h>
> -#include <pthread.h>
> -#include <sys/mman.h>
> -#include <inttypes.h>
> -
> -#include "iwch.h"
> -#include "iwch-abi.h"
> -
> -int iwch_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
> -{
> -	struct ibv_query_device cmd;
> -	uint64_t raw_fw_ver;
> -	unsigned major, minor, sub_minor;
> -	int ret;
> -
> -	ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, 
> -	  			   sizeof cmd);
> -	if (ret)
> -		return ret;
> -
> -	major = (raw_fw_ver >> 32) & 0xffff;
> -	minor = (raw_fw_ver >> 16) & 0xffff;
> -	sub_minor = raw_fw_ver & 0xffff;
> -
> -	snprintf(attr->fw_ver, sizeof attr->fw_ver,
> -		 "%d.%d.%d", major, minor, sub_minor);
> -
> -	return 0;
> -}
> -
> -int iwch_query_port(struct ibv_context *context, uint8_t port,
> -		    struct ibv_port_attr *attr)
> -{
> -	struct ibv_query_port cmd;
> -
> -	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
> -}
> -
> -struct ibv_pd *iwch_alloc_pd(struct ibv_context *context)
> -{
> -	struct ibv_alloc_pd cmd;
> -	struct uiwch_alloc_pd_resp resp;
> -	struct iwch_pd *pd;
> -
> -	pd = malloc(sizeof *pd);
> -	if (!pd)
> -		return NULL;
> -
> -	if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd,
> -			     &resp.ibv_resp, sizeof resp)) {
> -		free(pd);
> -		return NULL;
> -	}
> -
> -	return &pd->ibv_pd;
> -}
> -
> -int iwch_free_pd(struct ibv_pd *pd)
> -{
> -	int ret;
> -
> -	ret = ibv_cmd_dealloc_pd(pd);
> -	if (ret)
> -		return ret;
> -
> -	free(pd);
> -	return 0;
> -}
> -
> -struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
> -			   uint64_t hca_va, int access)
> -{
> -	struct iwch_mr *mhp;
> -	struct ibv_reg_mr cmd;
> -	struct uiwch_reg_mr_resp resp;
> -	struct iwch_device *dev = to_iwch_dev(pd->context->device);
> -
> -	PDBG("%s addr %p length %ld hca_va %p\n", __func__, addr, length,
> -	     hca_va);
> -
> -	mhp = malloc(sizeof *mhp);
> -	if (!mhp)
> -		return NULL;
> -
> -	if (ibv_cmd_reg_mr(pd, addr, length, hca_va,
> -			   access, &mhp->vmr, &cmd, sizeof(cmd),
> -			   &resp.ibv_resp, sizeof resp)) {
> -		free(mhp);
> -		return NULL;
> -	}
> -
> -	mhp->va_fbo = hca_va;
> -	mhp->page_size = iwch_page_shift - 12;
> -	mhp->pbl_addr = resp.pbl_addr;
> -	mhp->len = length;
> -
> -	PDBG("%s stag 0x%x va_fbo 0x%" PRIx64
> -             " page_size %d pbl_addr 0x%x len %d\n",
> -	     __func__, mhp->vmr.ibv_mr.rkey, mhp->va_fbo,
> -	     mhp->page_size, mhp->pbl_addr, mhp->len);
> -
> -	pthread_spin_lock(&dev->lock);
> -	dev->mmid2ptr[t3_mmid(mhp->vmr.ibv_mr.lkey)] = mhp;
> -	pthread_spin_unlock(&dev->lock);
> -	
> -	return &mhp->vmr.ibv_mr;
> -}
> -
> -int iwch_dereg_mr(struct verbs_mr *vmr)
> -{
> -	int ret;
> -	struct iwch_device *dev = to_iwch_dev(vmr->ibv_mr.pd->context->device);
> -
> -	ret = ibv_cmd_dereg_mr(vmr);
> -	if (ret)
> -		return ret;
> -
> -	pthread_spin_lock(&dev->lock);
> -	dev->mmid2ptr[t3_mmid(vmr->ibv_mr.lkey)] = NULL;
> -	pthread_spin_unlock(&dev->lock);
> -
> -	free(to_iwch_mr(vmr));
> -	
> -	return 0;
> -}
> -
> -struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
> -			      struct ibv_comp_channel *channel, int comp_vector)
> -{
> -	struct uiwch_create_cq cmd;
> -	struct uiwch_create_cq_resp resp;
> -	struct iwch_cq *chp;
> -	struct iwch_device *dev = to_iwch_dev(context->device);
> -	int ret;
> -
> -	chp = calloc(1, sizeof *chp);
> -	if (!chp) {
> -		return NULL;
> -	}
> -
> -	cmd.user_rptr_addr = (uint64_t)(unsigned long)&chp->cq.rptr;
> -	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
> -				&chp->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
> -				&resp.ibv_resp, sizeof resp);
> -	if (ret)
> -		goto err1;
> -
> -	pthread_spin_init(&chp->lock, PTHREAD_PROCESS_PRIVATE);
> -	chp->rhp = dev;
> -	chp->cq.cqid = resp.cqid;
> -	chp->cq.size_log2 = resp.size_log2;
> -	if (dev->abi_version == 0)
> -		chp->cq.memsize = PAGE_ALIGN((1UL << chp->cq.size_log2) *
> -					     sizeof(struct t3_cqe));
> -	else
> -		chp->cq.memsize = resp.memsize;
> -	chp->cq.queue = mmap(NULL, t3_cq_memsize(&chp->cq),
> -			     PROT_READ|PROT_WRITE, MAP_SHARED, context->cmd_fd,
> -			     resp.key);
> -	if (chp->cq.queue == MAP_FAILED)
> -		goto err2;
> -
> -	chp->cq.sw_queue = calloc(t3_cq_depth(&chp->cq), sizeof(struct t3_cqe));
> -	if (!chp->cq.sw_queue)
> -		goto err3;
> -
> -	PDBG("%s cqid 0x%x physaddr %" PRIx64 " va %p memsize %d\n", 
> -	       __FUNCTION__, chp->cq.cqid, resp.physaddr, chp->cq.queue, 
> -	       t3_cq_memsize(&chp->cq));
> -	
> -	pthread_spin_lock(&dev->lock);
> -	dev->cqid2ptr[chp->cq.cqid] = chp;
> -	pthread_spin_unlock(&dev->lock);
> -
> -	return &chp->ibv_cq;
> -err3:
> -	munmap(chp->cq.queue, t3_cq_memsize(&chp->cq));
> -err2:
> -	(void)ibv_cmd_destroy_cq(&chp->ibv_cq);
> -err1:
> -	free(chp);
> -	return NULL;
> -}
> -
> -int iwch_resize_cq(struct ibv_cq *ibcq, int cqe)
> -{
> -#ifdef notyet
> -	int ret;
> -	struct ibv_resize_cq cmd;
> -	struct iwch_cq *chp = to_iwch_cq(ibcq);
> -
> -	pthread_spin_lock(&chp->lock);
> -	ret = ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd);
> -	/* remap and realloc swcq here */
> -	pthread_spin_unlock(&chp->lock);
> -	return ret;
> -#else
> -	return -ENOSYS;
> -#endif
> -}
> -
> -int iwch_destroy_cq(struct ibv_cq *ibcq)
> -{
> -	int ret;
> -	struct iwch_cq *chp = to_iwch_cq(ibcq);
> -	void *cqva = chp->cq.queue;
> -	unsigned size = t3_cq_memsize(&chp->cq);
> -	struct iwch_device *dev = to_iwch_dev(ibcq->context->device);
> -
> -	munmap(cqva, size);
> -	ret = ibv_cmd_destroy_cq(ibcq);
> -	if (ret) {
> -		return ret;
> -	}
> -
> -	pthread_spin_lock(&dev->lock);
> -	dev->cqid2ptr[chp->cq.cqid] = NULL;
> -	pthread_spin_unlock(&dev->lock);
> -
> -	free(chp->cq.sw_queue);
> -	free(chp);
> -	return 0;
> -}
> -
> -struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
> -				struct ibv_srq_init_attr *attr)
> -{
> -	return NULL;
> -}
> -
> -int iwch_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, 
> -		    int attr_mask)
> -{
> -	return -ENOSYS;
> -}
> -
> -int iwch_destroy_srq(struct ibv_srq *srq)
> -{
> -	return -ENOSYS;
> -}
> -
> -int iwch_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, 
> -		       struct ibv_recv_wr **bad_wr)
> -{
> -	return -ENOSYS;
> -}
> -
> -struct ibv_qp *iwch_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
> -{
> -	struct uiwch_create_qp cmd;
> -	struct uiwch_create_qp_resp resp;
> -	struct iwch_qp *qhp;
> -	struct iwch_device *dev = to_iwch_dev(pd->context->device);
> -	int ret;
> -	void *dbva;
> -
> -	PDBG("%s enter qp\n", __FUNCTION__);
> -	qhp = calloc(1, sizeof *qhp);
> -	if (!qhp)
> -		goto err1;
> -
> -	ret = ibv_cmd_create_qp(pd, &qhp->ibv_qp, attr, &cmd.ibv_cmd, 
> -				sizeof cmd, &resp.ibv_resp, sizeof resp);
> -	if (ret)
> -		goto err2;
> -
> -	PDBG("%s qpid 0x%x physaddr %" PRIx64 " doorbell %" PRIx64 
> -	       " size %d sq_size %d rq_size %d\n",
> -		__FUNCTION__, resp.qpid, resp.physaddr, resp.doorbell,
> -		1 << resp.size_log2, 1 << resp.sq_size_log2, 
> -		1 << resp.rq_size_log2);
> -
> -	qhp->rhp = dev;
> -	qhp->wq.qpid = resp.qpid;
> -	qhp->wq.size_log2 = resp.size_log2;
> -	qhp->wq.sq_size_log2 = resp.sq_size_log2;
> -	qhp->wq.rq_size_log2 = resp.rq_size_log2;
> -	pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE);
> -	dbva = mmap(NULL, iwch_page_size, PROT_WRITE, MAP_SHARED, 
> -		    pd->context->cmd_fd, resp.db_key & ~(iwch_page_mask));
> -	if (dbva == MAP_FAILED)
> -		goto err3;
> -
> -	qhp->wq.doorbell = dbva + (resp.db_key & (iwch_page_mask));
> -	qhp->wq.queue = mmap(NULL, t3_wq_memsize(&qhp->wq),
> -			    PROT_READ|PROT_WRITE, MAP_SHARED, 
> -			    pd->context->cmd_fd, resp.key);
> -	if (qhp->wq.queue == MAP_FAILED)
> -		goto err4;
> -
> -	qhp->wq.rq = calloc(t3_rq_depth(&qhp->wq), sizeof (uint64_t));
> -	if (!qhp->wq.rq) 
> -		goto err5;
> -
> -	qhp->wq.sq = calloc(t3_sq_depth(&qhp->wq), sizeof (struct t3_swsq));
> -	if (!qhp->wq.sq) 
> -		goto err6;
> -
> -	PDBG("%s dbva %p wqva %p wq memsize %d\n", __FUNCTION__, 
> -	     qhp->wq.doorbell, qhp->wq.queue, t3_wq_memsize(&qhp->wq));
> -
> -	qhp->sq_sig_all = attr->sq_sig_all;
> -
> -	pthread_spin_lock(&dev->lock);
> -	dev->qpid2ptr[qhp->wq.qpid] = qhp;
> -	pthread_spin_unlock(&dev->lock);
> -
> -	return &qhp->ibv_qp;
> -err6:
> -	free(qhp->wq.rq);
> -err5:
> -	munmap((void *)qhp->wq.queue, t3_wq_memsize(&qhp->wq));
> -err4:
> -	munmap((void *)dbva, iwch_page_size);
> -err3:
> -	(void)ibv_cmd_destroy_qp(&qhp->ibv_qp);
> -err2:
> -	free(qhp);
> -err1:
> -	return NULL;
> -}
> -
> -static void reset_qp(struct iwch_qp *qhp)
> -{
> -	PDBG("%s enter qp %p\n", __FUNCTION__, qhp);
> -	qhp->wq.wptr = 0;
> -	qhp->wq.rq_wptr = qhp->wq.rq_rptr = 0;
> -	qhp->wq.sq_wptr = qhp->wq.sq_rptr = 0;
> -	qhp->wq.error = 0;
> -	qhp->wq.oldest_read = NULL;
> -	memset(qhp->wq.queue, 0, t3_wq_memsize(&qhp->wq));
> -}
> -
> -int iwch_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
> -		   int attr_mask)
> -{
> -	struct ibv_modify_qp cmd = {};
> -	struct iwch_qp *qhp = to_iwch_qp(ibqp);
> -	int ret;
> -
> -	PDBG("%s enter qp %p new state %d\n", __FUNCTION__, ibqp, attr_mask & IBV_QP_STATE ? attr->qp_state : -1);
> -	pthread_spin_lock(&qhp->lock);
> -	if (t3b_device(qhp->rhp) && t3_wq_in_error(&qhp->wq))
> -		iwch_flush_qp(qhp);
> -	ret = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof cmd);
> -	if (!ret && (attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET)
> -		reset_qp(qhp);
> -	pthread_spin_unlock(&qhp->lock);
> -	return ret;
> -}
> -
> -int iwch_destroy_qp(struct ibv_qp *ibqp)
> -{
> -	int ret;
> -	struct iwch_qp *qhp = to_iwch_qp(ibqp);
> -	struct iwch_device *dev = to_iwch_dev(ibqp->context->device);
> -	void *dbva, *wqva;
> -	unsigned wqsize;
> -
> -	PDBG("%s enter qp %p\n", __FUNCTION__, ibqp);
> -	if (t3b_device(dev)) {
> -		pthread_spin_lock(&qhp->lock);
> -		iwch_flush_qp(qhp);
> -		pthread_spin_unlock(&qhp->lock);
> -	}
> -
> -	dbva = (void *)((unsigned long)qhp->wq.doorbell & ~(iwch_page_mask));
> -	wqva = qhp->wq.queue;
> -	wqsize = t3_wq_memsize(&qhp->wq);
> -
> -	munmap(dbva, iwch_page_size);
> -	munmap(wqva, wqsize);
> -	ret = ibv_cmd_destroy_qp(ibqp);
> -	if (ret) {
> -		return ret;
> -	}
> -
> -	pthread_spin_lock(&dev->lock);
> -	dev->qpid2ptr[qhp->wq.qpid] = NULL;
> -	pthread_spin_unlock(&dev->lock);
> -
> -	free(qhp->wq.rq);
> -	free(qhp->wq.sq);
> -	free(qhp);
> -	return 0;
> -}
> -
> -int iwch_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
> -		  int attr_mask, struct ibv_qp_init_attr *init_attr)
> -{
> -	return -ENOSYS;
> -}
> -
> -struct ibv_ah *iwch_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
> -{
> -	return NULL;
> -}
> -
> -int iwch_destroy_ah(struct ibv_ah *ah)
> -{
> -	return -ENOSYS;
> -}
> -
> -int iwch_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
> -{
> -	return -ENOSYS;
> -}
> -
> -int iwch_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
> -{
> -	return -ENOSYS;
> -}
> -
> -void t3b_async_event(struct ibv_context *context,
> -		     struct ibv_async_event *event)
> -{
> -	PDBG("%s type %d obj %p\n", __FUNCTION__, event->event_type, 
> -	     event->element.cq);
> -
> -	switch (event->event_type) {
> -	case IBV_EVENT_CQ_ERR:
> -		break;
> -	case IBV_EVENT_QP_FATAL:
> -	case IBV_EVENT_QP_REQ_ERR:
> -	case IBV_EVENT_QP_ACCESS_ERR:
> -	case IBV_EVENT_PATH_MIG_ERR: {
> -		struct iwch_qp *qhp = to_iwch_qp(event->element.qp);
> -		pthread_spin_lock(&qhp->lock);
> -		iwch_flush_qp(qhp);
> -		pthread_spin_unlock(&qhp->lock);
> -		break;
> -	}
> -	case IBV_EVENT_SQ_DRAINED:
> -	case IBV_EVENT_PATH_MIG:
> -	case IBV_EVENT_COMM_EST:
> -	case IBV_EVENT_QP_LAST_WQE_REACHED:
> -	default:
> -		break;
> -	}
> -}
> -- 
> 2.20.1
> 
> 
> 
>From bcd621da7417a0a3696d7cbfc4dc2cb656862c44 Mon Sep 17 00:00:00 2001
From: Potnuri Bharat Teja <bharat@xxxxxxxxxxx>
Date: Mon, 21 Oct 2019 14:01:25 +0530
Subject: [PATCH rdma-core 1/2] libcxgb3: remove libcxgb3 from rdma-core

remove userspace provider for iw_cxgb3 after removing it from kernel.

Signed-off-by: Potnuri Bharat Teja <bharat@xxxxxxxxxxx>
---
 CMakeLists.txt                     |   1 -
 MAINTAINERS                        |   5 -
 README.md                          |   1 -
 debian/control                     |  10 +-
 debian/copyright                   |   3 +-
 kernel-boot/rdma-description.rules |   1 -
 kernel-boot/rdma-hw-modules.rules  |   1 -
 kernel-headers/CMakeLists.txt      |   2 -
 kernel-headers/rdma/cxgb3-abi.h    |  82 ----
 libibverbs/verbs.h                 |   1 -
 providers/cxgb3/CMakeLists.txt     |   6 -
 providers/cxgb3/cq.c               | 442 -----------------
 providers/cxgb3/cxio_wr.h          | 758 -----------------------------
 providers/cxgb3/firmware_exports.h | 148 ------
 providers/cxgb3/iwch-abi.h         |  51 --
 providers/cxgb3/iwch.c             | 269 ----------
 providers/cxgb3/iwch.h             | 218 ---------
 providers/cxgb3/qp.c               | 560 ---------------------
 providers/cxgb3/verbs.c            | 476 ------------------
 redhat/rdma-core.spec              |   3 -
 redhat/rdma.kernel-init            |   4 -
 suse/rdma-core.spec                |   2 -
 22 files changed, 4 insertions(+), 3040 deletions(-)
 delete mode 100644 kernel-headers/rdma/cxgb3-abi.h
 delete mode 100644 providers/cxgb3/CMakeLists.txt
 delete mode 100644 providers/cxgb3/cq.c
 delete mode 100644 providers/cxgb3/cxio_wr.h
 delete mode 100644 providers/cxgb3/firmware_exports.h
 delete mode 100644 providers/cxgb3/iwch-abi.h
 delete mode 100644 providers/cxgb3/iwch.c
 delete mode 100644 providers/cxgb3/iwch.h
 delete mode 100644 providers/cxgb3/qp.c
 delete mode 100644 providers/cxgb3/verbs.c

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fc17ef36cf24..cdc8d5444e70 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -616,7 +616,6 @@ add_subdirectory(librdmacm/man)
 # Providers
 if (HAVE_COHERENT_DMA)
 add_subdirectory(providers/bnxt_re)
-add_subdirectory(providers/cxgb3) # NO SPARSE
 add_subdirectory(providers/cxgb4) # NO SPARSE
 add_subdirectory(providers/efa)
 add_subdirectory(providers/efa/man)
diff --git a/MAINTAINERS b/MAINTAINERS
index e47b97b2ff70..8030891d96a4 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -51,11 +51,6 @@ M:	Devesh Sharma  <Devesh.sharma@xxxxxxxxxxxx>
 S:	Supported
 F:	providers/bnxt_re/
 
-CXGB3 USERSPACE PROVIDER (for iw_cxgb3.ko)
-M:	Steve Wise <swise@xxxxxxxxxxxxxxxxxxxxx>
-S:	Supported
-F:	providers/cxgb3/
-
 CXGB4 USERSPACE PROVIDER (for iw_cxgb4.ko)
 M:	Steve Wise <swise@xxxxxxxxxxxxxxxxxxxxx>
 S:	Supported
diff --git a/README.md b/README.md
index 451ff7fcb36e..a96351933a27 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,6 @@ under the providers/ directory. Support for the following Kernel RDMA drivers
 is included:
 
  - efa.ko
- - iw_cxgb3.ko
  - iw_cxgb4.ko
  - hfi1.ko
  - hns-roce.ko
diff --git a/debian/control b/debian/control
index f8c3b1550e14..65ee3844c42f 100644
--- a/debian/control
+++ b/debian/control
@@ -66,21 +66,18 @@ Package: ibverbs-providers
 Architecture: linux-any
 Multi-Arch: same
 Depends: ${misc:Depends}, ${shlibs:Depends}
-Provides: libcxgb3-1,
-          libefa1,
+Provides: libefa1,
           libipathverbs1,
           libmlx4-1,
           libmlx5-1,
           libmthca1,
           libnes1
-Replaces: libcxgb3-1 (<< 15),
-          libipathverbs1 (<< 15),
+Replaces: libipathverbs1 (<< 15),
           libmlx4-1 (<< 15),
           libmlx5-1 (<< 15),
           libmthca1 (<< 15),
           libnes1 (<< 15)
-Breaks: libcxgb3-1 (<< 15),
-        libipathverbs1 (<< 15),
+Breaks: libipathverbs1 (<< 15),
         libmlx4-1 (<< 15),
         libmlx5-1 (<< 15),
         libmthca1 (<< 15),
@@ -99,7 +96,6 @@ Description: User space provider drivers for libibverbs
  This package contains the user space verbs drivers:
  .
   - bnxt_re: Broadcom NetXtreme-E RoCE HCAs
-  - cxgb3: Chelsio T3 iWARP HCAs
   - cxgb4: Chelsio T4 iWARP HCAs
   - efa: Amazon Elastic Fabric Adapter
   - hfi1verbs: Intel Omni-Path HFI
diff --git a/debian/copyright b/debian/copyright
index c834859524e1..1af3cd8187cd 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -149,8 +149,7 @@ Files: providers/bnxt_re/*
 Copyright: 2015-2017, Broadcom Limited and/or its subsidiaries
 License: BSD-2-clause or GPL-2
 
-Files: providers/cxgb3/*
-       providers/cxgb4/*
+Files: providers/cxgb4/*
 Copyright: 2003-2016, Chelsio Communications, Inc.
 License: BSD-MIT or GPL-2
 
diff --git a/kernel-boot/rdma-description.rules b/kernel-boot/rdma-description.rules
index bb33dce40bd6..4ea59ba1977b 100644
--- a/kernel-boot/rdma-description.rules
+++ b/kernel-boot/rdma-description.rules
@@ -22,7 +22,6 @@ DRIVERS=="ib_qib", ENV{ID_RDMA_INFINIBAND}="1"
 DRIVERS=="hfi1", ENV{ID_RDMA_OPA}="1"
 
 # Hardware that supports iWarp
-DRIVERS=="cxgb3", ENV{ID_RDMA_IWARP}="1"
 DRIVERS=="cxgb4", ENV{ID_RDMA_IWARP}="1"
 DRIVERS=="i40e", ENV{ID_RDMA_IWARP}="1"
 DRIVERS=="nes", ENV{ID_RDMA_IWARP}="1"
diff --git a/kernel-boot/rdma-hw-modules.rules b/kernel-boot/rdma-hw-modules.rules
index dde0ab8dacac..da4bbe363ac4 100644
--- a/kernel-boot/rdma-hw-modules.rules
+++ b/kernel-boot/rdma-hw-modules.rules
@@ -8,7 +8,6 @@ SUBSYSTEM!="net", GOTO="rdma_hw_modules_end"
 # RDMA.
 ENV{ID_NET_DRIVER}=="be2net", RUN{builtin}+="kmod load ocrdma"
 ENV{ID_NET_DRIVER}=="bnxt_en", RUN{builtin}+="kmod load bnxt_re"
-ENV{ID_NET_DRIVER}=="cxgb3", RUN{builtin}+="kmod load iw_cxgb3"
 ENV{ID_NET_DRIVER}=="cxgb4", RUN{builtin}+="kmod load iw_cxgb4"
 ENV{ID_NET_DRIVER}=="hns", RUN{builtin}+="kmod load hns_roce"
 ENV{ID_NET_DRIVER}=="i40e", RUN{builtin}+="kmod load i40iw"
diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
index 50bc77e6ab6e..b3139b74a1d6 100644
--- a/kernel-headers/CMakeLists.txt
+++ b/kernel-headers/CMakeLists.txt
@@ -1,6 +1,5 @@
 publish_internal_headers(rdma
   rdma/bnxt_re-abi.h
-  rdma/cxgb3-abi.h
   rdma/cxgb4-abi.h
   rdma/efa-abi.h
   rdma/hns-abi.h
@@ -59,7 +58,6 @@ endfunction()
 # Transform the kernel ABIs used by the providers
 rdma_kernel_provider_abi(
   rdma/bnxt_re-abi.h
-  rdma/cxgb3-abi.h
   rdma/cxgb4-abi.h
   rdma/efa-abi.h
   rdma/hns-abi.h
diff --git a/kernel-headers/rdma/cxgb3-abi.h b/kernel-headers/rdma/cxgb3-abi.h
deleted file mode 100644
index 85aed672f43e..000000000000
--- a/kernel-headers/rdma/cxgb3-abi.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
-/*
- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef CXGB3_ABI_USER_H
-#define CXGB3_ABI_USER_H
-
-#include <linux/types.h>
-
-#define IWCH_UVERBS_ABI_VERSION	1
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __aligned_u64
- * instead.
- */
-struct iwch_create_cq_req {
-	__aligned_u64 user_rptr_addr;
-};
-
-struct iwch_create_cq_resp_v0 {
-	__aligned_u64 key;
-	__u32 cqid;
-	__u32 size_log2;
-};
-
-struct iwch_create_cq_resp {
-	__aligned_u64 key;
-	__u32 cqid;
-	__u32 size_log2;
-	__u32 memsize;
-	__u32 reserved;
-};
-
-struct iwch_create_qp_resp {
-	__aligned_u64 key;
-	__aligned_u64 db_key;
-	__u32 qpid;
-	__u32 size_log2;
-	__u32 sq_size_log2;
-	__u32 rq_size_log2;
-};
-
-struct iwch_reg_user_mr_resp {
-	__u32 pbl_addr;
-};
-
-struct iwch_alloc_pd_resp {
-	__u32 pdid;
-};
-
-#endif /* CXGB3_ABI_USER_H */
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index c411722b154f..12a33a99a968 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -2144,7 +2144,6 @@ struct ibv_device **ibv_get_device_list(int *num_devices);
 
 struct verbs_devices_ops;
 extern const struct verbs_device_ops verbs_provider_bnxt_re;
-extern const struct verbs_device_ops verbs_provider_cxgb3;
 extern const struct verbs_device_ops verbs_provider_cxgb4;
 extern const struct verbs_device_ops verbs_provider_efa;
 extern const struct verbs_device_ops verbs_provider_hfi1verbs;
diff --git a/providers/cxgb3/CMakeLists.txt b/providers/cxgb3/CMakeLists.txt
deleted file mode 100644
index a578105e7b28..000000000000
--- a/providers/cxgb3/CMakeLists.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-rdma_provider(cxgb3
-  cq.c
-  iwch.c
-  qp.c
-  verbs.c
-)
diff --git a/providers/cxgb3/cq.c b/providers/cxgb3/cq.c
deleted file mode 100644
index 6cb4fe74d064..000000000000
--- a/providers/cxgb3/cq.c
+++ /dev/null
@@ -1,442 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <config.h>
-
-#include <stdio.h>
-#include <pthread.h>
-#include <sys/errno.h>
-
-#include <infiniband/opcode.h>
-
-#include "iwch.h"
-#include "iwch-abi.h"
-
-int iwch_arm_cq(struct ibv_cq *ibcq, int solicited)
-{
-	int ret;
-	struct iwch_cq *chp = to_iwch_cq(ibcq);
-
-	pthread_spin_lock(&chp->lock);
-	ret = ibv_cmd_req_notify_cq(ibcq, solicited);
-	pthread_spin_unlock(&chp->lock);
-
-	return ret;
-}
-
-static inline void flush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq)
-{
-	struct t3_swsq *sqp;
-	uint32_t ptr = wq->sq_rptr;
-	int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr);
-	
-	sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
-	while (count--) {
-		if (!sqp->signaled) {
-			ptr++;
-			sqp = wq->sq + Q_PTR2IDX(ptr,  wq->sq_size_log2);
-		} else if (sqp->complete) {
-
-			/* 
-			 * Insert this completed cqe into the swcq.
-			 */
-			sqp->cqe.header |= htobe32(V_CQE_SWCQE(1));
-			*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) 
-				= sqp->cqe;
-			cq->sw_wptr++;
-			sqp->signaled = 0;
-			break;
-		} else
-			break;
-	}
-}
-
-static inline void create_read_req_cqe(struct t3_wq *wq,
-				       struct t3_cqe *hw_cqe,
-				       struct t3_cqe *read_cqe)
-{
-	CQE_WRID_SQ_WPTR(*read_cqe) = wq->oldest_read->sq_wptr;
-	read_cqe->len = wq->oldest_read->read_len;
-	read_cqe->header = htobe32(V_CQE_QPID(CQE_QPID(*hw_cqe)) |
-				 V_CQE_SWCQE(SW_CQE(*hw_cqe)) |
-				 V_CQE_OPCODE(T3_READ_REQ) |
-				 V_CQE_TYPE(1));
-}
-
-/*
- * Return a ptr to the next read wr in the SWSQ or NULL.
- */
-static inline void advance_oldest_read(struct t3_wq *wq)
-{
-
-	uint32_t rptr = wq->oldest_read - wq->sq + 1;
-	uint32_t wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2);
-
-	while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) {
-		wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
-
-		if (wq->oldest_read->opcode == T3_READ_REQ) {
-			return;
-		}
-		rptr++;
-	}
-	wq->oldest_read = NULL;
-}
-
-static inline int cxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq,
-		   struct t3_cqe *cqe, uint8_t *cqe_flushed,
-		   uint64_t *cookie)
-{
-	int ret = 0;
-	struct t3_cqe *hw_cqe, read_cqe;
-
-	*cqe_flushed = 0;
-	hw_cqe = cxio_next_cqe(cq);
-	udma_from_device_barrier();
-
-	/* 
-	 * Skip cqes not affiliated with a QP.
-	 */
-	if (wq == NULL) {
-		ret = -1;
-		goto skip_cqe;
-	}
-
-	/*
-	 * Gotta tweak READ completions:
-	 * 	1) the cqe doesn't contain the sq_wptr from the wr.
-	 *	2) opcode not reflected from the wr.
-	 *	3) read_len not reflected from the wr.
-	 *	4) cq_type is RQ_TYPE not SQ_TYPE.
-	 */
-	if (CQE_OPCODE(*hw_cqe) == T3_READ_RESP) {
-
-                /*
-		 * If this is an unsolicited read response to local stag 1, 
-		 * then the read was generated by the kernel driver as part 
-		 * of peer-2-peer connection setup.  So ignore the completion.
-		 */
-		if (CQE_WRID_STAG(*hw_cqe) == 1) {
-			if (CQE_STATUS(*hw_cqe))
-				wq->error = 1;
-			ret = -1;
-			goto skip_cqe;
-		}
-		
-		/* 
-	 	 * Don't write to the HWCQ, so create a new read req CQE 
-		 * in local memory.
-		 */
-		create_read_req_cqe(wq, hw_cqe, &read_cqe);
-		hw_cqe = &read_cqe;
-		advance_oldest_read(wq);
-	}
-
-	/* 
-	 * Errors.
-	 */
-	if (CQE_STATUS(*hw_cqe) || t3_wq_in_error(wq)) {
-		*cqe_flushed = t3_wq_in_error(wq);
-		t3_set_wq_in_error(wq);
-		goto proc_cqe;
-	}
-
-	/*
-	 * RECV completion.
-	 */
-	if (RQ_TYPE(*hw_cqe)) {
-
-		/* 
-		 * HW only validates 4 bits of MSN.  So we must validate that
-		 * the MSN in the SEND is the next expected MSN.  If its not,
-		 * then we complete this with TPT_ERR_MSN and mark the wq in 
-		 * error.
-		 */
-		if ((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1))) {
-			t3_set_wq_in_error(wq);
-			hw_cqe->header |= htobe32(V_CQE_STATUS(TPT_ERR_MSN));
-		}
-		goto proc_cqe;
-	}
-
-	/* 
- 	 * If we get here its a send completion.
-	 *
-	 * Handle out of order completion. These get stuffed
-	 * in the SW SQ. Then the SW SQ is walked to move any
-	 * now in-order completions into the SW CQ.  This handles
-	 * 2 cases:
-	 * 	1) reaping unsignaled WRs when the first subsequent
-	 *	   signaled WR is completed.
-	 *	2) out of order read completions.
-	 */
-	if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) {
-		struct t3_swsq *sqp;
-
-		sqp = wq->sq + 
-		      Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2);
-		sqp->cqe = *hw_cqe;
-		sqp->complete = 1;
-		ret = -1;
-		goto flush_wq;
-	}
-
-proc_cqe:
-	*cqe = *hw_cqe;
-
-	/*
-	 * Reap the associated WR(s) that are freed up with this
-	 * completion.
-	 */
-	if (SQ_TYPE(*hw_cqe)) {
-		wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe);
-		*cookie = (wq->sq + 
-			   Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2))->wr_id;
-		wq->sq_rptr++;
-	} else {
-		*cookie = *(wq->rq + Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2));
-		wq->rq_rptr++;
-	}
-
-flush_wq:
-	/*
-	 * Flush any completed cqes that are now in-order.
-	 */
-	flush_completed_wrs(wq, cq);
-
-skip_cqe:
-	if (SW_CQE(*hw_cqe)) {
-		PDBG("%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x\n", 
-		     __FUNCTION__, cq, cq->cqid, cq->sw_rptr);
-		++cq->sw_rptr;
-	} else {
-		PDBG("%s cq %p cqid 0x%x skip hw cqe sw_rptr 0x%x\n", 
-		     __FUNCTION__, cq, cq->cqid, cq->rptr);
-		++cq->rptr;
-	}
-
-	return ret;
-}
-
-/*
- * Get one cq entry from cxio and map it to openib.
- *
- * Returns:
- * 	0 			EMPTY;
- *	1			cqe returned
- *	-EAGAIN 		caller must try again
- * 	any other -errno	fatal error
- */
-static int iwch_poll_cq_one(struct iwch_device *rhp, struct iwch_cq *chp,
-		     struct ibv_wc *wc)
-{
-	struct iwch_qp *qhp = NULL;
-	struct t3_cqe cqe, *hw_cqe;
-	struct t3_wq *wq;
-	uint8_t cqe_flushed;
-	uint64_t cookie;
-	int ret = 1;
-
-	hw_cqe = cxio_next_cqe(&chp->cq);
-	udma_from_device_barrier();
-
-	if (!hw_cqe)
-		return 0;
-
-	qhp = rhp->qpid2ptr[CQE_QPID(*hw_cqe)];
-	if (!qhp)
-		wq = NULL;
-	else {
-		pthread_spin_lock(&qhp->lock);
-		wq = &(qhp->wq);
-	}
-	ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie);
-	if (ret) {
-		ret = -EAGAIN;
-		goto out;
-	}
-	ret = 1;
-
-	wc->wr_id = cookie;
-	wc->qp_num = qhp->wq.qpid;
-	wc->vendor_err = CQE_STATUS(cqe);
-	wc->wc_flags = 0;
-
-	PDBG("%s qpid 0x%x type %d opcode %d status 0x%x wrid hi 0x%x "
-	     "lo 0x%x cookie 0x%" PRIx64 "\n", 
-	     __FUNCTION__, CQE_QPID(cqe), CQE_TYPE(cqe),
-	     CQE_OPCODE(cqe), CQE_STATUS(cqe), CQE_WRID_HI(cqe),
-	     CQE_WRID_LOW(cqe), cookie);
-
-	if (CQE_TYPE(cqe) == 0) {
-		if (!CQE_STATUS(cqe))
-			wc->byte_len = CQE_LEN(cqe);
-		else
-			wc->byte_len = 0;
-		wc->opcode = IBV_WC_RECV;
-	} else {
-		switch (CQE_OPCODE(cqe)) {
-		case T3_RDMA_WRITE:
-			wc->opcode = IBV_WC_RDMA_WRITE;
-			break;
-		case T3_READ_REQ:
-			wc->opcode = IBV_WC_RDMA_READ;
-			wc->byte_len = CQE_LEN(cqe);
-			break;
-		case T3_SEND:
-		case T3_SEND_WITH_SE:
-			wc->opcode = IBV_WC_SEND;
-			break;
-		case T3_BIND_MW:
-			wc->opcode = IBV_WC_BIND_MW;
-			break;
-
-		/* these aren't supported yet */
-		case T3_SEND_WITH_INV:
-		case T3_SEND_WITH_SE_INV:
-		case T3_LOCAL_INV:
-		case T3_FAST_REGISTER:
-		default:
-			PDBG("%s Unexpected opcode %d CQID 0x%x QPID 0x%x\n", 
-			     __FUNCTION__, CQE_OPCODE(cqe), chp->cq.cqid, 
-			     CQE_QPID(cqe));
-			ret = -EINVAL;
-			goto out;
-		}
-	}
-
-	if (cqe_flushed) {
-		wc->status = IBV_WC_WR_FLUSH_ERR;
-	} else {
-		
-		switch (CQE_STATUS(cqe)) {
-		case TPT_ERR_SUCCESS:
-			wc->status = IBV_WC_SUCCESS;
-			break;
-		case TPT_ERR_STAG:
-			wc->status = IBV_WC_LOC_ACCESS_ERR;
-			break;
-		case TPT_ERR_PDID:
-			wc->status = IBV_WC_LOC_PROT_ERR;
-			break;
-		case TPT_ERR_QPID:
-		case TPT_ERR_ACCESS:
-			wc->status = IBV_WC_LOC_ACCESS_ERR;
-			break;
-		case TPT_ERR_WRAP:
-			wc->status = IBV_WC_GENERAL_ERR;
-			break;
-		case TPT_ERR_BOUND:
-			wc->status = IBV_WC_LOC_LEN_ERR;
-			break;
-		case TPT_ERR_INVALIDATE_SHARED_MR:
-		case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND:
-			wc->status = IBV_WC_MW_BIND_ERR;
-			break;
-		case TPT_ERR_CRC:
-		case TPT_ERR_MARKER:
-		case TPT_ERR_PDU_LEN_ERR:
-		case TPT_ERR_OUT_OF_RQE:
-		case TPT_ERR_DDP_VERSION:
-		case TPT_ERR_RDMA_VERSION:
-		case TPT_ERR_DDP_QUEUE_NUM:
-		case TPT_ERR_MSN:
-		case TPT_ERR_TBIT:
-		case TPT_ERR_MO:
-		case TPT_ERR_MSN_RANGE:
-		case TPT_ERR_IRD_OVERFLOW:
-		case TPT_ERR_OPCODE:
-			wc->status = IBV_WC_FATAL_ERR;
-			break;
-		case TPT_ERR_SWFLUSH:
-			wc->status = IBV_WC_WR_FLUSH_ERR;
-			break;
-		default:
-			PDBG("%s Unexpected status 0x%x CQID 0x%x QPID 0x%0x\n",
-			     __FUNCTION__, CQE_STATUS(cqe), chp->cq.cqid, 
-			     CQE_QPID(cqe));
-			ret = -EINVAL;
-		}
-	}
-out:
-	if (wq)
-		pthread_spin_unlock(&qhp->lock);
-	return ret;
-}
-
-int t3b_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
-{
-	struct iwch_device *rhp;
-	struct iwch_cq *chp;
-	int npolled;
-	int err = 0;
-
-	chp = to_iwch_cq(ibcq);
-	rhp = chp->rhp;
-
-	if (rhp->abi_version > 0 && t3_cq_in_error(&chp->cq)) {
-		t3_reset_cq_in_error(&chp->cq);
-		iwch_flush_qps(rhp);
-	}
-
-	pthread_spin_lock(&chp->lock);
-	for (npolled = 0; npolled < num_entries; ++npolled) {
-
-		/*
-	 	 * Because T3 can post CQEs that are out of order,
-	 	 * we might have to poll again after removing
-	 	 * one of these.  
-		 */
-		do {
-			err = iwch_poll_cq_one(rhp, chp, wc + npolled);
-		} while (err == -EAGAIN);
-		if (err <= 0)
-			break;
-	}
-	pthread_spin_unlock(&chp->lock);
-
-	if (err < 0)
-		return err;
-	else {
-		return npolled;
-	}
-}
-
-int t3a_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc *wc)
-{
-	int ret;
-	struct iwch_cq *chp = to_iwch_cq(ibcq);
-	
-	pthread_spin_lock(&chp->lock);
-	ret = ibv_cmd_poll_cq(ibcq, num_entries, wc);
-	pthread_spin_unlock(&chp->lock);
-	return ret;
-}
diff --git a/providers/cxgb3/cxio_wr.h b/providers/cxgb3/cxio_wr.h
deleted file mode 100644
index 042bd9414220..000000000000
--- a/providers/cxgb3/cxio_wr.h
+++ /dev/null
@@ -1,758 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef __CXIO_WR_H__
-#define __CXIO_WR_H__
-
-#include <stddef.h>
-#include <stdint.h>
-#include <endian.h>
-#include <util/udma_barrier.h>
-#include "firmware_exports.h"
-
-#define T3_MAX_NUM_QP (1<<15)
-#define T3_MAX_NUM_CQ (1<<15)
-#define T3_MAX_NUM_PD (1<<15)
-#define T3_MAX_NUM_STAG (1<<15)
-#define T3_MAX_SGE      4
-#define T3_MAX_INLINE  64
-
-#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
-#define Q_FULL(rptr,wptr,size_log2)  ( (((wptr)-(rptr))>>(size_log2)) && \
-				       ((rptr)!=(wptr)) )
-#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
-#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
-#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
-#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
-
-/* FIXME: Move me to a generic PCI mmio accessor */
-#define cpu_to_pci32(val) htole32(val)
-
-#define RING_DOORBELL(doorbell, QPID) { \
-	*doorbell = cpu_to_pci32(QPID); \
-}
-
-#define SEQ32_GE(x,y) (!( (((uint32_t) (x)) - ((uint32_t) (y))) & 0x80000000 ))
-
-enum t3_wr_flags {
-	T3_COMPLETION_FLAG = 0x01,
-	T3_NOTIFY_FLAG = 0x02,
-	T3_SOLICITED_EVENT_FLAG = 0x04,
-	T3_READ_FENCE_FLAG = 0x08,
-	T3_LOCAL_FENCE_FLAG = 0x10
-} __attribute__ ((packed));
-
-enum t3_wr_opcode {
-	T3_WR_BP = FW_WROPCODE_RI_BYPASS,
-	T3_WR_SEND = FW_WROPCODE_RI_SEND,
-	T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
-	T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
-	T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
-	T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
-	T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
-	T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
-	T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
-} __attribute__ ((packed));
-
-enum t3_rdma_opcode {
-	T3_RDMA_WRITE,		/* IETF RDMAP v1.0 ... */
-	T3_READ_REQ,
-	T3_READ_RESP,
-	T3_SEND,
-	T3_SEND_WITH_INV,
-	T3_SEND_WITH_SE,
-	T3_SEND_WITH_SE_INV,
-	T3_TERMINATE,
-	T3_RDMA_INIT,		/* CHELSIO RI specific ... */
-	T3_BIND_MW,
-	T3_FAST_REGISTER,
-	T3_LOCAL_INV,
-	T3_QP_MOD,
-	T3_BYPASS
-} __attribute__ ((packed));
-
-static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
-{
-	switch (wrop) {
-		case T3_WR_BP: return T3_BYPASS;
-		case T3_WR_SEND: return T3_SEND;
-		case T3_WR_WRITE: return T3_RDMA_WRITE;
-		case T3_WR_READ: return T3_READ_REQ;
-		case T3_WR_INV_STAG: return T3_LOCAL_INV;
-		case T3_WR_BIND: return T3_BIND_MW;
-		case T3_WR_INIT: return T3_RDMA_INIT;
-		case T3_WR_QP_MOD: return T3_QP_MOD;
-		default: break;
-	}
-	return -1;
-}
-
-
-/* Work request id */
-union t3_wrid {
-	struct {
-		uint32_t hi:32;
-		uint32_t low:32;
-	} id0;
-	uint64_t id1;
-};
-
-#define WRID(wrid)      	(wrid.id1)
-#define WRID_GEN(wrid)		(wrid.id0.wr_gen)
-#define WRID_IDX(wrid)		(wrid.id0.wr_idx)
-#define WRID_LO(wrid)		(wrid.id0.wr_lo)
-
-struct fw_riwrh {
-	uint32_t op_seop_flags;
-	uint32_t gen_tid_len;
-};
-
-#define S_FW_RIWR_OP		24
-#define M_FW_RIWR_OP		0xff
-#define V_FW_RIWR_OP(x)		((x) << S_FW_RIWR_OP)
-#define G_FW_RIWR_OP(x)   	((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
-
-#define S_FW_RIWR_SOPEOP	22
-#define M_FW_RIWR_SOPEOP	0x3
-#define V_FW_RIWR_SOPEOP(x)	((x) << S_FW_RIWR_SOPEOP)
-
-#define S_FW_RIWR_FLAGS		8
-#define M_FW_RIWR_FLAGS		0x3fffff
-#define V_FW_RIWR_FLAGS(x)	((x) << S_FW_RIWR_FLAGS)
-#define G_FW_RIWR_FLAGS(x)   	((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
-
-#define S_FW_RIWR_TID		8
-#define V_FW_RIWR_TID(x)	((x) << S_FW_RIWR_TID)
-
-#define S_FW_RIWR_LEN		0
-#define V_FW_RIWR_LEN(x)	((x) << S_FW_RIWR_LEN)
-
-#define S_FW_RIWR_GEN           31
-#define V_FW_RIWR_GEN(x)        ((x)  << S_FW_RIWR_GEN)
-
-struct t3_sge {
-	uint32_t stag;
-	uint32_t len;
-	uint64_t to;
-};
-
-/* If num_sgle is zero, flit 5+ contains immediate data.*/
-struct t3_send_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-
-	enum t3_rdma_opcode rdmaop:8;
-	uint32_t reserved:24;	/* 2 */
-	uint32_t rem_stag;	/* 2 */
-	uint32_t plen;		/* 3 */
-	uint32_t num_sgle;
-	struct t3_sge sgl[T3_MAX_SGE];	/* 4+ */
-};
-
-struct t3_local_inv_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	uint32_t stag;		/* 2 */
-	uint32_t reserved3;
-};
-
-struct t3_rdma_write_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	enum t3_rdma_opcode rdmaop:8;	/* 2 */
-	uint32_t reserved:24;	/* 2 */
-	uint32_t stag_sink;
-	uint64_t to_sink;	/* 3 */
-	uint32_t plen;		/* 4 */
-	uint32_t num_sgle;
-	struct t3_sge sgl[T3_MAX_SGE];	/* 5+ */
-};
-
-struct t3_rdma_read_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	enum t3_rdma_opcode rdmaop:8;	/* 2 */
-	uint32_t reserved:24;
-	uint32_t rem_stag;
-	uint64_t rem_to;	/* 3 */
-	uint32_t local_stag;	/* 4 */
-	uint32_t local_len;
-	uint64_t local_to;	/* 5 */
-};
-
-enum t3_addr_type {
-	T3_VA_BASED_TO = 0x0,
-	T3_ZERO_BASED_TO = 0x1
-} __attribute__ ((packed));
-
-enum t3_mem_perms {
-	T3_MEM_ACCESS_LOCAL_READ = 0x1,
-	T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
-	T3_MEM_ACCESS_REM_READ = 0x4,
-	T3_MEM_ACCESS_REM_WRITE = 0x8
-} __attribute__ ((packed));
-
-struct t3_bind_mw_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	uint32_t reserved:16;
-	enum t3_addr_type type:8;
-	enum t3_mem_perms perms:8;	/* 2 */
-	uint32_t mr_stag;
-	uint32_t mw_stag;	/* 3 */
-	uint32_t mw_len;
-	uint64_t mw_va;		/* 4 */
-	uint32_t mr_pbl_addr;	/* 5 */
-	uint32_t reserved2:24;
-	uint32_t mr_pagesz:8;
-};
-
-struct t3_receive_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	uint8_t pagesz[T3_MAX_SGE];
-	uint32_t num_sgle;		/* 2 */
-	struct t3_sge sgl[T3_MAX_SGE];	/* 3+ */
-	uint32_t pbl_addr[T3_MAX_SGE];
-};
-
-struct t3_bypass_wr {
-	struct fw_riwrh wrh;
-	union t3_wrid wrid;	/* 1 */
-};
-
-struct t3_modify_qp_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	uint32_t flags;		/* 2 */
-	uint32_t quiesce;	/* 2 */
-	uint32_t max_ird;	/* 3 */
-	uint32_t max_ord;	/* 3 */
-	uint64_t sge_cmd;	/* 4 */
-	uint64_t ctx1;		/* 5 */
-	uint64_t ctx0;		/* 6 */
-};
-
-enum t3_modify_qp_flags {
-	MODQP_QUIESCE  = 0x01,
-	MODQP_MAX_IRD  = 0x02,
-	MODQP_MAX_ORD  = 0x04,
-	MODQP_WRITE_EC = 0x08,
-	MODQP_READ_EC  = 0x10,
-};
-	
-
-enum t3_mpa_attrs {
-	uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
-	uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
-	uP_RI_MPA_CRC_ENABLE = 0x4,
-	uP_RI_MPA_IETF_ENABLE = 0x8
-} __attribute__ ((packed));
-
-enum t3_qp_caps {
-	uP_RI_QP_RDMA_READ_ENABLE = 0x01,
-	uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
-	uP_RI_QP_BIND_ENABLE = 0x04,
-	uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
-	uP_RI_QP_STAG0_ENABLE = 0x10
-} __attribute__ ((packed));
-
-struct t3_rdma_init_attr {
-	uint32_t tid;
-	uint32_t qpid;
-	uint32_t pdid;
-	uint32_t scqid;
-	uint32_t rcqid;
-	uint32_t rq_addr;
-	uint32_t rq_size;
-	enum t3_mpa_attrs mpaattrs;
-	enum t3_qp_caps qpcaps;
-	uint16_t tcp_emss;
-	uint32_t ord;
-	uint32_t ird;
-	uint64_t qp_dma_addr;
-	uint32_t qp_dma_size;
-	uint8_t rqes_posted;
-};
-
-struct t3_rdma_init_wr {
-	struct fw_riwrh wrh;	/* 0 */
-	union t3_wrid wrid;	/* 1 */
-	uint32_t qpid;		/* 2 */
-	uint32_t pdid;
-	uint32_t scqid;		/* 3 */
-	uint32_t rcqid;
-	uint32_t rq_addr;	/* 4 */
-	uint32_t rq_size;
-	enum t3_mpa_attrs mpaattrs:8;	/* 5 */
-	enum t3_qp_caps qpcaps:8;
-	uint32_t ulpdu_size:16;
-	uint32_t rqes_posted;	/* bits 31-1 - reservered */
-				/* bit     0 - set if RECV posted */
-	uint32_t ord;		/* 6 */
-	uint32_t ird;
-	uint64_t qp_dma_addr;	/* 7 */
-	uint32_t qp_dma_size;	/* 8 */
-	uint32_t rsvd;
-};
-
-union t3_wr {
-	struct t3_send_wr send;
-	struct t3_rdma_write_wr write;
-	struct t3_rdma_read_wr read;
-	struct t3_receive_wr recv;
-	struct t3_local_inv_wr local_inv;
-	struct t3_bind_mw_wr bind;
-	struct t3_bypass_wr bypass;
-	struct t3_rdma_init_wr init;
-	struct t3_modify_qp_wr qp_mod;
-	uint64_t flit[16];
-};
-
-#define T3_SQ_CQE_FLIT 	  13
-#define T3_SQ_COOKIE_FLIT 14
-
-#define T3_RQ_COOKIE_FLIT 13
-#define T3_RQ_CQE_FLIT 	  14
-
-static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
-				  enum t3_wr_flags flags, uint8_t genbit, 
-				  uint32_t tid, uint8_t len)
-{
-	wqe->op_seop_flags = htobe32(V_FW_RIWR_OP(op) |
-				   V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
-				   V_FW_RIWR_FLAGS(flags));
-	udma_to_device_barrier();
-	wqe->gen_tid_len = htobe32(V_FW_RIWR_GEN(genbit) | V_FW_RIWR_TID(tid) |
-				 V_FW_RIWR_LEN(len));
-	/* 2nd gen bit... */
-        ((union t3_wr *)wqe)->flit[15] = htobe64(genbit);
-}
-
-/*
- * T3 ULP2_TX commands
- */
-enum t3_utx_mem_op {
-	T3_UTX_MEM_READ = 2,
-	T3_UTX_MEM_WRITE = 3
-};
-
-/* T3 MC7 RDMA TPT entry format */
-
-enum tpt_mem_type {
-	TPT_NON_SHARED_MR = 0x0,
-	TPT_SHARED_MR = 0x1,
-	TPT_MW = 0x2,
-	TPT_MW_RELAXED_PROTECTION = 0x3
-};
-
-enum tpt_addr_type {
-	TPT_ZBTO = 0,
-	TPT_VATO = 1
-};
-
-enum tpt_mem_perm {
-	TPT_LOCAL_READ = 0x8,
-	TPT_LOCAL_WRITE = 0x4,
-	TPT_REMOTE_READ = 0x2,
-	TPT_REMOTE_WRITE = 0x1
-};
-
-struct tpt_entry {
-	uint32_t valid_stag_pdid;
-	uint32_t flags_pagesize_qpid;
-
-	uint32_t rsvd_pbl_addr;
-	uint32_t len;
-	uint32_t va_hi;
-	uint32_t va_low_or_fbo;
-
-	uint32_t rsvd_bind_cnt_or_pstag;
-	uint32_t rsvd_pbl_size;
-};
-
-#define S_TPT_VALID		31
-#define V_TPT_VALID(x)		((x) << S_TPT_VALID)
-#define F_TPT_VALID		V_TPT_VALID(1U)
-
-#define S_TPT_STAG_KEY		23
-#define M_TPT_STAG_KEY		0xFF
-#define V_TPT_STAG_KEY(x)	((x) << S_TPT_STAG_KEY)
-#define G_TPT_STAG_KEY(x)	(((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
-
-#define S_TPT_STAG_STATE	22
-#define V_TPT_STAG_STATE(x)	((x) << S_TPT_STAG_STATE)
-#define F_TPT_STAG_STATE	V_TPT_STAG_STATE(1U)
-
-#define S_TPT_STAG_TYPE		20
-#define M_TPT_STAG_TYPE		0x3
-#define V_TPT_STAG_TYPE(x)	((x) << S_TPT_STAG_TYPE)
-#define G_TPT_STAG_TYPE(x)	(((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
-
-#define S_TPT_PDID		0
-#define M_TPT_PDID		0xFFFFF
-#define V_TPT_PDID(x)		((x) << S_TPT_PDID)
-#define G_TPT_PDID(x)		(((x) >> S_TPT_PDID) & M_TPT_PDID)
-
-#define S_TPT_PERM		28
-#define M_TPT_PERM		0xF
-#define V_TPT_PERM(x)		((x) << S_TPT_PERM)
-#define G_TPT_PERM(x)		(((x) >> S_TPT_PERM) & M_TPT_PERM)
-
-#define S_TPT_REM_INV_DIS	27
-#define V_TPT_REM_INV_DIS(x)	((x) << S_TPT_REM_INV_DIS)
-#define F_TPT_REM_INV_DIS	V_TPT_REM_INV_DIS(1U)
-
-#define S_TPT_ADDR_TYPE		26
-#define V_TPT_ADDR_TYPE(x)	((x) << S_TPT_ADDR_TYPE)
-#define F_TPT_ADDR_TYPE		V_TPT_ADDR_TYPE(1U)
-
-#define S_TPT_MW_BIND_ENABLE	25
-#define V_TPT_MW_BIND_ENABLE(x)	((x) << S_TPT_MW_BIND_ENABLE)
-#define F_TPT_MW_BIND_ENABLE    V_TPT_MW_BIND_ENABLE(1U)
-
-#define S_TPT_PAGE_SIZE		20
-#define M_TPT_PAGE_SIZE		0x1F
-#define V_TPT_PAGE_SIZE(x)	((x) << S_TPT_PAGE_SIZE)
-#define G_TPT_PAGE_SIZE(x)	(((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
-
-#define S_TPT_PBL_ADDR		0
-#define M_TPT_PBL_ADDR		0x1FFFFFFF
-#define V_TPT_PBL_ADDR(x)	((x) << S_TPT_PBL_ADDR)
-#define G_TPT_PBL_ADDR(x)       (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
-
-#define S_TPT_QPID		0
-#define M_TPT_QPID		0xFFFFF
-#define V_TPT_QPID(x)		((x) << S_TPT_QPID)
-#define G_TPT_QPID(x)		(((x) >> S_TPT_QPID) & M_TPT_QPID)
-
-#define S_TPT_PSTAG		0
-#define M_TPT_PSTAG		0xFFFFFF
-#define V_TPT_PSTAG(x)		((x) << S_TPT_PSTAG)
-#define G_TPT_PSTAG(x)		(((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
-
-#define S_TPT_PBL_SIZE		0
-#define M_TPT_PBL_SIZE		0xFFFFF
-#define V_TPT_PBL_SIZE(x)	((x) << S_TPT_PBL_SIZE)
-#define G_TPT_PBL_SIZE(x)	(((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
-
-/*
- * CQE defs
- */
-struct t3_cqe {
-	uint32_t header:32;
-	uint32_t len:32;
-	uint32_t wrid_hi_stag:32;
-	uint32_t wrid_low_msn:32;
-};
-
-#define S_CQE_OOO	  31
-#define M_CQE_OOO	  0x1
-#define G_CQE_OOO(x)	  ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
-#define V_CEQ_OOO(x)	  ((x)<<S_CQE_OOO)
-
-#define S_CQE_QPID        12
-#define M_CQE_QPID        0x7FFFF
-#define G_CQE_QPID(x)     ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
-#define V_CQE_QPID(x) 	  ((x)<<S_CQE_QPID)
-
-#define S_CQE_SWCQE       11
-#define M_CQE_SWCQE       0x1
-#define G_CQE_SWCQE(x)    ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
-#define V_CQE_SWCQE(x) 	  ((x)<<S_CQE_SWCQE)
-
-#define S_CQE_GENBIT      10
-#define M_CQE_GENBIT      0x1
-#define G_CQE_GENBIT(x)   (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
-#define V_CQE_GENBIT(x)	  ((x)<<S_CQE_GENBIT)
-
-#define S_CQE_STATUS      5
-#define M_CQE_STATUS      0x1F
-#define G_CQE_STATUS(x)   ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
-#define V_CQE_STATUS(x)   ((x)<<S_CQE_STATUS)
-
-#define S_CQE_TYPE        4
-#define M_CQE_TYPE        0x1
-#define G_CQE_TYPE(x)     ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
-#define V_CQE_TYPE(x)     ((x)<<S_CQE_TYPE)
-
-#define S_CQE_OPCODE      0
-#define M_CQE_OPCODE      0xF
-#define G_CQE_OPCODE(x)   ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
-#define V_CQE_OPCODE(x)   ((x)<<S_CQE_OPCODE)
-
-#define SW_CQE(x)         (G_CQE_SWCQE(be32toh((x).header)))
-#define CQE_OOO(x)        (G_CQE_OOO(be32toh((x).header)))
-#define CQE_QPID(x)       (G_CQE_QPID(be32toh((x).header)))
-#define CQE_GENBIT(x)     (G_CQE_GENBIT(be32toh((x).header)))
-#define CQE_TYPE(x)       (G_CQE_TYPE(be32toh((x).header)))
-#define SQ_TYPE(x)	  (CQE_TYPE((x)))
-#define RQ_TYPE(x)	  (!CQE_TYPE((x)))
-#define CQE_STATUS(x)     (G_CQE_STATUS(be32toh((x).header)))
-#define CQE_OPCODE(x)     (G_CQE_OPCODE(be32toh((x).header)))
-
-#define CQE_LEN(x)        (be32toh((x).len))
-
-#define CQE_WRID_HI(x)    (be32toh((x).wrid_hi_stag))
-#define CQE_WRID_LOW(x)   (be32toh((x).wrid_low_msn))
-
-/* used for RQ completion processing */
-#define CQE_WRID_STAG(x)  (be32toh((x).wrid_hi_stag))
-#define CQE_WRID_MSN(x)   (be32toh((x).wrid_low_msn))
-
-/* used for SQ completion processing */
-#define CQE_WRID_SQ_WPTR(x)	((x).wrid_hi_stag)
-#define CQE_WRID_WPTR(x)   	((x).wrid_low_msn)
-
-#define TPT_ERR_SUCCESS                     0x0
-#define TPT_ERR_STAG                        0x1	 /* STAG invalid: either the */
-						 /* STAG is offlimt, being 0, */
-						 /* or STAG_key mismatch */
-#define TPT_ERR_PDID                        0x2	 /* PDID mismatch */
-#define TPT_ERR_QPID                        0x3	 /* QPID mismatch */
-#define TPT_ERR_ACCESS                      0x4	 /* Invalid access right */
-#define TPT_ERR_WRAP                        0x5	 /* Wrap error */
-#define TPT_ERR_BOUND                       0x6	 /* base and bounds voilation */
-#define TPT_ERR_INVALIDATE_SHARED_MR        0x7	 /* attempt to invalidate a  */
-						 /* shared memory region */
-#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8	 /* attempt to invalidate a  */
-						 /* shared memory region */
-#define TPT_ERR_ECC                         0x9	 /* ECC error detected */
-#define TPT_ERR_ECC_PSTAG                   0xA	 /* ECC error detected when  */
-						 /* reading PSTAG for a MW  */
-						 /* Invalidate */
-#define TPT_ERR_PBL_ADDR_BOUND              0xB	 /* pbl addr out of bounds:  */
-						 /* software error */
-#define TPT_ERR_SWFLUSH			    0xC	 /* SW FLUSHED */
-#define TPT_ERR_CRC                         0x10 /* CRC error */
-#define TPT_ERR_MARKER                      0x11 /* Marker error */
-#define TPT_ERR_PDU_LEN_ERR                 0x12 /* invalid PDU length */
-#define TPT_ERR_OUT_OF_RQE                  0x13 /* out of RQE */
-#define TPT_ERR_DDP_VERSION                 0x14 /* wrong DDP version */
-#define TPT_ERR_RDMA_VERSION                0x15 /* wrong RDMA version */
-#define TPT_ERR_OPCODE                      0x16 /* invalid rdma opcode */
-#define TPT_ERR_DDP_QUEUE_NUM               0x17 /* invalid ddp queue number */
-#define TPT_ERR_MSN                         0x18 /* MSN error */
-#define TPT_ERR_TBIT                        0x19 /* tag bit not set correctly */
-#define TPT_ERR_MO                          0x1A /* MO not 0 for TERMINATE  */
-						 /* or READ_REQ */
-#define TPT_ERR_MSN_GAP                     0x1B
-#define TPT_ERR_MSN_RANGE                   0x1C
-#define TPT_ERR_IRD_OVERFLOW                0x1D
-#define TPT_ERR_RQE_ADDR_BOUND              0x1E /* RQE addr out of bounds:  */
-						 /* software error */
-#define TPT_ERR_INTERNAL_ERR                0x1F /* internal error (opcode  */
-						 /* mismatch) */
-
-struct t3_swsq {
-	uint64_t 		wr_id;
-	struct t3_cqe 		cqe;
-	uint32_t		sq_wptr;
-	uint32_t		read_len;
-	int 			opcode;
-	int			complete;
-	int			signaled;	
-};
-
-/*
- * A T3 WQ implements both the SQ and RQ.
- */
-struct t3_wq {
-	union t3_wr *queue;		/* DMA Mapped work queue */
-	uint32_t error;			/* 1 once we go to ERROR */
-	uint32_t qpid;
-	uint32_t wptr;			/* idx to next available WR slot */
-	uint32_t size_log2;		/* total wq size */
-	struct t3_swsq *sq;		/* SW SQ */
-	struct t3_swsq *oldest_read;	/* tracks oldest pending read */
-	uint32_t sq_wptr;		/* sq_wptr - sq_rptr == count of */
-	uint32_t sq_rptr;		/* pending wrs */
-	uint32_t sq_size_log2;		/* sq size */
-	uint64_t *rq;			/* SW RQ (holds consumer wr_ids) */
-	uint32_t rq_wptr;		/* rq_wptr - rq_rptr == count of */
-	uint32_t rq_rptr;		/* pending wrs */
-	uint32_t rq_size_log2;		/* rq size */
-	volatile uint32_t *doorbell;	/* mapped adapter doorbell register */
-	int flushed;
-};
-
-struct t3_cq {
-	uint32_t cqid;
-	uint32_t rptr;
-	uint32_t wptr;
-	uint32_t size_log2;
-	struct t3_cqe *queue;
-	struct t3_cqe *sw_queue;
-	uint32_t sw_rptr;
-	uint32_t sw_wptr;
-	uint32_t memsize;
-};
-
-static inline unsigned t3_wq_depth(struct t3_wq *wq)
-{
-	return (1UL<<wq->size_log2);
-}
-
-static inline unsigned t3_sq_depth(struct t3_wq *wq)
-{
-	return (1UL<<wq->sq_size_log2);
-}
-
-static inline unsigned t3_rq_depth(struct t3_wq *wq)
-{
-	return (1UL<<wq->rq_size_log2);
-}
-
-static inline unsigned t3_cq_depth(struct t3_cq *cq)
-{
-	return (1UL<<cq->size_log2);
-}
-
-extern unsigned long iwch_page_size;
-extern unsigned long iwch_page_shift;
-extern unsigned long iwch_page_mask;
-
-#define PAGE_ALIGN(x) (((x) + iwch_page_mask) & ~iwch_page_mask)
-
-static inline unsigned t3_wq_memsize(struct t3_wq *wq)
-{
-	return PAGE_ALIGN((1UL<<wq->size_log2) * sizeof (union t3_wr));
-}
-
-static inline unsigned t3_cq_memsize(struct t3_cq *cq)
-{
-	return cq->memsize;
-}
-
-static inline unsigned t3_mmid(uint32_t stag)
-{
-	return (stag>>8);
-}
-
-struct t3_cq_status_page {
-	uint32_t cq_err;
-};
-
-static inline int t3_cq_in_error(struct t3_cq *cq)
-{
-	return ((struct t3_cq_status_page *)
-	       &cq->queue[1 << cq->size_log2])->cq_err;
-}
-
-static inline void t3_set_cq_in_error(struct t3_cq *cq)
-{
-	((struct t3_cq_status_page *)
-		&cq->queue[1 << cq->size_log2])->cq_err = 1;
-}
-
-static inline void t3_reset_cq_in_error(struct t3_cq *cq)
-{
-	((struct t3_cq_status_page *)
-		&cq->queue[1 << cq->size_log2])->cq_err = 0;
-}
-
-static inline int t3_wq_in_error(struct t3_wq *wq)
-{
-	/*
-	 * The kernel sets bit 0 in the first WR of the WQ memory
-	 * when the QP moves out of RTS...
-	 */
-        return (wq->queue->flit[13] & 1);
-}
-
-static inline void t3_set_wq_in_error(struct t3_wq *wq)
-{
-        wq->queue->flit[13] |= 1;
-}
-
-static inline int t3_wq_db_enabled(struct t3_wq *wq)
-{
-	return !(wq->queue->flit[13] & 2);
-}
-
-#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
-					 CQE_GENBIT(*cqe))
-
-static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
-{
-	struct t3_cqe *cqe;
-
-	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
-	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
-		return cqe;
-	return NULL;
-}
-
-static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
-{
-	struct t3_cqe *cqe;
-
-	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
-		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
-		return cqe;
-	}
-	return NULL;
-}
-
-static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
-{
-	struct t3_cqe *cqe;
-
-	if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
-		cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
-		return cqe;
-	}
-	cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
-	if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
-		return cqe;
-	return NULL;
-}
-
-/*
- * Return a ptr to the next read wr in the SWSQ or NULL.
- */
-static inline struct t3_swsq *next_read_wr(struct t3_wq *wq)
-{
-	uint32_t rptr = wq->oldest_read - wq->sq + 1;
-	int count = Q_COUNT(rptr, wq->sq_wptr);
-	struct t3_swsq *sqp;
-
-	while (count--) {
-		sqp = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2);
-
-		if (sqp->opcode == T3_READ_REQ)
-			return sqp;
-
-		rptr++;
-	}
-	return NULL;
-}
-#endif
diff --git a/providers/cxgb3/firmware_exports.h b/providers/cxgb3/firmware_exports.h
deleted file mode 100644
index 831140a4c8e3..000000000000
--- a/providers/cxgb3/firmware_exports.h
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Copyright (c) 2004-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef _FIRMWARE_EXPORTS_H_
-#define _FIRMWARE_EXPORTS_H_
-
-/* WR OPCODES supported by the firmware.
- */
-#define	FW_WROPCODE_FORWARD			0x01
-#define FW_WROPCODE_BYPASS			0x05
-
-#define FW_WROPCODE_TUNNEL_TX_PKT		0x03
-
-#define FW_WROPOCDE_ULPTX_DATA_SGL		0x00
-#define FW_WROPCODE_ULPTX_MEM_READ		0x02
-#define FW_WROPCODE_ULPTX_PKT			0x04
-#define FW_WROPCODE_ULPTX_INVALIDATE		0x06
-
-#define FW_WROPCODE_TUNNEL_RX_PKT		0x07
-
-#define FW_WROPCODE_TOE_GETTCB_RPL		0x08
-#define FW_WROPCODE_TOE_CLOSE_CON		0x09
-#define FW_WROPCODE_TOE_TP_ABORT_CON_REQ	0x0A
-#define FW_WROPCODE_TOE_HOST_ABORT_CON_RPL	0x0F
-#define FW_WROPCODE_TOE_HOST_ABORT_CON_REQ	0x0B
-#define FW_WROPCODE_TOE_TP_ABORT_CON_RPL	0x0C
-#define FW_WROPCODE_TOE_TX_DATA			0x0D
-#define FW_WROPCODE_TOE_TX_DATA_ACK		0x0E
-
-#define FW_WROPCODE_RI_RDMA_INIT		0x10
-#define FW_WROPCODE_RI_RDMA_WRITE		0x11
-#define FW_WROPCODE_RI_RDMA_READ_REQ		0x12
-#define FW_WROPCODE_RI_RDMA_READ_RESP		0x13
-#define FW_WROPCODE_RI_SEND			0x14
-#define FW_WROPCODE_RI_TERMINATE		0x15
-#define FW_WROPCODE_RI_RDMA_READ		0x16
-#define FW_WROPCODE_RI_RECEIVE			0x17
-#define FW_WROPCODE_RI_BIND_MW			0x18
-#define FW_WROPCODE_RI_FASTREGISTER_MR		0x19
-#define FW_WROPCODE_RI_LOCAL_INV		0x1A
-#define FW_WROPCODE_RI_MODIFY_QP		0x1B
-#define FW_WROPCODE_RI_BYPASS			0x1C
-
-#define FW_WROPOCDE_RSVD			0x1E
-
-#define FW_WROPCODE_SGE_EGRESSCONTEXT_RR	0x1F
-
-#define FW_WROPCODE_MNGT			0x1D
-#define FW_MNGTOPCODE_PKTSCHED_SET		0x00
-
-/* Maximum size of a WR sent from the host, limited by the SGE. 
- *
- * Note: WR coming from ULP or TP are only limited by CIM. 
- */
-#define FW_WR_SIZE			128
-
-/* Maximum number of outstanding WRs sent from the host. Value must be
- * programmed in the CTRL/TUNNEL/QP SGE Egress Context and used by TOM to 
- * limit the number of WRs per connection.
- */
-#ifndef N3
-# define FW_WR_NUM			16
-#else
-# define FW_WR_NUM			7
-#endif
-
-/* FW_TUNNEL_NUM corresponds to the number of supported TUNNEL Queues. These
- * queues must start at SGE Egress Context FW_TUNNEL_SGEEC_START and must
- * start at 'TID' (or 'uP Token') FW_TUNNEL_TID_START.
- *
- * Ingress Traffic (e.g. DMA completion credit)  for TUNNEL Queue[i] is sent 
- * to RESP Queue[i].
- */
-#define FW_TUNNEL_NUM                   8
-#define FW_TUNNEL_SGEEC_START           8
-#define FW_TUNNEL_TID_START             65544
-
-
-/* FW_CTRL_NUM corresponds to the number of supported CTRL Queues. These queues
- * must start at SGE Egress Context FW_CTRL_SGEEC_START and must start at 'TID'
- * (or 'uP Token') FW_CTRL_TID_START.
- *
- * Ingress Traffic for CTRL Queue[i] is sent to RESP Queue[i].
- */ 
-#define FW_CTRL_NUM			8
-#define FW_CTRL_SGEEC_START		65528
-#define FW_CTRL_TID_START		65536
-
-/* FW_TOE_NUM corresponds to the number of supported TOE Queues. These queues 
- * must start at SGE Egress Context FW_TOE_SGEEC_START. 
- * 
- * Note: the 'uP Token' in the SGE Egress Context fields is irrelevant for 
- * TOE Queues, as the host is responsible for providing the correct TID in
- * every WR.
- *
- * Ingress Trafffic for TOE Queue[i] is sent to RESP Queue[i].
- */
-#define FW_TOE_NUM			8
-#define FW_TOE_SGEEC_START		0
-
-/*
- * 
- */
-#define FW_RI_NUM			1
-#define FW_RI_SGEEC_START		65527
-#define FW_RI_TID_START			65552
-
-/*
- * The RX_PKT_TID 
- */
-#define FW_RX_PKT_NUM			1
-#define FW_RX_PKT_TID_START		65553
-
-/* FW_WRC_NUM corresponds to the number of Work Request Context that supported
- * by the firmware.
- */
-#define FW_WRC_NUM			(65536 + FW_TUNNEL_NUM + FW_CTRL_NUM +\
-					 FW_RI_NUM + FW_RX_PKT_NUM)
-
-#endif /* _FIRMWARE_EXPORTS_H_ */
diff --git a/providers/cxgb3/iwch-abi.h b/providers/cxgb3/iwch-abi.h
deleted file mode 100644
index 047f84b7ab63..000000000000
--- a/providers/cxgb3/iwch-abi.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef IWCH_ABI_H
-#define IWCH_ABI_H
-
-#include <stdint.h>
-#include <infiniband/kern-abi.h>
-#include <rdma/cxgb3-abi.h>
-#include <kernel-abi/cxgb3-abi.h>
-
-DECLARE_DRV_CMD(uiwch_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
-		empty, iwch_alloc_pd_resp);
-DECLARE_DRV_CMD(uiwch_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
-		iwch_create_cq_req, iwch_create_cq_resp);
-DECLARE_DRV_CMD(uiwch_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
-		empty, iwch_create_qp_resp);
-DECLARE_DRV_CMD(uiwch_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
-		empty, empty);
-DECLARE_DRV_CMD(uiwch_reg_mr, IB_USER_VERBS_CMD_REG_MR,
-		empty, iwch_reg_user_mr_resp);
-
-#endif				/* IWCH_ABI_H */
diff --git a/providers/cxgb3/iwch.c b/providers/cxgb3/iwch.c
deleted file mode 100644
index 6f3c8b9f1439..000000000000
--- a/providers/cxgb3/iwch.c
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/mman.h>
-#include <pthread.h>
-#include <string.h>
-
-#include "iwch.h"
-#include "iwch-abi.h"
-
-#define PCI_VENDOR_ID_CHELSIO		0x1425
-#define PCI_DEVICE_ID_CHELSIO_PE9000_2C	0x0020
-#define PCI_DEVICE_ID_CHELSIO_T302E	0x0021
-#define PCI_DEVICE_ID_CHELSIO_T310E	0x0022
-#define PCI_DEVICE_ID_CHELSIO_T320X	0x0023
-#define PCI_DEVICE_ID_CHELSIO_T302X	0x0024
-#define PCI_DEVICE_ID_CHELSIO_T320E	0x0025
-#define PCI_DEVICE_ID_CHELSIO_T310X	0x0026
-#define PCI_DEVICE_ID_CHELSIO_T3B10	0x0030
-#define PCI_DEVICE_ID_CHELSIO_T3B20	0x0031
-#define PCI_DEVICE_ID_CHELSIO_T3B02	0x0032
-#define PCI_DEVICE_ID_CHELSIO_T3C20	0x0035
-#define PCI_DEVICE_ID_CHELSIO_S320E	0x0036
-
-#define HCA(v, d, t)                                                           \
-	VERBS_PCI_MATCH(PCI_VENDOR_ID_##v, PCI_DEVICE_ID_CHELSIO_##d,          \
-			(void *)(CHELSIO_##t))
-static const struct verbs_match_ent hca_table[] = {
-	HCA(CHELSIO, PE9000_2C, T3B),
-	HCA(CHELSIO, T302E, T3A),
-	HCA(CHELSIO, T302X, T3A),
-	HCA(CHELSIO, T310E, T3A),
-	HCA(CHELSIO, T310X, T3A),
-	HCA(CHELSIO, T320E, T3A),
-	HCA(CHELSIO, T320X, T3A),
-	HCA(CHELSIO, T3B10, T3B),
-	HCA(CHELSIO, T3B20, T3B),
-	HCA(CHELSIO, T3B02, T3B),
-	HCA(CHELSIO, T3C20, T3B),
-	HCA(CHELSIO, S320E, T3B),
-	{},
-};
-
-static const struct verbs_context_ops iwch_ctx_common_ops = {
-	.query_device = iwch_query_device,
-	.query_port = iwch_query_port,
-	.alloc_pd = iwch_alloc_pd,
-	.dealloc_pd = iwch_free_pd,
-	.reg_mr = iwch_reg_mr,
-	.dereg_mr = iwch_dereg_mr,
-	.create_cq = iwch_create_cq,
-	.resize_cq = iwch_resize_cq,
-	.destroy_cq = iwch_destroy_cq,
-	.create_srq = iwch_create_srq,
-	.modify_srq = iwch_modify_srq,
-	.destroy_srq = iwch_destroy_srq,
-	.create_qp = iwch_create_qp,
-	.modify_qp = iwch_modify_qp,
-	.destroy_qp = iwch_destroy_qp,
-	.query_qp = iwch_query_qp,
-	.create_ah = iwch_create_ah,
-	.destroy_ah = iwch_destroy_ah,
-	.attach_mcast = iwch_attach_mcast,
-	.detach_mcast = iwch_detach_mcast,
-	.post_srq_recv = iwch_post_srq_recv,
-	.req_notify_cq = iwch_arm_cq,
-};
-
-static const struct verbs_context_ops iwch_ctx_t3a_ops = {
-	.poll_cq = t3a_poll_cq,
-	.post_recv = t3a_post_recv,
-	.post_send = t3a_post_send,
-};
-
-static const struct verbs_context_ops iwch_ctx_t3b_ops = {
-	.async_event = t3b_async_event,
-	.poll_cq = t3b_poll_cq,
-	.post_recv = t3b_post_recv,
-	.post_send = t3b_post_send,
-};
-
-unsigned long iwch_page_size;
-unsigned long iwch_page_shift;
-unsigned long iwch_page_mask;
-
-static struct verbs_context *iwch_alloc_context(struct ibv_device *ibdev,
-						int cmd_fd,
-						void *private_data)
-{
-	struct iwch_context *context;
-	struct ibv_get_context cmd;
-	struct uiwch_alloc_ucontext_resp resp;
-	struct iwch_device *rhp = to_iwch_dev(ibdev);
-
-	context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx,
-					       RDMA_DRIVER_CXGB3);
-	if (!context)
-		return NULL;
-
-	if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
-				&resp.ibv_resp, sizeof resp))
-		goto err_free;
-
-	verbs_set_ops(&context->ibv_ctx, &iwch_ctx_common_ops);
-
-	switch (rhp->hca_type) {
-	case CHELSIO_T3B:
-		PDBG("%s T3B device\n", __FUNCTION__);
-		verbs_set_ops(&context->ibv_ctx, &iwch_ctx_t3b_ops);
-		break;
-	case CHELSIO_T3A:
-		PDBG("%s T3A device\n", __FUNCTION__);
-		verbs_set_ops(&context->ibv_ctx, &iwch_ctx_t3a_ops);
-		break;
-	default:
-		PDBG("%s unknown hca type %d\n", __FUNCTION__, rhp->hca_type);
-		goto err_free;
-		break;
-	}	
-
-	return &context->ibv_ctx;
-
-err_free:
-	verbs_uninit_context(&context->ibv_ctx);
-	free(context);
-	return NULL;
-}
-
-static void iwch_free_context(struct ibv_context *ibctx)
-{
-	struct iwch_context *context = to_iwch_ctx(ibctx);
-
-	verbs_uninit_context(&context->ibv_ctx);
-	free(context);
-}
-
-static void iwch_uninit_device(struct verbs_device *verbs_device)
-{
-	struct iwch_device *dev = to_iwch_dev(&verbs_device->device);
-
-	free(dev);
-}
-
-static bool iwch_device_match(struct verbs_sysfs_dev *sysfs_dev)
-{
-	char value[32], *cp;
-	unsigned int fw_maj, fw_min;
-
-	/* Rely on the core code to match PCI devices */
-	if (!sysfs_dev->match)
-		return false;
-
-	/* 
-	 * Verify that the firmware major number matches.  Major number
-	 * mismatches are fatal.  Minor number mismatches are tolerated.
-	 */
-	if (ibv_get_fw_ver(value, sizeof(value), sysfs_dev))
-		return false;
-
-	cp = strtok(value+1, ".");
-	sscanf(cp, "%i", &fw_maj);
-	cp = strtok(NULL, ".");
-	sscanf(cp, "%i", &fw_min);
-
-	if (fw_maj < FW_MAJ) {
-		fprintf(stderr, "libcxgb3: Fatal firmware version mismatch.  "
-			"Firmware major number is %u and libcxgb3 needs %u.\n",
-			fw_maj, FW_MAJ);	
-		fflush(stderr);
-		return false;
-	}
-
-	DBGLOG("libcxgb3");
-
-	if ((signed int)fw_min < FW_MIN) {
-		PDBG("libcxgb3: non-fatal firmware version mismatch.  "
-			"Firmware minor number is %u and libcxgb3 needs %u.\n",
-			fw_min, FW_MIN);
-		fflush(stderr);
-	}
-
-	return true;
-}
-
-static struct verbs_device *iwch_device_alloc(struct verbs_sysfs_dev *sysfs_dev)
-{
-	struct iwch_device *dev;
-
-	dev = calloc(1, sizeof(*dev));
-	if (!dev)
-		return NULL;
-
-	pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE);
-	dev->hca_type = (uintptr_t)sysfs_dev->match->driver_data;
-	dev->abi_version = sysfs_dev->abi_ver;
-
-	iwch_page_size = sysconf(_SC_PAGESIZE);
-	iwch_page_shift = long_log2(iwch_page_size);
-	iwch_page_mask = iwch_page_size - 1;
-
-	dev->mmid2ptr = calloc(T3_MAX_NUM_STAG, sizeof(void *));
-	if (!dev->mmid2ptr) {
-		goto err1;
-	}
-	dev->qpid2ptr = calloc(T3_MAX_NUM_QP, sizeof(void *)); 
-	if (!dev->qpid2ptr) {
-		goto err2;
-	}
-	dev->cqid2ptr = calloc(T3_MAX_NUM_CQ, sizeof(void *));
-	if (!dev->cqid2ptr) 
-		goto err3;
-
-	return &dev->ibv_dev;
-
-err3:
-	free(dev->qpid2ptr);
-err2:
-	free(dev->mmid2ptr);
-err1:
-	free(dev);
-	return NULL;
-}
-
-static const struct verbs_device_ops iwch_dev_ops = {
-	.name = "cxgb3",
-	.match_min_abi_version = 0,
-	.match_max_abi_version = ABI_VERS,
-	.match_table = hca_table,
-	.match_device = iwch_device_match,
-	.alloc_device = iwch_device_alloc,
-	.uninit_device = iwch_uninit_device,
-	.alloc_context = iwch_alloc_context,
-	.free_context = iwch_free_context,
-};
-PROVIDER_DRIVER(cxgb3, iwch_dev_ops);
diff --git a/providers/cxgb3/iwch.h b/providers/cxgb3/iwch.h
deleted file mode 100644
index c7d85d3aab2e..000000000000
--- a/providers/cxgb3/iwch.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef IWCH_H
-#define IWCH_H
-
-#include <pthread.h>
-#include <inttypes.h>
-#include <stddef.h>
-
-#include <infiniband/driver.h>
-#include <util/udma_barrier.h>
-
-#include "cxio_wr.h"
-
-enum iwch_hca_type {
-	CHELSIO_T3A = 0,
-	CHELSIO_T3B = 1,
-};
-
-struct iwch_mr;
-
-#define ABI_VERS 1
-
-struct iwch_device {
-	struct verbs_device ibv_dev;
-	enum iwch_hca_type hca_type;
-	struct iwch_mr **mmid2ptr;
-	struct iwch_qp **qpid2ptr;
-	struct iwch_cq **cqid2ptr;
-	pthread_spinlock_t lock;
-	int abi_version;
-};
-
-static inline int t3b_device(struct iwch_device *dev)
-{
-	return (dev->hca_type == CHELSIO_T3B);
-}
-
-static inline int t3a_device(struct iwch_device *dev)
-{
-	return (dev->hca_type == CHELSIO_T3A);
-}
-
-struct iwch_context {
-	struct verbs_context ibv_ctx;
-};
-
-struct iwch_pd {
-	struct ibv_pd ibv_pd;
-};
-
-struct iwch_mr {
-	struct verbs_mr vmr;
-	uint64_t va_fbo;
-	uint32_t page_size;
-	uint32_t pbl_addr;
-	uint32_t len;
-};
-
-struct iwch_cq {
-	struct ibv_cq ibv_cq;
-	struct iwch_device *rhp;
-	struct t3_cq cq;
-	pthread_spinlock_t lock;
-};
-
-struct iwch_qp {
-	struct ibv_qp ibv_qp;
-	struct iwch_device *rhp;
-	struct t3_wq wq;
-	pthread_spinlock_t lock;
-	int sq_sig_all;
-};
-
-#define to_iwch_xxx(xxx, type)                                                 \
-	container_of(ib##xxx, struct iwch_##type, ibv_##xxx)
-
-static inline struct iwch_device *to_iwch_dev(struct ibv_device *ibdev)
-{
-	return container_of(ibdev, struct iwch_device, ibv_dev.device);
-}
-
-static inline struct iwch_context *to_iwch_ctx(struct ibv_context *ibctx)
-{
-	return container_of(ibctx, struct iwch_context, ibv_ctx.context);
-}
-
-static inline struct iwch_pd *to_iwch_pd(struct ibv_pd *ibpd)
-{
-	return to_iwch_xxx(pd, pd);
-}
-
-static inline struct iwch_cq *to_iwch_cq(struct ibv_cq *ibcq)
-{
-	return to_iwch_xxx(cq, cq);
-}
-
-static inline struct iwch_qp *to_iwch_qp(struct ibv_qp *ibqp)
-{
-	return to_iwch_xxx(qp, qp);
-}
-
-static inline struct iwch_mr *to_iwch_mr(struct verbs_mr *vmr)
-{
-	return container_of(vmr, struct iwch_mr, vmr);
-}
-
-static inline unsigned long long_log2(unsigned long x)
-{
-        unsigned long r = 0;
-        for (x >>= 1; x > 0; x >>= 1)
-                r++;
-        return r;
-}
-
-extern int iwch_query_device(struct ibv_context *context,
-			     struct ibv_device_attr *attr);
-extern int iwch_query_port(struct ibv_context *context, uint8_t port,
-			   struct ibv_port_attr *attr);
-
-extern struct ibv_pd *iwch_alloc_pd(struct ibv_context *context);
-extern int iwch_free_pd(struct ibv_pd *pd);
-
-extern struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
-				  uint64_t hca_va, int access);
-extern int iwch_dereg_mr(struct verbs_mr *mr);
-
-struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
-			      struct ibv_comp_channel *channel,
-			      int comp_vector);
-extern int iwch_resize_cq(struct ibv_cq *cq, int cqe);
-extern int iwch_destroy_cq(struct ibv_cq *cq);
-extern int t3a_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
-extern int t3b_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
-extern int iwch_arm_cq(struct ibv_cq *cq, int solicited);
-extern void iwch_cq_event(struct ibv_cq *cq);
-extern void iwch_init_cq_buf(struct iwch_cq *cq, int nent);
-
-extern struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
-				       struct ibv_srq_init_attr *attr);
-extern int iwch_modify_srq(struct ibv_srq *srq,
-			   struct ibv_srq_attr *attr,
-			   int mask);
-extern int iwch_destroy_srq(struct ibv_srq *srq);
-extern int iwch_post_srq_recv(struct ibv_srq *ibsrq,
-			      struct ibv_recv_wr *wr,
-			      struct ibv_recv_wr **bad_wr);
-
-extern struct ibv_qp *iwch_create_qp(struct ibv_pd *pd,
-				     struct ibv_qp_init_attr *attr);
-extern int iwch_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-			  int attr_mask);
-extern int iwch_destroy_qp(struct ibv_qp *qp);
-extern int iwch_query_qp(struct ibv_qp *qp,
-			 struct ibv_qp_attr *attr,
-			 int attr_mask,
-			 struct ibv_qp_init_attr *init_attr);
-extern void iwch_flush_qp(struct iwch_qp *qhp);
-extern void iwch_flush_qps(struct iwch_device *dev);
-extern int t3a_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
-			  struct ibv_send_wr **bad_wr);
-extern int t3b_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
-			  struct ibv_send_wr **bad_wr);
-extern int t3a_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
-			  struct ibv_recv_wr **bad_wr);
-extern int t3b_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
-			  struct ibv_recv_wr **bad_wr);
-extern struct ibv_ah *iwch_create_ah(struct ibv_pd *pd,
-			     struct ibv_ah_attr *ah_attr);
-extern int iwch_destroy_ah(struct ibv_ah *ah);
-extern int iwch_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
-			     uint16_t lid);
-extern int iwch_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid,
-			     uint16_t lid);
-extern void t3b_async_event(struct ibv_context *context,
-			    struct ibv_async_event *event);
-#ifdef DEBUG
-#include <syslog.h>
-#define DBGLOG(s) openlog(s, LOG_NDELAY|LOG_PID, LOG_LOCAL7)
-#define PDBG(fmt, args...) do {syslog(LOG_DEBUG, fmt, ##args);} while (0)
-#else
-#define DBGLOG(s) 
-#define PDBG(fmt, args...) do {} while (0)
-#endif
-
-#define FW_MAJ 5
-#define FW_MIN 0
-
-#endif				/* IWCH_H */
diff --git a/providers/cxgb3/qp.c b/providers/cxgb3/qp.c
deleted file mode 100644
index 4a1e7397cc96..000000000000
--- a/providers/cxgb3/qp.c
+++ /dev/null
@@ -1,560 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <config.h>
-
-#include <stdlib.h>
-#include <pthread.h>
-#include <string.h>
-
-#include "iwch.h"
-#include <stdio.h>
-
-#define ROUNDUP8(a) (((a) + 7) & ~7)
-
-static inline int iwch_build_rdma_send(union t3_wr *wqe, struct ibv_send_wr *wr,
-				       uint8_t *flit_cnt)
-{
-	int i;
-
-	if (wr->num_sge > T3_MAX_SGE)
-		return -1;
-	if (wr->send_flags & IBV_SEND_SOLICITED)
-		wqe->send.rdmaop = T3_SEND_WITH_SE;
-	else
-		wqe->send.rdmaop = T3_SEND;
-	wqe->send.rem_stag = 0;
-	wqe->send.reserved = 0;
-	if ((wr->send_flags & IBV_SEND_INLINE) || wr->num_sge == 0) {
-		uint8_t *datap;
-
-		wqe->send.plen = 0;
-		datap = (uint8_t *)&wqe->send.sgl[0];
-		wqe->send.num_sgle = 0;	/* indicates in-line data */
-		for (i = 0; i < wr->num_sge; i++) {
-			if ((wqe->send.plen + wr->sg_list[i].length) > 
-			    T3_MAX_INLINE)
-				return -1;
-			wqe->send.plen += wr->sg_list[i].length;
-			memcpy(datap, 
-			       (void *)(unsigned long)wr->sg_list[i].addr, 
-			       wr->sg_list[i].length);
-			datap += wr->sg_list[i].length;
-		}
-		*flit_cnt = 4 + (ROUNDUP8(wqe->send.plen) >> 3);
-		wqe->send.plen = htobe32(wqe->send.plen);
-	} else {
-		wqe->send.plen = 0;
-		for (i = 0; i < wr->num_sge; i++) {
-			if ((wqe->send.plen + wr->sg_list[i].length) < 
-			    wqe->send.plen) {
-				return -1;
-			}
-			wqe->send.plen += wr->sg_list[i].length;
-			wqe->send.sgl[i].stag =
-			    htobe32(wr->sg_list[i].lkey);
-			wqe->send.sgl[i].len =
-			    htobe32(wr->sg_list[i].length);
-			wqe->send.sgl[i].to = htobe64(wr->sg_list[i].addr);
-		}
-		wqe->send.plen = htobe32(wqe->send.plen);
-		wqe->send.num_sgle = htobe32(wr->num_sge);
-		*flit_cnt = 4 + ((wr->num_sge) << 1);
-	}
-	return 0;
-}
-
-static inline int iwch_build_rdma_write(union t3_wr *wqe, 
-					struct ibv_send_wr *wr,
-					uint8_t *flit_cnt)
-{
-	int i;
-
-	if (wr->num_sge > T3_MAX_SGE)
-		return -1;
-	wqe->write.rdmaop = T3_RDMA_WRITE;
-	wqe->write.reserved = 0;
-	wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey);
-	wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr);
-
-	wqe->write.num_sgle = wr->num_sge;
-
-	if ((wr->send_flags & IBV_SEND_INLINE) || wr->num_sge == 0) {
-		uint8_t *datap;
-
-		wqe->write.plen = 0;
-		datap = (uint8_t *)&wqe->write.sgl[0];
-		wqe->write.num_sgle = 0;	/* indicates in-line data */
-		for (i = 0; i < wr->num_sge; i++) {
-			if ((wqe->write.plen + wr->sg_list[i].length) >
-			    T3_MAX_INLINE)
-				return -1;
-			wqe->write.plen += wr->sg_list[i].length;
-			memcpy(datap, 
-			       (void *)(unsigned long)wr->sg_list[i].addr, 
-			       wr->sg_list[i].length);
-			datap += wr->sg_list[i].length;
-		}
-		*flit_cnt = 5 + (ROUNDUP8(wqe->write.plen) >> 3);
-		wqe->write.plen = htobe32(wqe->write.plen);
-	} else {
-		wqe->write.plen = 0;
-		for (i = 0; i < wr->num_sge; i++) {
-			if ((wqe->write.plen + wr->sg_list[i].length) < 
-			    wqe->write.plen) {
-				return -1;
-			}
-			wqe->write.plen += wr->sg_list[i].length;
-			wqe->write.sgl[i].stag =
-			    htobe32(wr->sg_list[i].lkey);
-			wqe->write.sgl[i].len =
-			    htobe32(wr->sg_list[i].length);
-			wqe->write.sgl[i].to =
-			    htobe64(wr->sg_list[i].addr);
-		}
-		wqe->write.plen = htobe32(wqe->write.plen);
-		wqe->write.num_sgle = htobe32(wr->num_sge);
-		*flit_cnt = 5 + ((wr->num_sge) << 1);
-	}
-	return 0;
-}
-
-static inline int iwch_build_rdma_read(union t3_wr *wqe, struct ibv_send_wr *wr,
-				       uint8_t *flit_cnt)
-{
-	if (wr->num_sge > 1)
-		return -1;
-	wqe->read.rdmaop = T3_READ_REQ;
-	wqe->read.reserved = 0;
-	if (wr->num_sge == 1 && wr->sg_list[0].length > 0) {
-		wqe->read.rem_stag = htobe32(wr->wr.rdma.rkey);
-		wqe->read.rem_to = htobe64(wr->wr.rdma.remote_addr);
-		wqe->read.local_stag = htobe32(wr->sg_list[0].lkey);
-		wqe->read.local_len = htobe32(wr->sg_list[0].length);
-		wqe->read.local_to = htobe64(wr->sg_list[0].addr);
-	} else {
-
-		/* build passable 0B read request */
-		wqe->read.rem_stag = 2;
-		wqe->read.rem_to = 2;
-		wqe->read.local_stag = 2;
-		wqe->read.local_len = 0;
-		wqe->read.local_to = 2;
-	}
-	*flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3;
-	return 0;
-}
-
-int t3b_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 
-		  struct ibv_send_wr **bad_wr)
-{
-	int err = 0;
-	uint8_t t3_wr_flit_cnt;
-	enum t3_wr_opcode t3_wr_opcode = 0;
-	enum t3_wr_flags t3_wr_flags;
-	struct iwch_qp *qhp;
-	uint32_t idx;
-	union t3_wr *wqe;
-	uint32_t num_wrs;
-	struct t3_swsq *sqp;
-
-	qhp = to_iwch_qp(ibqp);
-	pthread_spin_lock(&qhp->lock);
-	if (t3_wq_in_error(&qhp->wq)) {
-		iwch_flush_qp(qhp);
-		pthread_spin_unlock(&qhp->lock);
-		return -1;
-	}
-	num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 
-		  qhp->wq.sq_size_log2);
-	if (num_wrs <= 0) {
-		pthread_spin_unlock(&qhp->lock);
-		return -1;
-	}
-	while (wr) {
-		if (num_wrs == 0) {
-			err = -1;
-			*bad_wr = wr;
-			break;
-		}
-		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
-		wqe = (union t3_wr *) (qhp->wq.queue + idx);
-		t3_wr_flags = 0;
-		if (wr->send_flags & IBV_SEND_SOLICITED)
-			t3_wr_flags |= T3_SOLICITED_EVENT_FLAG;
-		if (wr->send_flags & IBV_SEND_FENCE)
-			t3_wr_flags |= T3_READ_FENCE_FLAG;
-		if ((wr->send_flags & IBV_SEND_SIGNALED) || qhp->sq_sig_all)
-			t3_wr_flags |= T3_COMPLETION_FLAG;
-		sqp = qhp->wq.sq + 
-		      Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2);
-		switch (wr->opcode) {
-		case IBV_WR_SEND:
-			t3_wr_opcode = T3_WR_SEND;
-			err = iwch_build_rdma_send(wqe, wr, &t3_wr_flit_cnt);
-			break;
-		case IBV_WR_RDMA_WRITE:
-			t3_wr_opcode = T3_WR_WRITE;
-			err = iwch_build_rdma_write(wqe, wr, &t3_wr_flit_cnt);
-			break;
-		case IBV_WR_RDMA_READ:
-			t3_wr_opcode = T3_WR_READ;
-			t3_wr_flags = 0;
-			err = iwch_build_rdma_read(wqe, wr, &t3_wr_flit_cnt);
-			if (err)
-				break;
-			sqp->read_len = wqe->read.local_len;
-			if (!qhp->wq.oldest_read)
-				qhp->wq.oldest_read = sqp;
-			break;
-		default:
-			PDBG("%s post of type=%d TBD!\n", __FUNCTION__, 
-			     wr->opcode);
-			err = -1;
-		}
-		if (err) {
-			*bad_wr = wr;
-			break;
-		}
-		wqe->send.wrid.id0.hi = qhp->wq.sq_wptr;
-		sqp->wr_id = wr->wr_id;
-		sqp->opcode = wr2opcode(t3_wr_opcode);
-		sqp->sq_wptr = qhp->wq.sq_wptr;
-		sqp->complete = 0;
-		sqp->signaled = (wr->send_flags & IBV_SEND_SIGNALED);
-
-		build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags,
-			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
-			       0, t3_wr_flit_cnt);
-		PDBG("%s cookie 0x%" PRIx64 
-		     " wq idx 0x%x swsq idx %ld opcode %d\n", 
-		     __FUNCTION__, wr->wr_id, idx, 
-		     Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2),
-		     sqp->opcode);
-		wr = wr->next;
-		num_wrs--;
-		++(qhp->wq.wptr);
-		++(qhp->wq.sq_wptr);
-	}
-	pthread_spin_unlock(&qhp->lock);
-	if (t3_wq_db_enabled(&qhp->wq))
-		RING_DOORBELL(qhp->wq.doorbell, qhp->wq.qpid);
-	return err;
-}
-
-int t3a_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 
-		   struct ibv_send_wr **bad_wr)
-{
-	int ret;
-	struct iwch_qp *qhp = to_iwch_qp(ibqp);
-
-	pthread_spin_lock(&qhp->lock);
-	ret = ibv_cmd_post_send(ibqp, wr, bad_wr);
-	pthread_spin_unlock(&qhp->lock);
-	return ret;
-}
-
-static inline int iwch_build_rdma_recv(struct iwch_device *rhp,
-				       union t3_wr *wqe, 
-				       struct ibv_recv_wr *wr)
-{
-	int i;
-	if (wr->num_sge > T3_MAX_SGE)
-		return -1;
-
-	wqe->recv.num_sgle = htobe32(wr->num_sge);
-	for (i = 0; i < wr->num_sge; i++) {
-		wqe->recv.sgl[i].stag = htobe32(wr->sg_list[i].lkey);
-		wqe->recv.sgl[i].len = htobe32(wr->sg_list[i].length);
-		wqe->recv.sgl[i].to = htobe64(wr->sg_list[i].addr);
-	}
-	for (; i < T3_MAX_SGE; i++) {
-		wqe->recv.sgl[i].stag = 0;
-		wqe->recv.sgl[i].len = 0;
-		wqe->recv.sgl[i].to = 0;
-	}
-	return 0;
-}
-
-static void insert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq)
-{
-	struct t3_cqe cqe;
-
-	PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __FUNCTION__, 
-	     wq, cq, cq->sw_rptr, cq->sw_wptr);
-	memset(&cqe, 0, sizeof(cqe));
-	cqe.header = V_CQE_STATUS(TPT_ERR_SWFLUSH) | 
-		     V_CQE_OPCODE(T3_SEND) | 
-		     V_CQE_TYPE(0) |
-		     V_CQE_SWCQE(1) |
-		     V_CQE_QPID(wq->qpid) | 
-		     V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, cq->size_log2));
-	cqe.header = htobe32(cqe.header);
-	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
-	cq->sw_wptr++;
-}
-
-static void flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count)
-{
-	uint32_t ptr;
-
-	/* flush RQ */
-	PDBG("%s rq_rptr 0x%x rq_wptr 0x%x skip count %u\n", __FUNCTION__, 
-	     wq->rq_rptr, wq->rq_wptr, count);
-	ptr = wq->rq_rptr + count;
-	while (ptr++ != wq->rq_wptr) {
-		insert_recv_cqe(wq, cq);
-	}
-}
-
-static void insert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq, 
-		          struct t3_swsq *sqp)
-{
-	struct t3_cqe cqe;
-
-	PDBG("%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x\n", __FUNCTION__, 
-	     wq, cq, cq->sw_rptr, cq->sw_wptr);
-	memset(&cqe, 0, sizeof(cqe));
-	cqe.header = V_CQE_STATUS(TPT_ERR_SWFLUSH) | 
-		     V_CQE_OPCODE(sqp->opcode) |
-		     V_CQE_TYPE(1) |
-		     V_CQE_SWCQE(1) |
-		     V_CQE_QPID(wq->qpid) | 
-		     V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, cq->size_log2));
-	cqe.header = htobe32(cqe.header);
-	CQE_WRID_SQ_WPTR(cqe) = sqp->sq_wptr;
-
-	*(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe;
-	cq->sw_wptr++;
-}
-
-static void flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count)
-{
-	uint32_t ptr;
-	struct t3_swsq *sqp;
-
-	ptr = wq->sq_rptr + count;
-	sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
-	while (ptr != wq->sq_wptr) {
-		insert_sq_cqe(wq, cq, sqp);
-		ptr++;
-		sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2);
-	}
-}
-
-/* 
- * Move all CQEs from the HWCQ into the SWCQ.
- */
-static void flush_hw_cq(struct t3_cq *cq)
-{
-	struct t3_cqe *cqe, *swcqe;
-
-	PDBG("%s cq %p cqid 0x%x\n", __FUNCTION__, cq, cq->cqid);
-	cqe = cxio_next_hw_cqe(cq);
-	while (cqe) {
-		PDBG("%s flushing hwcq rptr 0x%x to swcq wptr 0x%x\n", 
-		     __FUNCTION__, cq->rptr, cq->sw_wptr);
-		swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2);
-		*swcqe = *cqe;
-		swcqe->header |= htobe32(V_CQE_SWCQE(1));
-		cq->sw_wptr++;
-		cq->rptr++;
-		cqe = cxio_next_hw_cqe(cq);
-	}
-}
-
-static void count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
-{
-	struct t3_cqe *cqe;
-	uint32_t ptr;
-
-	*count = 0;
-	ptr = cq->sw_rptr;
-	while (!Q_EMPTY(ptr, cq->sw_wptr)) {
-		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
-		if ((SQ_TYPE(*cqe) || 
-		     (CQE_OPCODE(*cqe) == T3_READ_RESP && CQE_WRID_STAG(*cqe) != 1)) &&
-		    (CQE_QPID(*cqe) == wq->qpid))
-			(*count)++;
-		ptr++;
-	}	
-	PDBG("%s cq %p count %d\n", __FUNCTION__, cq, *count);
-}
-
-static void count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count)
-{
-	struct t3_cqe *cqe;
-	uint32_t ptr;
-
-	*count = 0;
-	ptr = cq->sw_rptr;
-	while (!Q_EMPTY(ptr, cq->sw_wptr)) {
-		cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2));
-		if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) && 
-		    (CQE_QPID(*cqe) == wq->qpid))
-			(*count)++;
-		ptr++;
-	}	
-	PDBG("%s cq %p count %d\n", __FUNCTION__, cq, *count);
-}
-
-/*
- * Assumes qhp lock is held.
- */
-void iwch_flush_qp(struct iwch_qp *qhp)
-{
-	struct iwch_cq *rchp, *schp;
-	int count;
-
-	if (qhp->wq.flushed)
-		return;
-
-	rchp = qhp->rhp->cqid2ptr[to_iwch_cq(qhp->ibv_qp.recv_cq)->cq.cqid];
-	schp = qhp->rhp->cqid2ptr[to_iwch_cq(qhp->ibv_qp.send_cq)->cq.cqid];
-	
-	PDBG("%s qhp %p rchp %p schp %p\n", __FUNCTION__, qhp, rchp, schp);
-	qhp->wq.flushed = 1;
-
-#ifdef notyet
-	/* take a ref on the qhp since we must release the lock */
-	atomic_inc(&qhp->refcnt);
-#endif
-	pthread_spin_unlock(&qhp->lock);
-
-	/* locking heirarchy: cq lock first, then qp lock. */
-	pthread_spin_lock(&rchp->lock);
-	pthread_spin_lock(&qhp->lock);
-	flush_hw_cq(&rchp->cq);
-	count_rcqes(&rchp->cq, &qhp->wq, &count);
-	flush_rq(&qhp->wq, &rchp->cq, count);
-	pthread_spin_unlock(&qhp->lock);
-	pthread_spin_unlock(&rchp->lock);
-
-	/* locking heirarchy: cq lock first, then qp lock. */
-	pthread_spin_lock(&schp->lock);
-	pthread_spin_lock(&qhp->lock);
-	flush_hw_cq(&schp->cq);
-	count_scqes(&schp->cq, &qhp->wq, &count);
-	flush_sq(&qhp->wq, &schp->cq, count);
-	pthread_spin_unlock(&qhp->lock);
-	pthread_spin_unlock(&schp->lock);
-
-#ifdef notyet
-	/* deref */
-	if (atomic_dec_and_test(&qhp->refcnt))
-                wake_up(&qhp->wait);
-#endif
-	pthread_spin_lock(&qhp->lock);
-}
-
-void iwch_flush_qps(struct iwch_device *dev)
-{
-	int i;
-
-	pthread_spin_lock(&dev->lock);
-	for (i=0; i < T3_MAX_NUM_QP; i++) {
-		struct iwch_qp *qhp = dev->qpid2ptr[i];
-		if (qhp) {
-			if (!qhp->wq.flushed && t3_wq_in_error(&qhp->wq)) {
-				pthread_spin_lock(&qhp->lock);
-				iwch_flush_qp(qhp);
-				pthread_spin_unlock(&qhp->lock);
-			}
-		}
-	}
-	pthread_spin_unlock(&dev->lock);
-
-}
-
-int t3b_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
-		   struct ibv_recv_wr **bad_wr)
-{
-	int err = 0;
-	struct iwch_qp *qhp;
-	uint32_t idx;
-	union t3_wr *wqe;
-	uint32_t num_wrs;
-
-	qhp = to_iwch_qp(ibqp);
-	pthread_spin_lock(&qhp->lock);
-	if (t3_wq_in_error(&qhp->wq)) {
-		iwch_flush_qp(qhp);
-		pthread_spin_unlock(&qhp->lock);
-		return -1;
-	}
-	num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 
-			    qhp->wq.rq_size_log2) - 1;
-	if (!wr) {
-		pthread_spin_unlock(&qhp->lock);
-		return -1;
-	}
-	while (wr) {
-		idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2);
-		wqe = (union t3_wr *) (qhp->wq.queue + idx);
-		if (num_wrs)
-			err = iwch_build_rdma_recv(qhp->rhp, wqe, wr);
-		else
-			err = -1;
-		if (err) {
-			*bad_wr = wr;
-			break;
-		}
-		qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, qhp->wq.rq_size_log2)] = 
-			wr->wr_id;
-		build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG,
-			       Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2),
-			       0, sizeof(struct t3_receive_wr) >> 3);
-		PDBG("%s cookie 0x%" PRIx64 
-		     " idx 0x%x rq_wptr 0x%x rw_rptr 0x%x "
-		     "wqe %p \n", __FUNCTION__, wr->wr_id, idx, 
-		     qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe);
-		++(qhp->wq.rq_wptr);
-		++(qhp->wq.wptr);
-		wr = wr->next;
-		num_wrs--;
-	}
-	pthread_spin_unlock(&qhp->lock);
-	if (t3_wq_db_enabled(&qhp->wq))
-		RING_DOORBELL(qhp->wq.doorbell, qhp->wq.qpid);
-	return err;
-}
-
-int t3a_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
-		   struct ibv_recv_wr **bad_wr)
-{
-	int ret;
-	struct iwch_qp *qhp = to_iwch_qp(ibqp);
-
-	pthread_spin_lock(&qhp->lock);
-	ret = ibv_cmd_post_recv(ibqp, wr, bad_wr);
-	pthread_spin_unlock(&qhp->lock);
-	return ret;
-}
diff --git a/providers/cxgb3/verbs.c b/providers/cxgb3/verbs.c
deleted file mode 100644
index 39a44192e29c..000000000000
--- a/providers/cxgb3/verbs.c
+++ /dev/null
@@ -1,476 +0,0 @@
-/*
- * Copyright (c) 2006-2007 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include <config.h>
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <errno.h>
-#include <pthread.h>
-#include <sys/mman.h>
-#include <inttypes.h>
-
-#include "iwch.h"
-#include "iwch-abi.h"
-
-int iwch_query_device(struct ibv_context *context, struct ibv_device_attr *attr)
-{
-	struct ibv_query_device cmd;
-	uint64_t raw_fw_ver;
-	unsigned major, minor, sub_minor;
-	int ret;
-
-	ret = ibv_cmd_query_device(context, attr, &raw_fw_ver, &cmd, 
-	  			   sizeof cmd);
-	if (ret)
-		return ret;
-
-	major = (raw_fw_ver >> 32) & 0xffff;
-	minor = (raw_fw_ver >> 16) & 0xffff;
-	sub_minor = raw_fw_ver & 0xffff;
-
-	snprintf(attr->fw_ver, sizeof attr->fw_ver,
-		 "%d.%d.%d", major, minor, sub_minor);
-
-	return 0;
-}
-
-int iwch_query_port(struct ibv_context *context, uint8_t port,
-		    struct ibv_port_attr *attr)
-{
-	struct ibv_query_port cmd;
-
-	return ibv_cmd_query_port(context, port, attr, &cmd, sizeof cmd);
-}
-
-struct ibv_pd *iwch_alloc_pd(struct ibv_context *context)
-{
-	struct ibv_alloc_pd cmd;
-	struct uiwch_alloc_pd_resp resp;
-	struct iwch_pd *pd;
-
-	pd = malloc(sizeof *pd);
-	if (!pd)
-		return NULL;
-
-	if (ibv_cmd_alloc_pd(context, &pd->ibv_pd, &cmd, sizeof cmd,
-			     &resp.ibv_resp, sizeof resp)) {
-		free(pd);
-		return NULL;
-	}
-
-	return &pd->ibv_pd;
-}
-
-int iwch_free_pd(struct ibv_pd *pd)
-{
-	int ret;
-
-	ret = ibv_cmd_dealloc_pd(pd);
-	if (ret)
-		return ret;
-
-	free(pd);
-	return 0;
-}
-
-struct ibv_mr *iwch_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
-			   uint64_t hca_va, int access)
-{
-	struct iwch_mr *mhp;
-	struct ibv_reg_mr cmd;
-	struct uiwch_reg_mr_resp resp;
-	struct iwch_device *dev = to_iwch_dev(pd->context->device);
-
-	PDBG("%s addr %p length %ld hca_va %p\n", __func__, addr, length,
-	     hca_va);
-
-	mhp = malloc(sizeof *mhp);
-	if (!mhp)
-		return NULL;
-
-	if (ibv_cmd_reg_mr(pd, addr, length, hca_va,
-			   access, &mhp->vmr, &cmd, sizeof(cmd),
-			   &resp.ibv_resp, sizeof resp)) {
-		free(mhp);
-		return NULL;
-	}
-
-	mhp->va_fbo = hca_va;
-	mhp->page_size = iwch_page_shift - 12;
-	mhp->pbl_addr = resp.pbl_addr;
-	mhp->len = length;
-
-	PDBG("%s stag 0x%x va_fbo 0x%" PRIx64
-             " page_size %d pbl_addr 0x%x len %d\n",
-	     __func__, mhp->vmr.ibv_mr.rkey, mhp->va_fbo,
-	     mhp->page_size, mhp->pbl_addr, mhp->len);
-
-	pthread_spin_lock(&dev->lock);
-	dev->mmid2ptr[t3_mmid(mhp->vmr.ibv_mr.lkey)] = mhp;
-	pthread_spin_unlock(&dev->lock);
-	
-	return &mhp->vmr.ibv_mr;
-}
-
-int iwch_dereg_mr(struct verbs_mr *vmr)
-{
-	int ret;
-	struct iwch_device *dev = to_iwch_dev(vmr->ibv_mr.pd->context->device);
-
-	ret = ibv_cmd_dereg_mr(vmr);
-	if (ret)
-		return ret;
-
-	pthread_spin_lock(&dev->lock);
-	dev->mmid2ptr[t3_mmid(vmr->ibv_mr.lkey)] = NULL;
-	pthread_spin_unlock(&dev->lock);
-
-	free(to_iwch_mr(vmr));
-	
-	return 0;
-}
-
-struct ibv_cq *iwch_create_cq(struct ibv_context *context, int cqe,
-			      struct ibv_comp_channel *channel, int comp_vector)
-{
-	struct uiwch_create_cq cmd;
-	struct uiwch_create_cq_resp resp;
-	struct iwch_cq *chp;
-	struct iwch_device *dev = to_iwch_dev(context->device);
-	int ret;
-
-	chp = calloc(1, sizeof *chp);
-	if (!chp) {
-		return NULL;
-	}
-
-	cmd.user_rptr_addr = (uint64_t)(unsigned long)&chp->cq.rptr;
-	ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector,
-				&chp->ibv_cq, &cmd.ibv_cmd, sizeof cmd,
-				&resp.ibv_resp, sizeof resp);
-	if (ret)
-		goto err1;
-
-	pthread_spin_init(&chp->lock, PTHREAD_PROCESS_PRIVATE);
-	chp->rhp = dev;
-	chp->cq.cqid = resp.cqid;
-	chp->cq.size_log2 = resp.size_log2;
-	if (dev->abi_version == 0)
-		chp->cq.memsize = PAGE_ALIGN((1UL << chp->cq.size_log2) *
-					     sizeof(struct t3_cqe));
-	else
-		chp->cq.memsize = resp.memsize;
-	chp->cq.queue = mmap(NULL, t3_cq_memsize(&chp->cq),
-			     PROT_READ|PROT_WRITE, MAP_SHARED, context->cmd_fd,
-			     resp.key);
-	if (chp->cq.queue == MAP_FAILED)
-		goto err2;
-
-	chp->cq.sw_queue = calloc(t3_cq_depth(&chp->cq), sizeof(struct t3_cqe));
-	if (!chp->cq.sw_queue)
-		goto err3;
-
-	PDBG("%s cqid 0x%x physaddr %" PRIx64 " va %p memsize %d\n", 
-	       __FUNCTION__, chp->cq.cqid, resp.physaddr, chp->cq.queue, 
-	       t3_cq_memsize(&chp->cq));
-	
-	pthread_spin_lock(&dev->lock);
-	dev->cqid2ptr[chp->cq.cqid] = chp;
-	pthread_spin_unlock(&dev->lock);
-
-	return &chp->ibv_cq;
-err3:
-	munmap(chp->cq.queue, t3_cq_memsize(&chp->cq));
-err2:
-	(void)ibv_cmd_destroy_cq(&chp->ibv_cq);
-err1:
-	free(chp);
-	return NULL;
-}
-
-int iwch_resize_cq(struct ibv_cq *ibcq, int cqe)
-{
-#ifdef notyet
-	int ret;
-	struct ibv_resize_cq cmd;
-	struct iwch_cq *chp = to_iwch_cq(ibcq);
-
-	pthread_spin_lock(&chp->lock);
-	ret = ibv_cmd_resize_cq(ibcq, cqe, &cmd, sizeof cmd);
-	/* remap and realloc swcq here */
-	pthread_spin_unlock(&chp->lock);
-	return ret;
-#else
-	return -ENOSYS;
-#endif
-}
-
-int iwch_destroy_cq(struct ibv_cq *ibcq)
-{
-	int ret;
-	struct iwch_cq *chp = to_iwch_cq(ibcq);
-	void *cqva = chp->cq.queue;
-	unsigned size = t3_cq_memsize(&chp->cq);
-	struct iwch_device *dev = to_iwch_dev(ibcq->context->device);
-
-	munmap(cqva, size);
-	ret = ibv_cmd_destroy_cq(ibcq);
-	if (ret) {
-		return ret;
-	}
-
-	pthread_spin_lock(&dev->lock);
-	dev->cqid2ptr[chp->cq.cqid] = NULL;
-	pthread_spin_unlock(&dev->lock);
-
-	free(chp->cq.sw_queue);
-	free(chp);
-	return 0;
-}
-
-struct ibv_srq *iwch_create_srq(struct ibv_pd *pd,
-				struct ibv_srq_init_attr *attr)
-{
-	return NULL;
-}
-
-int iwch_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *attr, 
-		    int attr_mask)
-{
-	return -ENOSYS;
-}
-
-int iwch_destroy_srq(struct ibv_srq *srq)
-{
-	return -ENOSYS;
-}
-
-int iwch_post_srq_recv(struct ibv_srq *ibsrq, struct ibv_recv_wr *wr, 
-		       struct ibv_recv_wr **bad_wr)
-{
-	return -ENOSYS;
-}
-
-struct ibv_qp *iwch_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
-{
-	struct uiwch_create_qp cmd;
-	struct uiwch_create_qp_resp resp;
-	struct iwch_qp *qhp;
-	struct iwch_device *dev = to_iwch_dev(pd->context->device);
-	int ret;
-	void *dbva;
-
-	PDBG("%s enter qp\n", __FUNCTION__);
-	qhp = calloc(1, sizeof *qhp);
-	if (!qhp)
-		goto err1;
-
-	ret = ibv_cmd_create_qp(pd, &qhp->ibv_qp, attr, &cmd.ibv_cmd, 
-				sizeof cmd, &resp.ibv_resp, sizeof resp);
-	if (ret)
-		goto err2;
-
-	PDBG("%s qpid 0x%x physaddr %" PRIx64 " doorbell %" PRIx64 
-	       " size %d sq_size %d rq_size %d\n",
-		__FUNCTION__, resp.qpid, resp.physaddr, resp.doorbell,
-		1 << resp.size_log2, 1 << resp.sq_size_log2, 
-		1 << resp.rq_size_log2);
-
-	qhp->rhp = dev;
-	qhp->wq.qpid = resp.qpid;
-	qhp->wq.size_log2 = resp.size_log2;
-	qhp->wq.sq_size_log2 = resp.sq_size_log2;
-	qhp->wq.rq_size_log2 = resp.rq_size_log2;
-	pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE);
-	dbva = mmap(NULL, iwch_page_size, PROT_WRITE, MAP_SHARED, 
-		    pd->context->cmd_fd, resp.db_key & ~(iwch_page_mask));
-	if (dbva == MAP_FAILED)
-		goto err3;
-
-	qhp->wq.doorbell = dbva + (resp.db_key & (iwch_page_mask));
-	qhp->wq.queue = mmap(NULL, t3_wq_memsize(&qhp->wq),
-			    PROT_READ|PROT_WRITE, MAP_SHARED, 
-			    pd->context->cmd_fd, resp.key);
-	if (qhp->wq.queue == MAP_FAILED)
-		goto err4;
-
-	qhp->wq.rq = calloc(t3_rq_depth(&qhp->wq), sizeof (uint64_t));
-	if (!qhp->wq.rq) 
-		goto err5;
-
-	qhp->wq.sq = calloc(t3_sq_depth(&qhp->wq), sizeof (struct t3_swsq));
-	if (!qhp->wq.sq) 
-		goto err6;
-
-	PDBG("%s dbva %p wqva %p wq memsize %d\n", __FUNCTION__, 
-	     qhp->wq.doorbell, qhp->wq.queue, t3_wq_memsize(&qhp->wq));
-
-	qhp->sq_sig_all = attr->sq_sig_all;
-
-	pthread_spin_lock(&dev->lock);
-	dev->qpid2ptr[qhp->wq.qpid] = qhp;
-	pthread_spin_unlock(&dev->lock);
-
-	return &qhp->ibv_qp;
-err6:
-	free(qhp->wq.rq);
-err5:
-	munmap((void *)qhp->wq.queue, t3_wq_memsize(&qhp->wq));
-err4:
-	munmap((void *)dbva, iwch_page_size);
-err3:
-	(void)ibv_cmd_destroy_qp(&qhp->ibv_qp);
-err2:
-	free(qhp);
-err1:
-	return NULL;
-}
-
-static void reset_qp(struct iwch_qp *qhp)
-{
-	PDBG("%s enter qp %p\n", __FUNCTION__, qhp);
-	qhp->wq.wptr = 0;
-	qhp->wq.rq_wptr = qhp->wq.rq_rptr = 0;
-	qhp->wq.sq_wptr = qhp->wq.sq_rptr = 0;
-	qhp->wq.error = 0;
-	qhp->wq.oldest_read = NULL;
-	memset(qhp->wq.queue, 0, t3_wq_memsize(&qhp->wq));
-}
-
-int iwch_modify_qp(struct ibv_qp *ibqp, struct ibv_qp_attr *attr,
-		   int attr_mask)
-{
-	struct ibv_modify_qp cmd = {};
-	struct iwch_qp *qhp = to_iwch_qp(ibqp);
-	int ret;
-
-	PDBG("%s enter qp %p new state %d\n", __FUNCTION__, ibqp, attr_mask & IBV_QP_STATE ? attr->qp_state : -1);
-	pthread_spin_lock(&qhp->lock);
-	if (t3b_device(qhp->rhp) && t3_wq_in_error(&qhp->wq))
-		iwch_flush_qp(qhp);
-	ret = ibv_cmd_modify_qp(ibqp, attr, attr_mask, &cmd, sizeof cmd);
-	if (!ret && (attr_mask & IBV_QP_STATE) && attr->qp_state == IBV_QPS_RESET)
-		reset_qp(qhp);
-	pthread_spin_unlock(&qhp->lock);
-	return ret;
-}
-
-int iwch_destroy_qp(struct ibv_qp *ibqp)
-{
-	int ret;
-	struct iwch_qp *qhp = to_iwch_qp(ibqp);
-	struct iwch_device *dev = to_iwch_dev(ibqp->context->device);
-	void *dbva, *wqva;
-	unsigned wqsize;
-
-	PDBG("%s enter qp %p\n", __FUNCTION__, ibqp);
-	if (t3b_device(dev)) {
-		pthread_spin_lock(&qhp->lock);
-		iwch_flush_qp(qhp);
-		pthread_spin_unlock(&qhp->lock);
-	}
-
-	dbva = (void *)((unsigned long)qhp->wq.doorbell & ~(iwch_page_mask));
-	wqva = qhp->wq.queue;
-	wqsize = t3_wq_memsize(&qhp->wq);
-
-	munmap(dbva, iwch_page_size);
-	munmap(wqva, wqsize);
-	ret = ibv_cmd_destroy_qp(ibqp);
-	if (ret) {
-		return ret;
-	}
-
-	pthread_spin_lock(&dev->lock);
-	dev->qpid2ptr[qhp->wq.qpid] = NULL;
-	pthread_spin_unlock(&dev->lock);
-
-	free(qhp->wq.rq);
-	free(qhp->wq.sq);
-	free(qhp);
-	return 0;
-}
-
-int iwch_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
-		  int attr_mask, struct ibv_qp_init_attr *init_attr)
-{
-	return -ENOSYS;
-}
-
-struct ibv_ah *iwch_create_ah(struct ibv_pd *pd, struct ibv_ah_attr *attr)
-{
-	return NULL;
-}
-
-int iwch_destroy_ah(struct ibv_ah *ah)
-{
-	return -ENOSYS;
-}
-
-int iwch_attach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
-{
-	return -ENOSYS;
-}
-
-int iwch_detach_mcast(struct ibv_qp *qp, const union ibv_gid *gid, uint16_t lid)
-{
-	return -ENOSYS;
-}
-
-void t3b_async_event(struct ibv_context *context,
-		     struct ibv_async_event *event)
-{
-	PDBG("%s type %d obj %p\n", __FUNCTION__, event->event_type, 
-	     event->element.cq);
-
-	switch (event->event_type) {
-	case IBV_EVENT_CQ_ERR:
-		break;
-	case IBV_EVENT_QP_FATAL:
-	case IBV_EVENT_QP_REQ_ERR:
-	case IBV_EVENT_QP_ACCESS_ERR:
-	case IBV_EVENT_PATH_MIG_ERR: {
-		struct iwch_qp *qhp = to_iwch_qp(event->element.qp);
-		pthread_spin_lock(&qhp->lock);
-		iwch_flush_qp(qhp);
-		pthread_spin_unlock(&qhp->lock);
-		break;
-	}
-	case IBV_EVENT_SQ_DRAINED:
-	case IBV_EVENT_PATH_MIG:
-	case IBV_EVENT_COMM_EST:
-	case IBV_EVENT_QP_LAST_WQE_REACHED:
-	default:
-		break;
-	}
-}
diff --git a/redhat/rdma-core.spec b/redhat/rdma-core.spec
index 731a4e10ff7b..f796246ce192 100644
--- a/redhat/rdma-core.spec
+++ b/redhat/rdma-core.spec
@@ -128,8 +128,6 @@ Summary: A library and drivers for direct userspace use of RDMA (InfiniBand/iWAR
 Requires(post): /sbin/ldconfig
 Requires(postun): /sbin/ldconfig
 Requires: %{name}%{?_isa} = %{version}-%{release}
-Provides: libcxgb3 = %{version}-%{release}
-Obsoletes: libcxgb3 < %{version}-%{release}
 Provides: libcxgb4 = %{version}-%{release}
 Obsoletes: libcxgb4 < %{version}-%{release}
 Provides: libefa = %{version}-%{release}
@@ -162,7 +160,6 @@ fast path operations.
 
 Device-specific plug-in ibverbs userspace drivers are included:
 
-- libcxgb3: Chelsio T3 iWARP HCA
 - libcxgb4: Chelsio T4 iWARP HCA
 - libefa: Amazon Elastic Fabric Adapter
 - libhfi1: Intel Omni-Path HFI
diff --git a/redhat/rdma.kernel-init b/redhat/rdma.kernel-init
index 6f50e72fcc3b..c7444a1c8d77 100644
--- a/redhat/rdma.kernel-init
+++ b/redhat/rdma.kernel-init
@@ -125,10 +125,6 @@ load_hardware_modules()
 	load_modules mlx5_ib
 	RC+=$?
     fi
-    if is_loaded cxgb3 -a ! is_loaded iw_cxgb3; then
-	load_modules iw_cxgb3
-	RC+=$?
-    fi
     if is_loaded cxgb4 -a ! is_loaded iw_cxgb4; then
 	load_modules iw_cxgb4
 	RC+=$?
diff --git a/suse/rdma-core.spec b/suse/rdma-core.spec
index 3221657d0682..c6562f1d2ca3 100644
--- a/suse/rdma-core.spec
+++ b/suse/rdma-core.spec
@@ -178,7 +178,6 @@ RDMA core development libraries and headers.
 Summary:        Library & drivers for direct userspace use of InfiniBand/iWARP/RoCE hardware
 Group:          System/Libraries
 Requires:       %{name}%{?_isa} = %{version}-%{release}
-Obsoletes:      libcxgb3-rdmav2 < %{version}-%{release}
 Obsoletes:      libcxgb4-rdmav2 < %{version}-%{release}
 Obsoletes:      libefa-rdmav2 < %{version}-%{release}
 Obsoletes:      libhfi1verbs-rdmav2 < %{version}-%{release}
@@ -208,7 +207,6 @@ fast path operations.
 
 Device-specific plug-in ibverbs userspace drivers are included:
 
-- libcxgb3: Chelsio T3 iWARP HCA
 - libcxgb4: Chelsio T4 iWARP HCA
 - libefa: Amazon Elastic Fabric Adapter
 - libhfi1: Intel Omni-Path HFI
-- 
2.18.0.232.gb7bd9486b055

>From 33cad2f67d735cf49b02f00a38b5c69f600979af Mon Sep 17 00:00:00 2001
From: Potnuri Bharat Teja <bharat@xxxxxxxxxxx>
Date: Mon, 21 Oct 2019 14:55:45 +0530
Subject: [PATCH rdma-core 2/2] Update kernel headers

To commit ?? ("RDMA/uapi: Fix and re-organize the usage of
rdma_driver_id")

Signed-off-by: Potnuri Bharat Teja <bharat@xxxxxxxxxxx>
---
 kernel-headers/rdma/ib_user_ioctl_verbs.h  | 22 ++++++++++++++++++++++
 kernel-headers/rdma/rdma_user_ioctl_cmds.h | 22 ----------------------
 2 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h
index 72c7fc75f960..9019b2d906ea 100644
--- a/kernel-headers/rdma/ib_user_ioctl_verbs.h
+++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h
@@ -173,4 +173,26 @@ struct ib_uverbs_query_port_resp_ex {
 	__u8  reserved[6];
 };
 
+enum rdma_driver_id {
+	RDMA_DRIVER_UNKNOWN,
+	RDMA_DRIVER_MLX5,
+	RDMA_DRIVER_MLX4,
+	RDMA_DRIVER_CXGB3,
+	RDMA_DRIVER_CXGB4,
+	RDMA_DRIVER_MTHCA,
+	RDMA_DRIVER_BNXT_RE,
+	RDMA_DRIVER_OCRDMA,
+	RDMA_DRIVER_NES,
+	RDMA_DRIVER_I40IW,
+	RDMA_DRIVER_VMW_PVRDMA,
+	RDMA_DRIVER_QEDR,
+	RDMA_DRIVER_HNS,
+	RDMA_DRIVER_USNIC,
+	RDMA_DRIVER_RXE,
+	RDMA_DRIVER_HFI1,
+	RDMA_DRIVER_QIB,
+	RDMA_DRIVER_EFA,
+	RDMA_DRIVER_SIW,
+};
+
 #endif
diff --git a/kernel-headers/rdma/rdma_user_ioctl_cmds.h b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
index b8bb285f6b2a..7b1ec806f8f9 100644
--- a/kernel-headers/rdma/rdma_user_ioctl_cmds.h
+++ b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
@@ -84,26 +84,4 @@ struct ib_uverbs_ioctl_hdr {
 	struct ib_uverbs_attr  attrs[0];
 };
 
-enum rdma_driver_id {
-	RDMA_DRIVER_UNKNOWN,
-	RDMA_DRIVER_MLX5,
-	RDMA_DRIVER_MLX4,
-	RDMA_DRIVER_CXGB3,
-	RDMA_DRIVER_CXGB4,
-	RDMA_DRIVER_MTHCA,
-	RDMA_DRIVER_BNXT_RE,
-	RDMA_DRIVER_OCRDMA,
-	RDMA_DRIVER_NES,
-	RDMA_DRIVER_I40IW,
-	RDMA_DRIVER_VMW_PVRDMA,
-	RDMA_DRIVER_QEDR,
-	RDMA_DRIVER_HNS,
-	RDMA_DRIVER_USNIC,
-	RDMA_DRIVER_RXE,
-	RDMA_DRIVER_HFI1,
-	RDMA_DRIVER_QIB,
-	RDMA_DRIVER_EFA,
-	RDMA_DRIVER_SIW,
-};
-
 #endif
-- 
2.18.0.232.gb7bd9486b055


[Index of Archives]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Photo]     [Yosemite News]     [Yosemite Photos]     [Linux Kernel]     [Linux SCSI]     [XFree86]

  Powered by Linux