From: Muneendra <muneendra.kumar@xxxxxxxxxxxx>
This driver introduces Fibre channel as the transport protocol
for providing RDMA support.
It takes inspiration from the Soft RoCE driver originally introduced by
System Fabric Works and later developed by Mellanox.
Eventhough it is inspired from the Soft RoCE driver, the underlying
transport layer is FC-NVMe (short for 'NVMe over fibre channel').
The request, response and completion state machines in the driver
have been heavily modified to adapt to the Exchange based Data
transfer mechanism of Fibre channel.
Some notes on the architecture and design:
rdma_rfc, implements the RDMA transport and registers with the ib_core as a
kernel verbs provider. It also implements the IO layer.
rdma_rfc registers with ib_core as RoCEv2 provider and sends the RDMA work
requests by encasulating each request in a RFC object which is sent over an
FC-NVMe exchange. This function enables user to send RDMA work requests over
Fibre channel while using IPv4 addresses as connection identifier.
rdma_rfc also registers as a software NIC with netdev stack to send and
receive ethernet packets over FC-NVMe.
The modules are configured by entries in /sys. There is a configuration script
(rfc_cfg) that simplifies the use of this interface. rfc_cfg is part of the
rfc user space code, librfc.
The use of rfc verbs in user space requires the inclusion of librfc as a device
specific plug-in to libibverbs. librfc is packaged separately.
Architecture:
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+-----------------------------------------------------------+
| Application |
+-----------------------------------------------------------+
+-----------------------------------+
| libibverbs |
User +-----------------------------------+
+--------+ +------+ +---------------+
| librfc | |librxe| | libmlx4 |
+--------+ +------+ +---------------+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+--------------+ +------------+
| Sockets | | CMA |
+--------------+ +------------+
+------------+
| CM |
+------------+
+--------------+ +-------------------------------+
| TCP/IP | | ib_core |
+--------------+ +-------------------------------+
+-------------------+ +-----------+ +----------------+
Kernel | rdma_rfc | | rdma_rxe | | HW RoCE driver |
+-------------------+ +-----------+ +----------------+
+--------------------------+
| FC-NVMe driver |
+--------------------------+
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The driver components and a non asci chart of the module can be found at a
pdf [1]
The design is similar but responder ,completion state machines are getting
executed in the fc-nvme workqueue contexts .
This reduced the number of context switches and locks during the data path.
Currently only IPv4 based sessions are supported.
[1] -https://github.com/brocade/RDMAoverFC/blob/master/RDMA%20over%20FC.pdf
Signed-off-by: Muneendra <muneendra.kumar@xxxxxxxxxxxx>
---
Documentation/infiniband/rfc/README.txt | 147 +++
Documentation/infiniband/rfc/RFC_testplan.txt | 20 +
Documentation/infiniband/rfc/RFC_validation.txt | 105 ++
drivers/infiniband/Kconfig | 1 +
drivers/infiniband/sw/Makefile | 1 +
drivers/infiniband/sw/rfc/Kconfig | 26 +
drivers/infiniband/sw/rfc/Makefile | 24 +
drivers/infiniband/sw/rfc/rfc.c | 382 ++++++
drivers/infiniband/sw/rfc/rfc.h | 115 ++
drivers/infiniband/sw/rfc/rfc_av.c | 94 ++
drivers/infiniband/sw/rfc/rfc_comp.c | 771 ++++++++++++
drivers/infiniband/sw/rfc/rfc_cq.c | 187 +++
drivers/infiniband/sw/rfc/rfc_hdr.h | 985 ++++++++++++++++
drivers/infiniband/sw/rfc/rfc_hw_counters.c | 80 ++
drivers/infiniband/sw/rfc/rfc_hw_counters.h | 63 +
drivers/infiniband/sw/rfc/rfc_loc.h | 289 +++++
drivers/infiniband/sw/rfc/rfc_mcast.c | 192 +++
drivers/infiniband/sw/rfc/rfc_mmap.c | 175 +++
drivers/infiniband/sw/rfc/rfc_mr.c | 850 +++++++++++++
drivers/infiniband/sw/rfc/rfc_net.c | 393 ++++++
drivers/infiniband/sw/rfc/rfc_net.h | 55 +
drivers/infiniband/sw/rfc/rfc_obj.c | 820 +++++++++++++
drivers/infiniband/sw/rfc/rfc_obj.h | 108 ++
drivers/infiniband/sw/rfc/rfc_opcode.c | 964 +++++++++++++++
drivers/infiniband/sw/rfc/rfc_opcode.h | 131 ++
drivers/infiniband/sw/rfc/rfc_param.h | 169 +++
drivers/infiniband/sw/rfc/rfc_pool.c | 501 ++++++++
drivers/infiniband/sw/rfc/rfc_pool.h | 167 +++
drivers/infiniband/sw/rfc/rfc_qp.c | 876 ++++++++++++++
drivers/infiniband/sw/rfc/rfc_queue.c | 214 ++++
drivers/infiniband/sw/rfc/rfc_queue.h | 181 +++
drivers/infiniband/sw/rfc/rfc_recv.c | 262 ++++
drivers/infiniband/sw/rfc/rfc_req.c | 768 ++++++++++++
drivers/infiniband/sw/rfc/rfc_resp.c | 1444 +++++++++++++++++++++++
drivers/infiniband/sw/rfc/rfc_srq.c | 181 +++
drivers/infiniband/sw/rfc/rfc_sysfs.c | 165 +++
drivers/infiniband/sw/rfc/rfc_task.c | 190 +++
drivers/infiniband/sw/rfc/rfc_task.h | 100 ++
drivers/infiniband/sw/rfc/rfc_tb.c | 795 +++++++++++++
drivers/infiniband/sw/rfc/rfc_verbs.c | 1323 +++++++++++++++++++++
drivers/infiniband/sw/rfc/rfc_verbs.h | 465 ++++++++
drivers/nvme/target/fc.c | 94 +-
drivers/nvme/target/io-cmd.c | 44 +-
include/linux/nvme-fc-driver.h | 6 +
include/uapi/rdma/rdma_user_ioctl_cmds.h | 2 +
include/uapi/rdma/rdma_user_rfc.h | 179 +++
46 files changed, 15095 insertions(+), 9 deletions(-)
create mode 100644 Documentation/infiniband/rfc/README.txt
create mode 100644 Documentation/infiniband/rfc/RFC_testplan.txt
create mode 100644 Documentation/infiniband/rfc/RFC_validation.txt
create mode 100644 drivers/infiniband/sw/rfc/Kconfig
create mode 100644 drivers/infiniband/sw/rfc/Makefile
create mode 100644 drivers/infiniband/sw/rfc/rfc.c
create mode 100644 drivers/infiniband/sw/rfc/rfc.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_av.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_comp.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_cq.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_hdr.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_hw_counters.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_hw_counters.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_loc.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_mcast.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_mmap.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_mr.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_net.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_net.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_obj.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_obj.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_opcode.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_opcode.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_param.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_pool.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_pool.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_qp.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_queue.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_queue.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_recv.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_req.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_resp.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_srq.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_sysfs.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_task.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_task.h
create mode 100644 drivers/infiniband/sw/rfc/rfc_tb.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_verbs.c
create mode 100644 drivers/infiniband/sw/rfc/rfc_verbs.h
create mode 100644 include/uapi/rdma/rdma_user_rfc.h
diff --git a/Documentation/infiniband/rfc/README.txt b/Documentation/infiniband/rfc/README.txt
new file mode 100644
index 0000000..c6f81ae
--- /dev/null
+++ b/Documentation/infiniband/rfc/README.txt
@@ -0,0 +1,147 @@
+****************************** Introduction ***************************
+
+This patchset introduces 'rdma_rfc' driver for Software implemention of
+RDMA over fibre channel (hereafter referred as 'RFC').
+
+Some background on the driver:
+
+This driver introduces Fibre channel as the transport protocol
+for providing RDMA support.
+It takes inspiration from the Soft RoCE driver originally introduced by
+System Fabric Works and later developed by Mellanox.
+
+Eventhough it is inspired from the Soft RoCE driver, the underlying
+transport layer is FC-NVMe (short for 'NVMe over fibre channel').
+The request, response and completion state machines in the driver
+have been heavily modified to adapt to the Exchange based Data
+transfer mechanism of Fibre channel.
+
+
+Some notes on the architecture and design:
+
+rdma_rfc, implements the RDMA transport and registers with the ib_core as a
+kernel verbs provider. It also implements the IO layer.
+rdma_rfc registers with ib_core as RoCEv2 provider and sends the RDMA work
+requests by encasulating each request in a RFC object which is sent over an
+FC-NVMe exchange. This function enables user to send RDMA work requests over
+Fibre channel while using IPv4 addresses as connection identifier.
+
+rdma_rfc also registers as a software NIC with netdev stack to send and
+receive ethernet packets over FC-NVMe.
+
+The modules are configured by entries in /sys. There is a configuration script
+(rfc_cfg) that simplifies the use of this interface. rfc_cfg is part of the
+rfc user space code, librfc.
+
+The use of rfc verbs in user space requires the inclusion of librfc as a device
+specific plug-in to libibverbs. librfc is packaged separately.
+
+
+Architecture:
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ +-----------------------------------------------------------+
+ | Application |
+ +-----------------------------------------------------------+
+ +-----------------------------------+
+ | libibverbs |
+User +-----------------------------------+
+ +--------+ +------+ +---------------+
+ | librfc | |librxe| | libmlx4 |
+ +--------+ +------+ +---------------+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ +--------------+ +------------+
+ | Sockets | | CMA |
+ +--------------+ +------------+
+ +------------+
+ | CM |
+ +------------+
+
+ +--------------+ +-------------------------------+
+ | TCP/IP | | ib_core |
+ +--------------+ +-------------------------------+
+ +-------------------+ +-----------+ +----------------+
+Kernel | rdma_rfc | | rdma_rxe | | HW RoCE driver |
+ +-------------------+ +-----------+ +----------------+
+ +--------------------------+
+ | FC-NVMe driver |
+ +--------------------------+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The driver components and a non asci chart of the module can be found at a
+pdf [1]
+The design is similar but responder ,completion state machines are getting
+executed in the fc-nvme workqueue contexts .
+This reduced the number of context switches and locks during the data path.
+
+Currently only IPv4 based sessions are supported.
+
+
+[1] -https://github.com/brocade/RDMAoverFC/blob/master/RDMA%20over%20FC.pdf
+
+
+*********************** Build Instructions **********************************
+1. Compile and install kernel:
+ - Clone kernel git:
+ git clone https://github.com/torvalds/linux.git
+ - Configure the kernel:
+ cd linux
+ cp /boot/config-$(uname –r) .confi
+ make menuconfig. (Need to enable CONFIG_RDMA_RFC.)
+ Need to enable CONFIG_INFINIBAND_ADDR_TRANS=y and
+ CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS=y in new config file
+ - Compile kernel using standard make process.
+ - Verify that the new kernel entry is added (e.g. to grub);
+ if not, need to add it manually.
+ - Boot with new kernel.
+
+4. Compile and Install rdma-core package containing librfc from
+ the following source:
+ https://github.com/linux-rdma/rdma-core.git
+
+5. Configure RFC:
+ - Load rdma_rfc kernel module using the rfc_cfg script included
+ in the rdma-core package:
+
+ rfc_cfg start (this might require sudo or root privileges)
+ (Verify that /dev/rfcblk device is created after this step.)
+ - Create RFC device over network interface created by rdma_rfc:
+ rfc_cfg add rfcnet0
+ - Use the status command to display the current configuration:
+ rfc_cfg status
+
+If configured successfully, you should see output similar to the following:
+ Name Link Driver Speed NMTU IPv4_addr RDEV RMTU
+ rfcnet0 yes plat_net rfc0 1024(3)
+
+Now you have an Infiniband device called rfc0.
+To use this device to run RDMA applications, user must do the following
+NVMe-over-FC configurations:
+
+******************* Configuring NVMe over FC *********************************
+
+Note: the NVMe FC HBA has to be in dual mode (i.e. initiator and target mode)
+
+(Below example uses Emulex scripts.
+Need to follow similar process for Qlogic adapters)
+
+1. Download the latest lpfc_nvme_scripts package from Emulex site.
+
+2. Configure nvme targets
+ cd /root/Downloads/lpfc-nvme-scripts-11.4.189.0/nvme_scripts
+ ./lpfc_nvmet_list.sh
+ ./lpfc_nvmet_namespace.sh test /dev/rfcblk
+ ./lpfc_nvmet_port.sh lpfc test
+ ./lpfc_nvmet_list.sh
+
+3. Connect the nvme target from remote machines
+ nvme connect-all --transport=fc --host-traddr=nn-0x<node wwn>:
+ pn-0x<port wwn> --traddr=nn-0x<node wwn>:pn-0x<port wwn>
+
+ ex. nvme connect-all --transport=fc --host-traddr=
+ nn-0x200000109b22eb3f:pn-0x100000109b22eb3f
+ --traddr=nn-0x200000109b22eb6f:pn-0x100000109b22eb6f
+
+ - Verify that the target is visible
+ nvme list
diff --git a/Documentation/infiniband/rfc/RFC_testplan.txt b/Documentation/infiniband/rfc/RFC_testplan.txt
new file mode 100644
index 0000000..26d66e0
--- /dev/null
+++ b/Documentation/infiniband/rfc/RFC_testplan.txt
@@ -0,0 +1,20 @@
+*******************************************************************************
+ RFC Test Plan
+*******************************************************************************
+
+List of features:
+
+Test type Descrption
+RDMA RDMA Traffic
+RDMACM RDMACM Traffic
+
+List of public tests/tools that can be used to check RFC driver:
+
+Test Name Test Description
+pingpong Uses ibv_*_pingpong examples that comes with libibverbs
+rdma_cm_bw RDMA CM bandwidth test
+rdma_cm_lat RDMA CM latency test
+rping Establishes a reliable RDMA connection between two nodes
+ using the librdmacm
+ucmatose RDMA CM connection and simple ping-pong test
+perftests Uses ib_* performance tests that come with OFED package.
diff --git a/Documentation/infiniband/rfc/RFC_validation.txt b/Documentation/infiniband/rfc/RFC_validation.txt
new file mode 100644
index 0000000..7278cfe
--- /dev/null
+++ b/Documentation/infiniband/rfc/RFC_validation.txt
@@ -0,0 +1,105 @@
+*******************************************************************************
+ validate that rfc is working
+*******************************************************************************
+
+
+To validate that rfc is working need to do the following steps:
+
+Validate that kernel modules are loaded using lsmod:
+1- Check that rfc drive is loaded:
+
+# lsmod | grep rfc
+rdma_rfc 151552 0
+ib_core 225280 14 ib_iser,rdma_rfc,ib_cm,rdma_cm,ib_umad,ib_srp,
+ ib_isert,ib_uverbs,rpcrdma,ib_ipoib,iw_cm,
+ ib_srpt,ib_ucm,rdma_ucm
+nvmet_fc 28672 2 rdma_rfc,lpfc
+nvmet 53248 2 rdma_rfc,nvmet_fc
+
+2- Check that ib_uverbs is loaded:
+
+# lsmod | grep ib_uverbs
+ib_uverbs 47211 0
+
+If the rfc driver is not loaded use rfc_cfg script that comes with librfc
+(userspace library) to load it.
+
+Validate that userspace applications are working:
+1- Check that rfc device can be accessed using ibv_devinfo (included in libibverbs-
+utils) and need to see that port state is in PORT_ACTIVE:
+
+# ibv_devinfo -d rfc0
+hca_id: rfc0
+transport: InfiniBand (0)
+fw_ver: 0.0.0
+node_guid: 0202:c9ff:fe07:d511
+sys_image_guid: 0000:0000:0000:0000
+vendor_id: 0x0000
+vendor_part_id: 0
+hw_ver: 0x0
+phys_port_cnt: 1
+ port: 1
+ state: PORT_ACTIVE (4)
+ max_mtu: 4096 (5)
+ active_mtu: 1024 (3)
+ sm_lid: 0
+ port_lid: 0
+ port_lmc: 0x00
+ link_layer: Ethernet
+
+
+
+2. Check that NVMe subsystem is configured
+ - Verify that remote NVMe target is visible using "nvme list" and
+ "nvme list_rdma"
+
+Validate that traffic over rfc device is working:
+Send traffic using ibv_*_pingpong tests (included in libibverbs-utils):
+
+server:
+# ibv_rc_pingpong -g 0 -d rfc0 -i 1
+ local address: LID 0x0000, QPN 0x000011, PSN 0x660fec,
+ GID fe80::202:c9ff:fe07:d511
+ remote address: LID 0x0000, QPN 0x000011, PSN 0x1c8285,
+ GID fe80::202:c9ff:fe07:d411
+8192000 bytes in 0.03 seconds = 1894.16 Mbit/sec
+1000 iters in 0.03 seconds = 34.60 usec/iter
+
+Client:
+# ibv_rc_pingpong -g 0 -d rfc0 -i 1 192.168.2.107
+ local address: LID 0x0000, QPN 0x000011, PSN 0x1c8285,
+ GID fe80::202:c9ff:fe07:d411
+ remote address: LID 0x0000, QPN 0x000011, PSN 0x660fec,
+ GID fe80::202:c9ff:fe07:d511
+8192000 bytes in 0.03 seconds = 1888.59 Mbit/sec
+1000 iters in 0.03 seconds = 34.70 usec/iter
+Send traffic using rping (RDMACM):
+
+server:
+# rping -s -a 192.168.2.107 -v -C 10
+server ping data: rdma-ping-0: ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnop
+server ping data: rdma-ping-1: BCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopq
+server ping data: rdma-ping-2: CDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr
+server ping data: rdma-ping-3: DEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs
+server ping data: rdma-ping-4: EFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrst
+server ping data: rdma-ping-5: FGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstu
+server ping data: rdma-ping-6: GHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuv
+server ping data: rdma-ping-7: HIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw
+server ping data: rdma-ping-8: IJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwx
+server ping data: rdma-ping-9: JKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxy
+server DISCONNECT EVENT...
+wait for RDMA_READ_ADV state 10
+
+client:
+# rping -c -a 192.168.2.108 -v -C 10
+ping data: rdma-ping-0: ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr
+ping data: rdma-ping-1: BCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs
+ping data: rdma-ping-2: CDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrst
+ping data: rdma-ping-3: DEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstu
+ping data: rdma-ping-4: EFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuv
+ping data: rdma-ping-5: FGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw
+ping data: rdma-ping-6: GHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwx
+ping data: rdma-ping-7: HIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxy
+ping data: rdma-ping-8: IJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz
+ping data: rdma-ping-9: JKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyzA
+client DISCONNECT EVENT...
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index ee270e0..7954e52 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -98,6 +98,7 @@ source "drivers/infiniband/ulp/isert/Kconfig"
source "drivers/infiniband/ulp/opa_vnic/Kconfig"
source "drivers/infiniband/sw/rdmavt/Kconfig"
source "drivers/infiniband/sw/rxe/Kconfig"
+source "drivers/infiniband/sw/rfc/Kconfig"
source "drivers/infiniband/hw/hfi1/Kconfig"
diff --git a/drivers/infiniband/sw/Makefile b/drivers/infiniband/sw/Makefile
index 8b095b2..4ffc988 100644
--- a/drivers/infiniband/sw/Makefile
+++ b/drivers/infiniband/sw/Makefile
@@ -1,2 +1,3 @@
obj-$(CONFIG_INFINIBAND_RDMAVT) += rdmavt/
obj-$(CONFIG_RDMA_RXE) += rxe/
+obj-$(CONFIG_RDMA_RFC) += rfc/
diff --git a/drivers/infiniband/sw/rfc/Kconfig b/drivers/infiniband/sw/rfc/Kconfig
new file mode 100644
index 0000000..4e6ca8b
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/Kconfig
@@ -0,0 +1,26 @@
+config RDMA_RFC
+ tristate "Software RDMA over FC (RFC) driver"
+ depends on INET && PCI && INFINIBAND
+ select NET_UDP_TUNNEL
+ select CRYPTO_CRC32
+ select DMA_VIRT_OPS
+ ---help---
+ This driver implements the InfiniBand RDMA transport over
+ the Linux network stack. It enables a system with a
+ standard Ethernet adapter to interoperate with a RoCE
+ adapter or with another system running the RXE driver.
+ Documentation on InfiniBand and RoCE can be downloaded at
+ www.infinibandta.org and www.openfabrics.org. (See also
+ siw which is a similar software driver for iWARP.)
+
+ The driver is split into two layers, one interfaces with the
+ Linux RDMA stack and implements a kernel or user space
+ verbs API. The user space verbs API requires a support
+ library named librfc which is loaded by the generic user
+ space verbs API, libibverbs. The other layer interfaces
+ with the Linux network stack at layer 3.
+
+ To configure and work with soft-RoCE driver please use the
+ following wiki page under "configure Soft-RoCE (RXE)" section:
+
+ https://github.com/linux-rdma/rdma-core/blob/master/Documentation/rfc.md
diff --git a/drivers/infiniband/sw/rfc/Makefile b/drivers/infiniband/sw/rfc/Makefile
new file mode 100644
index 0000000..d4e61e0
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/Makefile
@@ -0,0 +1,24 @@
+obj-$(CONFIG_RDMA_RFC) += rdma_rfc.o
+
+rdma_rfc-y := \
+ rfc.o \
+ rfc_comp.o \
+ rfc_req.o \
+ rfc_resp.o \
+ rfc_recv.o \
+ rfc_pool.o \
+ rfc_queue.o \
+ rfc_verbs.o \
+ rfc_av.o \
+ rfc_srq.o \
+ rfc_qp.o \
+ rfc_cq.o \
+ rfc_mr.o \
+ rfc_opcode.o \
+ rfc_mmap.o \
+ rfc_task.o \
+ rfc_net.o \
+ rfc_sysfs.o \
+ rfc_hw_counters.o \
+ rfc_obj.o \
+ rfc_tb.o
diff --git a/drivers/infiniband/sw/rfc/rfc.c b/drivers/infiniband/sw/rfc/rfc.c
new file mode 100644
index 0000000..298f5d6
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc.c
@@ -0,0 +1,382 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <net/addrconf.h>
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_obj.h"
+
+MODULE_AUTHOR("Muneendra Kumar Mandala, Amit Kumar Tyagi, Anand Nataraja Sundaram");
+MODULE_DESCRIPTION("RDMA over FC transport");
+MODULE_LICENSE("Dual BSD/GPL");
+
+/* free resources for all ports on a device */
+static void rfc_cleanup_ports(struct rfc_dev *rfc)
+{
+ kfree(rfc->port.pkey_tbl);
+ rfc->port.pkey_tbl = NULL;
+
+}
+
+/* free resources for a rfc device all objects created for this device must
+ * have been destroyed
+ */
+static void rfc_cleanup(struct rfc_dev *rfc)
+{
+ rfc_pool_cleanup(&rfc->uc_pool);
+ rfc_pool_cleanup(&rfc->pd_pool);
+ rfc_pool_cleanup(&rfc->ah_pool);
+ rfc_pool_cleanup(&rfc->srq_pool);
+ rfc_pool_cleanup(&rfc->qp_pool);
+ rfc_pool_cleanup(&rfc->cq_pool);
+ rfc_pool_cleanup(&rfc->mr_pool);
+ rfc_pool_cleanup(&rfc->mw_pool);
+ rfc_pool_cleanup(&rfc->mc_grp_pool);
+ rfc_pool_cleanup(&rfc->mc_elem_pool);
+
+ rfc_cleanup_ports(rfc);
+
+ crypto_free_shash(rfc->tfm);
+}
+
+/* called when all references have been dropped */
+void rfc_release(struct kref *kref)
+{
+ struct rfc_dev *rfc = container_of(kref, struct rfc_dev, ref_cnt);
+
+ rfc_cleanup(rfc);
+ ib_dealloc_device(&rfc->ib_dev);
+}
+
+/* initialize rfc device parameters */
+static void rfc_init_device_param(struct rfc_dev *rfc)
+{
+ rfc->max_inline_data = RFC_MAX_INLINE_DATA;
+
+ rfc->attr.fw_ver = RFC_FW_VER;
+ rfc->attr.max_mr_size = RFC_MAX_MR_SIZE;
+ rfc->attr.page_size_cap = RFC_PAGE_SIZE_CAP;
+ rfc->attr.vendor_id = RFC_VENDOR_ID;
+ rfc->attr.vendor_part_id = RFC_VENDOR_PART_ID;
+ rfc->attr.hw_ver = RFC_HW_VER;
+ rfc->attr.max_qp = RFC_MAX_QP;
+ rfc->attr.max_qp_wr = RFC_MAX_QP_WR;
+ rfc->attr.device_cap_flags = RFC_DEVICE_CAP_FLAGS;
+ rfc->attr.max_sge = RFC_MAX_SGE;
+ rfc->attr.max_sge_rd = RFC_MAX_SGE_RD;
+ rfc->attr.max_cq = RFC_MAX_CQ;
+ rfc->attr.max_cqe = (1 << RFC_MAX_LOG_CQE) - 1;
+ rfc->attr.max_mr = RFC_MAX_MR;
+ rfc->attr.max_pd = RFC_MAX_PD;
+ rfc->attr.max_qp_rd_atom = RFC_MAX_QP_RD_ATOM;
+ rfc->attr.max_ee_rd_atom = RFC_MAX_EE_RD_ATOM;
+ rfc->attr.max_res_rd_atom = RFC_MAX_RES_RD_ATOM;
+ rfc->attr.max_qp_init_rd_atom = RFC_MAX_QP_INIT_RD_ATOM;
+ rfc->attr.max_ee_init_rd_atom = RFC_MAX_EE_INIT_RD_ATOM;
+ rfc->attr.atomic_cap = RFC_ATOMIC_CAP;
+ rfc->attr.max_ee = RFC_MAX_EE;
+ rfc->attr.max_rdd = RFC_MAX_RDD;
+ rfc->attr.max_mw = RFC_MAX_MW;
+ rfc->attr.max_raw_ipv6_qp = RFC_MAX_RAW_IPV6_QP;
+ rfc->attr.max_raw_ethy_qp = RFC_MAX_RAW_ETHY_QP;
+ rfc->attr.max_mcast_grp = RFC_MAX_MCAST_GRP;
+ rfc->attr.max_mcast_qp_attach = RFC_MAX_MCAST_QP_ATTACH;
+ rfc->attr.max_total_mcast_qp_attach = RFC_MAX_TOT_MCAST_QP_ATTACH;
+ rfc->attr.max_ah = RFC_MAX_AH;
+ rfc->attr.max_fmr = RFC_MAX_FMR;
+ rfc->attr.max_map_per_fmr = RFC_MAX_MAP_PER_FMR;
+ rfc->attr.max_srq = RFC_MAX_SRQ;
+ rfc->attr.max_srq_wr = RFC_MAX_SRQ_WR;
+ rfc->attr.max_srq_sge = RFC_MAX_SRQ_SGE;
+ rfc->attr.max_fast_reg_page_list_len = RFC_MAX_FMR_PAGE_LIST_LEN;
+ rfc->attr.max_pkeys = RFC_MAX_PKEYS;
+ rfc->attr.local_ca_ack_delay = RFC_LOCAL_CA_ACK_DELAY;
+
+ rfc->max_ucontext = RFC_MAX_UCONTEXT;
+}
+
+/* initialize port attributes */
+static int rfc_init_port_param(struct rfc_port *port)
+{
+ port->attr.state = RFC_PORT_STATE;
+ port->attr.max_mtu = RFC_PORT_MAX_MTU;
+ port->attr.active_mtu = RFC_PORT_ACTIVE_MTU;
+ port->attr.gid_tbl_len = RFC_PORT_GID_TBL_LEN;
+ port->attr.port_cap_flags = RFC_PORT_PORT_CAP_FLAGS;
+ port->attr.max_msg_sz = RFC_PORT_MAX_MSG_SZ;
+ port->attr.bad_pkey_cntr = RFC_PORT_BAD_PKEY_CNTR;
+ port->attr.qkey_viol_cntr = RFC_PORT_QKEY_VIOL_CNTR;
+ port->attr.pkey_tbl_len = RFC_PORT_PKEY_TBL_LEN;
+ port->attr.lid = RFC_PORT_LID;
+ port->attr.sm_lid = RFC_PORT_SM_LID;
+ port->attr.lmc = RFC_PORT_LMC;
+ port->attr.max_vl_num = RFC_PORT_MAX_VL_NUM;
+ port->attr.sm_sl = RFC_PORT_SM_SL;
+ port->attr.subnet_timeout = RFC_PORT_SUBNET_TIMEOUT;
+ port->attr.init_type_reply = RFC_PORT_INIT_TYPE_REPLY;
+ port->attr.active_width = RFC_PORT_ACTIVE_WIDTH;
+ port->attr.active_speed = RFC_PORT_ACTIVE_SPEED;
+ port->attr.phys_state = RFC_PORT_PHYS_STATE;
+ port->mtu_cap =
+ ib_mtu_enum_to_int(RFC_PORT_ACTIVE_MTU);
+ port->subnet_prefix = cpu_to_be64(RFC_PORT_SUBNET_PREFIX);
+
+ return 0;
+}
+
+/* initialize port state, note IB convention that HCA ports are always
+ * numbered from 1
+ */
+static int rfc_init_ports(struct rfc_dev *rfc)
+{
+ struct rfc_port *port = &rfc->port;
+
+ rfc_init_port_param(port);
+
+ if (!port->attr.pkey_tbl_len || !port->attr.gid_tbl_len)
+ return -EINVAL;
+
+ port->pkey_tbl = kcalloc(port->attr.pkey_tbl_len,
+ sizeof(*port->pkey_tbl), GFP_KERNEL);
+
+ if (!port->pkey_tbl)
+ return -ENOMEM;
+
+ port->pkey_tbl[0] = 0xffff;
+ addrconf_addr_eui48((unsigned char *)&port->port_guid,
+ rfc->ndev->dev_addr);
+
+ spin_lock_init(&port->port_lock);
+
+ return 0;
+}
+
+/* init pools of managed objects */
+static int rfc_init_pools(struct rfc_dev *rfc)
+{
+ int err;
+
+ err = rfc_pool_init(rfc, &rfc->uc_pool, RFC_TYPE_UC,
+ rfc->max_ucontext);
+ if (err)
+ goto err1;
+
+ err = rfc_pool_init(rfc, &rfc->pd_pool, RFC_TYPE_PD,
+ rfc->attr.max_pd);
+ if (err)
+ goto err2;
+
+ err = rfc_pool_init(rfc, &rfc->ah_pool, RFC_TYPE_AH,
+ rfc->attr.max_ah);
+ if (err)
+ goto err3;
+
+ err = rfc_pool_init(rfc, &rfc->srq_pool, RFC_TYPE_SRQ,
+ rfc->attr.max_srq);
+ if (err)
+ goto err4;
+
+ err = rfc_pool_init(rfc, &rfc->qp_pool, RFC_TYPE_QP,
+ rfc->attr.max_qp);
+ if (err)
+ goto err5;
+
+ err = rfc_pool_init(rfc, &rfc->cq_pool, RFC_TYPE_CQ,
+ rfc->attr.max_cq);
+ if (err)
+ goto err6;
+
+ err = rfc_pool_init(rfc, &rfc->mr_pool, RFC_TYPE_MR,
+ rfc->attr.max_mr);
+ if (err)
+ goto err7;
+
+ err = rfc_pool_init(rfc, &rfc->mw_pool, RFC_TYPE_MW,
+ rfc->attr.max_mw);
+ if (err)
+ goto err8;
+
+ err = rfc_pool_init(rfc, &rfc->mc_grp_pool, RFC_TYPE_MC_GRP,
+ rfc->attr.max_mcast_grp);
+ if (err)
+ goto err9;
+
+ err = rfc_pool_init(rfc, &rfc->mc_elem_pool, RFC_TYPE_MC_ELEM,
+ rfc->attr.max_total_mcast_qp_attach);
+ if (err)
+ goto err10;
+
+ return 0;
+
+err10:
+ rfc_pool_cleanup(&rfc->mc_grp_pool);
+err9:
+ rfc_pool_cleanup(&rfc->mw_pool);
+err8:
+ rfc_pool_cleanup(&rfc->mr_pool);
+err7:
+ rfc_pool_cleanup(&rfc->cq_pool);
+err6:
+ rfc_pool_cleanup(&rfc->qp_pool);
+err5:
+ rfc_pool_cleanup(&rfc->srq_pool);
+err4:
+ rfc_pool_cleanup(&rfc->ah_pool);
+err3:
+ rfc_pool_cleanup(&rfc->pd_pool);
+err2:
+ rfc_pool_cleanup(&rfc->uc_pool);
+err1:
+ return err;
+}
+
+/* initialize rfc device state */
+static int rfc_init(struct rfc_dev *rfc)
+{
+ int err;
+
+ /* init default device parameters */
+ rfc_init_device_param(rfc);
+
+ err = rfc_init_ports(rfc);
+ if (err)
+ goto err1;
+
+ err = rfc_init_pools(rfc);
+ if (err)
+ goto err2;
+
+ /* init pending mmap list */
+ spin_lock_init(&rfc->mmap_offset_lock);
+ spin_lock_init(&rfc->pending_lock);
+ INIT_LIST_HEAD(&rfc->pending_mmaps);
+ INIT_LIST_HEAD(&rfc->list);
+
+ mutex_init(&rfc->usdev_lock);
+
+ return 0;
+
+err2:
+ rfc_cleanup_ports(rfc);
+err1:
+ return err;
+}
+
+int rfc_set_mtu(struct rfc_dev *rfc, unsigned int ndev_mtu)
+{
+ struct rfc_port *port = &rfc->port;
+ enum ib_mtu mtu;
+
+ mtu = eth_mtu_int_to_enum(ndev_mtu);
+
+ /* Make sure that new MTU in range */
+ mtu = mtu ? min_t(enum ib_mtu, mtu, RFC_PORT_MAX_MTU) : IB_MTU_256;
+
+ port->attr.active_mtu = mtu;
+ port->mtu_cap = ib_mtu_enum_to_int(mtu);
+
+ return 0;
+}
+EXPORT_SYMBOL(rfc_set_mtu);
+
+/* called by ifc layer to create new rfc device.
+ * The caller should allocate memory for rfc by calling ib_alloc_device.
+ */
+int rfc_add(struct rfc_dev *rfc, unsigned int mtu)
+{
+ int err;
+
+ kref_init(&rfc->ref_cnt);
+
+ err = rfc_init(rfc);
+ if (err)
+ goto err1;
+
+ err = rfc_set_mtu(rfc, mtu);
+ if (err)
+ goto err1;
+
+ err = rfc_register_device(rfc);
+ if (err)
+ goto err1;
+ rfc_update_ip_addr(rfc->ndev->name);
+ strcpy(rfc->objname, "rfcblk");
+ return 0;
+
+err1:
+ rfc_dev_put(rfc);
+ return err;
+}
+EXPORT_SYMBOL(rfc_add);
+
+/* called by the ifc layer to remove a device */
+void rfc_remove(struct rfc_dev *rfc)
+{
+ rfc_unregister_device(rfc);
+
+ rfc_dev_put(rfc);
+}
+EXPORT_SYMBOL(rfc_remove);
+
+static int __init rfc_module_init(void)
+{
+ int err;
+
+ /* initialize slab caches for managed objects */
+ err = rfc_cache_init();
+ if (err) {
+ pr_err("unable to init object pools\n");
+ return err;
+ }
+
+ err = rfc_obj_init();
+ if (err)
+ return err;
+
+ pr_info("loaded\n");
+ return 0;
+}
+
+static void __exit rfc_module_exit(void)
+{
+ rfc_remove_all();
+ rfc_obj_exit();
+ rfc_cache_exit();
+
+ pr_info("unloaded\n");
+}
+
+late_initcall(rfc_module_init);
+module_exit(rfc_module_exit);
diff --git a/drivers/infiniband/sw/rfc/rfc.h b/drivers/infiniband/sw/rfc/rfc.h
new file mode 100644
index 0000000..7742a0e
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc.h
@@ -0,0 +1,115 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_H
+#define RFC_H
+
+#ifdef pr_fmt
+#undef pr_fmt
+#endif
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/module.h>
+#include <linux/crc32.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_pack.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_cache.h>
+#include <rdma/ib_addr.h>
+#include <crypto/hash.h>
+
+#include "rfc_net.h"
+#include "rfc_opcode.h"
+#include "rfc_hdr.h"
+#include "rfc_param.h"
+#include "rfc_verbs.h"
+#include "rfc_loc.h"
+
+/*
+ * Version 1 and Version 2 are identical on 64 bit machines, but on 32 bit
+ * machines Version 2 has a different struct layout.
+ */
+#define RFC_UVERBS_ABI_VERSION 2
+
+#define IB_PHYS_STATE_LINK_UP (5)
+#define IB_PHYS_STATE_LINK_DOWN (3)
+
+#define RFC_ROCE_V2_SPORT (0xc000)
+
+static inline u32 rfc_crc32(struct rfc_dev *rfc,
+ u32 crc, void *next, size_t len)
+{
+ u32 retval;
+ int err;
+
+ SHASH_DESC_ON_STACK(shash, rfc->tfm);
+
+ shash->tfm = rfc->tfm;
+ shash->flags = 0;
+ *(u32 *)shash_desc_ctx(shash) = crc;
+ err = crypto_shash_update(shash, next, len);
+ if (unlikely(err)) {
+ pr_warn_ratelimited("failed crc calculation, err: %d\n", err);
+ return crc32_le(crc, next, len);
+ }
+
+ retval = *(u32 *)shash_desc_ctx(shash);
+ barrier_data(shash_desc_ctx(shash));
+ return retval;
+}
+
+int rfc_set_mtu(struct rfc_dev *rfc, unsigned int dev_mtu);
+
+int rfc_add(struct rfc_dev *rfc, unsigned int mtu);
+void rfc_remove(struct rfc_dev *rfc);
+void rfc_remove_all(void);
+
+u64 rfc_rcv_hdr(struct rfc_recv_data *rcv, struct sg_table **rsgp);
+u64 rfc_rcv_done(struct rfc_recv_data *rcv);
+
+static inline void rfc_dev_put(struct rfc_dev *rfc)
+{
+ kref_put(&rfc->ref_cnt, rfc_release);
+}
+struct rfc_dev *net_to_rfc(struct net_device *ndev);
+struct rfc_dev *get_rfc_by_name(const char *name);
+
+void rfc_port_up(struct rfc_dev *rfc);
+void rfc_port_down(struct rfc_dev *rfc);
+
+#endif /* RFC_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_av.c b/drivers/infiniband/sw/rfc/rfc_av.c
new file mode 100644
index 0000000..eb704e1
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_av.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+
+int rfc_av_chk_attr(struct rfc_dev *rfc, struct rdma_ah_attr *attr)
+{
+ struct rfc_port *port;
+
+ port = &rfc->port;
+
+ if (rdma_ah_get_ah_flags(attr) & IB_AH_GRH) {
+ u8 sgid_index = rdma_ah_read_grh(attr)->sgid_index;
+
+ if (sgid_index > port->attr.gid_tbl_len) {
+ pr_warn("invalid sgid index = %d\n", sgid_index);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+void rfc_av_from_attr(u8 port_num, struct rfc_av *av,
+ struct rdma_ah_attr *attr)
+{
+ memset(av, 0, sizeof(*av));
+ memcpy(&av->grh, rdma_ah_read_grh(attr),
+ sizeof(*rdma_ah_read_grh(attr)));
+ av->port_num = port_num;
+}
+
+void rfc_av_to_attr(struct rfc_av *av, struct rdma_ah_attr *attr)
+{
+ attr->type = RDMA_AH_ATTR_TYPE_ROCE;
+ memcpy(rdma_ah_retrieve_grh(attr), &av->grh, sizeof(av->grh));
+ rdma_ah_set_ah_flags(attr, IB_AH_GRH);
+ rdma_ah_set_port_num(attr, av->port_num);
+}
+
+void rfc_av_fill_ip_info(struct rfc_av *av,
+ struct rdma_ah_attr *attr,
+ struct ib_gid_attr *sgid_attr,
+ union ib_gid *sgid)
+{
+ rdma_gid2ip((struct sockaddr *)&av->sgid_addr, sgid);
+ rdma_gid2ip((struct sockaddr *)&av->dgid_addr,
+ &rdma_ah_read_grh(attr)->dgid);
+ av->network_type = ib_gid_to_network_type(sgid_attr->gid_type, sgid);
+}
+
+struct rfc_av *rfc_get_av(struct rfc_obj_info *obj)
+{
+ if (!obj || !obj->qp)
+ return NULL;
+
+ if (qp_type(obj->qp) == IB_QPT_RC || qp_type(obj->qp) == IB_QPT_UC)
+ return &obj->qp->pri_av;
+
+ return (obj->wqe) ? &obj->wqe->av : NULL;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_comp.c b/drivers/infiniband/sw/rfc/rfc_comp.c
new file mode 100644
index 0000000..3e80e87
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_comp.c
@@ -0,0 +1,771 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+#include "rfc_task.h"
+#include <linux/nvme.h>
+
+enum comp_state {
+ COMPST_GET_ACK,
+ COMPST_GET_WQE,
+ COMPST_COMP_WQE,
+ COMPST_COMP_ACK,
+ COMPST_CHECK_PSN,
+ COMPST_CHECK_ACK,
+ COMPST_READ,
+ COMPST_ATOMIC,
+ COMPST_WRITE_SEND,
+ COMPST_UPDATE_COMP,
+ COMPST_ERROR_RETRY,
+ COMPST_RNR_RETRY,
+ COMPST_ERROR,
+ COMPST_EXIT, /* We have an issue, and we want to rerun the completer */
+ COMPST_DONE, /* The completer finished successflly */
+};
+
+static char *comp_state_name[] = {
+ [COMPST_GET_ACK] = "GET ACK",
+ [COMPST_GET_WQE] = "GET WQE",
+ [COMPST_COMP_WQE] = "COMP WQE",
+ [COMPST_COMP_ACK] = "COMP ACK",
+ [COMPST_CHECK_PSN] = "CHECK PSN",
+ [COMPST_CHECK_ACK] = "CHECK ACK",
+ [COMPST_READ] = "READ",
+ [COMPST_ATOMIC] = "ATOMIC",
+ [COMPST_WRITE_SEND] = "WRITE/SEND",
+ [COMPST_UPDATE_COMP] = "UPDATE COMP",
+ [COMPST_ERROR_RETRY] = "ERROR RETRY",
+ [COMPST_RNR_RETRY] = "RNR RETRY",
+ [COMPST_ERROR] = "ERROR",
+ [COMPST_EXIT] = "EXIT",
+ [COMPST_DONE] = "DONE",
+};
+
+static unsigned long rnrnak_usec[32] = {
+ [IB_RNR_TIMER_655_36] = 655360,
+ [IB_RNR_TIMER_000_01] = 10,
+ [IB_RNR_TIMER_000_02] = 20,
+ [IB_RNR_TIMER_000_03] = 30,
+ [IB_RNR_TIMER_000_04] = 40,
+ [IB_RNR_TIMER_000_06] = 60,
+ [IB_RNR_TIMER_000_08] = 80,
+ [IB_RNR_TIMER_000_12] = 120,
+ [IB_RNR_TIMER_000_16] = 160,
+ [IB_RNR_TIMER_000_24] = 240,
+ [IB_RNR_TIMER_000_32] = 320,
+ [IB_RNR_TIMER_000_48] = 480,
+ [IB_RNR_TIMER_000_64] = 640,
+ [IB_RNR_TIMER_000_96] = 960,
+ [IB_RNR_TIMER_001_28] = 1280,
+ [IB_RNR_TIMER_001_92] = 1920,
+ [IB_RNR_TIMER_002_56] = 2560,
+ [IB_RNR_TIMER_003_84] = 3840,
+ [IB_RNR_TIMER_005_12] = 5120,
+ [IB_RNR_TIMER_007_68] = 7680,
+ [IB_RNR_TIMER_010_24] = 10240,
+ [IB_RNR_TIMER_015_36] = 15360,
+ [IB_RNR_TIMER_020_48] = 20480,
+ [IB_RNR_TIMER_030_72] = 30720,
+ [IB_RNR_TIMER_040_96] = 40960,
+ [IB_RNR_TIMER_061_44] = 61410,
+ [IB_RNR_TIMER_081_92] = 81920,
+ [IB_RNR_TIMER_122_88] = 122880,
+ [IB_RNR_TIMER_163_84] = 163840,
+ [IB_RNR_TIMER_245_76] = 245760,
+ [IB_RNR_TIMER_327_68] = 327680,
+ [IB_RNR_TIMER_491_52] = 491520,
+};
+
+static inline unsigned long rnrnak_jiffies(u8 timeout)
+{
+ return max_t(unsigned long,
+ usecs_to_jiffies(rnrnak_usec[timeout]), 1);
+}
+
+static enum ib_wc_opcode wr_to_wc_opcode(enum ib_wr_opcode opcode)
+{
+ switch (opcode) {
+ case IB_WR_RDMA_WRITE: return IB_WC_RDMA_WRITE;
+ case IB_WR_RDMA_WRITE_WITH_IMM: return IB_WC_RDMA_WRITE;
+ case IB_WR_SEND: return IB_WC_SEND;
+ case IB_WR_SEND_WITH_IMM: return IB_WC_SEND;
+ case IB_WR_RDMA_READ: return IB_WC_RDMA_READ;
+ case IB_WR_ATOMIC_CMP_AND_SWP: return IB_WC_COMP_SWAP;
+ case IB_WR_ATOMIC_FETCH_AND_ADD: return IB_WC_FETCH_ADD;
+ case IB_WR_LSO: return IB_WC_LSO;
+ case IB_WR_SEND_WITH_INV: return IB_WC_SEND;
+ case IB_WR_RDMA_READ_WITH_INV: return IB_WC_RDMA_READ;
+ case IB_WR_LOCAL_INV: return IB_WC_LOCAL_INV;
+ case IB_WR_REG_MR: return IB_WC_REG_MR;
+
+ default:
+ return 0xff;
+ }
+}
+
+
+void retransmit_timer(struct timer_list *t)
+{
+ struct rfc_qp *qp = from_timer(qp, t, retrans_timer);
+
+ if (qp->valid) {
+ qp->comp.timeout = 1;
+ rfc_run_task(&qp->comp.task, 1);
+ }
+}
+
+void retransmit_timer_direct(struct timer_list *t)
+{
+ pr_debug("RFC: retransmit timer fired\n");
+}
+
+void rfc_comp_queue_send_obj(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ list_add_tail(&obj->list, &qp->resp_objs);
+
+ rfc_run_task(&qp->comp.task, 1);
+}
+
+static inline enum comp_state get_wqe_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe **wqe_p)
+{
+ struct rfc_send_wqe *wqe;
+ enum comp_state rstate = 0;
+
+ /* we come here whether or not we found a response object to see if
+ * there are any posted WQEs
+ */
+ wqe = queue_head(qp->sq.queue);
+ *wqe_p = wqe;
+
+ /* no WQE or requester has not started it yet */
+ if (!wqe || wqe->state == wqe_state_posted) {
+ rstate = (obj ? COMPST_DONE : COMPST_EXIT);
+ goto done;
+ }
+ /* WQE does not require an ack */
+ if (wqe->state == wqe_state_done) {
+
+ if (qp_type(qp) == IB_QPT_RC)
+ rstate = COMPST_COMP_ACK;
+ else
+ rstate = COMPST_COMP_WQE;
+
+ goto done;
+ }
+
+ /* WQE caused an error */
+ if (wqe->state == wqe_state_error) {
+ rstate = COMPST_ERROR;
+ goto done;
+ }
+
+ /* we have a WQE, if we also have an ack check its PSN */
+ rstate = (obj ? COMPST_CHECK_PSN : COMPST_EXIT);
+
+done:
+ return rstate;
+}
+
+static inline void reset_retry_counters(struct rfc_qp *qp)
+{
+ qp->comp.retry_cnt = qp->attr.retry_cnt;
+ qp->comp.rnr_retry = qp->attr.rnr_retry;
+}
+
+static inline enum comp_state check_psn(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe)
+{
+ s32 diff;
+
+ /* check to see if response is past the oldest WQE. if it is, complete
+ * send/write or error read/atomic
+ */
+ diff = psn_compare(obj->psn, wqe->last_psn);
+ if (diff > 0) {
+ if (wqe->state == wqe_state_pending) {
+
+ if (wqe->mask & WR_ATOMIC_OR_READ_MASK)
+ return COMPST_ERROR_RETRY;
+
+ reset_retry_counters(qp);
+ return COMPST_COMP_WQE;
+ } else {
+ return COMPST_DONE;
+ }
+ }
+
+ /* compare response object to expected response */
+ diff = psn_compare(obj->psn, qp->comp.psn);
+ if (diff < 0) {
+ /* response is most likely a retried object if it matches an
+ * uncompleted WQE go complete it else ignore it
+ */
+ if (obj->psn == wqe->last_psn)
+ return COMPST_COMP_ACK;
+ else
+ return COMPST_DONE;
+ } else if ((diff > 0) && (wqe->mask & WR_ATOMIC_OR_READ_MASK)) {
+ return COMPST_DONE;
+ } else {
+ return COMPST_CHECK_ACK;
+ }
+}
+
+
+inline enum comp_state check_ack_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe,
+ u8 syn,
+ u32 msn)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+
+
+ if (!obj->status) { // IO SUCCESS
+ if (obj->mask & RFC_ATMETH_MASK) {
+ reset_retry_counters(qp);
+ return COMPST_ATOMIC;
+ }
+ syn = AETH_ACK;
+ msn = 0;
+ }
+ if (obj->status && obj->status != NVME_SC_INTERNAL) {
+ pr_warn("unexpected IO error: status %d\n", obj->status);
+ wqe->status = IB_WC_REM_OP_ERR;
+ return COMPST_ERROR;
+ }
+
+ /* Handle IO errors/Aborts */
+ switch (syn & AETH_TYPE_MASK) {
+ case AETH_ACK:
+ reset_retry_counters(qp);
+ return COMPST_COMP_ACK;
+
+ case AETH_RNR_NAK:
+ rfc_counter_inc(rfc, RFC_CNT_RCV_RNR);
+ return COMPST_RNR_RETRY;
+
+ case AETH_NAK:
+ switch (syn) {
+ case AETH_NAK_PSN_SEQ_ERROR:
+ /* a nak implicitly acks all objects with psns
+ * before
+ */
+ if (psn_compare(obj->psn, qp->comp.psn) > 0) {
+ rfc_counter_inc(rfc,
+ RFC_CNT_RCV_SEQ_ERR);
+ // TBD: check responder state m/c
+ qp->comp.psn = (msn) & 0x00ffffff;
+ if (qp->req.wait_psn) {
+ qp->req.wait_psn = 0;
+ rfc_run_task(&qp->req.task, 1);
+ }
+ }
+ return COMPST_ERROR_RETRY;
+
+ case AETH_NAK_INVALID_REQ:
+ wqe->status = IB_WC_REM_INV_REQ_ERR;
+ return COMPST_ERROR;
+
+ case AETH_NAK_REM_ACC_ERR:
+ wqe->status = IB_WC_REM_ACCESS_ERR;
+ return COMPST_ERROR;
+
+ case AETH_NAK_REM_OP_ERR:
+ wqe->status = IB_WC_REM_OP_ERR;
+ return COMPST_ERROR;
+
+ default:
+ pr_warn("unexpected nak %x\n", syn);
+ wqe->status = IB_WC_REM_OP_ERR;
+ return COMPST_ERROR;
+ }
+
+ default:
+ return COMPST_ERROR;
+ }
+
+}
+
+static struct sg_table *rfc_comp_alloc_sgl(int nents)
+{
+ struct sg_table *temprsg = NULL;
+ int npages;
+
+ temprsg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+
+ if (unlikely(!temprsg))
+ return NULL;
+
+ npages = SG_MAX_SINGLE_ALLOC;
+ if (sg_alloc_table(temprsg, npages, GFP_KERNEL)) {
+ kfree(temprsg);
+ return NULL;
+ }
+ temprsg->orig_nents = SG_MAX_SINGLE_ALLOC;
+ temprsg->nents = 0;
+ return temprsg;
+}
+
+static inline enum comp_state rfc_comp_get_read_sgl(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe,
+ struct sg_table **rsg)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct sg_table *temprsg = NULL;
+ int ret;
+
+ temprsg = rfc_comp_alloc_sgl(SG_MAX_SINGLE_ALLOC);
+ *rsg = temprsg;
+ ret = copy_sgl(rfc, qp->pd, IB_ACCESS_LOCAL_WRITE,
+ &wqe->dma, temprsg,
+ payload_size(obj), to_mem_obj, NULL);
+ if (ret)
+ return COMPST_ERROR;
+
+ if (wqe->dma.resid == 0 && (obj->mask & RFC_END_MASK))
+ return COMPST_COMP_ACK;
+ else
+ return COMPST_UPDATE_COMP;
+}
+static inline enum comp_state find_comp_next_state(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe)
+{
+ if (wqe->dma.resid == 0 && (obj->mask & RFC_END_MASK))
+ return COMPST_COMP_ACK;
+ else {
+ return COMPST_COMP_ACK;
+ // return COMPST_UPDATE_COMP;
+ }
+}
+
+static void make_send_cqe(struct rfc_qp *qp, struct rfc_send_wqe *wqe,
+ struct rfc_cqe *cqe)
+{
+ memset(cqe, 0, sizeof(*cqe));
+
+ if (!qp->is_user) {
+ struct ib_wc *wc = &cqe->ibwc;
+
+ wc->wr_id = wqe->wr.wr_id;
+ wc->status = wqe->status;
+ wc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+ if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+ wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+ wc->wc_flags = IB_WC_WITH_IMM;
+ wc->byte_len = wqe->dma.length;
+ wc->qp = &qp->ibqp;
+ } else {
+ struct ib_uverbs_wc *uwc = &cqe->uibwc;
+
+ uwc->wr_id = wqe->wr.wr_id;
+ uwc->status = wqe->status;
+ uwc->opcode = wr_to_wc_opcode(wqe->wr.opcode);
+ if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
+ wqe->wr.opcode == IB_WR_SEND_WITH_IMM)
+ uwc->wc_flags = IB_WC_WITH_IMM;
+ uwc->byte_len = wqe->dma.length;
+ uwc->qp_num = qp->ibqp.qp_num;
+ }
+}
+
+/*
+ * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS
+ * ---------8<---------8<-------------
+ * ...Note that if a completion error occurs, a Work Completion
+ * will always be generated, even if the signaling
+ * indicator requests an Unsignaled Completion.
+ * ---------8<---------8<-------------
+ */
+void rfc_comp_do_complete(struct rfc_qp *qp, struct rfc_send_wqe *wqe)
+{
+ struct rfc_cqe cqe;
+ unsigned int sq_idx = 0;
+
+ sq_idx = consumer_index(qp->sq.queue);
+
+ if ((qp->sq_sig_type == IB_SIGNAL_ALL_WR) ||
+ (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
+ wqe->status != IB_WC_SUCCESS) {
+ make_send_cqe(qp, wqe, &cqe);
+ advance_consumer(qp->sq.queue);
+ rfc_cq_post(qp->scq, &cqe, 0);
+ } else {
+ advance_consumer(qp->sq.queue);
+ }
+
+ /*
+ * we completed something so let req run again
+ * if it is trying to fence
+ */
+ if (qp->req.wait_fence) {
+ qp->req.wait_fence = 0;
+ rfc_run_task(&qp->req.task, 1);
+ }
+}
+
+static inline enum comp_state complete_ack_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe)
+{
+ unsigned long flags;
+
+ if (wqe->has_rd_atomic) {
+ wqe->has_rd_atomic = 0;
+ atomic_inc(&qp->req.rd_atomic);
+ if (qp->req.need_rd_atomic) {
+ qp->comp.timeout_retry = 0;
+ qp->req.need_rd_atomic = 0;
+ rfc_run_task(&qp->req.task, 1);
+ }
+ }
+
+ if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
+ /* state_lock used by requester & completer */
+ spin_lock_irqsave(&qp->state_lock, flags);
+ if ((qp->req.state == QP_STATE_DRAIN) &&
+ (qp->comp.psn == qp->req.psn)) {
+ qp->req.state = QP_STATE_DRAINED;
+ spin_unlock_irqrestore(&qp->state_lock, flags);
+
+ if (qp->ibqp.event_handler) {
+ struct ib_event ev;
+
+ ev.device = qp->ibqp.device;
+ ev.element.qp = &qp->ibqp;
+ ev.event = IB_EVENT_SQ_DRAINED;
+ qp->ibqp.event_handler(&ev,
+ qp->ibqp.qp_context);
+ }
+ } else {
+ spin_unlock_irqrestore(&qp->state_lock, flags);
+ }
+ }
+
+
+ rfc_comp_do_complete(qp, wqe);
+
+ return COMPST_UPDATE_COMP;
+}
+
+
+static inline enum comp_state complete_wqe_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_send_wqe *wqe)
+{
+ qp->comp.opcode = -1;
+
+ if (obj) {
+ if (psn_compare(obj->psn, qp->comp.psn) >= 0)
+ qp->comp.psn = (obj->psn + 1) & BTH_PSN_MASK;
+
+ if (qp->req.wait_psn) {
+ qp->req.wait_psn = 0;
+ rfc_run_task(&qp->req.task, 1);
+ }
+ }
+
+ rfc_comp_do_complete(qp, wqe);
+
+ return COMPST_DONE;
+}
+
+static void rfc_drain_resp_objs(struct rfc_qp *qp, bool notify)
+{
+ struct rfc_send_wqe *wqe;
+
+ while ((wqe = queue_head(qp->sq.queue))) {
+ if (notify) {
+ wqe->status = IB_WC_WR_FLUSH_ERR;
+ rfc_comp_do_complete(qp, wqe);
+ } else {
+ advance_consumer(qp->sq.queue);
+ }
+ }
+}
+
+static inline enum comp_state get_comp_obj(struct rfc_qp *qp,
+ struct rfc_obj_info **obj_p)
+{
+ *obj_p = NULL;
+ if (!list_empty(&qp->resp_objs)) {
+ *obj_p = list_first_entry(&qp->resp_objs,
+ struct rfc_obj_info, list);
+ list_del(&((*obj_p)->list));
+ }
+
+ return COMPST_GET_WQE;
+}
+
+int rfc_completer_direct(void *arg)
+{
+ struct rfc_qp *qp = (struct rfc_qp *)arg;
+ struct rfc_obj_info *obj = NULL;
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct rfc_send_wqe *wqe = NULL;
+ enum comp_state state;
+ int status = 0;
+ u64 resp = 0, atomic_orig = 0;
+ u8 syn = 0;
+ u32 msn = 0;
+
+ rfc_add_ref(qp);
+
+ if (!qp->valid || qp->req.state == QP_STATE_ERROR ||
+ qp->req.state == QP_STATE_RESET) {
+ rfc_drain_resp_objs(qp, qp->valid &&
+ qp->req.state == QP_STATE_ERROR);
+ goto exit;
+ }
+
+ if (qp->comp.timeout) {
+ qp->comp.timeout_retry = 1;
+ qp->comp.timeout = 0;
+ } else {
+ qp->comp.timeout_retry = 0;
+ }
+
+ if (qp->req.need_retry)
+ goto exit;
+
+ state = COMPST_GET_ACK;
+
+ while (1) {
+ pr_debug("qp#%d state = %s\n", qp_num(qp),
+ comp_state_name[state]);
+ switch (state) {
+ case COMPST_GET_ACK:
+ get_comp_obj(qp, &obj);
+ if (obj)
+ qp->comp.timeout_retry = 0;
+ state = COMPST_GET_WQE;
+ break;
+
+ case COMPST_GET_WQE:
+// state = get_wqe(qp, obj, &wqe);
+ if (obj) {
+ wqe = obj->wqe;
+ if (qp_type(qp) == IB_QPT_RC)
+ state = COMPST_CHECK_ACK;
+ else
+ state = COMPST_COMP_WQE;
+ } else {
+ wqe = NULL;
+ state = COMPST_EXIT;
+ }
+ break;
+
+ case COMPST_CHECK_PSN:
+ state = check_psn(qp, obj, wqe);
+ break;
+
+ case COMPST_CHECK_ACK:
+ // Both obj and wqe are available now
+ status = obj->status;
+ resp = obj->resp;
+ syn = ((resp & AETH_SYN_MASK) >> 24);
+ msn = resp & AETH_MSN_MASK;
+ state = check_ack_direct(qp, obj, wqe, syn, msn);
+ break;
+
+ case COMPST_ATOMIC:
+ atomic_orig = resp;
+
+ copy_data(obj->rfc, qp->pd, IB_ACCESS_LOCAL_WRITE,
+ &obj->wqe->dma, &atomic_orig,
+ sizeof(u64), to_mem_obj, NULL);
+ state = COMPST_COMP_ACK;
+ break;
+
+ case COMPST_COMP_ACK:
+ if (obj->mask & RFC_END_MASK) {
+ if (obj->wqe->last_psn != obj->psn) {
+ /*
+ * Identify an incomplete READ/WRITE from
+ * dma.resid
+ */
+ } else {
+ obj->wqe->state = wqe_state_done;
+ /*
+ * Send CQE to SCQ and complete the
+ * send wqe processing.
+ */
+ state = complete_ack_direct(qp, obj,
+ wqe);
+ }
+ }
+ // No need to update comp.psn for duplicate object
+ if (psn_compare(obj->psn, qp->comp.psn) >= 0)
+ state = COMPST_UPDATE_COMP;
+ else
+ state = COMPST_DONE;
+ break;
+
+ case COMPST_COMP_WQE:
+ state = complete_wqe_direct(qp, obj, wqe);
+ break;
+
+ case COMPST_UPDATE_COMP:
+ if (obj->mask & RFC_END_MASK)
+ qp->comp.opcode = -1;
+ else
+ qp->comp.opcode = obj->opcode;
+
+ qp->comp.psn = (obj->psn + 1) & BTH_PSN_MASK;
+
+ if (qp->req.wait_psn) {
+ qp->req.wait_psn = 0;
+ rfc_run_task(&qp->req.task, 1);
+ }
+
+ state = COMPST_DONE;
+ break;
+
+ case COMPST_DONE:
+ kfree(obj);
+ obj = NULL;
+ state = COMPST_EXIT;
+ break;
+
+ case COMPST_EXIT:
+ if (qp->comp.timeout_retry && wqe &&
+ (wqe->state == wqe_state_pending ||
+ (wqe->state == wqe_state_processing &&
+ wqe->dma.length > wqe->dma.resid))) {
+ state = COMPST_ERROR_RETRY;
+ break;
+ }
+ /* re reset the timeout counter if
+ * (1) QP is type RC
+ * (2) the QP is alive
+ * (3) there is a object sent by the requester that
+ * might be acked (we still might get spurious
+ * timeouts but try to keep them as few as possible)
+ * (4) the timeout parameter is set
+ */
+ if ((qp_type(qp) == IB_QPT_RC) &&
+ (qp->req.state == QP_STATE_READY) &&
+ (psn_compare(qp->req.psn, qp->comp.psn) > 0) &&
+ qp->qp_timeout_jiffies) {
+ mod_timer(&qp->retrans_timer,
+ jiffies + qp->qp_timeout_jiffies);
+ }
+ goto exit;
+
+ case COMPST_ERROR_RETRY:
+
+ /* there is nothing to retry in this case */
+ if (!wqe || (wqe->state == wqe_state_posted))
+ goto exit;
+
+ if (qp->comp.retry_cnt > 0) {
+ qp->comp.retry_cnt--;
+
+ /* no point in retrying if we have already
+ * seen the last ack that the requester could
+ * have caused
+ */
+ if (psn_compare(qp->req.psn,
+ qp->comp.psn) > 0) {
+ /* tell the requester to retry the
+ * send queue next time around
+ */
+ rfc_counter_inc(rfc,
+ RFC_CNT_COMP_RETRY);
+ qp->req.need_retry = 1;
+ rfc_run_task(&qp->req.task, 1);
+ }
+
+ goto exit;
+
+ } else {
+ rfc_counter_inc(rfc, RFC_CNT_RETRY_EXCEEDED);
+ wqe->status = IB_WC_RETRY_EXC_ERR;
+ state = COMPST_ERROR;
+ }
+ break;
+
+ case COMPST_RNR_RETRY:
+ if (qp->comp.rnr_retry > 0) {
+ qp->comp.rnr_retry--;
+
+ qp->req.need_retry = 1;
+ pr_debug("qp#%d set rnr nak timer\n",
+ qp_num(qp));
+ mod_timer(&qp->rnr_nak_timer,
+ jiffies + rnrnak_jiffies(syn
+ & ~AETH_TYPE_MASK));
+ goto exit;
+ } else {
+ rfc_counter_inc(rfc,
+ RFC_CNT_RNR_RETRY_EXCEEDED);
+ wqe->status = IB_WC_RNR_RETRY_EXC_ERR;
+ state = COMPST_ERROR;
+ }
+ break;
+
+ case COMPST_ERROR:
+ WARN_ON_ONCE(wqe->status == IB_WC_SUCCESS);
+ rfc_comp_do_complete(qp, wqe);
+ rfc_qp_error(qp);
+
+ goto exit;
+
+ default:
+ pr_err("RFC: completer moved to error state. qp#%d\n",
+ qp_num(qp));
+ rfc_qp_error(qp);
+
+ goto exit;
+ }
+ }
+
+exit:
+ /* we come here if we are done with processing and want the task to
+ * exit from the loop calling us
+ */
+ kfree(obj);
+ obj = NULL;
+ rfc_drop_ref(qp);
+ return -EAGAIN;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_cq.c b/drivers/infiniband/sw/rfc/rfc_cq.c
new file mode 100644
index 0000000..e597f9f
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_cq.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+int rfc_cq_chk_attr(struct rfc_dev *rfc, struct rfc_cq *cq,
+ int cqe, int comp_vector)
+{
+ int count;
+
+ if (cqe <= 0) {
+ pr_warn("cqe(%d) <= 0\n", cqe);
+ goto err1;
+ }
+
+ if (cqe > rfc->attr.max_cqe) {
+ pr_warn("cqe(%d) > max_cqe(%d)\n",
+ cqe, rfc->attr.max_cqe);
+ goto err1;
+ }
+
+ if (cq) {
+ count = queue_count(cq->queue);
+ if (cqe < count) {
+ pr_warn("cqe(%d) < current # elements in queue (%d)",
+ cqe, count);
+ goto err1;
+ }
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static void rfc_send_complete(unsigned long data)
+{
+ struct rfc_cq *cq = (struct rfc_cq *)data;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+ if (cq->is_dying) {
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+ return;
+ }
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+ cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
+}
+
+int rfc_cq_from_init(struct rfc_dev *rfc, struct rfc_cq *cq, int cqe,
+ int comp_vector, struct ib_ucontext *context,
+ struct rfc_create_cq_resp __user *uresp)
+{
+ int err;
+
+ cq->queue = rfc_queue_init(rfc, &cqe,
+ sizeof(struct rfc_cqe));
+ if (!cq->queue) {
+ pr_warn("unable to create cq\n");
+ return -ENOMEM;
+ }
+
+ err = do_mmap_info(rfc, uresp ? &uresp->mi : NULL, context,
+ cq->queue->buf, cq->queue->buf_size, &cq->queue->ip);
+ if (err) {
+ kvfree(cq->queue->buf);
+ kfree(cq->queue);
+ return err;
+ }
+
+ if (uresp)
+ cq->is_user = 1;
+
+ cq->is_dying = false;
+
+ tasklet_init(&cq->comp_task, rfc_send_complete, (unsigned long)cq);
+
+ spin_lock_init(&cq->cq_lock);
+ cq->ibcq.cqe = cqe;
+ return 0;
+}
+
+int rfc_cq_resize_queue(struct rfc_cq *cq, int cqe,
+ struct rfc_resize_cq_resp __user *uresp)
+{
+ int err;
+
+ err = rfc_queue_resize(cq->queue, (unsigned int *)&cqe,
+ sizeof(struct rfc_cqe),
+ cq->queue->ip ? cq->queue->ip->context : NULL,
+ uresp ? &uresp->mi : NULL, NULL, &cq->cq_lock);
+ if (!err)
+ cq->ibcq.cqe = cqe;
+
+ return err;
+}
+
+int rfc_cq_post(struct rfc_cq *cq, struct rfc_cqe *cqe, int solicited)
+{
+ struct ib_event ev;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+
+ if (unlikely(queue_full(cq->queue))) {
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+ if (cq->ibcq.event_handler) {
+ ev.device = cq->ibcq.device;
+ ev.element.cq = &cq->ibcq;
+ ev.event = IB_EVENT_CQ_ERR;
+ cq->ibcq.event_handler(&ev, cq->ibcq.cq_context);
+ }
+
+ return -EBUSY;
+ }
+
+ memcpy(producer_addr(cq->queue), cqe, sizeof(*cqe));
+
+ /* make sure all changes to the CQ are written before we update the
+ * producer pointer
+ */
+ smp_wmb();
+
+ advance_producer(cq->queue);
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+ if ((cq->notify == IB_CQ_NEXT_COMP) ||
+ (cq->notify == IB_CQ_SOLICITED && solicited)) {
+ cq->notify = 0;
+ tasklet_schedule(&cq->comp_task);
+ }
+
+ return 0;
+}
+
+void rfc_cq_disable(struct rfc_cq *cq)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+ cq->is_dying = true;
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+}
+
+void rfc_cq_cleanup(struct rfc_pool_entry *arg)
+{
+ struct rfc_cq *cq = container_of(arg, typeof(*cq), pelem);
+
+ if (cq->queue)
+ rfc_queue_cleanup(cq->queue);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_hdr.h b/drivers/infiniband/sw/rfc/rfc_hdr.h
new file mode 100644
index 0000000..279ffdb
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_hdr.h
@@ -0,0 +1,985 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_HDR_H
+#define RFC_HDR_H
+
+struct rfc_obj_info {
+ struct rfc_dev *rfc; /* device that owns object */
+ struct rfc_qp *qp; /* qp that owns object */
+ struct rfc_send_wqe *wqe; /* send wqe */
+ struct list_head list;
+ u8 *hdr; /* points to bth */
+ u32 mask; /* useful info about obj */
+ u32 psn; /* bth psn of object */
+ u16 pkey_index; /* partition of obj */
+ u32 paylen; /* length of bth - icrc */
+ u8 port_num; /* port obj received on */
+ u8 opcode; /* bth opcode of object */
+ u8 offset; /* bth offset from obj->hdr */
+ __be32 saddr; /* SRC IP of received obj */
+ __be32 daddr; /* DST IP of received obj */
+ int status; /* status of RFC IO */
+ u64 resp; /* Response of RFC IO */
+ struct page *inline_page; /* page for inline data */
+};
+
+
+
+/*
+ * IBA header types and methods
+ *
+ * Some of these are for reference and completeness only since
+ * rfc does not currently support RD transport
+ * most of this could be moved into IB core. ib_pack.h has
+ * part of this but is incomplete
+ *
+ * Header specific routines to insert/extract values to/from headers
+ * the routines that are named __hhh_(set_)fff() take a pointer to a
+ * hhh header and get(set) the fff field. The routines named
+ * hhh_(set_)fff take a object info struct and find the
+ * header and field based on the opcode in the object.
+ * Conversion to/from network byte order from cpu order is also done.
+ */
+
+#define RFC_ICRC_SIZE (4)
+#define RFC_MAX_HDR_LENGTH (80)
+
+/******************************************************************************
+ * Base Transport Header
+ ******************************************************************************/
+struct rfc_bth {
+ u8 opcode;
+ u8 flags;
+ __be16 pkey;
+ __be32 qpn;
+ __be32 apsn;
+};
+
+#define BTH_TVER (0)
+#define BTH_DEF_PKEY (0xffff)
+
+#define BTH_SE_MASK (0x80)
+#define BTH_MIG_MASK (0x40)
+#define BTH_PAD_MASK (0x30)
+#define BTH_TVER_MASK (0x0f)
+#define BTH_FECN_MASK (0x80000000)
+#define BTH_BECN_MASK (0x40000000)
+#define BTH_RESV6A_MASK (0x3f000000)
+#define BTH_QPN_MASK (0x00ffffff)
+#define BTH_ACK_MASK (0x80000000)
+#define BTH_RESV7_MASK (0x7f000000)
+#define BTH_PSN_MASK (0x00ffffff)
+
+static inline u8 __bth_opcode(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return bth->opcode;
+}
+
+static inline void __bth_set_opcode(void *arg, u8 opcode)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->opcode = opcode;
+}
+
+static inline u8 __bth_se(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return 0 != (BTH_SE_MASK & bth->flags);
+}
+
+static inline void __bth_set_se(void *arg, int se)
+{
+ struct rfc_bth *bth = arg;
+
+ if (se)
+ bth->flags |= BTH_SE_MASK;
+ else
+ bth->flags &= ~BTH_SE_MASK;
+}
+
+static inline u8 __bth_mig(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return 0 != (BTH_MIG_MASK & bth->flags);
+}
+
+static inline void __bth_set_mig(void *arg, u8 mig)
+{
+ struct rfc_bth *bth = arg;
+
+ if (mig)
+ bth->flags |= BTH_MIG_MASK;
+ else
+ bth->flags &= ~BTH_MIG_MASK;
+}
+
+static inline u8 __bth_pad(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return (BTH_PAD_MASK & bth->flags) >> 4;
+}
+
+static inline void __bth_set_pad(void *arg, u8 pad)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->flags = (BTH_PAD_MASK & (pad << 4)) |
+ (~BTH_PAD_MASK & bth->flags);
+}
+
+static inline u8 __bth_tver(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return BTH_TVER_MASK & bth->flags;
+}
+
+static inline void __bth_set_tver(void *arg, u8 tver)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->flags = (BTH_TVER_MASK & tver) |
+ (~BTH_TVER_MASK & bth->flags);
+}
+
+static inline u16 __bth_pkey(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return be16_to_cpu(bth->pkey);
+}
+
+static inline void __bth_set_pkey(void *arg, u16 pkey)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->pkey = cpu_to_be16(pkey);
+}
+
+static inline u32 __bth_qpn(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return BTH_QPN_MASK & be32_to_cpu(bth->qpn);
+}
+
+static inline void __bth_set_qpn(void *arg, u32 qpn)
+{
+ struct rfc_bth *bth = arg;
+ u32 resvqpn = be32_to_cpu(bth->qpn);
+
+ bth->qpn = cpu_to_be32((BTH_QPN_MASK & qpn) |
+ (~BTH_QPN_MASK & resvqpn));
+}
+
+static inline int __bth_fecn(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return 0 != (cpu_to_be32(BTH_FECN_MASK) & bth->qpn);
+}
+
+static inline void __bth_set_fecn(void *arg, int fecn)
+{
+ struct rfc_bth *bth = arg;
+
+ if (fecn)
+ bth->qpn |= cpu_to_be32(BTH_FECN_MASK);
+ else
+ bth->qpn &= ~cpu_to_be32(BTH_FECN_MASK);
+}
+
+static inline int __bth_becn(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return 0 != (cpu_to_be32(BTH_BECN_MASK) & bth->qpn);
+}
+
+static inline void __bth_set_becn(void *arg, int becn)
+{
+ struct rfc_bth *bth = arg;
+
+ if (becn)
+ bth->qpn |= cpu_to_be32(BTH_BECN_MASK);
+ else
+ bth->qpn &= ~cpu_to_be32(BTH_BECN_MASK);
+}
+
+static inline u8 __bth_resv6a(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return (BTH_RESV6A_MASK & be32_to_cpu(bth->qpn)) >> 24;
+}
+
+static inline void __bth_set_resv6a(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->qpn = cpu_to_be32(~BTH_RESV6A_MASK);
+}
+
+static inline int __bth_ack(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return 0 != (cpu_to_be32(BTH_ACK_MASK) & bth->apsn);
+}
+
+static inline void __bth_set_ack(void *arg, int ack)
+{
+ struct rfc_bth *bth = arg;
+
+ if (ack)
+ bth->apsn |= cpu_to_be32(BTH_ACK_MASK);
+ else
+ bth->apsn &= ~cpu_to_be32(BTH_ACK_MASK);
+}
+
+static inline void __bth_set_resv7(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ bth->apsn &= ~cpu_to_be32(BTH_RESV7_MASK);
+}
+
+static inline u32 __bth_psn(void *arg)
+{
+ struct rfc_bth *bth = arg;
+
+ return BTH_PSN_MASK & be32_to_cpu(bth->apsn);
+}
+
+static inline void __bth_set_psn(void *arg, u32 psn)
+{
+ struct rfc_bth *bth = arg;
+ u32 apsn = be32_to_cpu(bth->apsn);
+
+ bth->apsn = cpu_to_be32((BTH_PSN_MASK & psn) |
+ (~BTH_PSN_MASK & apsn));
+}
+
+static inline u8 bth_opcode(struct rfc_obj_info *obj)
+{
+ return __bth_opcode(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_opcode(struct rfc_obj_info *obj, u8 opcode)
+{
+ __bth_set_opcode(obj->hdr + obj->offset, opcode);
+}
+
+static inline u8 bth_se(struct rfc_obj_info *obj)
+{
+ return __bth_se(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_se(struct rfc_obj_info *obj, int se)
+{
+ __bth_set_se(obj->hdr + obj->offset, se);
+}
+
+static inline u8 bth_mig(struct rfc_obj_info *obj)
+{
+ return __bth_mig(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_mig(struct rfc_obj_info *obj, u8 mig)
+{
+ __bth_set_mig(obj->hdr + obj->offset, mig);
+}
+
+static inline u8 bth_pad(struct rfc_obj_info *obj)
+{
+ return __bth_pad(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_pad(struct rfc_obj_info *obj, u8 pad)
+{
+ __bth_set_pad(obj->hdr + obj->offset, pad);
+}
+
+static inline u8 bth_tver(struct rfc_obj_info *obj)
+{
+ return __bth_tver(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_tver(struct rfc_obj_info *obj, u8 tver)
+{
+ __bth_set_tver(obj->hdr + obj->offset, tver);
+}
+
+static inline u16 bth_pkey(struct rfc_obj_info *obj)
+{
+ return __bth_pkey(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_pkey(struct rfc_obj_info *obj, u16 pkey)
+{
+ __bth_set_pkey(obj->hdr + obj->offset, pkey);
+}
+
+static inline u32 bth_qpn(struct rfc_obj_info *obj)
+{
+ return __bth_qpn(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_qpn(struct rfc_obj_info *obj, u32 qpn)
+{
+ __bth_set_qpn(obj->hdr + obj->offset, qpn);
+}
+
+static inline int bth_fecn(struct rfc_obj_info *obj)
+{
+ return __bth_fecn(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_fecn(struct rfc_obj_info *obj, int fecn)
+{
+ __bth_set_fecn(obj->hdr + obj->offset, fecn);
+}
+
+static inline int bth_becn(struct rfc_obj_info *obj)
+{
+ return __bth_becn(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_becn(struct rfc_obj_info *obj, int becn)
+{
+ __bth_set_becn(obj->hdr + obj->offset, becn);
+}
+
+static inline u8 bth_resv6a(struct rfc_obj_info *obj)
+{
+ return __bth_resv6a(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_resv6a(struct rfc_obj_info *obj)
+{
+ __bth_set_resv6a(obj->hdr + obj->offset);
+}
+
+static inline int bth_ack(struct rfc_obj_info *obj)
+{
+ return __bth_ack(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_ack(struct rfc_obj_info *obj, int ack)
+{
+ __bth_set_ack(obj->hdr + obj->offset, ack);
+}
+
+static inline void bth_set_resv7(struct rfc_obj_info *obj)
+{
+ __bth_set_resv7(obj->hdr + obj->offset);
+}
+
+static inline u32 bth_psn(struct rfc_obj_info *obj)
+{
+ return __bth_psn(obj->hdr + obj->offset);
+}
+
+static inline void bth_set_psn(struct rfc_obj_info *obj, u32 psn)
+{
+ __bth_set_psn(obj->hdr + obj->offset, psn);
+}
+
+static inline void bth_init(struct rfc_obj_info *obj, u8 opcode, int se,
+ int mig, int pad, u16 pkey, u32 qpn, int ack_req,
+ u32 psn)
+{
+ struct rfc_bth *bth = (struct rfc_bth *)(obj->hdr + obj->offset);
+
+ bth->opcode = opcode;
+ bth->flags = (pad << 4) & BTH_PAD_MASK;
+ if (se)
+ bth->flags |= BTH_SE_MASK;
+ if (mig)
+ bth->flags |= BTH_MIG_MASK;
+ bth->pkey = cpu_to_be16(pkey);
+ bth->qpn = cpu_to_be32(qpn & BTH_QPN_MASK);
+ psn &= BTH_PSN_MASK;
+ if (ack_req)
+ psn |= BTH_ACK_MASK;
+ bth->apsn = cpu_to_be32(psn);
+}
+
+/******************************************************************************
+ * Reliable Datagram Extended Transport Header
+ ******************************************************************************/
+struct rfc_rdeth {
+ __be32 een;
+};
+
+#define RDETH_EEN_MASK (0x00ffffff)
+
+static inline u8 __rdeth_een(void *arg)
+{
+ struct rfc_rdeth *rdeth = arg;
+
+ return RDETH_EEN_MASK & be32_to_cpu(rdeth->een);
+}
+
+static inline void __rdeth_set_een(void *arg, u32 een)
+{
+ struct rfc_rdeth *rdeth = arg;
+
+ rdeth->een = cpu_to_be32(RDETH_EEN_MASK & een);
+}
+
+static inline u8 rdeth_een(struct rfc_obj_info *obj)
+{
+ return __rdeth_een(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RDETH]);
+}
+
+static inline void rdeth_set_een(struct rfc_obj_info *obj, u32 een)
+{
+ __rdeth_set_een(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RDETH], een);
+}
+
+/******************************************************************************
+ * Datagram Extended Transport Header
+ ******************************************************************************/
+struct rfc_deth {
+ __be32 qkey;
+ __be32 sqp;
+};
+
+#define GSI_QKEY (0x80010000)
+#define DETH_SQP_MASK (0x00ffffff)
+
+static inline u32 __deth_qkey(void *arg)
+{
+ struct rfc_deth *deth = arg;
+
+ return be32_to_cpu(deth->qkey);
+}
+
+static inline void __deth_set_qkey(void *arg, u32 qkey)
+{
+ struct rfc_deth *deth = arg;
+
+ deth->qkey = cpu_to_be32(qkey);
+}
+
+static inline u32 __deth_sqp(void *arg)
+{
+ struct rfc_deth *deth = arg;
+
+ return DETH_SQP_MASK & be32_to_cpu(deth->sqp);
+}
+
+static inline void __deth_set_sqp(void *arg, u32 sqp)
+{
+ struct rfc_deth *deth = arg;
+
+ deth->sqp = cpu_to_be32(DETH_SQP_MASK & sqp);
+}
+
+static inline u32 deth_qkey(struct rfc_obj_info *obj)
+{
+ return __deth_qkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_DETH]);
+}
+
+static inline void deth_set_qkey(struct rfc_obj_info *obj, u32 qkey)
+{
+ __deth_set_qkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_DETH], qkey);
+}
+
+static inline u32 deth_sqp(struct rfc_obj_info *obj)
+{
+ return __deth_sqp(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_DETH]);
+}
+
+static inline void deth_set_sqp(struct rfc_obj_info *obj, u32 sqp)
+{
+ __deth_set_sqp(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_DETH], sqp);
+}
+
+/******************************************************************************
+ * RDMA Extended Transport Header
+ ******************************************************************************/
+struct rfc_reth {
+ __be64 va;
+ __be32 rkey;
+ __be32 len;
+};
+
+static inline u64 __reth_va(void *arg)
+{
+ struct rfc_reth *reth = arg;
+
+ return be64_to_cpu(reth->va);
+}
+
+static inline void __reth_set_va(void *arg, u64 va)
+{
+ struct rfc_reth *reth = arg;
+
+ reth->va = cpu_to_be64(va);
+}
+
+static inline u32 __reth_rkey(void *arg)
+{
+ struct rfc_reth *reth = arg;
+
+ return be32_to_cpu(reth->rkey);
+}
+
+static inline void __reth_set_rkey(void *arg, u32 rkey)
+{
+ struct rfc_reth *reth = arg;
+
+ reth->rkey = cpu_to_be32(rkey);
+}
+
+static inline u32 __reth_len(void *arg)
+{
+ struct rfc_reth *reth = arg;
+
+ return be32_to_cpu(reth->len);
+}
+
+static inline void __reth_set_len(void *arg, u32 len)
+{
+ struct rfc_reth *reth = arg;
+
+ reth->len = cpu_to_be32(len);
+}
+
+static inline u64 reth_va(struct rfc_obj_info *obj)
+{
+ return __reth_va(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH]);
+}
+
+static inline void reth_set_va(struct rfc_obj_info *obj, u64 va)
+{
+ __reth_set_va(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH], va);
+}
+
+static inline u32 reth_rkey(struct rfc_obj_info *obj)
+{
+ return __reth_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH]);
+}
+
+static inline void reth_set_rkey(struct rfc_obj_info *obj, u32 rkey)
+{
+ __reth_set_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH], rkey);
+}
+
+static inline u32 reth_len(struct rfc_obj_info *obj)
+{
+ return __reth_len(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH]);
+}
+
+static inline void reth_set_len(struct rfc_obj_info *obj, u32 len)
+{
+ __reth_set_len(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_RETH], len);
+}
+
+/******************************************************************************
+ * Atomic Extended Transport Header
+ ******************************************************************************/
+struct rfc_atmeth {
+ __be64 va;
+ __be32 rkey;
+ __be64 swap_add;
+ __be64 comp;
+} __attribute__((__packed__));
+
+static inline u64 __atmeth_va(void *arg)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ return be64_to_cpu(atmeth->va);
+}
+
+static inline void __atmeth_set_va(void *arg, u64 va)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ atmeth->va = cpu_to_be64(va);
+}
+
+static inline u32 __atmeth_rkey(void *arg)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ return be32_to_cpu(atmeth->rkey);
+}
+
+static inline void __atmeth_set_rkey(void *arg, u32 rkey)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ atmeth->rkey = cpu_to_be32(rkey);
+}
+
+static inline u64 __atmeth_swap_add(void *arg)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ return be64_to_cpu(atmeth->swap_add);
+}
+
+static inline void __atmeth_set_swap_add(void *arg, u64 swap_add)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ atmeth->swap_add = cpu_to_be64(swap_add);
+}
+
+static inline u64 __atmeth_comp(void *arg)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ return be64_to_cpu(atmeth->comp);
+}
+
+static inline void __atmeth_set_comp(void *arg, u64 comp)
+{
+ struct rfc_atmeth *atmeth = arg;
+
+ atmeth->comp = cpu_to_be64(comp);
+}
+
+static inline u64 atmeth_va(struct rfc_obj_info *obj)
+{
+ return __atmeth_va(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH]);
+}
+
+static inline void atmeth_set_va(struct rfc_obj_info *obj, u64 va)
+{
+ __atmeth_set_va(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH], va);
+}
+
+static inline u32 atmeth_rkey(struct rfc_obj_info *obj)
+{
+ return __atmeth_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH]);
+}
+
+static inline void atmeth_set_rkey(struct rfc_obj_info *obj, u32 rkey)
+{
+ __atmeth_set_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH], rkey);
+}
+
+static inline u64 atmeth_swap_add(struct rfc_obj_info *obj)
+{
+ return __atmeth_swap_add(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH]);
+}
+
+static inline void atmeth_set_swap_add(struct rfc_obj_info *obj, u64 swap_add)
+{
+ __atmeth_set_swap_add(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH], swap_add);
+}
+
+static inline u64 atmeth_comp(struct rfc_obj_info *obj)
+{
+ return __atmeth_comp(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH]);
+}
+
+static inline void atmeth_set_comp(struct rfc_obj_info *obj, u64 comp)
+{
+ __atmeth_set_comp(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMETH], comp);
+}
+
+/******************************************************************************
+ * Ack Extended Transport Header
+ ******************************************************************************/
+struct rfc_aeth {
+ __be32 smsn;
+};
+
+#define AETH_SYN_MASK (0xff000000)
+#define AETH_MSN_MASK (0x00ffffff)
+
+enum aeth_syndrome {
+ AETH_TYPE_MASK = 0xe0,
+ AETH_ACK = 0x00,
+ AETH_RNR_NAK = 0x20,
+ AETH_RSVD = 0x40,
+ AETH_NAK = 0x60,
+ AETH_ACK_UNLIMITED = 0x1f,
+ AETH_NAK_PSN_SEQ_ERROR = 0x60,
+ AETH_NAK_INVALID_REQ = 0x61,
+ AETH_NAK_REM_ACC_ERR = 0x62,
+ AETH_NAK_REM_OP_ERR = 0x63,
+ AETH_NAK_INV_RD_REQ = 0x64,
+};
+
+static inline u8 __aeth_syn(void *arg)
+{
+ struct rfc_aeth *aeth = arg;
+
+ return (AETH_SYN_MASK & be32_to_cpu(aeth->smsn)) >> 24;
+}
+
+static inline void __aeth_set_syn(void *arg, u8 syn)
+{
+ struct rfc_aeth *aeth = arg;
+ u32 smsn = be32_to_cpu(aeth->smsn);
+
+ aeth->smsn = cpu_to_be32((AETH_SYN_MASK & (syn << 24)) |
+ (~AETH_SYN_MASK & smsn));
+}
+
+static inline u32 __aeth_msn(void *arg)
+{
+ struct rfc_aeth *aeth = arg;
+
+ return AETH_MSN_MASK & be32_to_cpu(aeth->smsn);
+}
+
+static inline void __aeth_set_msn(void *arg, u32 msn)
+{
+ struct rfc_aeth *aeth = arg;
+ u32 smsn = be32_to_cpu(aeth->smsn);
+
+ aeth->smsn = cpu_to_be32((AETH_MSN_MASK & msn) |
+ (~AETH_MSN_MASK & smsn));
+}
+
+static inline u8 aeth_syn(struct rfc_obj_info *obj)
+{
+ return __aeth_syn(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_AETH]);
+}
+
+static inline void aeth_set_syn(struct rfc_obj_info *obj, u8 syn)
+{
+ __aeth_set_syn(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_AETH], syn);
+}
+
+static inline u32 aeth_msn(struct rfc_obj_info *obj)
+{
+ return __aeth_msn(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_AETH]);
+}
+
+static inline void aeth_set_msn(struct rfc_obj_info *obj, u32 msn)
+{
+ __aeth_set_msn(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_AETH], msn);
+}
+
+/******************************************************************************
+ * Atomic Ack Extended Transport Header
+ ******************************************************************************/
+struct rfc_atmack {
+ __be64 orig;
+};
+
+static inline u64 __atmack_orig(void *arg)
+{
+ struct rfc_atmack *atmack = arg;
+
+ return be64_to_cpu(atmack->orig);
+}
+
+static inline void __atmack_set_orig(void *arg, u64 orig)
+{
+ struct rfc_atmack *atmack = arg;
+
+ atmack->orig = cpu_to_be64(orig);
+}
+
+static inline u64 atmack_orig(struct rfc_obj_info *obj)
+{
+ return __atmack_orig(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMACK]);
+}
+
+static inline void atmack_set_orig(struct rfc_obj_info *obj, u64 orig)
+{
+ __atmack_set_orig(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_ATMACK], orig);
+}
+
+/******************************************************************************
+ * Immediate Extended Transport Header
+ ******************************************************************************/
+struct rfc_immdt {
+ __be32 imm;
+};
+
+static inline __be32 __immdt_imm(void *arg)
+{
+ struct rfc_immdt *immdt = arg;
+
+ return immdt->imm;
+}
+
+static inline void __immdt_set_imm(void *arg, __be32 imm)
+{
+ struct rfc_immdt *immdt = arg;
+
+ immdt->imm = imm;
+}
+
+static inline __be32 immdt_imm(struct rfc_obj_info *obj)
+{
+ return __immdt_imm(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_IMMDT]);
+}
+
+static inline void immdt_set_imm(struct rfc_obj_info *obj, __be32 imm)
+{
+ __immdt_set_imm(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_IMMDT], imm);
+}
+
+/******************************************************************************
+ * Invalidate Extended Transport Header
+ ******************************************************************************/
+struct rfc_ieth {
+ __be32 rkey;
+};
+
+static inline u32 __ieth_rkey(void *arg)
+{
+ struct rfc_ieth *ieth = arg;
+
+ return be32_to_cpu(ieth->rkey);
+}
+
+static inline void __ieth_set_rkey(void *arg, u32 rkey)
+{
+ struct rfc_ieth *ieth = arg;
+
+ ieth->rkey = cpu_to_be32(rkey);
+}
+
+static inline u32 ieth_rkey(struct rfc_obj_info *obj)
+{
+ return __ieth_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_IETH]);
+}
+
+static inline void ieth_set_rkey(struct rfc_obj_info *obj, u32 rkey)
+{
+ __ieth_set_rkey(obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_IETH], rkey);
+}
+
+enum rfc_hdr_length {
+ RFC_BTH_BYTES = sizeof(struct rfc_bth),
+ RFC_DETH_BYTES = sizeof(struct rfc_deth),
+ RFC_IMMDT_BYTES = sizeof(struct rfc_immdt),
+ RFC_RETH_BYTES = sizeof(struct rfc_reth),
+ RFC_AETH_BYTES = sizeof(struct rfc_aeth),
+ RFC_ATMACK_BYTES = sizeof(struct rfc_atmack),
+ RFC_ATMETH_BYTES = sizeof(struct rfc_atmeth),
+ RFC_IETH_BYTES = sizeof(struct rfc_ieth),
+ RFC_RDETH_BYTES = sizeof(struct rfc_rdeth),
+};
+
+static inline size_t header_size(struct rfc_obj_info *obj)
+{
+ return obj->offset + rfc_opcode[obj->opcode].length;
+}
+
+static inline void *payload_addr(struct rfc_obj_info *obj)
+{
+ return obj->hdr + obj->offset
+ + rfc_opcode[obj->opcode].offset[RFC_PAYLOAD];
+}
+
+static inline size_t payload_size(struct rfc_obj_info *obj)
+{
+ return obj->paylen;
+}
+
+struct rsg_page_info {
+
+ uint32_t offset;
+ uint32_t len;
+};
+
+struct rsg_info {
+
+ int nents;
+ struct rsg_page_info pginfo[0];
+};
+
+#define ROFC_MTU 0x10000 // 64k
+
+#define RFC_RECV_DATA_HDR_SZ (RFC_BTH_BYTES + RFC_AETH_BYTES + \
+ RFC_RETH_BYTES + RFC_ATMETH_BYTES)
+
+struct rfc_recv_data {
+
+ struct list_head rcv_obj;
+ struct rfc_obj_info obj;
+ u8 data[RFC_RECV_DATA_HDR_SZ];
+ int data_valid;
+ uint32_t protocol;
+ uint32_t len;
+ uint32_t transport_header;
+ uint32_t network_header;
+ uint32_t hdr_len;
+ struct page *eth_page;
+
+};
+
+#endif /* RFC_HDR_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_hw_counters.c b/drivers/infiniband/sw/rfc/rfc_hw_counters.c
new file mode 100644
index 0000000..f90ce1e
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_hw_counters.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_hw_counters.h"
+
+static const char * const rfc_counter_name[] = {
+ [RFC_CNT_SENT_PKTS] = "sent_objs",
+ [RFC_CNT_RCVD_PKTS] = "rcvd_objs",
+ [RFC_CNT_DUP_REQ] = "duplicate_request",
+ [RFC_CNT_OUT_OF_SEQ_REQ] = "out_of_sequence",
+ [RFC_CNT_RCV_RNR] = "rcvd_rnr_err",
+ [RFC_CNT_SND_RNR] = "send_rnr_err",
+ [RFC_CNT_RCV_SEQ_ERR] = "rcvd_seq_err",
+ [RFC_CNT_COMPLETER_SCHED] = "ack_deffered",
+ [RFC_CNT_RETRY_EXCEEDED] = "retry_exceeded_err",
+ [RFC_CNT_RNR_RETRY_EXCEEDED] = "retry_rnr_exceeded_err",
+ [RFC_CNT_COMP_RETRY] = "completer_retry_err",
+ [RFC_CNT_SEND_ERR] = "send_err",
+};
+
+int rfc_ib_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port, int index)
+{
+ struct rfc_dev *dev = to_rdev(ibdev);
+ unsigned int cnt;
+
+ if (!port || !stats)
+ return -EINVAL;
+
+ for (cnt = 0; cnt < ARRAY_SIZE(rfc_counter_name); cnt++)
+ stats->value[cnt] = dev->stats_counters[cnt];
+
+ return ARRAY_SIZE(rfc_counter_name);
+}
+
+struct rdma_hw_stats *rfc_ib_alloc_hw_stats(struct ib_device *ibdev,
+ u8 port_num)
+{
+ BUILD_BUG_ON(ARRAY_SIZE(rfc_counter_name) != RFC_NUM_OF_COUNTERS);
+ /* We support only per port stats */
+ if (!port_num)
+ return NULL;
+
+ return rdma_alloc_hw_stats_struct(rfc_counter_name,
+ ARRAY_SIZE(rfc_counter_name),
+ RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_hw_counters.h b/drivers/infiniband/sw/rfc/rfc_hw_counters.h
new file mode 100644
index 0000000..ffc7684
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_hw_counters.h
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_HW_COUNTERS_H
+#define RFC_HW_COUNTERS_H
+
+/*
+ * when adding counters to enum also add
+ * them to rfc_counter_name[] vector.
+ */
+enum rfc_counters {
+ RFC_CNT_SENT_PKTS,
+ RFC_CNT_RCVD_PKTS,
+ RFC_CNT_DUP_REQ,
+ RFC_CNT_OUT_OF_SEQ_REQ,
+ RFC_CNT_RCV_RNR,
+ RFC_CNT_SND_RNR,
+ RFC_CNT_RCV_SEQ_ERR,
+ RFC_CNT_COMPLETER_SCHED,
+ RFC_CNT_RETRY_EXCEEDED,
+ RFC_CNT_RNR_RETRY_EXCEEDED,
+ RFC_CNT_COMP_RETRY,
+ RFC_CNT_SEND_ERR,
+ RFC_NUM_OF_COUNTERS
+};
+
+struct rdma_hw_stats *rfc_ib_alloc_hw_stats(struct ib_device *ibdev,
+ u8 port_num);
+int rfc_ib_get_hw_stats(struct ib_device *ibdev,
+ struct rdma_hw_stats *stats,
+ u8 port, int index);
+#endif /* RFC_HW_COUNTERS_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_loc.h b/drivers/infiniband/sw/rfc/rfc_loc.h
new file mode 100644
index 0000000..630508d
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_loc.h
@@ -0,0 +1,289 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_LOC_H
+#define RFC_LOC_H
+
+/* rfc_av.c */
+
+int rfc_av_chk_attr(struct rfc_dev *rfc, struct rdma_ah_attr *attr);
+
+void rfc_av_from_attr(u8 port_num, struct rfc_av *av,
+ struct rdma_ah_attr *attr);
+
+void rfc_av_to_attr(struct rfc_av *av, struct rdma_ah_attr *attr);
+
+void rfc_av_fill_ip_info(struct rfc_av *av,
+ struct rdma_ah_attr *attr,
+ struct ib_gid_attr *sgid_attr,
+ union ib_gid *sgid);
+
+struct rfc_av *rfc_get_av(struct rfc_obj_info *obj);
+
+/* rfc_cq.c */
+int rfc_cq_chk_attr(struct rfc_dev *rfc, struct rfc_cq *cq,
+ int cqe, int comp_vector);
+
+int rfc_cq_from_init(struct rfc_dev *rfc, struct rfc_cq *cq, int cqe,
+ int comp_vector, struct ib_ucontext *context,
+ struct rfc_create_cq_resp __user *uresp);
+
+int rfc_cq_resize_queue(struct rfc_cq *cq, int new_cqe,
+ struct rfc_resize_cq_resp __user *uresp);
+
+int rfc_cq_post(struct rfc_cq *cq, struct rfc_cqe *cqe, int solicited);
+
+void rfc_cq_disable(struct rfc_cq *cq);
+
+void rfc_cq_cleanup(struct rfc_pool_entry *arg);
+
+/* rfc_mmap.c */
+struct rfc_mmap_info {
+ struct list_head pending_mmaps;
+ struct ib_ucontext *context;
+ struct kref ref;
+ void *obj;
+
+ struct mminfo info;
+};
+
+void rfc_mmap_release(struct kref *ref);
+
+struct rfc_mmap_info *rfc_create_mmap_info(struct rfc_dev *dev,
+ u32 size,
+ struct ib_ucontext *context,
+ void *obj);
+
+int rfc_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
+/* rfc_mr.c */
+enum copy_direction {
+ to_mem_obj,
+ from_mem_obj,
+};
+
+int rfc_mem_init_dma(struct rfc_dev *rfc, struct rfc_pd *pd,
+ int access, struct rfc_mem *mem);
+
+int rfc_mem_init_user(struct rfc_dev *rfc, struct rfc_pd *pd, u64 start,
+ u64 length, u64 iova, int access, struct ib_udata *udata,
+ struct rfc_mem *mr);
+
+int rfc_mem_init_fast(struct rfc_dev *rfc, struct rfc_pd *pd,
+ int max_pages, struct rfc_mem *mem);
+
+int rfc_mem_copy(struct rfc_mem *mem, u64 iova, void *addr,
+ int length, enum copy_direction dir, u32 *crcp);
+int rfc_mem_copy_sgl(struct rfc_mem *mem, u64 iova, struct sg_table *rsg,
+ int length, enum copy_direction dir, u32 *crcp);
+
+int copy_data(struct rfc_dev *rfc, struct rfc_pd *pd, int access,
+ struct rfc_dma_info *dma, void *addr, int length,
+ enum copy_direction dir, u32 *crcp);
+int copy_sgl(struct rfc_dev *rfc, struct rfc_pd *pd, int access,
+ struct rfc_dma_info *dma, struct sg_table *rsg, int length,
+ enum copy_direction dir, u32 *crcp);
+
+void *iova_to_vaddr(struct rfc_mem *mem, u64 iova, int length);
+
+enum lookup_type {
+ lookup_local,
+ lookup_remote,
+};
+
+struct rfc_mem *lookup_mem(struct rfc_pd *pd, int access, u32 key,
+ enum lookup_type type);
+
+int mem_check_range(struct rfc_mem *mem, u64 iova, size_t length);
+
+int rfc_mem_map_pages(struct rfc_dev *rfc, struct rfc_mem *mem,
+ u64 *page, int num_pages, u64 iova);
+
+void rfc_mem_cleanup(struct rfc_pool_entry *arg);
+
+int advance_dma_data(struct rfc_dma_info *dma, unsigned int length);
+
+/* rfc_obj.c */
+int rfc_send(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ struct sg_table *rsg);
+struct sg_table *rfc_init_object(struct rfc_dev *rfc, struct rfc_av *av,
+ int paylen, struct rfc_obj_info *obj);
+int rfc_prepare(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ struct sg_table *rsg, u32 *crc);
+enum rdma_link_layer rfc_link_layer(struct rfc_dev *rfc, unsigned int port_num);
+const char *rfc_parent_name(struct rfc_dev *rfc, unsigned int port_num);
+struct device *rfc_dma_device(struct rfc_dev *rfc);
+
+/* rfc_qp.c */
+int rfc_qp_chk_init(struct rfc_dev *rfc, struct ib_qp_init_attr *init);
+
+int rfc_qp_from_init(struct rfc_dev *rfc, struct rfc_qp *qp, struct rfc_pd *pd,
+ struct ib_qp_init_attr *init,
+ struct rfc_create_qp_resp __user *uresp,
+ struct ib_pd *ibpd);
+
+int rfc_qp_to_init(struct rfc_qp *qp, struct ib_qp_init_attr *init);
+
+int rfc_qp_chk_attr(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct ib_qp_attr *attr, int mask);
+
+int rfc_qp_from_attr(struct rfc_qp *qp, struct ib_qp_attr *attr,
+ int mask, struct ib_udata *udata);
+
+int rfc_qp_to_attr(struct rfc_qp *qp, struct ib_qp_attr *attr, int mask);
+
+void rfc_qp_error(struct rfc_qp *qp);
+
+void rfc_qp_destroy(struct rfc_qp *qp);
+
+void rfc_qp_cleanup(struct rfc_pool_entry *arg);
+
+static inline int qp_num(struct rfc_qp *qp)
+{
+ return qp->ibqp.qp_num;
+}
+
+static inline enum ib_qp_type qp_type(struct rfc_qp *qp)
+{
+ return qp->ibqp.qp_type;
+}
+
+static inline enum ib_qp_state qp_state(struct rfc_qp *qp)
+{
+ return qp->attr.qp_state;
+}
+
+static inline int qp_mtu(struct rfc_qp *qp)
+{
+ if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
+ return qp->attr.path_mtu;
+ else
+ return RFC_PORT_MAX_MTU;
+}
+
+static inline int rcv_wqe_size(int max_sge)
+{
+ return sizeof(struct rfc_recv_wqe) +
+ max_sge * sizeof(struct ib_sge);
+}
+
+void free_rd_atomic_resource(struct rfc_qp *qp, struct resp_res *res);
+
+static inline void rfc_advance_resp_resource(struct rfc_qp *qp)
+{
+ qp->resp.res_head++;
+ if (unlikely(qp->resp.res_head == qp->attr.max_dest_rd_atomic))
+ qp->resp.res_head = 0;
+}
+
+void retransmit_timer(struct timer_list *t);
+void retransmit_timer_direct(struct timer_list *t);
+void rnr_nak_timer(struct timer_list *t);
+
+/* rfc_srq.c */
+#define IB_SRQ_INIT_MASK (~IB_SRQ_LIMIT)
+
+int rfc_srq_chk_attr(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_attr *attr, enum ib_srq_attr_mask mask);
+
+int rfc_srq_from_init(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_init_attr *init,
+ struct ib_ucontext *context,
+ struct rfc_create_srq_resp __user *uresp);
+
+int rfc_srq_from_attr(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
+ struct rfc_modify_srq_cmd *ucmd);
+
+void rfc_release(struct kref *kref);
+
+void rfc_drain_recv_queue(struct rfc_qp *qp, bool notify);
+int rfc_completer_direct(void *arg);
+void rfc_comp_do_complete(struct rfc_qp *qp, struct rfc_send_wqe *wqe);
+int rfc_requester(void *arg);
+u64 rfc_responder_done_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj, u64 *resp);
+u64 rfc_responder_hdr(struct rfc_dev *blkrfc, struct rfc_qp *qp,
+ struct rfc_recv_data *rcv, struct sg_table **rsgp);
+
+void rfc_comp_queue_send_obj(struct rfc_dev *rfc,
+ struct rfc_qp *qp, struct rfc_obj_info *obj);
+
+u64 rfc_resp_queue_done(struct rfc_dev *rfc,
+ struct rfc_qp *qp, struct rfc_recv_data *rcv);
+
+
+static inline unsigned int wr_opcode_mask(int opcode, struct rfc_qp *qp)
+{
+ return rfc_wr_opcode_info[opcode].mask[qp->ibqp.qp_type];
+}
+
+
+static inline int rfc_xmit_object(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct sg_table *rsg)
+{
+ int err;
+ int is_request = obj->mask & RFC_REQ_MASK;
+
+ if ((is_request && (qp->req.state != QP_STATE_READY)) ||
+ (!is_request && (qp->resp.state != QP_STATE_READY))) {
+ pr_info("Packet dropped. QP is not in ready state\n");
+ pr_err(" %s():Packet dropped. QP is not in ready state\n",
+ __func__);
+ goto drop;
+ }
+
+ if (obj->mask & RFC_LOOPBACK_MASK)
+ pr_err(" %s(): Loopback not Expected!!!!!\n", __func__);
+ else
+ err = rfc_send(rfc, obj, rsg);
+
+ if (err) {
+ rfc->xmit_errors++;
+ rfc_counter_inc(rfc, RFC_CNT_SEND_ERR);
+ return err;
+ }
+
+ rfc_counter_inc(rfc, RFC_CNT_SENT_PKTS);
+ goto done;
+
+drop:
+ err = 0;
+done:
+ return err;
+}
+
+#endif /* RFC_LOC_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_mcast.c b/drivers/infiniband/sw/rfc/rfc_mcast.c
new file mode 100644
index 0000000..0b0afb6
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_mcast.c
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+
+int rfc_mcast_get_grp(struct rfc_dev *rfc, union ib_gid *mgid,
+ struct rfc_mc_grp **grp_p)
+{
+ int err;
+ struct rfc_mc_grp *grp;
+
+ if (rfc->attr.max_mcast_qp_attach == 0) {
+ err = -EINVAL;
+ goto err1;
+ }
+
+ grp = rfc_pool_get_key(&rfc->mc_grp_pool, mgid);
+ if (grp)
+ goto done;
+
+ grp = rfc_alloc(&rfc->mc_grp_pool);
+ if (!grp) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ INIT_LIST_HEAD(&grp->qp_list);
+ spin_lock_init(&grp->mcg_lock);
+ grp->rfc = rfc;
+
+ rfc_add_key(grp, mgid);
+
+// err = rfc_mcast_add(rfc, mgid);
+ if (err)
+ goto err2;
+
+done:
+ *grp_p = grp;
+ return 0;
+
+err2:
+ rfc_drop_ref(grp);
+err1:
+ return err;
+}
+
+int rfc_mcast_add_grp_elem(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct rfc_mc_grp *grp)
+{
+ int err;
+ struct rfc_mc_elem *elem;
+
+ /* check to see of the qp is already a member of the group */
+ spin_lock_bh(&qp->grp_lock);
+ spin_lock_bh(&grp->mcg_lock);
+ list_for_each_entry(elem, &grp->qp_list, qp_list) {
+ if (elem->qp == qp) {
+ err = 0;
+ goto out;
+ }
+ }
+
+ if (grp->num_qp >= rfc->attr.max_mcast_qp_attach) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ elem = rfc_alloc(&rfc->mc_elem_pool);
+ if (!elem) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /* each qp holds a ref on the grp */
+ rfc_add_ref(grp);
+
+ grp->num_qp++;
+ elem->qp = qp;
+ elem->grp = grp;
+
+ list_add(&elem->qp_list, &grp->qp_list);
+ list_add(&elem->grp_list, &qp->grp_list);
+
+ err = 0;
+out:
+ spin_unlock_bh(&grp->mcg_lock);
+ spin_unlock_bh(&qp->grp_lock);
+ return err;
+}
+
+int rfc_mcast_drop_grp_elem(struct rfc_dev *rfc, struct rfc_qp *qp,
+ union ib_gid *mgid)
+{
+ struct rfc_mc_grp *grp;
+ struct rfc_mc_elem *elem, *tmp;
+
+ grp = rfc_pool_get_key(&rfc->mc_grp_pool, mgid);
+ if (!grp)
+ goto err1;
+
+ spin_lock_bh(&qp->grp_lock);
+ spin_lock_bh(&grp->mcg_lock);
+
+ list_for_each_entry_safe(elem, tmp, &grp->qp_list, qp_list) {
+ if (elem->qp == qp) {
+ list_del(&elem->qp_list);
+ list_del(&elem->grp_list);
+ grp->num_qp--;
+
+ spin_unlock_bh(&grp->mcg_lock);
+ spin_unlock_bh(&qp->grp_lock);
+ rfc_drop_ref(elem);
+ rfc_drop_ref(grp); /* ref held by QP */
+ rfc_drop_ref(grp); /* ref from get_key */
+ return 0;
+ }
+ }
+
+ spin_unlock_bh(&grp->mcg_lock);
+ spin_unlock_bh(&qp->grp_lock);
+ rfc_drop_ref(grp); /* ref from get_key */
+err1:
+ return -EINVAL;
+}
+
+void rfc_drop_all_mcast_groups(struct rfc_qp *qp)
+{
+ struct rfc_mc_grp *grp;
+ struct rfc_mc_elem *elem;
+
+ while (1) {
+ spin_lock_bh(&qp->grp_lock);
+ if (list_empty(&qp->grp_list)) {
+ spin_unlock_bh(&qp->grp_lock);
+ break;
+ }
+ elem = list_first_entry(&qp->grp_list, struct rfc_mc_elem,
+ grp_list);
+ list_del(&elem->grp_list);
+ spin_unlock_bh(&qp->grp_lock);
+
+ grp = elem->grp;
+ spin_lock_bh(&grp->mcg_lock);
+ list_del(&elem->qp_list);
+ grp->num_qp--;
+ spin_unlock_bh(&grp->mcg_lock);
+ rfc_drop_ref(grp);
+ rfc_drop_ref(elem);
+ }
+}
+
+void rfc_mc_cleanup(struct rfc_pool_entry *arg)
+{
+ struct rfc_mc_grp *grp = container_of(arg, typeof(*grp), pelem);
+ struct rfc_dev *rfc = grp->rfc;
+
+ rfc_drop_key(grp);
+ rfc_mcast_delete(rfc, &grp->mgid);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_mmap.c b/drivers/infiniband/sw/rfc/rfc_mmap.c
new file mode 100644
index 0000000..15b4192
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_mmap.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/errno.h>
+#include <asm/pgtable.h>
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+void rfc_mmap_release(struct kref *ref)
+{
+ struct rfc_mmap_info *ip = container_of(ref,
+ struct rfc_mmap_info, ref);
+ struct rfc_dev *rfc = to_rdev(ip->context->device);
+
+ spin_lock_bh(&rfc->pending_lock);
+
+ if (!list_empty(&ip->pending_mmaps))
+ list_del(&ip->pending_mmaps);
+
+ spin_unlock_bh(&rfc->pending_lock);
+
+ vfree(ip->obj); /* buf */
+ kfree(ip);
+}
+
+/*
+ * open and close keep track of how many times the memory region is mapped,
+ * to avoid releasing it.
+ */
+static void rfc_vma_open(struct vm_area_struct *vma)
+{
+ struct rfc_mmap_info *ip = vma->vm_private_data;
+
+ kref_get(&ip->ref);
+}
+
+static void rfc_vma_close(struct vm_area_struct *vma)
+{
+ struct rfc_mmap_info *ip = vma->vm_private_data;
+
+ kref_put(&ip->ref, rfc_mmap_release);
+}
+
+static const struct vm_operations_struct rfc_vm_ops = {
+ .open = rfc_vma_open,
+ .close = rfc_vma_close,
+};
+
+/**
+ * rfc_mmap - create a new mmap region
+ * @context: the IB user context of the process making the mmap() call
+ * @vma: the VMA to be initialized
+ * Return zero if the mmap is OK. Otherwise, return an errno.
+ */
+int rfc_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+ struct rfc_dev *rfc = to_rdev(context->device);
+ unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+ unsigned long size = vma->vm_end - vma->vm_start;
+ struct rfc_mmap_info *ip, *pp;
+ int ret;
+
+ /*
+ * Search the device's list of objects waiting for a mmap call.
+ * Normally, this list is very short since a call to create a
+ * CQ, QP, or SRQ is soon followed by a call to mmap().
+ */
+ spin_lock_bh(&rfc->pending_lock);
+ list_for_each_entry_safe(ip, pp, &rfc->pending_mmaps, pending_mmaps) {
+ if (context != ip->context || (__u64)offset != ip->info.offset)
+ continue;
+
+ /* Don't allow a mmap larger than the object. */
+ if (size > ip->info.size) {
+ pr_err("mmap region is larger than the object!\n");
+ spin_unlock_bh(&rfc->pending_lock);
+ ret = -EINVAL;
+ goto done;
+ }
+
+ goto found_it;
+ }
+ pr_warn("unable to find pending mmap info\n");
+ spin_unlock_bh(&rfc->pending_lock);
+ ret = -EINVAL;
+ goto done;
+
+found_it:
+ list_del_init(&ip->pending_mmaps);
+ spin_unlock_bh(&rfc->pending_lock);
+
+ ret = remap_vmalloc_range(vma, ip->obj, 0);
+ if (ret) {
+ pr_err("err %d from remap_vmalloc_range\n", ret);
+ goto done;
+ }
+
+ vma->vm_ops = &rfc_vm_ops;
+ vma->vm_private_data = ip;
+ rfc_vma_open(vma);
+done:
+ return ret;
+}
+
+/*
+ * Allocate information for rfc_mmap
+ */
+struct rfc_mmap_info *rfc_create_mmap_info(struct rfc_dev *rfc,
+ u32 size,
+ struct ib_ucontext *context,
+ void *obj)
+{
+ struct rfc_mmap_info *ip;
+
+ ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+ if (!ip)
+ return NULL;
+
+ size = PAGE_ALIGN(size);
+
+ spin_lock_bh(&rfc->mmap_offset_lock);
+
+ if (rfc->mmap_offset == 0)
+ rfc->mmap_offset = ALIGN(PAGE_SIZE, SHMLBA);
+
+ ip->info.offset = rfc->mmap_offset;
+ rfc->mmap_offset += ALIGN(size, SHMLBA);
+
+ spin_unlock_bh(&rfc->mmap_offset_lock);
+
+ INIT_LIST_HEAD(&ip->pending_mmaps);
+ ip->info.size = size;
+ ip->context = context;
+ ip->obj = obj;
+ kref_init(&ip->ref);
+
+ return ip;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_mr.c b/drivers/infiniband/sw/rfc/rfc_mr.c
new file mode 100644
index 0000000..75b559b
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_mr.c
@@ -0,0 +1,850 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+
+/*
+ * lfsr (linear feedback shift register) with period 255
+ */
+static u8 rfc_get_key(void)
+{
+ static u32 key = 1;
+
+ key = key << 1;
+
+ key |= (0 != (key & 0x100)) ^ (0 != (key & 0x10))
+ ^ (0 != (key & 0x80)) ^ (0 != (key & 0x40));
+
+ key &= 0xff;
+
+ return key;
+}
+
+int mem_check_range(struct rfc_mem *mem, u64 iova, size_t length)
+{
+ switch (mem->type) {
+ case RFC_MEM_TYPE_DMA:
+ return 0;
+
+ case RFC_MEM_TYPE_MR:
+ case RFC_MEM_TYPE_FMR:
+ if (iova < mem->iova ||
+ length > mem->length ||
+ iova > mem->iova + mem->length - length)
+ return -EFAULT;
+ return 0;
+
+ default:
+ return -EFAULT;
+ }
+}
+
+#define IB_ACCESS_REMOTE (IB_ACCESS_REMOTE_READ \
+ | IB_ACCESS_REMOTE_WRITE \
+ | IB_ACCESS_REMOTE_ATOMIC)
+
+static void rfc_mem_init(int access, struct rfc_mem *mem)
+{
+ u32 lkey = mem->pelem.index << 8 | rfc_get_key();
+ u32 rkey = (access & IB_ACCESS_REMOTE) ? lkey : 0;
+
+ if (mem->pelem.pool->type == RFC_TYPE_MR) {
+ mem->ibmr.lkey = lkey;
+ mem->ibmr.rkey = rkey;
+ }
+
+ mem->lkey = lkey;
+ mem->rkey = rkey;
+ mem->state = RFC_MEM_STATE_INVALID;
+ mem->type = RFC_MEM_TYPE_NONE;
+ mem->map_shift = ilog2(RFC_BUF_PER_MAP);
+}
+
+void rfc_mem_cleanup(struct rfc_pool_entry *arg)
+{
+ struct rfc_mem *mem = container_of(arg, typeof(*mem), pelem);
+ int i;
+
+ if (mem->umem)
+ ib_umem_release(mem->umem);
+
+ if (mem->map) {
+ for (i = 0; i < mem->num_map; i++)
+ kfree(mem->map[i]);
+
+ kfree(mem->map);
+ }
+}
+
+static int rfc_mem_alloc(struct rfc_dev *rfc, struct rfc_mem *mem, int num_buf)
+{
+ int i;
+ int num_map;
+ struct rfc_map **map = mem->map;
+
+ num_map = (num_buf + RFC_BUF_PER_MAP - 1) / RFC_BUF_PER_MAP;
+
+ mem->map = kmalloc_array(num_map, sizeof(*map), GFP_KERNEL);
+ if (!mem->map)
+ goto err1;
+
+ for (i = 0; i < num_map; i++) {
+ mem->map[i] = kmalloc(sizeof(**map), GFP_KERNEL);
+ if (!mem->map[i])
+ goto err2;
+ }
+
+ BUILD_BUG_ON(!is_power_of_2(RFC_BUF_PER_MAP));
+
+ mem->map_shift = ilog2(RFC_BUF_PER_MAP);
+ mem->map_mask = RFC_BUF_PER_MAP - 1;
+
+ mem->num_buf = num_buf;
+ mem->num_map = num_map;
+ mem->max_buf = num_map * RFC_BUF_PER_MAP;
+
+ return 0;
+
+err2:
+ for (i--; i >= 0; i--)
+ kfree(mem->map[i]);
+
+ kfree(mem->map);
+err1:
+ return -ENOMEM;
+}
+
+int rfc_mem_init_dma(struct rfc_dev *rfc, struct rfc_pd *pd,
+ int access, struct rfc_mem *mem)
+{
+ rfc_mem_init(access, mem);
+
+ mem->pd = pd;
+ mem->access = access;
+ mem->state = RFC_MEM_STATE_VALID;
+ mem->type = RFC_MEM_TYPE_DMA;
+
+ return 0;
+}
+
+int rfc_mem_init_user(struct rfc_dev *rfc, struct rfc_pd *pd, u64 start,
+ u64 length, u64 iova, int access, struct ib_udata *udata,
+ struct rfc_mem *mem)
+{
+ int entry;
+ struct rfc_map **map;
+ struct rfc_phys_buf *buf = NULL;
+ struct ib_umem *umem;
+ struct scatterlist *sg;
+ int num_buf;
+ void *vaddr;
+ int err;
+
+ umem = ib_umem_get(pd->ibpd.uobject->context, start, length, access, 0);
+ if (IS_ERR(umem)) {
+ pr_warn("err %d from rfc_umem_get\n",
+ (int)PTR_ERR(umem));
+ err = -EINVAL;
+ goto err1;
+ }
+
+ mem->umem = umem;
+ num_buf = umem->nmap;
+
+ rfc_mem_init(access, mem);
+
+ err = rfc_mem_alloc(rfc, mem, num_buf);
+ if (err) {
+ pr_warn("err %d from rfc_mem_alloc\n", err);
+ ib_umem_release(umem);
+ goto err1;
+ }
+
+ mem->page_shift = umem->page_shift;
+ mem->page_mask = BIT(umem->page_shift) - 1;
+
+ num_buf = 0;
+ map = mem->map;
+ if (length > 0) {
+ buf = map[0]->buf;
+
+ for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
+ vaddr = page_address(sg_page(sg));
+ if (!vaddr) {
+ pr_warn("null vaddr\n");
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ buf->addr = (uintptr_t)vaddr;
+ buf->size = BIT(umem->page_shift);
+ num_buf++;
+ buf++;
+
+ if (num_buf >= RFC_BUF_PER_MAP) {
+ map++;
+ buf = map[0]->buf;
+ num_buf = 0;
+ }
+ }
+ }
+
+ mem->pd = pd;
+ mem->umem = umem;
+ mem->access = access;
+ mem->length = length;
+ mem->iova = iova;
+ mem->va = start;
+ mem->offset = ib_umem_offset(umem);
+ mem->state = RFC_MEM_STATE_VALID;
+ mem->type = RFC_MEM_TYPE_MR;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+int rfc_mem_init_fast(struct rfc_dev *rfc, struct rfc_pd *pd,
+ int max_pages, struct rfc_mem *mem)
+{
+ int err;
+
+ rfc_mem_init(0, mem);
+
+ /* In fastreg, we also set the rkey */
+ mem->ibmr.rkey = mem->ibmr.lkey;
+
+ err = rfc_mem_alloc(rfc, mem, max_pages);
+ if (err)
+ goto err1;
+
+ mem->pd = pd;
+ mem->max_buf = max_pages;
+ mem->state = RFC_MEM_STATE_FREE;
+ mem->type = RFC_MEM_TYPE_MR;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+
+static void lookup_iova(
+ struct rfc_mem *mem,
+ u64 iova,
+ int *m_out,
+ int *n_out,
+ size_t *offset_out)
+{
+ size_t offset = iova - mem->iova + mem->offset;
+ int map_index;
+ int buf_index;
+ u64 length;
+
+ if (likely(mem->page_shift)) {
+ *offset_out = offset & mem->page_mask;
+ offset >>= mem->page_shift;
+ *n_out = offset & mem->map_mask;
+ *m_out = offset >> mem->map_shift;
+ } else {
+ map_index = 0;
+ buf_index = 0;
+
+ length = mem->map[map_index]->buf[buf_index].size;
+
+ while (offset >= length) {
+ offset -= length;
+ buf_index++;
+
+ if (buf_index == RFC_BUF_PER_MAP) {
+ map_index++;
+ buf_index = 0;
+ }
+ length = mem->map[map_index]->buf[buf_index].size;
+ }
+
+ *m_out = map_index;
+ *n_out = buf_index;
+ *offset_out = offset;
+ }
+}
+
+void rfc_lookup_iova(
+ struct rfc_mem *mem,
+ u64 iova,
+ int *m_out,
+ int *n_out,
+ size_t *offset_out)
+{
+ lookup_iova(mem, iova, m_out, n_out, offset_out);
+}
+
+void *iova_to_vaddr(struct rfc_mem *mem, u64 iova, int length)
+{
+ size_t offset;
+ int m, n;
+ void *addr;
+
+ if (mem->state != RFC_MEM_STATE_VALID) {
+ pr_warn("mem not in valid state\n");
+ addr = NULL;
+ goto out;
+ }
+
+ if (!mem->map) {
+ addr = (void *)(uintptr_t)iova;
+ goto out;
+ }
+
+ if (mem_check_range(mem, iova, length)) {
+ pr_warn("range violation\n");
+ addr = NULL;
+ goto out;
+ }
+
+ lookup_iova(mem, iova, &m, &n, &offset);
+
+ if (offset + length > mem->map[m]->buf[n].size) {
+ pr_warn("crosses page boundary\n");
+ addr = NULL;
+ goto out;
+ }
+
+ addr = (void *)(uintptr_t)mem->map[m]->buf[n].addr + offset;
+
+out:
+ return addr;
+}
+
+/* copy data from a range (vaddr, vaddr+length-1) to or from
+ * a mem object starting at iova. Compute incremental value of
+ * crc32 if crcp is not zero. caller must hold a reference to mem
+ */
+int rfc_mem_copy(struct rfc_mem *mem, u64 iova, void *addr, int length,
+ enum copy_direction dir, u32 *crcp)
+{
+ int err;
+ int bytes;
+ u8 *va;
+ struct rfc_map **map;
+ struct rfc_phys_buf *buf;
+ int m;
+ int i;
+ size_t offset;
+ u32 crc = crcp ? (*crcp) : 0;
+
+ if (length == 0)
+ return 0;
+
+ if (mem->type == RFC_MEM_TYPE_DMA) {
+ u8 *src, *dest;
+
+ src = (dir == to_mem_obj) ?
+ addr : ((void *)(uintptr_t)iova);
+
+ dest = (dir == to_mem_obj) ?
+ ((void *)(uintptr_t)iova) : addr;
+
+ if (crcp)
+ *crcp = rfc_crc32(to_rdev(mem->pd->ibpd.device),
+ *crcp, src, length);
+
+ memcpy(dest, src, length);
+
+ return 0;
+ }
+
+ WARN_ON_ONCE(!mem->map);
+
+ err = mem_check_range(mem, iova, length);
+ if (err) {
+ err = -EFAULT;
+ goto err1;
+ }
+
+ lookup_iova(mem, iova, &m, &i, &offset);
+
+ map = mem->map + m;
+ buf = map[0]->buf + i;
+
+ while (length > 0) {
+ u8 *src, *dest;
+
+ va = (u8 *)(uintptr_t)buf->addr + offset;
+ src = (dir == to_mem_obj) ? addr : va;
+ dest = (dir == to_mem_obj) ? va : addr;
+
+ bytes = buf->size - offset;
+
+ if (bytes > length)
+ bytes = length;
+
+ if (crcp)
+ crc = rfc_crc32(to_rdev(mem->pd->ibpd.device),
+ crc, src, bytes);
+
+ memcpy(dest, src, bytes);
+
+ length -= bytes;
+ addr += bytes;
+
+ offset = 0;
+ buf++;
+ i++;
+
+ if (i == RFC_BUF_PER_MAP) {
+ i = 0;
+ map++;
+ buf = map[0]->buf;
+ }
+ }
+
+ if (crcp)
+ *crcp = crc;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+/* copy data in or out of a wqe, i.e. sg list
+ * under the control of a dma descriptor
+ */
+int copy_data(
+ struct rfc_dev *rfc,
+ struct rfc_pd *pd,
+ int access,
+ struct rfc_dma_info *dma,
+ void *addr,
+ int length,
+ enum copy_direction dir,
+ u32 *crcp)
+{
+ int bytes;
+ struct rfc_sge *sge = &dma->sge[dma->cur_sge];
+ int offset = dma->sge_offset;
+ int resid = dma->resid;
+ struct rfc_mem *mem = NULL;
+ u64 iova;
+ int err;
+
+ if (length == 0)
+ return 0;
+
+ if (length > resid) {
+ err = -EINVAL;
+ goto err2;
+ }
+
+ if (sge->length && (offset < sge->length)) {
+ mem = lookup_mem(pd, access, sge->lkey, lookup_local);
+ if (!mem) {
+ err = -EINVAL;
+ goto err1;
+ }
+ }
+
+ while (length > 0) {
+ bytes = length;
+
+ if (offset >= sge->length) {
+ if (mem) {
+ rfc_drop_ref(mem);
+ mem = NULL;
+ }
+ sge++;
+ dma->cur_sge++;
+ offset = 0;
+
+ if (dma->cur_sge >= dma->num_sge) {
+ err = -ENOSPC;
+ goto err2;
+ }
+
+ if (sge->length) {
+ mem = lookup_mem(pd, access, sge->lkey,
+ lookup_local);
+ if (!mem) {
+ err = -EINVAL;
+ goto err1;
+ }
+ } else {
+ continue;
+ }
+ }
+
+ if (bytes > sge->length - offset)
+ bytes = sge->length - offset;
+
+ if (bytes > 0) {
+ iova = sge->addr + offset;
+
+ err = rfc_mem_copy(mem, iova, addr, bytes, dir, crcp);
+ if (err)
+ goto err2;
+
+ offset += bytes;
+ resid -= bytes;
+ length -= bytes;
+ addr += bytes;
+ }
+ }
+
+ dma->sge_offset = offset;
+ dma->resid = resid;
+
+ if (mem)
+ rfc_drop_ref(mem);
+
+ return 0;
+
+err2:
+ if (mem)
+ rfc_drop_ref(mem);
+err1:
+ return err;
+}
+
+int rfc_mem_copy_sgl(struct rfc_mem *mem, u64 iova, struct sg_table *rsg,
+ int length, enum copy_direction dir, u32 *crcp)
+{
+ int err;
+ int bytes;
+ u8 *va;
+ struct rfc_map **map;
+ struct rfc_phys_buf *buf;
+ int m;
+ int i;
+ size_t offset;
+ u32 crc = crcp ? (*crcp) : 0;
+
+ if (length == 0)
+ return 0;
+
+ if (mem->type == RFC_MEM_TYPE_DMA) {
+ u8 *src, *dest;
+
+ if (dir == to_mem_obj) {
+ dest = ((void *)(uintptr_t)iova);
+ sg_set_page(&rsg->sgl[rsg->nents++],
+ virt_to_page(dest), length,
+ offset_in_page(dest));
+ } else { // dir == from_mem_obj
+ src = ((void *)(uintptr_t)iova);
+ sg_set_page(&rsg->sgl[rsg->nents++],
+ virt_to_page(src), length,
+ offset_in_page(src));
+ }
+
+ return 0;
+ }
+
+ WARN_ON_ONCE(!mem->map);
+
+ err = mem_check_range(mem, iova, length);
+ if (err) {
+ err = -EFAULT;
+ goto err1;
+ }
+
+ lookup_iova(mem, iova, &m, &i, &offset);
+
+ map = mem->map + m;
+ buf = map[0]->buf + i;
+
+ while (length > 0) {
+
+ u8 *src, *dest;
+
+ va = (u8 *)(uintptr_t)buf->addr + offset;
+
+ bytes = buf->size - offset;
+
+ if (bytes > length)
+ bytes = length;
+
+ if (dir == to_mem_obj) {
+ dest = ((void *)(uintptr_t)va);
+ sg_set_page(&rsg->sgl[rsg->nents++],
+ virt_to_page(buf->addr), bytes, offset);
+ } else {
+ src = ((void *)(uintptr_t)va);
+ sg_set_page(&rsg->sgl[rsg->nents++],
+ virt_to_page(buf->addr), bytes, offset);
+ }
+
+
+ length -= bytes;
+
+ offset = 0;
+ buf++;
+ i++;
+
+ if (i == RFC_BUF_PER_MAP) {
+ i = 0;
+ map++;
+ buf = map[0]->buf;
+ }
+ }
+
+ if (crcp)
+ *crcp = crc;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+/* copy data in or out of a wqe, i.e. sg list
+ * under the control of a dma descriptor
+ */
+int copy_sgl(
+ struct rfc_dev *rfc,
+ struct rfc_pd *pd,
+ int access,
+ struct rfc_dma_info *dma,
+ struct sg_table *rsg,
+ int length,
+ enum copy_direction dir,
+ u32 *crcp)
+{
+ int bytes;
+ struct rfc_sge *sge = &dma->sge[dma->cur_sge];
+ int offset = dma->sge_offset;
+ int resid = dma->resid;
+ struct rfc_mem *mem = NULL;
+ u64 iova;
+ int err;
+
+ if (length == 0)
+ return 0;
+
+ if (length > resid) {
+ err = -EINVAL;
+ goto err2;
+ }
+
+ if (sge->length && (offset < sge->length)) {
+ mem = lookup_mem(pd, access, sge->lkey, lookup_local);
+ if (!mem) {
+ err = -EINVAL;
+ goto err1;
+ }
+ }
+
+ while (length > 0) {
+ bytes = length;
+
+ if (offset >= sge->length) {
+ if (mem) {
+ rfc_drop_ref(mem);
+ mem = NULL;
+ }
+ sge++;
+ dma->cur_sge++;
+ offset = 0;
+
+ if (dma->cur_sge >= dma->num_sge) {
+ err = -ENOSPC;
+ goto err2;
+ }
+
+ if (sge->length) {
+ mem = lookup_mem(pd, access, sge->lkey,
+ lookup_local);
+ if (!mem) {
+ err = -EINVAL;
+ goto err1;
+ }
+ } else {
+ continue;
+ }
+ }
+
+ if (bytes > sge->length - offset)
+ bytes = sge->length - offset;
+
+ if (bytes > 0) {
+ iova = sge->addr + offset;
+
+ err = rfc_mem_copy_sgl(mem, iova,
+ rsg, bytes, dir, crcp);
+ if (err)
+ goto err2;
+
+ offset += bytes;
+ resid -= bytes;
+ length -= bytes;
+ }
+ }
+
+ dma->sge_offset = offset;
+ dma->resid = resid;
+
+ if (mem)
+ rfc_drop_ref(mem);
+
+ return 0;
+
+err2:
+ if (mem)
+ rfc_drop_ref(mem);
+err1:
+ return err;
+}
+
+int advance_dma_data(struct rfc_dma_info *dma, unsigned int length)
+{
+ struct rfc_sge *sge = &dma->sge[dma->cur_sge];
+ int offset = dma->sge_offset;
+ int resid = dma->resid;
+
+ while (length) {
+ unsigned int bytes;
+
+ if (offset >= sge->length) {
+ sge++;
+ dma->cur_sge++;
+ offset = 0;
+ if (dma->cur_sge >= dma->num_sge)
+ return -ENOSPC;
+ }
+
+ bytes = length;
+
+ if (bytes > sge->length - offset)
+ bytes = sge->length - offset;
+
+ offset += bytes;
+ resid -= bytes;
+ length -= bytes;
+ }
+
+ dma->sge_offset = offset;
+ dma->resid = resid;
+
+ return 0;
+}
+
+/* (1) find the mem (mr or mw) corresponding to lkey/rkey
+ * depending on lookup_type
+ * (2) verify that the (qp) pd matches the mem pd
+ * (3) verify that the mem can support the requested access
+ * (4) verify that mem state is valid
+ */
+struct rfc_mem *lookup_mem(struct rfc_pd *pd, int access, u32 key,
+ enum lookup_type type)
+{
+ struct rfc_mem *mem;
+ struct rfc_dev *rfc = to_rdev(pd->ibpd.device);
+ int index = key >> 8;
+
+ if (index >= RFC_MIN_MR_INDEX && index <= RFC_MAX_MR_INDEX) {
+ mem = rfc_pool_get_index(&rfc->mr_pool, index);
+ if (!mem)
+ goto err1;
+ } else {
+ goto err1;
+ }
+
+ if ((type == lookup_local && mem->lkey != key) ||
+ (type == lookup_remote && mem->rkey != key))
+ goto err2;
+
+ if (mem->pd != pd)
+ goto err2;
+
+ if (access && !(access & mem->access))
+ goto err2;
+
+ if (mem->state != RFC_MEM_STATE_VALID)
+ goto err2;
+
+ return mem;
+
+err2:
+ rfc_drop_ref(mem);
+err1:
+ return NULL;
+}
+
+int rfc_mem_map_pages(struct rfc_dev *rfc, struct rfc_mem *mem,
+ u64 *page, int num_pages, u64 iova)
+{
+ int i;
+ int num_buf;
+ int err;
+ struct rfc_map **map;
+ struct rfc_phys_buf *buf;
+ int page_size;
+
+ if (num_pages > mem->max_buf) {
+ err = -EINVAL;
+ goto err1;
+ }
+
+ num_buf = 0;
+ page_size = 1 << mem->page_shift;
+ map = mem->map;
+ buf = map[0]->buf;
+
+ for (i = 0; i < num_pages; i++) {
+ buf->addr = *page++;
+ buf->size = page_size;
+ buf++;
+ num_buf++;
+
+ if (num_buf == RFC_BUF_PER_MAP) {
+ map++;
+ buf = map[0]->buf;
+ num_buf = 0;
+ }
+ }
+
+ mem->iova = iova;
+ mem->va = iova;
+ mem->length = num_pages << mem->page_shift;
+ mem->state = RFC_MEM_STATE_VALID;
+
+ return 0;
+
+err1:
+ return err;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_net.c b/drivers/infiniband/sw/rfc/rfc_net.c
new file mode 100644
index 0000000..e850a39
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_net.c
@@ -0,0 +1,393 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+
+#include <linux/types.h>
+#include <linux/string.h>
+/* Disk on RAM Driver */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/stat.h>
+#include <linux/errno.h>
+#include <linux/major.h>
+#include <linux/blkpg.h>
+#include <linux/types.h>
+#include <linux/genhd.h>
+#include <linux/blkdev.h>
+#include <linux/errno.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/ip.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/skbuff.h>
+#include <linux/ethtool.h>
+#include <net/checksum.h>
+#include <linux/platform_device.h>
+#include "rfc_obj.h"
+
+
+#define PLAT_NAME "plat_net"
+
+
+static DEFINE_SPINLOCK(rfc_net_lock);
+#define LOCK(x, y) spin_lock_irqsave((x), (y))
+#define UNLOCK(x, y) spin_unlock_irqrestore((x), (y))
+#define LOCK_INIT(x) spin_lock_init((x))
+
+struct net_device *rfc_net_dev;
+struct rfc_skb_blk_push {
+ struct page *page1;
+ u32 daddr;
+ int len;
+ struct list_head active_push_reqs;
+};
+struct list_head skb_blk_push_list;
+struct rfc_net_private {
+ struct napi_struct napi;
+ struct net_device *dev;
+};
+
+/* MAC address storage */
+static unsigned char rfc_net_mac_addr[6] = {
+ 0xA0, 0x36, 0x9F, 0x21, 0x62, 0x80
+
+};
+//struct page *xmit_page = NULL;
+struct work_struct net_blk_work;
+static struct workqueue_struct *net_blk_wrk_st;
+
+static void rfc_net_xfer_complete(struct bio *bio)
+{
+ complete((struct completion *)bio->bi_private);
+}
+
+static netdev_tx_t rfc_net_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+ struct iphdr *ip_header;
+ unsigned int *len;
+ unsigned long flags = 0;
+ u8 *buffer;
+
+ struct rfc_skb_blk_push *rfc_skb = kmalloc(
+ sizeof(struct rfc_skb_blk_push), GFP_KERNEL);
+
+ rfc_skb->page1 = alloc_pages(GFP_ATOMIC, 0);
+
+ ip_header = (struct iphdr *)skb_network_header(skb);
+ rfc_skb->daddr = (unsigned int)ip_header->daddr;
+
+ buffer = page_address(rfc_skb->page1);
+ memset(buffer, 0x0, OBJ_SIZE);
+ /* Make sure no one else owns the skb */
+ skb_orphan(skb);
+
+ memcpy(buffer, skb->data, skb->len);
+ /*storing the skb len in a predefined offset*/
+ len = (unsigned int *)(buffer + OBJ_SIZE - sizeof(unsigned int) - 1);
+ *len = skb->len;
+
+ LOCK(&rfc_net_lock, flags);
+ list_add_tail(&rfc_skb->active_push_reqs, &skb_blk_push_list);
+
+ UNLOCK(&rfc_net_lock, flags);
+ queue_work(net_blk_wrk_st, &net_blk_work);
+ /* Free skb */
+ dev_kfree_skb(skb);
+ return NETDEV_TX_OK;
+}
+
+/*pushes the object to the n/w thorugh blk*/
+static void push_data_skb_blk(struct work_struct *work)
+{
+ struct block_device *bdev1 = NULL;
+ struct completion event;
+ struct bio *bio;
+ struct rfc_skb_blk_push *rfc_skb;
+ unsigned long flags = 0;
+ char *dev_path;
+
+ LOCK(&rfc_net_lock, flags);
+ while (!list_empty(&skb_blk_push_list)) {
+ rfc_skb = list_first_entry(&skb_blk_push_list,
+ struct rfc_skb_blk_push, active_push_reqs);
+
+ UNLOCK(&rfc_net_lock, flags);
+ dev_path = dev_path_get_by_list(rfc_skb->daddr);
+ if (!dev_path) {
+ pr_err("\n%s dev_path_get_by_list failed %d\n",
+ __func__, rfc_skb->daddr);
+ return;
+ }
+ bdev1 = blkdev_get_by_path(dev_path,
+ FMODE_READ|FMODE_WRITE, NULL);
+ if (!bdev1) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return;
+ }
+ bio = bio_alloc(GFP_NOIO, 4);
+ bio_set_dev(bio, bdev1);
+ bio->bi_iter.bi_sector = 0xa5;
+ bio_add_page(bio, rfc_skb->page1, OBJ_SIZE, 0);
+ init_completion(&event);
+ bio->bi_private = &event;
+ bio->bi_end_io = rfc_net_xfer_complete;
+ bio->bi_opf = REQ_OP_WRITE;
+ submit_bio(bio);
+
+ wait_for_completion(&event);
+ bio_put(bio);
+ blkdev_put(bdev1, FMODE_READ|FMODE_WRITE);
+ LOCK(&rfc_net_lock, flags);
+ list_del(&rfc_skb->active_push_reqs);
+ UNLOCK(&rfc_net_lock, flags);
+ __free_pages(rfc_skb->page1, 0);
+ kfree(rfc_skb);
+ LOCK(&rfc_net_lock, flags);
+ }
+ UNLOCK(&rfc_net_lock, flags);
+
+}
+
+/*pushes the object received through blk to the n/w */
+void push_data_blk_skb(char *buffer)
+{
+ struct sk_buff *skb;
+ int len = 0;
+ int *data_len;
+
+ /*get the data len from the predefined offset*/
+ data_len = (unsigned int *)
+ (buffer + OBJ_SIZE - sizeof(unsigned int) - 1);
+
+ len = *data_len;
+ /* Get an skb for object reception */
+ skb = dev_alloc_skb(len + 2);
+ if (!skb) {
+ pr_err("%s(): dev_alloc_skb failed:len=0x%x\n", __func__, len);
+ return;
+ }
+ /* Make sure header is aligned to 16 bytes */
+ skb_reserve(skb, 2);
+ /* Copy object payload into skb */
+ memcpy((void *)skb_put(skb, len), buffer, len);
+ /* Write metadata, and then pass to the receive level */
+ skb->dev = rfc_net_dev;
+ skb->protocol = eth_type_trans(skb, rfc_net_dev);
+ skb->ip_summed = CHECKSUM_UNNECESSARY; /* don't check it */
+ netif_receive_skb(skb);
+}
+EXPORT_SYMBOL(push_data_blk_skb);
+
+static int rfc_net_rx_poll(struct napi_struct *napi, int budget)
+{
+ return 0;
+}
+
+static u32 rfc_net_always_on(struct net_device *dev)
+{
+ return 1;
+}
+static int rfc_net_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
+{
+ cmd->supported = 0;
+ cmd->advertising = 0;
+ cmd->speed = 1000;
+ return 0;
+}
+static const struct ethtool_ops rfc_net_ethtool_ops = {
+ .get_link = rfc_net_always_on,
+ .get_settings = rfc_net_get_settings
+};
+
+static int rfc_net_dev_init(struct net_device *dev)
+{
+ return 0;
+}
+
+static void rfc_net_dev_free(struct net_device *dev)
+{
+ free_netdev(dev);
+}
+
+
+static int rfc_net_open(struct net_device *dev)
+{
+ struct rfc_net_private *rfc_net;
+
+ /* Flush Rx ring of objs received while down */
+ rfc_net = netdev_priv(dev);
+
+ /* (Re)enable NAPI and Rx interrupt */
+ napi_enable(&rfc_net->napi);
+
+ return 0;
+}
+
+static int rfc_net_close(struct net_device *dev)
+{
+ struct rfc_net_private *rfc_net;
+
+ rfc_net = netdev_priv(dev);
+ napi_disable(&rfc_net->napi);
+
+ return 0;
+}
+
+static const struct net_device_ops rfc_net_ops = {
+ .ndo_open = rfc_net_open,
+ .ndo_stop = rfc_net_close,
+ .ndo_init = rfc_net_dev_init,
+ .ndo_start_xmit = rfc_net_xmit,
+};
+
+static int rfc_net_init(struct platform_device *pdev)
+{
+ int err = 0;
+ struct rfc_net_private *rfc_net;
+ int ret = 0;
+
+ /* Alloc our device and unit number */
+ rfc_net_dev = alloc_etherdev(sizeof(struct rfc_net_private));
+ if (!rfc_net_dev) {
+ pr_err("%s: alloc_etherdev failed !\n", __func__);
+ return -ENODEV;
+ }
+
+ /* Setup MAC address */
+ memcpy(rfc_net_dev->dev_addr, rfc_net_mac_addr,
+ sizeof(rfc_net_mac_addr));
+ rfc_net_dev->addr_len = ETH_ALEN;
+
+ /* Init some basics */
+ ether_setup(rfc_net_dev);
+ rfc_net_dev->ethtool_ops = &rfc_net_ethtool_ops;
+ rfc_net_dev->netdev_ops = &rfc_net_ops;
+ rfc_net_dev->mtu = 1024;
+ rfc_net_dev->type = 1;
+ rfc_net_dev->flags = rfc_net_dev->flags | IFF_NOARP;
+ strcpy(rfc_net_dev->name, "rfcnet0");
+ ret = dev_alloc_name(rfc_net_dev, rfc_net_dev->name);
+ if (ret < 0) {
+ pr_err("%s(): dev_alloc_name failed!\n", __func__);
+ free_netdev(rfc_net_dev);
+ return ret;
+ }
+ /* Register our device with kernel */
+ SET_NETDEV_DEV(rfc_net_dev, &pdev->dev);
+ err = register_netdev(rfc_net_dev);
+ if (err) {
+ pr_err("%s: register_netdev() failed!\n", __func__);
+ free_netdev(rfc_net_dev);
+ return err;
+ }
+
+ /* Set up private data */
+ rfc_net = netdev_priv(rfc_net_dev);
+ rfc_net->dev = rfc_net_dev;
+
+
+
+ /* Hook into NAPI */
+ netif_napi_add(rfc_net->dev, &rfc_net->napi, rfc_net_rx_poll, 64);
+ net_blk_wrk_st = create_workqueue("net_blk_wrk_st");
+
+ INIT_WORK(&net_blk_work, push_data_skb_blk);
+ INIT_LIST_HEAD(&skb_blk_push_list);
+
+ return 0;
+}
+static void rfc_net_uninit(void)
+{
+ unregister_netdev(rfc_net_dev);
+ rfc_net_dev_free(rfc_net_dev);
+ destroy_workqueue(net_blk_wrk_st);
+}
+
+
+static int plat_net_probe(struct platform_device *pdev)
+{
+ rfc_net_init(pdev);
+ return 0;
+}
+static struct platform_driver plat_net_driver = {
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = PLAT_NAME,
+ },
+ .probe = plat_net_probe,
+};
+
+
+static struct platform_device *plat_net_pdev;
+static u64 plat_net_dmamask = DMA_BIT_MASK(32);
+int rfc_net_link_init(void)
+{
+ int err;
+
+ err = platform_driver_register(&plat_net_driver);
+ if (err)
+ goto exit;
+
+ plat_net_pdev = platform_device_alloc(PLAT_NAME, -1);
+ if (!plat_net_pdev) {
+ pr_err("Device allocation failed\n");
+ err = -ENOMEM;
+ goto exit_driver_unregister;
+ }
+ plat_net_pdev->dev.dma_mask = &plat_net_dmamask;
+ plat_net_pdev->dev.coherent_dma_mask = plat_net_dmamask;
+
+ err = platform_device_add(plat_net_pdev);
+ if (err) {
+ pr_err("Device addition failed (%d)\n", err);
+ goto exit_device_put;
+ }
+
+ return 0;
+
+exit_device_put:
+ platform_device_put(plat_net_pdev);
+exit_driver_unregister:
+ platform_driver_unregister(&plat_net_driver);
+exit:
+ return err;
+}
+
+void rfc_net_link_uninit(void)
+{
+ rfc_net_uninit();
+ platform_driver_unregister(&plat_net_driver);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_net.h b/drivers/infiniband/sw/rfc/rfc_net.h
new file mode 100644
index 0000000..08fc1e0
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_net.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_NET_H
+#define RFC_NET_H
+
+#include <net/sock.h>
+#include <net/if_inet6.h>
+#include <linux/module.h>
+
+struct rfc_recv_sockets {
+ struct socket *sk4;
+ struct socket *sk6;
+};
+
+extern struct rfc_recv_sockets recv_sockets;
+extern struct notifier_block rfc_net_notifier;
+
+struct rfc_dev *rfc_net_add(struct net_device *ndev);
+
+int rfc_obj_init(void);
+void rfc_obj_exit(void);
+#endif /* RFC_NET_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_obj.c b/drivers/infiniband/sw/rfc/rfc_obj.c
new file mode 100644
index 0000000..474a3c1
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_obj.c
@@ -0,0 +1,820 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/netdevice.h>
+#include <linux/if.h>
+#include <rdma/ib_addr.h>
+
+#include "rfc.h"
+#include "rfc_net.h"
+#include "rfc_loc.h"
+#include "rfc_obj.h"
+#include <linux/nvme.h>
+
+#define RFC_BLKDEV_NAME_MAX 64
+
+#define SGL_LEN(obj) ((obj)->paylen + rfc_opcode[(obj)->opcode].length)
+
+static LIST_HEAD(rfc_dev_list);
+static DEFINE_SPINLOCK(dev_list_lock); /* spinlock for device list */
+
+struct rfc_obj_info g_cb = {0};
+
+struct rfc_dev *net_to_rfc(struct net_device *ndev)
+{
+ struct rfc_dev *rfc;
+ struct rfc_dev *found = NULL;
+
+ spin_lock_bh(&dev_list_lock);
+ list_for_each_entry(rfc, &rfc_dev_list, list) {
+ if (rfc->ndev == ndev) {
+ found = rfc;
+ break;
+ }
+ }
+ spin_unlock_bh(&dev_list_lock);
+
+ return found;
+}
+
+struct rfc_dev *get_rfc_by_name(const char *name)
+{
+ struct rfc_dev *rfc;
+ struct rfc_dev *found = NULL;
+
+ spin_lock_bh(&dev_list_lock);
+ list_for_each_entry(rfc, &rfc_dev_list, list) {
+ if (!strcmp(name, rfc->ib_dev.name)) {
+ found = rfc;
+ break;
+ }
+ }
+ spin_unlock_bh(&dev_list_lock);
+ return found;
+}
+
+struct rfc_dev *get_rfc_by_objname(const char *objname)
+{
+ struct rfc_dev *rfc;
+ struct rfc_dev *found = NULL;
+
+ spin_lock_bh(&dev_list_lock);
+ list_for_each_entry(rfc, &rfc_dev_list, list) {
+ if (!strcmp(objname, rfc->objname)) {
+ found = rfc;
+ break;
+ }
+ }
+ spin_unlock_bh(&dev_list_lock);
+ return found;
+}
+
+struct rfc_recv_sockets recv_sockets;
+
+struct device *rfc_dma_device(struct rfc_dev *rfc)
+{
+ struct net_device *ndev;
+
+ ndev = rfc->ndev;
+
+ if (is_vlan_dev(ndev))
+ ndev = vlan_dev_real_dev(ndev);
+
+ return ndev->dev.parent;
+}
+
+
+int rfc_prepare(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ struct sg_table *rsg, u32 *crc)
+{
+ int err = 0;
+
+ *crc = 0xABCD;
+
+ return err;
+}
+
+static int rfc_send_blk(struct sg_table *rsg, int len, void *data);
+
+static int find_sgl_length(struct sg_table *rsg)
+{
+ struct scatterlist *sgl = rsg->sgl, *tmpsg;
+ int i, length = 0;
+
+ for_each_sg(sgl, tmpsg, rsg->nents, i) {
+ length += tmpsg->length;
+ }
+ return length;
+}
+
+void print_sgl(struct sg_table *rsg)
+{
+ struct scatterlist *sgl = rsg->sgl, *tmpsg;
+ int i = 0;
+
+ for_each_sg(sgl, tmpsg, rsg->nents, i) {
+ pr_debug("\tSGL %d offset %d length %d\n", i, tmpsg->offset,
+ tmpsg->length);
+ }
+}
+
+int rfc_send(struct rfc_dev *rfc,
+ struct rfc_obj_info *obj,
+ struct sg_table *rsg)
+{
+ struct rfc_av *av;
+ int err;
+
+ av = rfc_get_av(obj);
+
+ // Check whether middle SGLs contain any incomplete page
+ if (qp_type(obj->qp) == IB_QPT_GSI) {
+ struct page *page = NULL, *hdr_page = NULL;
+ struct scatterlist *sg_list_start, *sg;
+ int len = 0, i, ret, hdr_len;
+ char *buf;
+ __be32 *addrp = NULL;
+ __be32 *addrp1 = NULL;
+
+ page = alloc_page(GFP_KERNEL);
+ sg_list_start = rsg->sgl;
+ ret = rsg->nents;
+ buf = page_to_virt(page);
+ print_sgl(rsg);
+ for_each_sg(sg_list_start, sg, ret, i) {
+ if (i == 0)
+ continue;
+ memcpy(buf, sg_virt(sg), sg->length);
+ len += sg->length;
+ buf += sg->length;
+ }
+
+ // copy the src and dstips at the end
+ addrp = (__be32 *)buf;
+ addrp1 = addrp;
+ *addrp = av->sgid_addr._sockaddr_in.sin_addr.s_addr;
+ addrp1++;
+ *addrp1 = av->dgid_addr._sockaddr_in.sin_addr.s_addr;
+ len += 2 * sizeof(__be32);
+ // Copy the first page details and free the old sg_table
+ hdr_page = sg_page(rsg->sgl);
+ hdr_len = rfc_opcode[obj->opcode].length;
+ sg_free_table(rsg);
+ if (sg_alloc_table(rsg, SG_MAX_SINGLE_ALLOC, GFP_KERNEL)) {
+ kfree(rsg);
+ pr_err("%s(): ERROR: SG Allocation failure\n",
+ __func__);
+ return -ENOMEM;
+ }
+ rsg->nents = 0;
+ rsg->orig_nents = SG_MAX_SINGLE_ALLOC;
+ sg_set_page(&rsg->sgl[rsg->nents++], hdr_page, hdr_len, 0);
+ sg_set_page(&rsg->sgl[rsg->nents++], page, len, 0);
+ }
+ if (av->network_type == RDMA_NETWORK_IPV4) {
+ int len = find_sgl_length(rsg);
+
+ print_sgl(rsg);
+ err = rfc_send_blk(rsg, len, obj);
+ } else if (av->network_type == RDMA_NETWORK_IPV6) {
+ int len = find_sgl_length(rsg);
+
+ print_sgl(rsg);
+ err = rfc_send_blk(rsg, len, obj);
+ } else {
+ int len = find_sgl_length(rsg);
+
+ print_sgl(rsg);
+ err = rfc_send_blk(rsg, len, obj);
+ if (unlikely(err != 0)) {
+ pr_err("ERROR in sending rfc_send_blk\n");
+ return -EINVAL;
+ }
+ }
+ return 0;
+
+}
+
+int rfc_send_data_free(void *data, struct sg_table *rsg)
+{
+ struct rfc_obj_info *obj = (struct rfc_obj_info *)data;
+
+ if (bth_qpn(obj) == IB_QPT_GSI || obj->mask & RFC_ATMETH_MASK)
+ __free_pages(sg_page(&rsg->sgl[1]), 0);
+
+ if (obj->mask & RFC_WRITE_OR_SEND) {
+ if (obj->inline_page) {
+ __free_pages(obj->inline_page, 0);
+ obj->inline_page = NULL;
+ }
+ }
+ __free_pages(sg_page(&rsg->sgl[0]), 0);
+ sg_free_table(rsg);
+ kfree(rsg);
+ return 0;
+}
+
+int rfc_send_data_done(void *data, struct sg_table *rsg, u64 resp, int status)
+{
+ struct rfc_obj_info *obj = (struct rfc_obj_info *)data;
+ struct rfc_dev *rfc = obj->rfc;
+ struct rfc_port *port = &rfc->port;
+ struct rfc_qp *qp = NULL;
+ u32 qpn = bth_qpn(obj);
+ int index = 0;
+
+ rfc_send_data_free(data, rsg);
+ index = (qpn == 0) ? port->qp_smi_index : ((qpn == 1) ?
+ port->qp_gsi_index : qpn);
+ qp = rfc_pool_get_index(&rfc->qp_pool, index);
+ if (!qp) {
+ pr_err("%s(): qp#%d index %d does not exit\n", __func__,
+ qpn, index);
+ kfree(obj);
+ return 0;
+ }
+
+ obj->status = status;
+ obj->resp = resp;
+ if (qp_type(obj->qp) == IB_QPT_GSI)
+ obj->wqe->state = wqe_state_done;
+
+ // queue the obj to completer list
+ rfc_comp_queue_send_obj(obj->rfc, obj->qp, obj);
+
+ // Run completer
+ rfc_completer_direct(obj->qp);
+
+ return 0;
+}
+EXPORT_SYMBOL(rfc_send_data_done);
+
+static inline int addr_same(struct rfc_dev *rfc, struct rfc_av *av)
+{
+ return rfc->port.port_guid == av->grh.dgid.global.interface_id;
+}
+
+struct sg_table *rfc_init_object(struct rfc_dev *rfc, struct rfc_av *av,
+ int paylen, struct rfc_obj_info *obj)
+{
+ unsigned int hdr_len;
+ struct sg_table *rsg = NULL;
+ int npages = 0;
+ struct page *page;
+
+ hdr_len = 0;
+
+ npages = SG_MAX_SINGLE_ALLOC;
+ rsg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+
+ if (unlikely(!rsg))
+ return NULL;
+
+ if (sg_alloc_table(rsg, npages, GFP_KERNEL)) {
+ kfree(rsg);
+ return NULL;
+ }
+
+ page = alloc_page(GFP_KERNEL);
+ sg_set_page(rsg->sgl, page, rfc_opcode[obj->opcode].length, 0);
+ rsg->nents = 1;
+ rsg->orig_nents = npages;
+
+
+ if (obj->mask & RFC_ATMETH_MASK) {
+ struct page *page2 = alloc_page(GFP_KERNEL);
+
+ sg_set_page(&rsg->sgl[rsg->nents++], page2,
+ sizeof(struct rfc_atmeth), 0);
+ }
+ obj->rfc = rfc;
+ obj->port_num = 1;
+ obj->hdr = page_to_virt(page);
+ obj->mask |= RFC_GRH_MASK;
+
+ memset(obj->hdr, 0, rfc_opcode[obj->opcode].length);
+
+ return rsg;
+}
+
+/*
+ * this is required by rfc_cfg to match rfc devices in
+ * /sys/class/infiniband up with their underlying ethernet devices
+ */
+const char *rfc_parent_name(struct rfc_dev *rfc, unsigned int port_num)
+{
+ return rfc->ndev->name;
+}
+
+enum rdma_link_layer rfc_link_layer(struct rfc_dev *rfc, unsigned int port_num)
+{
+ return IB_LINK_LAYER_ETHERNET;
+}
+
+struct rfc_dev *rfc_net_add(struct net_device *ndev)
+{
+ int err;
+ struct rfc_dev *rfc = NULL;
+
+ rfc = (struct rfc_dev *)ib_alloc_device(sizeof(*rfc));
+ if (!rfc)
+ return NULL;
+
+ rfc->ndev = ndev;
+
+ err = rfc_add(rfc, ndev->mtu);
+ if (err) {
+ ib_dealloc_device(&rfc->ib_dev);
+ return NULL;
+ }
+
+ spin_lock_bh(&dev_list_lock);
+ list_add_tail(&rfc->list, &rfc_dev_list);
+ spin_unlock_bh(&dev_list_lock);
+ return rfc;
+}
+
+void rfc_remove_all(void)
+{
+ spin_lock_bh(&dev_list_lock);
+ while (!list_empty(&rfc_dev_list)) {
+ struct rfc_dev *rfc =
+ list_first_entry(&rfc_dev_list, struct rfc_dev, list);
+
+ list_del(&rfc->list);
+ spin_unlock_bh(&dev_list_lock);
+ rfc_remove(rfc);
+ spin_lock_bh(&dev_list_lock);
+ }
+ spin_unlock_bh(&dev_list_lock);
+}
+EXPORT_SYMBOL(rfc_remove_all);
+
+static void rfc_port_event(struct rfc_dev *rfc,
+ enum ib_event_type event)
+{
+ struct ib_event ev;
+
+ ev.device = &rfc->ib_dev;
+ ev.element.port_num = 1;
+ ev.event = event;
+
+ ib_dispatch_event(&ev);
+}
+
+/* Caller must hold net_info_lock */
+void rfc_port_up(struct rfc_dev *rfc)
+{
+ struct rfc_port *port;
+
+ port = &rfc->port;
+ port->attr.state = IB_PORT_ACTIVE;
+ port->attr.phys_state = IB_PHYS_STATE_LINK_UP;
+
+ rfc_port_event(rfc, IB_EVENT_PORT_ACTIVE);
+ pr_info("set %s active\n", rfc->ib_dev.name);
+}
+
+/* Caller must hold net_info_lock */
+void rfc_port_down(struct rfc_dev *rfc)
+{
+ struct rfc_port *port;
+
+ port = &rfc->port;
+ port->attr.state = IB_PORT_DOWN;
+ port->attr.phys_state = IB_PHYS_STATE_LINK_DOWN;
+
+ rfc_port_event(rfc, IB_EVENT_PORT_ERR);
+ pr_info("set %s down\n", rfc->ib_dev.name);
+}
+
+static int rfc_notify(struct notifier_block *not_blk,
+ unsigned long event,
+ void *arg)
+{
+ struct net_device *ndev = netdev_notifier_info_to_dev(arg);
+ struct rfc_dev *rfc = net_to_rfc(ndev);
+
+ if (!rfc)
+ goto out;
+
+ switch (event) {
+ case NETDEV_UNREGISTER:
+ list_del(&rfc->list);
+ rfc_remove(rfc);
+ break;
+ case NETDEV_UP:
+ rfc_port_up(rfc);
+ break;
+ case NETDEV_DOWN:
+ rfc_port_down(rfc);
+ break;
+ case NETDEV_CHANGEMTU:
+ pr_info("%s changed mtu to %d\n", ndev->name, ndev->mtu);
+ rfc_set_mtu(rfc, ndev->mtu);
+ break;
+ case NETDEV_REBOOT:
+ case NETDEV_CHANGE:
+ case NETDEV_GOING_DOWN:
+ case NETDEV_CHANGEADDR:
+ case NETDEV_CHANGENAME:
+ case NETDEV_FEAT_CHANGE:
+ default:
+ pr_info("ignoring netdev event = %ld for %s\n",
+ event, ndev->name);
+ break;
+ }
+out:
+ return NOTIFY_OK;
+}
+
+struct notifier_block rfc_net_notifier = {
+ .notifier_call = rfc_notify,
+};
+
+static int rfc_send_blk(struct sg_table *rsg, int len, void *data)
+
+{
+ struct rfc_obj_info *obj = (struct rfc_obj_info *)data;
+ struct rdma_hdr_common chdr;
+ struct rfc_bth *bth = (struct rfc_bth *)obj->hdr;
+ struct rfc_av *av = rfc_get_av(obj);
+ __be32 daddr = av->dgid_addr._sockaddr_in.sin_addr.s_addr;
+
+ // Note: All values are in be32
+ memset(&chdr, 0, sizeof(struct rdma_hdr_common));
+ chdr.opcode = bth->opcode;
+ chdr.flags = bth->flags;
+ chdr.qpn = bth->qpn;
+ chdr.apsn = bth->apsn;
+ if (obj->mask & RFC_RETH_MASK) {
+ struct rfc_reth *reth = (struct rfc_reth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_RETH]);
+
+ chdr.rkey_AETH_IMMDT.rkey = reth->rkey;
+ chdr.deth_va.va = reth->va;
+
+ if (obj->mask & RFC_IETH_MASK) {
+ pr_debug("RFC: IETH not supported with RETH\n");
+ goto send;
+ }
+ if (obj->mask & RFC_DETH_MASK) {
+ pr_debug("RFC: DETH not supported with RETH\n");
+ goto send;
+ }
+ if (obj->mask & RFC_AETH_MASK) {
+ pr_debug("RFC: AETH not supported with RETH\n");
+ goto send;
+ }
+ if (obj->mask & RFC_ATMACK_MASK) {
+ pr_debug("RFC: ATMACK not supported with RETH\n");
+ goto send;
+ }
+ } else if (obj->mask & RFC_IETH_MASK) {
+ struct rfc_ieth *ieth = (struct rfc_ieth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_IETH]);
+
+ chdr.rkey_AETH_IMMDT.rkey = ieth->rkey;
+ } else if (obj->mask & RFC_AETH_MASK) {
+ struct rfc_aeth *aeth = (struct rfc_aeth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_AETH]);
+
+ chdr.rkey_AETH_IMMDT.AETH = aeth->smsn;
+ }
+ if (obj->mask & RFC_IMMDT_MASK) {
+ struct rfc_immdt *immdt = (struct rfc_immdt *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_IMMDT]);
+
+ chdr.pkey = __cpu_to_le16(__be32_to_cpu(immdt->imm));
+ }
+ if (obj->mask & RFC_DETH_MASK) {
+ struct rfc_deth *deth = (struct rfc_deth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_DETH]);
+
+ chdr.deth_va.dth.qkey = deth->qkey;
+ chdr.deth_va.dth.sqp = deth->sqp;
+ }
+ if (obj->mask & RFC_ATMACK_MASK) {
+ struct rfc_atmack *atmack = (struct rfc_atmack *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_ATMACK]);
+
+ chdr.deth_va.atm_orig = atmack->orig; // be64 to be64
+ }
+
+send:
+ rfc_sendobject(&chdr, rsg, len, daddr, data,
+ (obj->mask & RFC_READ_MASK)?RFC_OP_READ:RFC_OP_WRITE);
+ return 0;
+}
+
+static void print_rcv_sgl(struct sg_table *rsg)
+{
+ struct scatterlist *sgl = rsg->sgl, *tmpsg;
+ int i = 0;
+
+ for_each_sg(sgl, tmpsg, rsg->nents, i) {
+ pr_debug("\tSGL %d offset %d length %d\n", i, tmpsg->offset,
+ tmpsg->length);
+ }
+}
+
+static int build_rdma_hdr_from_nvme(char *buf, int len,
+ struct rfc_obj_info *obj)
+{
+ struct rdma_hdr_common *chdrp = (struct rdma_hdr_common *)buf;
+ struct rfc_bth *bth = (struct rfc_bth *)obj->hdr;
+ u32 qpn;
+
+ // Note: All values are in be32
+ bth->opcode = chdrp->opcode;
+ bth->flags = chdrp->flags;
+ bth->pkey = chdrp->pkey;
+ bth->qpn = chdrp->qpn;
+ bth->apsn = chdrp->apsn;
+
+ obj->offset = 0;
+ obj->opcode = bth_opcode(obj);
+ obj->psn = bth_psn(obj);
+ qpn = bth_qpn(obj);
+ obj->mask = rfc_opcode[obj->opcode].mask;
+ if (obj->mask & RFC_RETH_MASK) {
+ struct rfc_reth *reth = (struct rfc_reth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_RETH]);
+
+ reth->rkey = chdrp->rkey_AETH_IMMDT.rkey;
+ reth->va = chdrp->deth_va.va;
+ reth->len = cpu_to_be32(len);
+ } else if (obj->mask & RFC_IETH_MASK) {
+ struct rfc_ieth *ieth = (struct rfc_ieth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_IETH]);
+
+ ieth->rkey = chdrp->rkey_AETH_IMMDT.rkey;
+ } else if (obj->mask & RFC_AETH_MASK) {
+ struct rfc_aeth *aeth = (struct rfc_aeth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_AETH]);
+
+ aeth->smsn = chdrp->rkey_AETH_IMMDT.AETH;
+ }
+ if (obj->mask & RFC_IMMDT_MASK) {
+ struct rfc_immdt *immdt = (struct rfc_immdt *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_IMMDT]);
+
+ immdt->imm = __cpu_to_be32(__le16_to_cpu(chdrp->pkey));
+ }
+ if (obj->mask & RFC_DETH_MASK) {
+ struct rfc_deth *deth = (struct rfc_deth *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_DETH]);
+
+ deth->qkey = chdrp->deth_va.dth.qkey;
+ deth->sqp = chdrp->deth_va.dth.sqp;
+ }
+ if (obj->mask & RFC_ATMACK_MASK) {
+ struct rfc_atmack *atmack = (struct rfc_atmack *)
+ (obj->hdr + rfc_opcode[obj->opcode].offset[RFC_ATMACK]);
+
+ atmack->orig = chdrp->deth_va.atm_orig; // be64 to be64
+ }
+ return 0;
+}
+
+static inline u32 chdr_qpn(void *buf)
+{
+ struct rdma_hdr_common *chdrp = (struct rdma_hdr_common *)buf;
+
+ // Note: All values are in be32
+ return chdrp->qpn;
+}
+
+u64 rfc_get_target_sgl(void *buf, int len, struct sg_table **rsgp, char *name)
+{
+ struct rfc_dev *rfc = get_rfc_by_objname(name);
+ struct sg_table *rsg = NULL;
+ struct rfc_obj_info *obj = NULL;
+ struct rfc_bth *bth = NULL;
+ struct rfc_recv_data *rcv = NULL;
+ struct rfc_qp *qp = NULL;
+ struct rfc_port *port = NULL;
+ int bthlen = 0;
+ u64 resp = 0;
+ u32 qpn = 0;
+ int index = 0;
+
+ *rsgp = NULL;
+
+ if (!rfc) {
+ pr_err("%s(): ERROR : rfc not found for objname %s\n",
+ __func__, name);
+ goto drop;
+ }
+
+ port = &rfc->port;
+
+ qpn = be32_to_cpu(chdr_qpn(buf));
+
+ index = (qpn == 0) ?
+ port->qp_smi_index : ((qpn == 1) ?
+ port->qp_gsi_index : qpn);
+
+ qp = rfc_pool_get_index(&rfc->qp_pool, index);
+ if (!qp) {
+ pr_err("%s(): qp#%d index %d does not exit\n",
+ __func__, qpn, index);
+ goto drop;
+ }
+ rcv = &qp->rcv_data;
+ // Allocate for worst case ( 12 + 4 + 16 + 28 = 60 bytes header)
+ bthlen = sizeof(struct rfc_bth) +
+ sizeof(struct rfc_aeth) +
+ sizeof(struct rfc_reth) +
+ sizeof(struct rfc_atmeth);
+// rcv = kmalloc(sizeof (struct rfc_recv_data), GFP_KERNEL);
+ bth = (struct rfc_bth *)rcv->data;
+ memset(bth, 0, bthlen);
+ obj = &rcv->obj;
+ obj->hdr = (u8 *)bth;
+ build_rdma_hdr_from_nvme(buf, len, obj);
+
+ /* rfc_obj_info is available now */
+// rcv->data = obj->hdr;
+ rcv->transport_header = rcv->network_header = 0;
+ rcv->protocol = htons(ETH_P_IP);
+ rcv->len = len;
+ rcv->hdr_len = 0;
+
+ obj->rfc = rfc;
+ obj->port_num = 1;
+ obj->offset = 0;
+ obj->paylen = len;
+
+
+ resp = rfc_rcv_hdr(rcv, &rsg);
+ if (rsg) {
+ print_rcv_sgl(rsg);
+ /* if we have the rsg, we have obj->qp */
+ }
+ *rsgp = rsg;
+
+ return resp;
+drop:
+ return 0;
+
+}
+EXPORT_SYMBOL(rfc_get_target_sgl);
+
+u64 rfc_recv_blk(char *buf, int len, char *name)
+{
+ struct rfc_dev *rfc = get_rfc_by_objname(name);
+ struct rfc_obj_info *obj = NULL;
+ struct rfc_bth *bth = NULL;
+ struct rfc_recv_data *rcv = NULL;
+ struct rfc_qp *qp = NULL;
+ struct rfc_port *port = NULL;
+ u64 ret = 0;
+ u32 qpn = 0;
+ int bthlen = 0;
+ int index = 0;
+
+ if (!rfc) {
+ pr_err("RFC recv ERROR : no rfcdev for objname %s\n", name);
+ goto drop;
+ }
+
+ port = &rfc->port;
+
+ qpn = be32_to_cpu(chdr_qpn(buf));
+
+ index = (qpn == 0) ?
+ port->qp_smi_index : ((qpn == 1) ?
+ port->qp_gsi_index : qpn);
+
+ qp = rfc_pool_get_index(&rfc->qp_pool, index);
+ if (!qp) {
+ pr_err("%s(): qp#%d index %d does not exit\n",
+ __func__, qpn, index);
+ kfree(obj);
+ return 0;
+ }
+ rcv = &qp->rcv_data;
+
+ bth = (struct rfc_bth *)rcv->data;
+ obj = &rcv->obj;
+ obj->hdr = (u8 *)bth;
+ /* rcv->data_valid might be 0 in case a CMD came with 0 data. */
+ if (rcv->data_valid == 0) {
+ bthlen = sizeof(struct rfc_bth) +
+ sizeof(struct rfc_aeth) +
+ sizeof(struct rfc_reth) +
+ sizeof(struct rfc_atmeth);
+ memset(bth, 0, bthlen);
+ build_rdma_hdr_from_nvme(buf, len, obj);
+ rcv->transport_header = rcv->network_header = 0;
+ rcv->protocol = htons(ETH_P_IP);
+ rcv->len = len;
+ rcv->hdr_len = 0;
+
+ obj->rfc = rfc;
+ obj->port_num = 1;
+ obj->offset = 0;
+ obj->paylen = len;
+ }
+ if (obj->mask & RFC_ATOMIC_MASK) {
+ /* copy the atomic header contents from eth_page */
+ struct rfc_atmeth *objatmeth, *atmeth;
+
+ atmeth = (struct rfc_atmeth *)page_address(rcv->eth_page);
+ objatmeth = (struct rfc_atmeth *)(obj->hdr +
+ rfc_opcode[obj->opcode].offset[RFC_ATMETH]);
+ // be64 to be64
+ memcpy(objatmeth, atmeth, sizeof(struct rfc_atmeth));
+ __free_pages(rcv->eth_page, 0);
+ }
+ qpn = bth_qpn(obj);
+ if (qpn == IB_QPT_GSI) {
+ // Extract the src and dest ip from the end of GSI data
+ __be32 *addrp = (__be32 *)((char *)page_address(rcv->eth_page)
+ + payload_size(obj) - sizeof(__be32));
+ obj->daddr = *addrp;
+ obj->saddr = *(addrp - 1);
+ obj->paylen = obj->paylen - 2 * sizeof(__be32);
+ rcv->len = obj->paylen;
+ // eth_page will be freed later after GSI data copy
+ }
+
+ ret = rfc_rcv_done(rcv);
+
+ rcv->data_valid = 0;
+ return ret;
+drop:
+ return 0;
+
+}
+EXPORT_SYMBOL(rfc_recv_blk);
+
+u64 rfc_target_data_done(void *hdr, int len, char *name)
+{
+ return rfc_recv_blk((char *)hdr, len, name);
+}
+EXPORT_SYMBOL(rfc_target_data_done);
+
+static int rfc_blk_init(void)
+{
+ // initialize the tb driver
+ rfcblk_init();
+
+ fc_register_rdma_alloc_tgt_pgs(rfc_get_target_sgl);
+ fc_register_rdma_dma_done(rfc_target_data_done);
+ return 0;
+}
+
+int rfc_obj_init(void)
+{
+ int err;
+
+ err = rfc_net_link_init();
+ err = rfc_blk_init();
+
+ if (err) {
+ pr_err("Failed to register netdev notifier\n");
+ goto err_out;
+ }
+ return 0;
+err_out:
+ return err;
+}
+void rfc_obj_exit(void)
+{
+ rfcblk_cleanup();
+ rfc_net_link_uninit();
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_obj.h b/drivers/infiniband/sw/rfc/rfc_obj.h
new file mode 100644
index 0000000..3147107
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_obj.h
@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+
+#ifndef RFC_OBJ_H
+#define RFC_OBJ_H
+
+#include <uapi/scsi/fc/fc_fs.h>
+#include <uapi/scsi/fc/fc_els.h>
+#include <linux/nvme-fc-driver.h>
+
+#define OBJ_SIZE 4096
+/*
+ * NVMe CMDIU structures for RDMA info
+ */
+struct deth_header {
+ __be32 qkey; /*Q key*/
+ __be32 sqp; /*Source queue pair number*/
+};
+
+union rdma_deth_va {
+ __be64 va;
+ __be64 atm_orig;
+ struct deth_header dth;
+};
+
+/* reth: rkey
+ * AETH :syndrome and msg seq number
+ * IMMDT: data
+ * for AETH and IMMDT there is no need to send reth
+ * payload so we can reuse this
+ */
+union rdma_hdr_data_types {
+ __be32 rkey;
+ __be32 AETH;
+ __be32 IMMDT;
+};
+
+struct atomic_header {
+ __be64 swap_add; /*Atomic extended transport header: swap_add*/
+ __be64 comp_data; /*Atomic extended transport header: comp_add*/
+};
+
+struct rdma_hdr_common {
+ union rdma_deth_va deth_va; /* Word 16,17 */
+ union rdma_hdr_data_types rkey_AETH_IMMDT; /* Word 18 */
+ __u8 opcode; /* Word 19 - beth : opcode */
+ __u8 flags; /* Word 19 - Beth : flags*/
+ __be16 pkey; /* Word 19 - beth : patition key */
+ __be32 qpn; /* Word 20 - beth : queue pair number */
+ __be32 apsn; /* Word 21 - beth : object sequence number */
+};
+
+#define RFC_OP_WRITE 1
+#define RFC_OP_READ 2
+
+/* ########### FUNCTION DECLARATIONS ############################## */
+/* rfc_net.c */
+u64 rfc_target_data_done(void *hdr, int len, char *name);
+int rfc_send_data_done(void *data, struct sg_table *rsg,
+ u64 resp, int status);
+
+/* rfc_net_link.c */
+int rfc_net_link_init(void);
+void rfc_net_link_uninit(void);
+void push_data_blk_skb(char *buffer);
+
+/* rfc_tb.c */
+int rfcblk_init(void);
+void rfcblk_cleanup(void);
+int rfc_sendobject(struct rdma_hdr_common *rdma_hdr, struct sg_table *rsg,
+ int len, __be32 daddr, void *data, int op);
+int rfc_update_ip_addr(char *dev_name);
+char *dev_path_get_by_list(u32 ipaddr);
+
+/* END rfc_tb.c */
+
+#endif /* RFC_OBJ_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_opcode.c b/drivers/infiniband/sw/rfc/rfc_opcode.c
new file mode 100644
index 0000000..f62974c
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_opcode.c
@@ -0,0 +1,964 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <rdma/ib_pack.h>
+#include "rfc_opcode.h"
+//#include "rfc_hdr.h"
+#include "rfc.h"
+
+/* useful information about work request opcodes and obj opcodes in
+ * table form
+ */
+struct rfc_wr_opcode_info rfc_wr_opcode_info[] = {
+ [IB_WR_RDMA_WRITE] = {
+ .name = "IB_WR_RDMA_WRITE",
+ .mask = {
+ [IB_QPT_RC] = WR_INLINE_MASK | WR_WRITE_MASK,
+ [IB_QPT_UC] = WR_INLINE_MASK | WR_WRITE_MASK,
+ },
+ },
+ [IB_WR_RDMA_WRITE_WITH_IMM] = {
+ .name = "IB_WR_RDMA_WRITE_WITH_IMM",
+ .mask = {
+ [IB_QPT_RC] = WR_INLINE_MASK | WR_WRITE_MASK,
+ [IB_QPT_UC] = WR_INLINE_MASK | WR_WRITE_MASK,
+ },
+ },
+ [IB_WR_SEND] = {
+ .name = "IB_WR_SEND",
+ .mask = {
+ [IB_QPT_SMI] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_GSI] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_RC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UD] = WR_INLINE_MASK | WR_SEND_MASK,
+ },
+ },
+ [IB_WR_SEND_WITH_IMM] = {
+ .name = "IB_WR_SEND_WITH_IMM",
+ .mask = {
+ [IB_QPT_SMI] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_GSI] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_RC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UD] = WR_INLINE_MASK | WR_SEND_MASK,
+ },
+ },
+ [IB_WR_RDMA_READ] = {
+ .name = "IB_WR_RDMA_READ",
+ .mask = {
+ [IB_QPT_RC] = WR_READ_MASK,
+ },
+ },
+ [IB_WR_ATOMIC_CMP_AND_SWP] = {
+ .name = "IB_WR_ATOMIC_CMP_AND_SWP",
+ .mask = {
+ [IB_QPT_RC] = WR_ATOMIC_MASK,
+ },
+ },
+ [IB_WR_ATOMIC_FETCH_AND_ADD] = {
+ .name = "IB_WR_ATOMIC_FETCH_AND_ADD",
+ .mask = {
+ [IB_QPT_RC] = WR_ATOMIC_MASK,
+ },
+ },
+ [IB_WR_LSO] = {
+ .name = "IB_WR_LSO",
+ .mask = {
+ /* not supported */
+ },
+ },
+ [IB_WR_SEND_WITH_INV] = {
+ .name = "IB_WR_SEND_WITH_INV",
+ .mask = {
+ [IB_QPT_RC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UC] = WR_INLINE_MASK | WR_SEND_MASK,
+ [IB_QPT_UD] = WR_INLINE_MASK | WR_SEND_MASK,
+ },
+ },
+ [IB_WR_RDMA_READ_WITH_INV] = {
+ .name = "IB_WR_RDMA_READ_WITH_INV",
+ .mask = {
+ [IB_QPT_RC] = WR_READ_MASK,
+ },
+ },
+ [IB_WR_LOCAL_INV] = {
+ .name = "IB_WR_LOCAL_INV",
+ .mask = {
+ [IB_QPT_RC] = WR_REG_MASK,
+ },
+ },
+ [IB_WR_REG_MR] = {
+ .name = "IB_WR_REG_MR",
+ .mask = {
+ [IB_QPT_RC] = WR_REG_MASK,
+ },
+ },
+};
+
+struct rfc_opcode_info rfc_opcode[RFC_NUM_OPCODE] = {
+ [IB_OPCODE_RC_SEND_FIRST] = {
+ .name = "IB_OPCODE_RC_SEND_FIRST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_RWR_MASK
+ | RFC_SEND_MASK | RFC_START_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_MIDDLE] = {
+ .name = "IB_OPCODE_RC_SEND_MIDDLE]",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_SEND_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_LAST] = {
+ .name = "IB_OPCODE_RC_SEND_LAST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_COMP_MASK
+ | RFC_SEND_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_SEND_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_ONLY] = {
+ .name = "IB_OPCODE_RC_SEND_ONLY",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_COMP_MASK
+ | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_FIRST] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_FIRST",
+ .mask = RFC_RETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_MIDDLE] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_MIDDLE",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_LAST] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_LAST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_ONLY] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY",
+ .mask = RFC_RETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_RETH_MASK | RFC_IMMDT_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_READ_REQUEST] = {
+ .name = "IB_OPCODE_RC_RDMA_READ_REQUEST",
+ .mask = RFC_RETH_MASK | RFC_REQ_MASK | RFC_READ_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST] = {
+ .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST",
+ .mask = RFC_AETH_MASK | RFC_PAYLOAD_MASK | RFC_ACK_MASK
+ | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_AETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE] = {
+ .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE",
+ .mask = RFC_PAYLOAD_MASK | RFC_ACK_MASK | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST] = {
+ .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST",
+ .mask = RFC_AETH_MASK | RFC_PAYLOAD_MASK | RFC_ACK_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_AETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY] = {
+ .name = "IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY",
+ .mask = RFC_AETH_MASK | RFC_PAYLOAD_MASK | RFC_ACK_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_AETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_ACKNOWLEDGE] = {
+ .name = "IB_OPCODE_RC_ACKNOWLEDGE",
+ .mask = RFC_AETH_MASK | RFC_ACK_MASK | RFC_START_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_AETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE] = {
+ .name = "IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE",
+ .mask = RFC_AETH_MASK | RFC_ATMACK_MASK | RFC_ACK_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMACK_BYTES + RFC_AETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_AETH] = RFC_BTH_BYTES,
+ [RFC_ATMACK] = RFC_BTH_BYTES
+ + RFC_AETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_ATMACK_BYTES + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_COMPARE_SWAP] = {
+ .name = "IB_OPCODE_RC_COMPARE_SWAP",
+ .mask = RFC_ATMETH_MASK | RFC_REQ_MASK | RFC_ATOMIC_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_ATMETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_ATMETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_FETCH_ADD] = {
+ .name = "IB_OPCODE_RC_FETCH_ADD",
+ .mask = RFC_ATMETH_MASK | RFC_REQ_MASK | RFC_ATOMIC_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_ATMETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_ATMETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE] = {
+ .name = "IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE",
+ .mask = RFC_IETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_SEND_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE] = {
+ .name = "IB_OPCODE_RC_SEND_ONLY_INV",
+ .mask = RFC_IETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IETH_BYTES,
+ }
+ },
+
+ /* UC */
+ [IB_OPCODE_UC_SEND_FIRST] = {
+ .name = "IB_OPCODE_UC_SEND_FIRST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_RWR_MASK
+ | RFC_SEND_MASK | RFC_START_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_SEND_MIDDLE] = {
+ .name = "IB_OPCODE_UC_SEND_MIDDLE",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_SEND_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_SEND_LAST] = {
+ .name = "IB_OPCODE_UC_SEND_LAST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_COMP_MASK
+ | RFC_SEND_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_SEND_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_SEND_ONLY] = {
+ .name = "IB_OPCODE_UC_SEND_ONLY",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_COMP_MASK
+ | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_FIRST] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_FIRST",
+ .mask = RFC_RETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_MIDDLE] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_MIDDLE",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_LAST] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_LAST",
+ .mask = RFC_PAYLOAD_MASK | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE",
+ .mask = RFC_IMMDT_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_IMMDT] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_ONLY] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY",
+ .mask = RFC_RETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_RETH_MASK | RFC_IMMDT_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_RETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RETH] = RFC_BTH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+
+ /* RD */
+ [IB_OPCODE_RD_SEND_FIRST] = {
+ .name = "IB_OPCODE_RD_SEND_FIRST",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_SEND_MIDDLE] = {
+ .name = "IB_OPCODE_RD_SEND_MIDDLE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_SEND_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_SEND_LAST] = {
+ .name = "IB_OPCODE_RD_SEND_LAST",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_COMP_MASK | RFC_SEND_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RD_SEND_LAST_WITH_IMMEDIATE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_IMMDT_MASK
+ | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_SEND_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_SEND_ONLY] = {
+ .name = "IB_OPCODE_RD_SEND_ONLY",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_SEND_MASK | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RD_SEND_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_IMMDT_MASK
+ | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_FIRST] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_FIRST",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_RETH_MASK
+ | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_RETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_MIDDLE] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_MIDDLE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_LAST] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_LAST",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_LAST_WITH_IMMEDIATE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_IMMDT_MASK
+ | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_ONLY] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_RETH_MASK
+ | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_WRITE_MASK | RFC_START_MASK
+ | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_RETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_RETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_RD_RDMA_WRITE_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_RETH_MASK
+ | RFC_IMMDT_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_WRITE_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_RETH_BYTES
+ + RFC_DETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_RETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_RETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_RETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_READ_REQUEST] = {
+ .name = "IB_OPCODE_RD_RDMA_READ_REQUEST",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_RETH_MASK
+ | RFC_REQ_MASK | RFC_READ_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_RETH_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_RETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RETH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST] = {
+ .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_FIRST",
+ .mask = RFC_RDETH_MASK | RFC_AETH_MASK
+ | RFC_PAYLOAD_MASK | RFC_ACK_MASK
+ | RFC_START_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_AETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE] = {
+ .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_MIDDLE",
+ .mask = RFC_RDETH_MASK | RFC_PAYLOAD_MASK | RFC_ACK_MASK
+ | RFC_MIDDLE_MASK,
+ .length = RFC_BTH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST] = {
+ .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_LAST",
+ .mask = RFC_RDETH_MASK | RFC_AETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_ACK_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_AETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY] = {
+ .name = "IB_OPCODE_RD_RDMA_READ_RESPONSE_ONLY",
+ .mask = RFC_RDETH_MASK | RFC_AETH_MASK | RFC_PAYLOAD_MASK
+ | RFC_ACK_MASK | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_AETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_ACKNOWLEDGE] = {
+ .name = "IB_OPCODE_RD_ACKNOWLEDGE",
+ .mask = RFC_RDETH_MASK | RFC_AETH_MASK | RFC_ACK_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_AETH_BYTES + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_AETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE] = {
+ .name = "IB_OPCODE_RD_ATOMIC_ACKNOWLEDGE",
+ .mask = RFC_RDETH_MASK | RFC_AETH_MASK | RFC_ATMACK_MASK
+ | RFC_ACK_MASK | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMACK_BYTES + RFC_AETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_AETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_ATMACK] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_AETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_COMPARE_SWAP] = {
+ .name = "RD_COMPARE_SWAP",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_ATMETH_MASK
+ | RFC_REQ_MASK | RFC_ATOMIC_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMETH_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_ATMETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES +
+ + RFC_ATMETH_BYTES
+ + RFC_DETH_BYTES +
+ + RFC_RDETH_BYTES,
+ }
+ },
+ [IB_OPCODE_RD_FETCH_ADD] = {
+ .name = "IB_OPCODE_RD_FETCH_ADD",
+ .mask = RFC_RDETH_MASK | RFC_DETH_MASK | RFC_ATMETH_MASK
+ | RFC_REQ_MASK | RFC_ATOMIC_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_ATMETH_BYTES + RFC_DETH_BYTES
+ + RFC_RDETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_RDETH] = RFC_BTH_BYTES,
+ [RFC_DETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES,
+ [RFC_ATMETH] = RFC_BTH_BYTES
+ + RFC_RDETH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES +
+ + RFC_ATMETH_BYTES
+ + RFC_DETH_BYTES +
+ + RFC_RDETH_BYTES,
+ }
+ },
+
+ /* UD */
+ [IB_OPCODE_UD_SEND_ONLY] = {
+ .name = "IB_OPCODE_UD_SEND_ONLY",
+ .mask = RFC_DETH_MASK | RFC_PAYLOAD_MASK | RFC_REQ_MASK
+ | RFC_COMP_MASK | RFC_RWR_MASK | RFC_SEND_MASK
+ | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_DETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_DETH] = RFC_BTH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_DETH_BYTES,
+ }
+ },
+ [IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE] = {
+ .name = "IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE",
+ .mask = RFC_DETH_MASK | RFC_IMMDT_MASK | RFC_PAYLOAD_MASK
+ | RFC_REQ_MASK | RFC_COMP_MASK | RFC_RWR_MASK
+ | RFC_SEND_MASK | RFC_START_MASK | RFC_END_MASK,
+ .length = RFC_BTH_BYTES + RFC_IMMDT_BYTES + RFC_DETH_BYTES,
+ .offset = {
+ [RFC_BTH] = 0,
+ [RFC_DETH] = RFC_BTH_BYTES,
+ [RFC_IMMDT] = RFC_BTH_BYTES
+ + RFC_DETH_BYTES,
+ [RFC_PAYLOAD] = RFC_BTH_BYTES
+ + RFC_DETH_BYTES
+ + RFC_IMMDT_BYTES,
+ }
+ },
+
+};
diff --git a/drivers/infiniband/sw/rfc/rfc_opcode.h b/drivers/infiniband/sw/rfc/rfc_opcode.h
new file mode 100644
index 0000000..79bffcc
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_opcode.h
@@ -0,0 +1,131 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_OPCODE_H
+#define RFC_OPCODE_H
+
+/*
+ * contains header bit mask definitions and header lengths
+ * declaration of the rfc_opcode_info struct and
+ * rfc_wr_opcode_info struct
+ */
+
+enum rfc_wr_mask {
+ WR_INLINE_MASK = BIT(0),
+ WR_ATOMIC_MASK = BIT(1),
+ WR_SEND_MASK = BIT(2),
+ WR_READ_MASK = BIT(3),
+ WR_WRITE_MASK = BIT(4),
+ WR_LOCAL_MASK = BIT(5),
+ WR_REG_MASK = BIT(6),
+
+ WR_READ_OR_WRITE_MASK = WR_READ_MASK | WR_WRITE_MASK,
+ WR_READ_WRITE_OR_SEND_MASK = WR_READ_OR_WRITE_MASK | WR_SEND_MASK,
+ WR_WRITE_OR_SEND_MASK = WR_WRITE_MASK | WR_SEND_MASK,
+ WR_ATOMIC_OR_READ_MASK = WR_ATOMIC_MASK | WR_READ_MASK,
+};
+
+#define WR_MAX_QPT (8)
+
+struct rfc_wr_opcode_info {
+ char *name;
+ enum rfc_wr_mask mask[WR_MAX_QPT];
+};
+
+extern struct rfc_wr_opcode_info rfc_wr_opcode_info[];
+
+enum rfc_hdr_type {
+ RFC_LRH,
+ RFC_GRH,
+ RFC_BTH,
+ RFC_RETH,
+ RFC_AETH,
+ RFC_ATMETH,
+ RFC_ATMACK,
+ RFC_IETH,
+ RFC_RDETH,
+ RFC_DETH,
+ RFC_IMMDT,
+ RFC_PAYLOAD,
+ NUM_HDR_TYPES
+};
+
+enum rfc_hdr_mask {
+ RFC_LRH_MASK = BIT(RFC_LRH),
+ RFC_GRH_MASK = BIT(RFC_GRH),
+ RFC_BTH_MASK = BIT(RFC_BTH),
+ RFC_IMMDT_MASK = BIT(RFC_IMMDT),
+ RFC_RETH_MASK = BIT(RFC_RETH),
+ RFC_AETH_MASK = BIT(RFC_AETH),
+ RFC_ATMETH_MASK = BIT(RFC_ATMETH),
+ RFC_ATMACK_MASK = BIT(RFC_ATMACK),
+ RFC_IETH_MASK = BIT(RFC_IETH),
+ RFC_RDETH_MASK = BIT(RFC_RDETH),
+ RFC_DETH_MASK = BIT(RFC_DETH),
+ RFC_PAYLOAD_MASK = BIT(RFC_PAYLOAD),
+
+ RFC_REQ_MASK = BIT(NUM_HDR_TYPES + 0),
+ RFC_ACK_MASK = BIT(NUM_HDR_TYPES + 1),
+ RFC_SEND_MASK = BIT(NUM_HDR_TYPES + 2),
+ RFC_WRITE_MASK = BIT(NUM_HDR_TYPES + 3),
+ RFC_READ_MASK = BIT(NUM_HDR_TYPES + 4),
+ RFC_ATOMIC_MASK = BIT(NUM_HDR_TYPES + 5),
+
+ RFC_RWR_MASK = BIT(NUM_HDR_TYPES + 6),
+ RFC_COMP_MASK = BIT(NUM_HDR_TYPES + 7),
+
+ RFC_START_MASK = BIT(NUM_HDR_TYPES + 8),
+ RFC_MIDDLE_MASK = BIT(NUM_HDR_TYPES + 9),
+ RFC_END_MASK = BIT(NUM_HDR_TYPES + 10),
+
+ RFC_LOOPBACK_MASK = BIT(NUM_HDR_TYPES + 12),
+
+ RFC_READ_OR_ATOMIC = (RFC_READ_MASK | RFC_ATOMIC_MASK),
+ RFC_WRITE_OR_SEND = (RFC_WRITE_MASK | RFC_SEND_MASK),
+};
+
+#define OPCODE_NONE (-1)
+#define RFC_NUM_OPCODE 256
+
+struct rfc_opcode_info {
+ char *name;
+ enum rfc_hdr_mask mask;
+ int length;
+ int offset[NUM_HDR_TYPES];
+};
+
+extern struct rfc_opcode_info rfc_opcode[RFC_NUM_OPCODE];
+
+#endif /* RFC_OPCODE_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_param.h b/drivers/infiniband/sw/rfc/rfc_param.h
new file mode 100644
index 0000000..f066ddf
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_param.h
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_PARAM_H
+#define RFC_PARAM_H
+
+static inline enum ib_mtu rfc_mtu_int_to_enum(int mtu)
+{
+ if (mtu < 256)
+ return 0;
+ else if (mtu < 512)
+ return IB_MTU_256;
+ else if (mtu < 1024)
+ return IB_MTU_512;
+ else if (mtu < 2048)
+ return IB_MTU_1024;
+ else if (mtu < 4096)
+ return IB_MTU_2048;
+ else
+ return IB_MTU_4096;
+}
+
+/* Find the IB mtu for a given network MTU. */
+static inline enum ib_mtu eth_mtu_int_to_enum(int mtu)
+{
+ mtu -= RFC_MAX_HDR_LENGTH;
+
+ return rfc_mtu_int_to_enum(mtu);
+}
+
+/* default/initial rfc device parameter settings */
+enum rfc_device_param {
+ RFC_FW_VER = 0,
+ RFC_MAX_MR_SIZE = -1ull,
+ RFC_PAGE_SIZE_CAP = 0xfffff000,
+ RFC_VENDOR_ID = 0,
+ RFC_VENDOR_PART_ID = 0,
+ RFC_HW_VER = 0,
+ RFC_MAX_QP = 0x10000,
+ RFC_MAX_QP_WR = 0x4000,
+ RFC_MAX_INLINE_DATA = 400,
+ RFC_DEVICE_CAP_FLAGS = IB_DEVICE_BAD_PKEY_CNTR
+ | IB_DEVICE_BAD_QKEY_CNTR
+ | IB_DEVICE_AUTO_PATH_MIG
+ | IB_DEVICE_CHANGE_PHY_PORT
+ | IB_DEVICE_UD_AV_PORT_ENFORCE
+ | IB_DEVICE_PORT_ACTIVE_EVENT
+ | IB_DEVICE_SYS_IMAGE_GUID
+ | IB_DEVICE_RC_RNR_NAK_GEN
+ | IB_DEVICE_SRQ_RESIZE
+ | IB_DEVICE_MEM_MGT_EXTENSIONS,
+ RFC_MAX_SGE = 32,
+ RFC_MAX_SGE_RD = 32,
+ RFC_MAX_CQ = 16384,
+ RFC_MAX_LOG_CQE = 15,
+ RFC_MAX_MR = 2 * 1024,
+ RFC_MAX_PD = 0x7ffc,
+ RFC_MAX_QP_RD_ATOM = 128,
+ RFC_MAX_EE_RD_ATOM = 0,
+ RFC_MAX_RES_RD_ATOM = 0x3f000,
+ RFC_MAX_QP_INIT_RD_ATOM = 128,
+ RFC_MAX_EE_INIT_RD_ATOM = 0,
+ RFC_ATOMIC_CAP = 1,
+ RFC_MAX_EE = 0,
+ RFC_MAX_RDD = 0,
+ RFC_MAX_MW = 0,
+ RFC_MAX_RAW_IPV6_QP = 0,
+ RFC_MAX_RAW_ETHY_QP = 0,
+ RFC_MAX_MCAST_GRP = 8192,
+ RFC_MAX_MCAST_QP_ATTACH = 56,
+ RFC_MAX_TOT_MCAST_QP_ATTACH = 0x70000,
+ RFC_MAX_AH = 100,
+ RFC_MAX_FMR = 0,
+ RFC_MAX_MAP_PER_FMR = 0,
+ RFC_MAX_SRQ = 960,
+ RFC_MAX_SRQ_WR = 0x4000,
+ RFC_MIN_SRQ_WR = 1,
+ RFC_MAX_SRQ_SGE = 27,
+ RFC_MIN_SRQ_SGE = 1,
+ RFC_MAX_FMR_PAGE_LIST_LEN = 512,
+ RFC_MAX_PKEYS = 64,
+ RFC_LOCAL_CA_ACK_DELAY = 15,
+
+ RFC_MAX_UCONTEXT = 512,
+
+ RFC_NUM_PORT = 1,
+
+ RFC_MIN_QP_INDEX = 16,
+ RFC_MAX_QP_INDEX = 0x00020000,
+
+ RFC_MIN_SRQ_INDEX = 0x00020001,
+ RFC_MAX_SRQ_INDEX = 0x00040000,
+
+ RFC_MIN_MR_INDEX = 0x00000001,
+ RFC_MAX_MR_INDEX = 0x00040000,
+ RFC_MIN_MW_INDEX = 0x00040001,
+ RFC_MAX_MW_INDEX = 0x00060000,
+ RFC_MAX_PKT_PER_ACK = 64,
+
+ RFC_MAX_UNACKED_PSNS = 128,
+
+ /* Delay before calling arbiter timer */
+ RFC_NSEC_ARB_TIMER_DELAY = 200,
+};
+
+/* default/initial rfc port parameters */
+enum rfc_port_param {
+ RFC_PORT_STATE = IB_PORT_DOWN,
+ RFC_PORT_MAX_MTU = IB_MTU_4096,
+ RFC_PORT_ACTIVE_MTU = IB_MTU_256,
+ RFC_PORT_GID_TBL_LEN = 1024,
+ RFC_PORT_PORT_CAP_FLAGS = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP,
+ RFC_PORT_MAX_MSG_SZ = 0x800000,
+ RFC_PORT_BAD_PKEY_CNTR = 0,
+ RFC_PORT_QKEY_VIOL_CNTR = 0,
+ RFC_PORT_LID = 0,
+ RFC_PORT_SM_LID = 0,
+ RFC_PORT_SM_SL = 0,
+ RFC_PORT_LMC = 0,
+ RFC_PORT_MAX_VL_NUM = 1,
+ RFC_PORT_SUBNET_TIMEOUT = 0,
+ RFC_PORT_INIT_TYPE_REPLY = 0,
+ RFC_PORT_ACTIVE_WIDTH = IB_WIDTH_1X,
+ RFC_PORT_ACTIVE_SPEED = 1,
+ RFC_PORT_PKEY_TBL_LEN = 64,
+ RFC_PORT_PHYS_STATE = 2,
+ RFC_PORT_SUBNET_PREFIX = 0xfe80000000000000ULL,
+};
+
+/* default/initial port info parameters */
+enum rfc_port_info_param {
+ RFC_PORT_INFO_VL_CAP = 4, /* 1-8 */
+ RFC_PORT_INFO_MTU_CAP = 5, /* 4096 */
+ RFC_PORT_INFO_OPER_VL = 1, /* 1 */
+};
+
+#endif /* RFC_PARAM_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_pool.c b/drivers/infiniband/sw/rfc/rfc_pool.c
new file mode 100644
index 0000000..531c496
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_pool.c
@@ -0,0 +1,501 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+
+/* info about object pools
+ * note that mr and mw share a single index space
+ * so that one can map an lkey to the correct type of object
+ */
+struct rfc_type_info rfc_type_info[RFC_NUM_TYPES] = {
+ [RFC_TYPE_UC] = {
+ .name = "rfc-uc",
+ .size = sizeof(struct rfc_ucontext),
+ },
+ [RFC_TYPE_PD] = {
+ .name = "rfc-pd",
+ .size = sizeof(struct rfc_pd),
+ },
+ [RFC_TYPE_AH] = {
+ .name = "rfc-ah",
+ .size = sizeof(struct rfc_ah),
+ .flags = RFC_POOL_ATOMIC,
+ },
+ [RFC_TYPE_SRQ] = {
+ .name = "rfc-srq",
+ .size = sizeof(struct rfc_srq),
+ .flags = RFC_POOL_INDEX,
+ .min_index = RFC_MIN_SRQ_INDEX,
+ .max_index = RFC_MAX_SRQ_INDEX,
+ },
+ [RFC_TYPE_QP] = {
+ .name = "rfc-qp",
+ .size = sizeof(struct rfc_qp),
+ .cleanup = rfc_qp_cleanup,
+ .flags = RFC_POOL_INDEX,
+ .min_index = RFC_MIN_QP_INDEX,
+ .max_index = RFC_MAX_QP_INDEX,
+ },
+ [RFC_TYPE_CQ] = {
+ .name = "rfc-cq",
+ .size = sizeof(struct rfc_cq),
+ .cleanup = rfc_cq_cleanup,
+ },
+ [RFC_TYPE_MR] = {
+ .name = "rfc-mr",
+ .size = sizeof(struct rfc_mem),
+ .cleanup = rfc_mem_cleanup,
+ .flags = RFC_POOL_INDEX,
+ .max_index = RFC_MAX_MR_INDEX,
+ .min_index = RFC_MIN_MR_INDEX,
+ },
+ [RFC_TYPE_MW] = {
+ .name = "rfc-mw",
+ .size = sizeof(struct rfc_mem),
+ .flags = RFC_POOL_INDEX,
+ .max_index = RFC_MAX_MW_INDEX,
+ .min_index = RFC_MIN_MW_INDEX,
+ },
+ [RFC_TYPE_MC_GRP] = {
+ .name = "rfc-mc_grp",
+ .size = sizeof(struct rfc_mc_grp),
+ // .cleanup = rfc_mc_cleanup,
+ .flags = RFC_POOL_KEY,
+ .key_offset = offsetof(struct rfc_mc_grp, mgid),
+ .key_size = sizeof(union ib_gid),
+ },
+ [RFC_TYPE_MC_ELEM] = {
+ .name = "rfc-mc_elem",
+ .size = sizeof(struct rfc_mc_elem),
+ .flags = RFC_POOL_ATOMIC,
+ },
+};
+
+static inline const char *pool_name(struct rfc_pool *pool)
+{
+ return rfc_type_info[pool->type].name;
+}
+
+static inline struct kmem_cache *pool_cache(struct rfc_pool *pool)
+{
+ return rfc_type_info[pool->type].cache;
+}
+
+int rfc_cache_init(void)
+{
+ int err;
+ int i;
+ size_t size;
+ struct rfc_type_info *type;
+
+ for (i = 0; i < RFC_NUM_TYPES; i++) {
+ type = &rfc_type_info[i];
+ size = ALIGN(type->size, RFC_POOL_ALIGN);
+ type->cache = kmem_cache_create(type->name, size,
+ RFC_POOL_ALIGN,
+ RFC_POOL_CACHE_FLAGS, NULL);
+ if (!type->cache) {
+ pr_err("Unable to init kmem cache for %s\n",
+ type->name);
+ err = -ENOMEM;
+ goto err1;
+ }
+ }
+
+ return 0;
+
+err1:
+ while (--i >= 0) {
+ kmem_cache_destroy(type->cache);
+ type->cache = NULL;
+ }
+
+ return err;
+}
+
+void rfc_cache_exit(void)
+{
+ int i;
+ struct rfc_type_info *type;
+
+ for (i = 0; i < RFC_NUM_TYPES; i++) {
+ type = &rfc_type_info[i];
+ kmem_cache_destroy(type->cache);
+ type->cache = NULL;
+ }
+}
+
+static int rfc_pool_init_index(struct rfc_pool *pool, u32 max, u32 min)
+{
+ int err = 0;
+ size_t size;
+
+ if ((max - min + 1) < pool->max_elem) {
+ pr_warn("not enough indices for max_elem\n");
+ err = -EINVAL;
+ goto out;
+ }
+
+ pool->max_index = max;
+ pool->min_index = min;
+
+ size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
+ pool->table = kmalloc(size, GFP_KERNEL);
+ if (!pool->table) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ pool->table_size = size;
+ bitmap_zero(pool->table, max - min + 1);
+
+out:
+ return err;
+}
+
+int rfc_pool_init(
+ struct rfc_dev *rfc,
+ struct rfc_pool *pool,
+ enum rfc_elem_type type,
+ unsigned int max_elem)
+{
+ int err = 0;
+ size_t size = rfc_type_info[type].size;
+
+ memset(pool, 0, sizeof(*pool));
+
+ pool->rfc = rfc;
+ pool->type = type;
+ pool->max_elem = max_elem;
+ pool->elem_size = ALIGN(size, RFC_POOL_ALIGN);
+ pool->flags = rfc_type_info[type].flags;
+ pool->tree = RB_ROOT;
+ pool->cleanup = rfc_type_info[type].cleanup;
+
+ atomic_set(&pool->num_elem, 0);
+
+ kref_init(&pool->ref_cnt);
+
+ spin_lock_init(&pool->pool_lock);
+
+ if (rfc_type_info[type].flags & RFC_POOL_INDEX) {
+ err = rfc_pool_init_index(pool,
+ rfc_type_info[type].max_index,
+ rfc_type_info[type].min_index);
+ if (err)
+ goto out;
+ }
+
+ if (rfc_type_info[type].flags & RFC_POOL_KEY) {
+ pool->key_offset = rfc_type_info[type].key_offset;
+ pool->key_size = rfc_type_info[type].key_size;
+ }
+
+ pool->state = rfc_pool_valid;
+
+out:
+ return err;
+}
+
+static void rfc_pool_release(struct kref *kref)
+{
+ struct rfc_pool *pool = container_of(kref, struct rfc_pool, ref_cnt);
+
+ pool->state = rfc_pool_invalid;
+ kfree(pool->table);
+}
+
+static void rfc_pool_put(struct rfc_pool *pool)
+{
+ kref_put(&pool->ref_cnt, rfc_pool_release);
+}
+
+int rfc_pool_cleanup(struct rfc_pool *pool)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ pool->state = rfc_pool_invalid;
+ if (atomic_read(&pool->num_elem) > 0)
+ pr_warn("%s pool destroyed with unfree'd elem\n",
+ pool_name(pool));
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+
+ rfc_pool_put(pool);
+
+ return 0;
+}
+
+static u32 alloc_index(struct rfc_pool *pool)
+{
+ u32 index;
+ u32 range = pool->max_index - pool->min_index + 1;
+
+ index = find_next_zero_bit(pool->table, range, pool->last);
+ if (index >= range)
+ index = find_first_zero_bit(pool->table, range);
+
+ WARN_ON_ONCE(index >= range);
+ set_bit(index, pool->table);
+ pool->last = index;
+ return index + pool->min_index;
+}
+
+static void insert_index(struct rfc_pool *pool, struct rfc_pool_entry *new)
+{
+ struct rb_node **link = &pool->tree.rb_node;
+ struct rb_node *parent = NULL;
+ struct rfc_pool_entry *elem;
+
+ while (*link) {
+ parent = *link;
+ elem = rb_entry(parent, struct rfc_pool_entry, node);
+
+ if (elem->index == new->index) {
+ pr_warn("element already exists!\n");
+ goto out;
+ }
+
+ if (elem->index > new->index)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+
+ rb_link_node(&new->node, parent, link);
+ rb_insert_color(&new->node, &pool->tree);
+out:
+ return;
+}
+
+static void insert_key(struct rfc_pool *pool, struct rfc_pool_entry *new)
+{
+ struct rb_node **link = &pool->tree.rb_node;
+ struct rb_node *parent = NULL;
+ struct rfc_pool_entry *elem;
+ int cmp;
+
+ while (*link) {
+ parent = *link;
+ elem = rb_entry(parent, struct rfc_pool_entry, node);
+
+ cmp = memcmp((u8 *)elem + pool->key_offset,
+ (u8 *)new + pool->key_offset, pool->key_size);
+
+ if (cmp == 0) {
+ pr_warn("key already exists!\n");
+ goto out;
+ }
+
+ if (cmp > 0)
+ link = &(*link)->rb_left;
+ else
+ link = &(*link)->rb_right;
+ }
+
+ rb_link_node(&new->node, parent, link);
+ rb_insert_color(&new->node, &pool->tree);
+out:
+ return;
+}
+
+void rfc_add_key(void *arg, void *key)
+{
+ struct rfc_pool_entry *elem = arg;
+ struct rfc_pool *pool = elem->pool;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
+ insert_key(pool, elem);
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+void rfc_drop_key(void *arg)
+{
+ struct rfc_pool_entry *elem = arg;
+ struct rfc_pool *pool = elem->pool;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ rb_erase(&elem->node, &pool->tree);
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+void rfc_add_index(void *arg)
+{
+ struct rfc_pool_entry *elem = arg;
+ struct rfc_pool *pool = elem->pool;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ elem->index = alloc_index(pool);
+ insert_index(pool, elem);
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+void rfc_drop_index(void *arg)
+{
+ struct rfc_pool_entry *elem = arg;
+ struct rfc_pool *pool = elem->pool;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ clear_bit(elem->index - pool->min_index, pool->table);
+ rb_erase(&elem->node, &pool->tree);
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+}
+
+void *rfc_alloc(struct rfc_pool *pool)
+{
+ struct rfc_pool_entry *elem;
+ unsigned long flags;
+
+ might_sleep_if(!(pool->flags & RFC_POOL_ATOMIC));
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+ if (pool->state != rfc_pool_valid) {
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+ return NULL;
+ }
+ kref_get(&pool->ref_cnt);
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+
+ kref_get(&pool->rfc->ref_cnt);
+
+ if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
+ goto out_put_pool;
+
+ elem = kmem_cache_zalloc(pool_cache(pool),
+ (pool->flags & RFC_POOL_ATOMIC) ?
+ GFP_ATOMIC : GFP_KERNEL);
+ if (!elem)
+ goto out_put_pool;
+
+ elem->pool = pool;
+ kref_init(&elem->ref_cnt);
+
+ return elem;
+
+out_put_pool:
+ atomic_dec(&pool->num_elem);
+ rfc_dev_put(pool->rfc);
+ rfc_pool_put(pool);
+ return NULL;
+}
+
+void rfc_elem_release(struct kref *kref)
+{
+ struct rfc_pool_entry *elem =
+ container_of(kref, struct rfc_pool_entry, ref_cnt);
+ struct rfc_pool *pool = elem->pool;
+
+ if (pool->cleanup)
+ pool->cleanup(elem);
+
+ kmem_cache_free(pool_cache(pool), elem);
+ atomic_dec(&pool->num_elem);
+ rfc_dev_put(pool->rfc);
+ rfc_pool_put(pool);
+}
+
+void *rfc_pool_get_index(struct rfc_pool *pool, u32 index)
+{
+ struct rb_node *node = NULL;
+ struct rfc_pool_entry *elem = NULL;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+
+ if (pool->state != rfc_pool_valid)
+ goto out;
+
+ node = pool->tree.rb_node;
+
+ while (node) {
+ elem = rb_entry(node, struct rfc_pool_entry, node);
+
+ if (elem->index > index)
+ node = node->rb_left;
+ else if (elem->index < index)
+ node = node->rb_right;
+ else
+ break;
+ }
+
+ if (node)
+ kref_get(&elem->ref_cnt);
+
+out:
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+ return node ? elem : NULL;
+}
+
+void *rfc_pool_get_key(struct rfc_pool *pool, void *key)
+{
+ struct rb_node *node = NULL;
+ struct rfc_pool_entry *elem = NULL;
+ int cmp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&pool->pool_lock, flags);
+
+ if (pool->state != rfc_pool_valid)
+ goto out;
+
+ node = pool->tree.rb_node;
+
+ while (node) {
+ elem = rb_entry(node, struct rfc_pool_entry, node);
+
+ cmp = memcmp((u8 *)elem + pool->key_offset,
+ key, pool->key_size);
+
+ if (cmp > 0)
+ node = node->rb_left;
+ else if (cmp < 0)
+ node = node->rb_right;
+ else
+ break;
+ }
+
+ if (node)
+ kref_get(&elem->ref_cnt);
+
+out:
+ spin_unlock_irqrestore(&pool->pool_lock, flags);
+ return node ? elem : NULL;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_pool.h b/drivers/infiniband/sw/rfc/rfc_pool.h
new file mode 100644
index 0000000..78de3e1
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_pool.h
@@ -0,0 +1,167 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_POOL_H
+#define RFC_POOL_H
+
+#define RFC_POOL_ALIGN (16)
+#define RFC_POOL_CACHE_FLAGS (0)
+
+enum rfc_pool_flags {
+ RFC_POOL_ATOMIC = BIT(0),
+ RFC_POOL_INDEX = BIT(1),
+ RFC_POOL_KEY = BIT(2),
+};
+
+enum rfc_elem_type {
+ RFC_TYPE_UC,
+ RFC_TYPE_PD,
+ RFC_TYPE_AH,
+ RFC_TYPE_SRQ,
+ RFC_TYPE_QP,
+ RFC_TYPE_CQ,
+ RFC_TYPE_MR,
+ RFC_TYPE_MW,
+ RFC_TYPE_MC_GRP,
+ RFC_TYPE_MC_ELEM,
+ RFC_NUM_TYPES, /* keep me last */
+};
+
+struct rfc_pool_entry;
+
+struct rfc_type_info {
+ const char *name;
+ size_t size;
+ void (*cleanup)(struct rfc_pool_entry *obj);
+ enum rfc_pool_flags flags;
+ u32 max_index;
+ u32 min_index;
+ size_t key_offset;
+ size_t key_size;
+ struct kmem_cache *cache;
+};
+
+extern struct rfc_type_info rfc_type_info[];
+
+enum rfc_pool_state {
+ rfc_pool_invalid,
+ rfc_pool_valid,
+};
+
+struct rfc_pool_entry {
+ struct rfc_pool *pool;
+ struct kref ref_cnt;
+ struct list_head list;
+
+ /* only used if indexed or keyed */
+ struct rb_node node;
+ u32 index;
+};
+
+struct rfc_pool {
+ struct rfc_dev *rfc;
+ spinlock_t pool_lock; /* pool spinlock */
+ size_t elem_size;
+ struct kref ref_cnt;
+ void (*cleanup)(struct rfc_pool_entry *obj);
+ enum rfc_pool_state state;
+ enum rfc_pool_flags flags;
+ enum rfc_elem_type type;
+
+ unsigned int max_elem;
+ atomic_t num_elem;
+
+ /* only used if indexed or keyed */
+ struct rb_root tree;
+ unsigned long *table;
+ size_t table_size;
+ u32 max_index;
+ u32 min_index;
+ u32 last;
+ size_t key_offset;
+ size_t key_size;
+};
+
+/* initialize slab caches for managed objects */
+int rfc_cache_init(void);
+
+/* cleanup slab caches for managed objects */
+void rfc_cache_exit(void);
+
+/* initialize a pool of objects with given limit on
+ * number of elements. gets parameters from rfc_type_info
+ * pool elements will be allocated out of a slab cache
+ */
+int rfc_pool_init(struct rfc_dev *rfc, struct rfc_pool *pool,
+ enum rfc_elem_type type, u32 max_elem);
+
+/* free resources from object pool */
+int rfc_pool_cleanup(struct rfc_pool *pool);
+
+/* allocate an object from pool */
+void *rfc_alloc(struct rfc_pool *pool);
+
+/* assign an index to an indexed object and insert object into
+ * pool's rb tree
+ */
+void rfc_add_index(void *elem);
+
+/* drop an index and remove object from rb tree */
+void rfc_drop_index(void *elem);
+
+/* assign a key to a keyed object and insert object into
+ * pool's rb tree
+ */
+void rfc_add_key(void *elem, void *key);
+
+/* remove elem from rb tree */
+void rfc_drop_key(void *elem);
+
+/* lookup an indexed object from index. takes a reference on object */
+void *rfc_pool_get_index(struct rfc_pool *pool, u32 index);
+
+/* lookup keyed object from key. takes a reference on the object */
+void *rfc_pool_get_key(struct rfc_pool *pool, void *key);
+
+/* cleanup an object when all references are dropped */
+void rfc_elem_release(struct kref *kref);
+
+/* take a reference on an object */
+#define rfc_add_ref(elem) kref_get(&(elem)->pelem.ref_cnt)
+
+/* drop a reference on an object */
+#define rfc_drop_ref(elem) kref_put(&(elem)->pelem.ref_cnt, rfc_elem_release)
+
+#endif /* RFC_POOL_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_qp.c b/drivers/infiniband/sw/rfc/rfc_qp.c
new file mode 100644
index 0000000..32b42cf
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_qp.c
@@ -0,0 +1,876 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include <linux/sched.h>
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+#include "rfc_task.h"
+
+char *rfc_qp_state_name[] = {
+ [QP_STATE_RESET] = "RESET",
+ [QP_STATE_INIT] = "INIT",
+ [QP_STATE_READY] = "READY",
+ [QP_STATE_DRAIN] = "DRAIN",
+ [QP_STATE_DRAINED] = "DRAINED",
+ [QP_STATE_ERROR] = "ERROR",
+};
+
+static int rfc_qp_chk_cap(struct rfc_dev *rfc, struct ib_qp_cap *cap,
+ int has_srq)
+{
+ if (cap->max_send_wr > rfc->attr.max_qp_wr) {
+ pr_warn("invalid send wr = %d > %d\n",
+ cap->max_send_wr, rfc->attr.max_qp_wr);
+ goto err1;
+ }
+
+ if (cap->max_send_sge > rfc->attr.max_sge) {
+ pr_warn("invalid send sge = %d > %d\n",
+ cap->max_send_sge, rfc->attr.max_sge);
+ goto err1;
+ }
+
+ if (!has_srq) {
+ if (cap->max_recv_wr > rfc->attr.max_qp_wr) {
+ pr_warn("invalid recv wr = %d > %d\n",
+ cap->max_recv_wr, rfc->attr.max_qp_wr);
+ goto err1;
+ }
+
+ if (cap->max_recv_sge > rfc->attr.max_sge) {
+ pr_warn("invalid recv sge = %d > %d\n",
+ cap->max_recv_sge, rfc->attr.max_sge);
+ goto err1;
+ }
+ }
+
+ if (cap->max_inline_data > rfc->max_inline_data) {
+ pr_warn("invalid max inline data = %d > %d\n",
+ cap->max_inline_data, rfc->max_inline_data);
+ goto err1;
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+int rfc_qp_chk_init(struct rfc_dev *rfc, struct ib_qp_init_attr *init)
+{
+ struct ib_qp_cap *cap = &init->cap;
+ struct rfc_port *port;
+ int port_num = init->port_num;
+
+ if (!init->recv_cq || !init->send_cq) {
+ pr_warn("missing cq\n");
+ goto err1;
+ }
+
+ if (rfc_qp_chk_cap(rfc, cap, !!init->srq))
+ goto err1;
+
+ if (init->qp_type == IB_QPT_SMI || init->qp_type == IB_QPT_GSI) {
+ if (port_num != 1) {
+ pr_warn("invalid port = %d\n", port_num);
+ goto err1;
+ }
+
+ port = &rfc->port;
+
+ if (init->qp_type == IB_QPT_SMI && port->qp_smi_index) {
+ pr_warn("SMI QP exists for port %d\n", port_num);
+ goto err1;
+ }
+
+ if (init->qp_type == IB_QPT_GSI && port->qp_gsi_index) {
+ pr_warn("GSI QP exists for port %d\n", port_num);
+ goto err1;
+ }
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static int alloc_rd_atomic_resources(struct rfc_qp *qp, unsigned int n)
+{
+ qp->resp.res_head = 0;
+ qp->resp.res_tail = 0;
+ qp->resp.resources = kcalloc(n, sizeof(struct resp_res), GFP_KERNEL);
+
+ if (!qp->resp.resources)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static void free_rd_atomic_resources(struct rfc_qp *qp)
+{
+ if (qp->resp.resources) {
+ int i;
+
+ for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
+ struct resp_res *res = &qp->resp.resources[i];
+
+ free_rd_atomic_resource(qp, res);
+ }
+ kfree(qp->resp.resources);
+ qp->resp.resources = NULL;
+ }
+}
+
+void free_rd_atomic_resource(struct rfc_qp *qp, struct resp_res *res)
+{
+ if (res->type == RFC_ATOMIC_MASK) {
+ rfc_drop_ref(qp);
+ } else if (res->type == RFC_READ_MASK) {
+ if (res->read.mr)
+ rfc_drop_ref(res->read.mr);
+ }
+ res->type = 0;
+}
+
+static void cleanup_rd_atomic_resources(struct rfc_qp *qp)
+{
+ int i;
+ struct resp_res *res;
+
+ if (qp->resp.resources) {
+ for (i = 0; i < qp->attr.max_dest_rd_atomic; i++) {
+ res = &qp->resp.resources[i];
+ free_rd_atomic_resource(qp, res);
+ }
+ }
+}
+
+static void rfc_qp_init_misc(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct ib_qp_init_attr *init)
+{
+ struct rfc_port *port;
+ u32 qpn;
+
+ qp->sq_sig_type = init->sq_sig_type;
+ qp->attr.path_mtu = 1;
+// qp->mtu = ib_mtu_enum_to_int(qp->attr.path_mtu);
+ qp->mtu = ROFC_MTU;
+
+ qpn = qp->pelem.index;
+ port = &rfc->port;
+
+ switch (init->qp_type) {
+ case IB_QPT_SMI:
+ qp->ibqp.qp_num = 0;
+ port->qp_smi_index = qpn;
+ qp->attr.port_num = init->port_num;
+ break;
+
+ case IB_QPT_GSI:
+ qp->ibqp.qp_num = 1;
+ port->qp_gsi_index = qpn;
+ qp->attr.port_num = init->port_num;
+ break;
+
+ default:
+ qp->ibqp.qp_num = qpn;
+ break;
+ }
+
+ INIT_LIST_HEAD(&qp->grp_list);
+
+ INIT_LIST_HEAD(&qp->req_objs);
+ INIT_LIST_HEAD(&qp->resp_objs);
+ INIT_LIST_HEAD(&qp->send_objs);
+
+
+ spin_lock_init(&qp->grp_lock);
+ spin_lock_init(&qp->state_lock);
+
+ atomic_set(&qp->ssn, 0);
+}
+
+static void rfc_work_requester(struct work_struct *_work)
+{
+ struct rfc_task *task = container_of(_work, struct rfc_task, work);
+
+ rfc_requester(task->arg);
+}
+
+static int rfc_qp_init_req(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct ib_qp_init_attr *init,
+ struct ib_ucontext *context,
+ struct rfc_create_qp_resp __user *uresp)
+{
+ int err;
+ int wqe_size;
+
+ err = sock_create_kern(&init_net, AF_INET, SOCK_DGRAM, 0, &qp->sk);
+ if (err < 0)
+ return err;
+ qp->sk->sk->sk_user_data = qp;
+
+ qp->sq.max_wr = init->cap.max_send_wr;
+ qp->sq.max_sge = init->cap.max_send_sge;
+ qp->sq.max_inline = init->cap.max_inline_data;
+
+ wqe_size = max_t(int, sizeof(struct rfc_send_wqe) +
+ qp->sq.max_sge * sizeof(struct ib_sge),
+ sizeof(struct rfc_send_wqe) +
+ qp->sq.max_inline);
+
+ qp->sq.queue = rfc_queue_init(rfc,
+ &qp->sq.max_wr,
+ wqe_size);
+ if (!qp->sq.queue)
+ return -ENOMEM;
+
+ err = do_mmap_info(rfc, uresp ? &uresp->sq_mi : NULL, context,
+ qp->sq.queue->buf, qp->sq.queue->buf_size,
+ &qp->sq.queue->ip);
+
+ if (err) {
+ kvfree(qp->sq.queue->buf);
+ kfree(qp->sq.queue);
+ return err;
+ }
+
+ qp->req.wqe_index = producer_index(qp->sq.queue);
+ qp->req.state = QP_STATE_RESET;
+ qp->req.opcode = -1;
+ qp->comp.opcode = -1;
+
+ spin_lock_init(&qp->sq.sq_lock);
+
+ rfc_init_task(rfc, &qp->req.task, qp,
+ rfc_requester, "req");
+ qp->req.task.queue = create_workqueue("requester");
+ INIT_WORK(&qp->req.task.work, rfc_work_requester);
+ rfc_init_task(rfc, &qp->comp.task, qp,
+ rfc_completer_direct, "comp");
+ qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
+ if (init->qp_type == IB_QPT_RC) {
+ timer_setup(&qp->rnr_nak_timer, rnr_nak_timer, 0);
+ // timer_setup(&qp->retrans_timer, retransmit_timer, 0);
+ timer_setup(&qp->retrans_timer, retransmit_timer_direct, 0);
+ }
+ return 0;
+}
+
+struct rfc_resp_arg_t {
+ struct rfc_qp *qp;
+ struct rfc_obj_info_t *obj;
+};
+static int rfc_qp_init_resp(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct ib_qp_init_attr *init,
+ struct ib_ucontext *context,
+ struct rfc_create_qp_resp __user *uresp)
+{
+ int err;
+ int wqe_size;
+
+ if (!qp->srq) {
+ qp->rq.max_wr = init->cap.max_recv_wr;
+ qp->rq.max_sge = init->cap.max_recv_sge;
+
+ wqe_size = rcv_wqe_size(qp->rq.max_sge);
+
+ pr_debug("qp#%d max_wr = %d, max_sge = %d, wqe_size = %d\n",
+ qp_num(qp), qp->rq.max_wr, qp->rq.max_sge, wqe_size);
+
+ qp->rq.queue = rfc_queue_init(rfc,
+ &qp->rq.max_wr,
+ wqe_size);
+ if (!qp->rq.queue)
+ return -ENOMEM;
+
+ err = do_mmap_info(rfc, uresp ? &uresp->rq_mi : NULL, context,
+ qp->rq.queue->buf, qp->rq.queue->buf_size,
+ &qp->rq.queue->ip);
+ if (err) {
+ kvfree(qp->rq.queue->buf);
+ kfree(qp->rq.queue);
+ return err;
+ }
+ }
+
+ spin_lock_init(&qp->rq.producer_lock);
+ spin_lock_init(&qp->rq.consumer_lock);
+
+
+ qp->resp.opcode = OPCODE_NONE;
+ qp->resp.msn = 0;
+ qp->resp.state = QP_STATE_RESET;
+
+ return 0;
+}
+
+/* called by the create qp verb */
+int rfc_qp_from_init(struct rfc_dev *rfc, struct rfc_qp *qp, struct rfc_pd *pd,
+ struct ib_qp_init_attr *init,
+ struct rfc_create_qp_resp __user *uresp,
+ struct ib_pd *ibpd)
+{
+ int err;
+ struct rfc_cq *rcq = to_rcq(init->recv_cq);
+ struct rfc_cq *scq = to_rcq(init->send_cq);
+ struct rfc_srq *srq = init->srq ? to_rsrq(init->srq) : NULL;
+ struct ib_ucontext *context = ibpd->uobject ? ibpd->uobject->context : NULL;
+
+ rfc_add_ref(pd);
+ rfc_add_ref(rcq);
+ rfc_add_ref(scq);
+ if (srq)
+ rfc_add_ref(srq);
+
+ qp->pd = pd;
+ qp->rcq = rcq;
+ qp->scq = scq;
+ qp->srq = srq;
+
+ rfc_qp_init_misc(rfc, qp, init);
+
+ err = rfc_qp_init_req(rfc, qp, init, context, uresp);
+ if (err)
+ goto err1;
+
+ err = rfc_qp_init_resp(rfc, qp, init, context, uresp);
+ if (err)
+ goto err2;
+
+ qp->attr.qp_state = IB_QPS_RESET;
+ qp->valid = 1;
+
+ return 0;
+
+err2:
+ rfc_queue_cleanup(qp->sq.queue);
+err1:
+ if (srq)
+ rfc_drop_ref(srq);
+ rfc_drop_ref(scq);
+ rfc_drop_ref(rcq);
+ rfc_drop_ref(pd);
+
+ return err;
+}
+
+/* called by the query qp verb */
+int rfc_qp_to_init(struct rfc_qp *qp, struct ib_qp_init_attr *init)
+{
+ init->event_handler = qp->ibqp.event_handler;
+ init->qp_context = qp->ibqp.qp_context;
+ init->send_cq = qp->ibqp.send_cq;
+ init->recv_cq = qp->ibqp.recv_cq;
+ init->srq = qp->ibqp.srq;
+
+ init->cap.max_send_wr = qp->sq.max_wr;
+ init->cap.max_send_sge = qp->sq.max_sge;
+ init->cap.max_inline_data = qp->sq.max_inline;
+
+ if (!qp->srq) {
+ init->cap.max_recv_wr = qp->rq.max_wr;
+ init->cap.max_recv_sge = qp->rq.max_sge;
+ }
+
+ init->sq_sig_type = qp->sq_sig_type;
+
+ init->qp_type = qp->ibqp.qp_type;
+ init->port_num = 1;
+
+ return 0;
+}
+
+/* called by the modify qp verb, this routine checks all the parameters before
+ * making any changes
+ */
+int rfc_qp_chk_attr(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct ib_qp_attr *attr, int mask)
+{
+ enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ?
+ attr->cur_qp_state : qp->attr.qp_state;
+ enum ib_qp_state new_state = (mask & IB_QP_STATE) ?
+ attr->qp_state : cur_state;
+
+ if (!ib_modify_qp_is_ok(cur_state, new_state, qp_type(qp), mask,
+ IB_LINK_LAYER_ETHERNET)) {
+ pr_warn("invalid mask or state for qp\n");
+ goto err1;
+ }
+
+ if (mask & IB_QP_STATE) {
+ if (cur_state == IB_QPS_SQD) {
+ if (qp->req.state == QP_STATE_DRAIN &&
+ new_state != IB_QPS_ERR)
+ goto err1;
+ }
+ }
+
+ if (mask & IB_QP_PORT) {
+ if (attr->port_num != 1) {
+ pr_warn("invalid port %d\n", attr->port_num);
+ goto err1;
+ }
+ }
+
+ if (mask & IB_QP_CAP && rfc_qp_chk_cap(rfc, &attr->cap, !!qp->srq))
+ goto err1;
+
+ if (mask & IB_QP_AV && rfc_av_chk_attr(rfc, &attr->ah_attr))
+ goto err1;
+
+ if (mask & IB_QP_ALT_PATH) {
+ if (rfc_av_chk_attr(rfc, &attr->alt_ah_attr))
+ goto err1;
+ if (attr->alt_port_num != 1) {
+ pr_warn("invalid alt port %d\n", attr->alt_port_num);
+ goto err1;
+ }
+ if (attr->alt_timeout > 31) {
+ pr_warn("invalid QP alt timeout %d > 31\n",
+ attr->alt_timeout);
+ goto err1;
+ }
+ }
+
+ if (mask & IB_QP_PATH_MTU) {
+ struct rfc_port *port = &rfc->port;
+
+ enum ib_mtu max_mtu = port->attr.max_mtu;
+ enum ib_mtu mtu = attr->path_mtu;
+
+ if (mtu > max_mtu) {
+ pr_debug("invalid mtu (%d) > (%d)\n",
+ ib_mtu_enum_to_int(mtu),
+ ib_mtu_enum_to_int(max_mtu));
+ goto err1;
+ }
+ }
+
+ if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ if (attr->max_rd_atomic > rfc->attr.max_qp_rd_atom) {
+ pr_warn("invalid max_rd_atomic %d > %d\n",
+ attr->max_rd_atomic,
+ rfc->attr.max_qp_rd_atom);
+ goto err1;
+ }
+ }
+
+ if (mask & IB_QP_TIMEOUT) {
+ if (attr->timeout > 31) {
+ pr_warn("invalid QP timeout %d > 31\n",
+ attr->timeout);
+ goto err1;
+ }
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+/* move the qp to the reset state */
+static void rfc_qp_reset(struct rfc_qp *qp)
+{
+ /* stop tasks from running */
+ rfc_disable_task(&qp->resp.task);
+
+ /* stop request/comp */
+ if (qp->sq.queue) {
+ if (qp_type(qp) == IB_QPT_RC)
+ rfc_disable_task(&qp->comp.task);
+ rfc_disable_task(&qp->req.task);
+ }
+
+ /* move qp to the reset state */
+ qp->req.state = QP_STATE_RESET;
+ qp->resp.state = QP_STATE_RESET;
+
+ /* let state machines reset themselves drain work and object queues
+ * etc.
+ */
+ __rfc_do_task(&qp->resp.task);
+
+ if (qp->sq.queue) {
+ __rfc_do_task(&qp->comp.task);
+ __rfc_do_task(&qp->req.task);
+ rfc_queue_reset(qp->sq.queue);
+ }
+
+ /* cleanup attributes */
+ atomic_set(&qp->ssn, 0);
+ qp->req.opcode = -1;
+ qp->req.need_retry = 0;
+ qp->req.noack_objs = 0;
+ qp->resp.msn = 0;
+ qp->resp.opcode = -1;
+ qp->resp.drop_msg = 0;
+ qp->resp.goto_error = 0;
+ qp->resp.sent_psn_nak = 0;
+
+ if (qp->resp.mr) {
+ rfc_drop_ref(qp->resp.mr);
+ qp->resp.mr = NULL;
+ }
+
+ cleanup_rd_atomic_resources(qp);
+
+ /* reenable tasks */
+ rfc_enable_task(&qp->resp.task);
+
+ if (qp->sq.queue) {
+ if (qp_type(qp) == IB_QPT_RC)
+ rfc_enable_task(&qp->comp.task);
+
+ rfc_enable_task(&qp->req.task);
+ }
+}
+
+/* drain the send queue */
+static void rfc_qp_drain(struct rfc_qp *qp)
+{
+ if (qp->sq.queue) {
+ if (qp->req.state != QP_STATE_DRAINED) {
+ qp->req.state = QP_STATE_DRAIN;
+ if (qp_type(qp) == IB_QPT_RC)
+ rfc_run_task(&qp->comp.task, 1);
+ else
+ __rfc_do_task(&qp->comp.task);
+ rfc_run_task(&qp->req.task, 1);
+ }
+ }
+}
+
+/* move the qp to the error state */
+void rfc_qp_error(struct rfc_qp *qp)
+{
+ qp->req.state = QP_STATE_ERROR;
+ qp->resp.state = QP_STATE_ERROR;
+ qp->attr.qp_state = IB_QPS_ERR;
+
+ /* drain work and object queues */
+ if (qp_type(qp) == IB_QPT_RC)
+ rfc_run_task(&qp->comp.task, 1);
+ else
+ __rfc_do_task(&qp->comp.task);
+ rfc_run_task(&qp->req.task, 1);
+}
+
+/* called by the modify qp verb */
+int rfc_qp_from_attr(struct rfc_qp *qp, struct ib_qp_attr *attr, int mask,
+ struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ union ib_gid sgid;
+ struct ib_gid_attr sgid_attr;
+
+ if (mask & IB_QP_MAX_QP_RD_ATOMIC) {
+ int max_rd_atomic = __roundup_pow_of_two(attr->max_rd_atomic);
+
+ qp->attr.max_rd_atomic = max_rd_atomic;
+ atomic_set(&qp->req.rd_atomic, max_rd_atomic);
+ }
+
+ if (mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+ int max_dest_rd_atomic =
+ __roundup_pow_of_two(attr->max_dest_rd_atomic);
+
+ qp->attr.max_dest_rd_atomic = max_dest_rd_atomic;
+
+ free_rd_atomic_resources(qp);
+
+ err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic);
+ if (err)
+ return err;
+ }
+
+ if (mask & IB_QP_CUR_STATE)
+ qp->attr.cur_qp_state = attr->qp_state;
+
+ if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
+ qp->attr.en_sqd_async_notify = attr->en_sqd_async_notify;
+
+ if (mask & IB_QP_ACCESS_FLAGS)
+ qp->attr.qp_access_flags = attr->qp_access_flags;
+
+ if (mask & IB_QP_PKEY_INDEX)
+ qp->attr.pkey_index = attr->pkey_index;
+
+ if (mask & IB_QP_PORT)
+ qp->attr.port_num = attr->port_num;
+
+ if (mask & IB_QP_QKEY)
+ qp->attr.qkey = attr->qkey;
+
+ if (mask & IB_QP_AV) {
+ ib_get_cached_gid(&rfc->ib_dev, 1,
+ rdma_ah_read_grh(&attr->ah_attr)->sgid_index,
+ &sgid, &sgid_attr);
+ rfc_av_from_attr(attr->port_num, &qp->pri_av, &attr->ah_attr);
+ rfc_av_fill_ip_info(&qp->pri_av, &attr->ah_attr,
+ &sgid_attr, &sgid);
+ if (sgid_attr.ndev)
+ dev_put(sgid_attr.ndev);
+ }
+
+ if (mask & IB_QP_ALT_PATH) {
+ u8 sgid_index =
+ rdma_ah_read_grh(&attr->alt_ah_attr)->sgid_index;
+
+ ib_get_cached_gid(&rfc->ib_dev, 1, sgid_index,
+ &sgid, &sgid_attr);
+
+ rfc_av_from_attr(attr->alt_port_num, &qp->alt_av,
+ &attr->alt_ah_attr);
+ rfc_av_fill_ip_info(&qp->alt_av, &attr->alt_ah_attr,
+ &sgid_attr, &sgid);
+ if (sgid_attr.ndev)
+ dev_put(sgid_attr.ndev);
+
+ qp->attr.alt_port_num = attr->alt_port_num;
+ qp->attr.alt_pkey_index = attr->alt_pkey_index;
+ qp->attr.alt_timeout = attr->alt_timeout;
+ }
+
+ if (mask & IB_QP_PATH_MTU) {
+ qp->attr.path_mtu = attr->path_mtu;
+// qp->mtu = ib_mtu_enum_to_int(attr->path_mtu);
+ qp->mtu = ROFC_MTU;
+ }
+
+ if (mask & IB_QP_TIMEOUT) {
+ qp->attr.timeout = attr->timeout;
+ if (attr->timeout == 0) {
+ qp->qp_timeout_jiffies = 0;
+ } else {
+ /* According to the spec, timeout = 4.096 * 2 ^ attr->timeout [us] */
+ int j = nsecs_to_jiffies(4096ULL << attr->timeout);
+
+ qp->qp_timeout_jiffies = j ? j : 1;
+ }
+ }
+
+ if (mask & IB_QP_RETRY_CNT) {
+ qp->attr.retry_cnt = attr->retry_cnt;
+ qp->comp.retry_cnt = attr->retry_cnt;
+ pr_debug("qp#%d set retry count = %d\n", qp_num(qp),
+ attr->retry_cnt);
+ }
+
+ if (mask & IB_QP_RNR_RETRY) {
+ qp->attr.rnr_retry = attr->rnr_retry;
+ qp->comp.rnr_retry = attr->rnr_retry;
+ pr_debug("qp#%d set rnr retry count = %d\n", qp_num(qp),
+ attr->rnr_retry);
+ }
+
+ if (mask & IB_QP_RQ_PSN) {
+ qp->attr.rq_psn = (attr->rq_psn & BTH_PSN_MASK);
+ qp->resp.psn = qp->attr.rq_psn;
+ pr_debug("qp#%d set resp psn = 0x%x\n", qp_num(qp),
+ qp->resp.psn);
+ }
+
+ if (mask & IB_QP_MIN_RNR_TIMER) {
+ qp->attr.min_rnr_timer = attr->min_rnr_timer;
+ pr_debug("qp#%d set min rnr timer = 0x%x\n", qp_num(qp),
+ attr->min_rnr_timer);
+ }
+
+ if (mask & IB_QP_SQ_PSN) {
+ qp->attr.sq_psn = (attr->sq_psn & BTH_PSN_MASK);
+ qp->req.psn = qp->attr.sq_psn;
+ qp->comp.psn = qp->attr.sq_psn;
+ pr_debug("qp#%d set req psn = 0x%x\n", qp_num(qp), qp->req.psn);
+ }
+
+ if (mask & IB_QP_PATH_MIG_STATE)
+ qp->attr.path_mig_state = attr->path_mig_state;
+
+ if (mask & IB_QP_DEST_QPN)
+ qp->attr.dest_qp_num = attr->dest_qp_num;
+
+ if (mask & IB_QP_STATE) {
+ qp->attr.qp_state = attr->qp_state;
+
+ switch (attr->qp_state) {
+ case IB_QPS_RESET:
+ pr_debug("qp#%d state -> RESET\n", qp_num(qp));
+ rfc_qp_reset(qp);
+ break;
+
+ case IB_QPS_INIT:
+ pr_debug("qp#%d state -> INIT\n", qp_num(qp));
+ qp->req.state = QP_STATE_INIT;
+ qp->resp.state = QP_STATE_INIT;
+ break;
+
+ case IB_QPS_RTR:
+ pr_debug("qp#%d state -> RTR\n", qp_num(qp));
+ qp->resp.state = QP_STATE_READY;
+ break;
+
+ case IB_QPS_RTS:
+ pr_debug("qp#%d state -> RTS\n", qp_num(qp));
+ qp->req.state = QP_STATE_READY;
+ break;
+
+ case IB_QPS_SQD:
+ pr_debug("qp#%d state -> SQD\n", qp_num(qp));
+ rfc_qp_drain(qp);
+ break;
+
+ case IB_QPS_SQE:
+ pr_warn("qp#%d state -> SQE !!?\n", qp_num(qp));
+ /* Not possible from modify_qp. */
+ break;
+
+ case IB_QPS_ERR:
+ pr_debug("qp#%d state -> ERR\n", qp_num(qp));
+ rfc_qp_error(qp);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+/* called by the query qp verb */
+int rfc_qp_to_attr(struct rfc_qp *qp, struct ib_qp_attr *attr, int mask)
+{
+ *attr = qp->attr;
+
+ attr->rq_psn = qp->resp.psn;
+ attr->sq_psn = qp->req.psn;
+
+ attr->cap.max_send_wr = qp->sq.max_wr;
+ attr->cap.max_send_sge = qp->sq.max_sge;
+ attr->cap.max_inline_data = qp->sq.max_inline;
+
+ if (!qp->srq) {
+ attr->cap.max_recv_wr = qp->rq.max_wr;
+ attr->cap.max_recv_sge = qp->rq.max_sge;
+ }
+
+ rfc_av_to_attr(&qp->pri_av, &attr->ah_attr);
+ rfc_av_to_attr(&qp->alt_av, &attr->alt_ah_attr);
+
+ if (qp->req.state == QP_STATE_DRAIN) {
+ attr->sq_draining = 1;
+ /* applications that get this state
+ * typically spin on it. yield the
+ * processor
+ */
+ cond_resched();
+ } else {
+ attr->sq_draining = 0;
+ }
+
+ pr_debug("attr->sq_draining = %d\n", attr->sq_draining);
+
+ return 0;
+}
+
+/* called by the destroy qp verb */
+void rfc_qp_destroy(struct rfc_qp *qp)
+{
+ qp->valid = 0;
+ qp->qp_timeout_jiffies = 0;
+ if (qp_type(qp) == IB_QPT_RC) {
+ del_timer_sync(&qp->retrans_timer);
+ del_timer_sync(&qp->rnr_nak_timer);
+ }
+
+ rfc_cleanup_task(&qp->req.task);
+ rfc_cleanup_task(&qp->comp.task);
+
+ /* flush out any receive wr's or pending requests */
+ __rfc_do_task(&qp->req.task);
+ if (qp->sq.queue) {
+ __rfc_do_task(&qp->comp.task);
+ __rfc_do_task(&qp->req.task);
+ }
+}
+
+/* called when the last reference to the qp is dropped */
+static void rfc_qp_do_cleanup(struct work_struct *work)
+{
+ struct rfc_qp *qp = container_of(work, typeof(*qp), cleanup_work.work);
+
+// rfc_drop_all_mcast_groups(qp);
+
+ if (qp->sq.queue)
+ rfc_queue_cleanup(qp->sq.queue);
+
+ if (qp->srq)
+ rfc_drop_ref(qp->srq);
+
+ if (qp->rq.queue)
+ rfc_queue_cleanup(qp->rq.queue);
+
+ if (qp->scq)
+ rfc_drop_ref(qp->scq);
+ if (qp->rcq)
+ rfc_drop_ref(qp->rcq);
+ if (qp->pd)
+ rfc_drop_ref(qp->pd);
+
+ if (qp->resp.mr) {
+ rfc_drop_ref(qp->resp.mr);
+ qp->resp.mr = NULL;
+ }
+
+ if (qp_type(qp) == IB_QPT_RC)
+ sk_dst_reset(qp->sk->sk);
+
+ free_rd_atomic_resources(qp);
+
+ kernel_sock_shutdown(qp->sk, SHUT_RDWR);
+ sock_release(qp->sk);
+}
+
+/* called when the last reference to the qp is dropped */
+void rfc_qp_cleanup(struct rfc_pool_entry *arg)
+{
+ struct rfc_qp *qp = container_of(arg, typeof(*qp), pelem);
+
+ execute_in_process_context(rfc_qp_do_cleanup, &qp->cleanup_work);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_queue.c b/drivers/infiniband/sw/rfc/rfc_queue.c
new file mode 100644
index 0000000..83dc137
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_queue.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must retailuce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/vmalloc.h>
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+int do_mmap_info(struct rfc_dev *rfc,
+ struct mminfo __user *outbuf,
+ struct ib_ucontext *context,
+ struct rfc_queue_buf *buf,
+ size_t buf_size,
+ struct rfc_mmap_info **ip_p)
+{
+ int err;
+ struct rfc_mmap_info *ip = NULL;
+
+ if (outbuf) {
+ ip = rfc_create_mmap_info(rfc, buf_size, context, buf);
+ if (!ip)
+ goto err1;
+
+ err = copy_to_user(outbuf, &ip->info, sizeof(ip->info));
+ if (err)
+ goto err2;
+
+ spin_lock_bh(&rfc->pending_lock);
+ list_add(&ip->pending_mmaps, &rfc->pending_mmaps);
+ spin_unlock_bh(&rfc->pending_lock);
+ }
+
+ *ip_p = ip;
+
+ return 0;
+
+err2:
+ kfree(ip);
+err1:
+ return -EINVAL;
+}
+
+inline void rfc_queue_reset(struct rfc_queue *q)
+{
+ /* queue is comprised from header and the memory
+ * of the actual queue. See "struct rfc_queue_buf" in rfc_queue.h
+ * reset only the queue itself and not the management header
+ */
+ memset(q->buf->data, 0, q->buf_size - sizeof(struct rfc_queue_buf));
+}
+
+struct rfc_queue *rfc_queue_init(struct rfc_dev *rfc,
+ int *num_elem,
+ unsigned int elem_size)
+{
+ struct rfc_queue *q;
+ size_t buf_size;
+ unsigned int num_slots;
+
+ /* num_elem == 0 is allowed, but uninteresting */
+ if (*num_elem < 0)
+ goto err1;
+
+ q = kmalloc(sizeof(*q), GFP_KERNEL);
+ if (!q)
+ goto err1;
+
+ q->rfc = rfc;
+
+ /* used in resize, only need to copy used part of queue */
+ q->elem_size = elem_size;
+
+ /* pad element up to at least a cacheline and always a power of 2 */
+ if (elem_size < cache_line_size())
+ elem_size = cache_line_size();
+ elem_size = roundup_pow_of_two(elem_size);
+
+ q->log2_elem_size = order_base_2(elem_size);
+
+ num_slots = *num_elem + 1;
+ num_slots = roundup_pow_of_two(num_slots);
+ q->index_mask = num_slots - 1;
+
+ buf_size = sizeof(struct rfc_queue_buf) + num_slots * elem_size;
+
+ q->buf = vmalloc_user(buf_size);
+ if (!q->buf)
+ goto err2;
+
+ q->buf->log2_elem_size = q->log2_elem_size;
+ q->buf->index_mask = q->index_mask;
+
+ q->buf_size = buf_size;
+
+ *num_elem = num_slots - 1;
+ return q;
+
+err2:
+ kfree(q);
+err1:
+ return NULL;
+}
+
+/* copies elements from original q to new q and then swaps the contents of the
+ * two q headers. This is so that if anyone is holding a pointer to q it will
+ * still work
+ */
+static int resize_finish(struct rfc_queue *q, struct rfc_queue *new_q,
+ unsigned int num_elem)
+{
+ if (!queue_empty(q) && (num_elem < queue_count(q)))
+ return -EINVAL;
+
+ while (!queue_empty(q)) {
+ memcpy(producer_addr(new_q), consumer_addr(q),
+ new_q->elem_size);
+ advance_producer(new_q);
+ advance_consumer(q);
+ }
+
+ swap(*q, *new_q);
+
+ return 0;
+}
+
+int rfc_queue_resize(struct rfc_queue *q,
+ unsigned int *num_elem_p,
+ unsigned int elem_size,
+ struct ib_ucontext *context,
+ struct mminfo __user *outbuf,
+ spinlock_t *producer_lock,
+ spinlock_t *consumer_lock)
+{
+ struct rfc_queue *new_q;
+ unsigned int num_elem = *num_elem_p;
+ int err;
+ unsigned long flags = 0, flags1;
+
+ new_q = rfc_queue_init(q->rfc, &num_elem, elem_size);
+ if (!new_q)
+ return -ENOMEM;
+
+ err = do_mmap_info(new_q->rfc, outbuf, context, new_q->buf,
+ new_q->buf_size, &new_q->ip);
+ if (err) {
+ vfree(new_q->buf);
+ kfree(new_q);
+ goto err1;
+ }
+
+ spin_lock_irqsave(consumer_lock, flags1);
+
+ if (producer_lock) {
+ spin_lock_irqsave(producer_lock, flags);
+ err = resize_finish(q, new_q, num_elem);
+ spin_unlock_irqrestore(producer_lock, flags);
+ } else {
+ err = resize_finish(q, new_q, num_elem);
+ }
+
+ spin_unlock_irqrestore(consumer_lock, flags1);
+
+ rfc_queue_cleanup(new_q); /* new/old dep on err */
+ if (err)
+ goto err1;
+
+ *num_elem_p = num_elem;
+ return 0;
+
+err1:
+ return err;
+}
+
+void rfc_queue_cleanup(struct rfc_queue *q)
+{
+ if (q->ip)
+ kref_put(&q->ip->ref, rfc_mmap_release);
+ else
+ vfree(q->buf);
+
+ kfree(q);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_queue.h b/drivers/infiniband/sw/rfc/rfc_queue.h
new file mode 100644
index 0000000..35c9d45
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_queue.h
@@ -0,0 +1,181 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_QUEUE_H
+#define RFC_QUEUE_H
+
+/* implements a simple circular buffer that can optionally be
+ * shared between user space and the kernel and can be resized
+
+ * the requested element size is rounded up to a power of 2
+ * and the number of elements in the buffer is also rounded
+ * up to a power of 2. Since the queue is empty when the
+ * producer and consumer indices match the maximum capacity
+ * of the queue is one less than the number of element slots
+ */
+
+/* this data structure is shared between user space and kernel
+ * space for those cases where the queue is shared. It contains
+ * the producer and consumer indices. Is also contains a copy
+ * of the queue size parameters for user space to use but the
+ * kernel must use the parameters in the rfc_queue struct
+ * this MUST MATCH the corresponding librfc struct
+ * for performance reasons arrange to have producer and consumer
+ * pointers in separate cache lines
+ * the kernel should always mask the indices to avoid accessing
+ * memory outside of the data area
+ */
+struct rfc_queue_buf {
+ __u32 log2_elem_size;
+ __u32 index_mask;
+ __u32 pad_1[30];
+ __u32 producer_index;
+ __u32 pad_2[31];
+ __u32 consumer_index;
+ __u32 pad_3[31];
+ __u8 data[0];
+};
+
+struct rfc_queue {
+ struct rfc_dev *rfc;
+ struct rfc_queue_buf *buf;
+ struct rfc_mmap_info *ip;
+ size_t buf_size;
+ size_t elem_size;
+ unsigned int log2_elem_size;
+ unsigned int index_mask;
+};
+
+int do_mmap_info(struct rfc_dev *rfc,
+ struct mminfo __user *outbuf,
+ struct ib_ucontext *context,
+ struct rfc_queue_buf *buf,
+ size_t buf_size,
+ struct rfc_mmap_info **ip_p);
+
+void rfc_queue_reset(struct rfc_queue *q);
+
+struct rfc_queue *rfc_queue_init(struct rfc_dev *rfc,
+ int *num_elem,
+ unsigned int elem_size);
+
+int rfc_queue_resize(struct rfc_queue *q,
+ unsigned int *num_elem_p,
+ unsigned int elem_size,
+ struct ib_ucontext *context,
+ struct mminfo __user *outbuf,
+ /* Protect producers while resizing queue */
+ spinlock_t *producer_lock,
+ /* Protect consumers while resizing queue */
+ spinlock_t *consumer_lock);
+
+void rfc_queue_cleanup(struct rfc_queue *queue);
+
+static inline int next_index(struct rfc_queue *q, int index)
+{
+ return (index + 1) & q->buf->index_mask;
+}
+
+static inline int queue_empty(struct rfc_queue *q)
+{
+ return ((q->buf->producer_index - q->buf->consumer_index)
+ & q->index_mask) == 0;
+}
+
+static inline int queue_full(struct rfc_queue *q)
+{
+ return ((q->buf->producer_index + 1 - q->buf->consumer_index)
+ & q->index_mask) == 0;
+}
+
+static inline void advance_producer(struct rfc_queue *q)
+{
+ q->buf->producer_index = (q->buf->producer_index + 1)
+ & q->index_mask;
+}
+
+static inline void advance_consumer(struct rfc_queue *q)
+{
+ q->buf->consumer_index = (q->buf->consumer_index + 1)
+ & q->index_mask;
+}
+
+static inline void *producer_addr(struct rfc_queue *q)
+{
+ return q->buf->data + ((q->buf->producer_index & q->index_mask)
+ << q->log2_elem_size);
+}
+
+static inline void *consumer_addr(struct rfc_queue *q)
+{
+ return q->buf->data + ((q->buf->consumer_index & q->index_mask)
+ << q->log2_elem_size);
+}
+
+static inline unsigned int producer_index(struct rfc_queue *q)
+{
+ return q->buf->producer_index;
+}
+
+static inline unsigned int consumer_index(struct rfc_queue *q)
+{
+ return q->buf->consumer_index;
+}
+
+static inline void *addr_from_index(struct rfc_queue *q, unsigned int index)
+{
+ return q->buf->data + ((index & q->index_mask)
+ << q->buf->log2_elem_size);
+}
+
+static inline unsigned int index_from_addr(const struct rfc_queue *q,
+ const void *addr)
+{
+ return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
+ & q->index_mask;
+}
+
+static inline unsigned int queue_count(const struct rfc_queue *q)
+{
+ return (q->buf->producer_index - q->buf->consumer_index)
+ & q->index_mask;
+}
+
+static inline void *queue_head(struct rfc_queue *q)
+{
+ return queue_empty(q) ? NULL : consumer_addr(q);
+}
+
+#endif /* RFC_QUEUE_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_recv.c b/drivers/infiniband/sw/rfc/rfc_recv.c
new file mode 100644
index 0000000..4743740
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_recv.c
@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+
+#include "rfc.h"
+#include "rfc_loc.h"
+
+static int check_type_state(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ struct rfc_qp *qp)
+{
+ if (unlikely(!qp->valid))
+ goto err1;
+
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ if (unlikely((obj->opcode & IB_OPCODE_RC) != 0)) {
+ pr_warn_ratelimited("bad qp type\n");
+ goto err1;
+ }
+ break;
+ case IB_QPT_UC:
+ if (unlikely(!(obj->opcode & IB_OPCODE_UC))) {
+ pr_warn_ratelimited("bad qp type\n");
+ goto err1;
+ }
+ break;
+ case IB_QPT_UD:
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ if (unlikely(!(obj->opcode & IB_OPCODE_UD))) {
+ pr_warn_ratelimited("bad qp type\n");
+ goto err1;
+ }
+ break;
+ default:
+ pr_warn_ratelimited("unsupported qp type\n");
+ goto err1;
+ }
+
+ if (obj->mask & RFC_REQ_MASK) {
+ if (unlikely(qp->resp.state != QP_STATE_READY))
+ goto err1;
+ } else if (unlikely(qp->req.state < QP_STATE_READY ||
+ qp->req.state > QP_STATE_DRAINED)) {
+ goto err1;
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static void set_qkey_viol_cntr(struct rfc_port *port)
+{
+ spin_lock_bh(&port->port_lock);
+ port->attr.qkey_viol_cntr = min((u32)0xffff,
+ port->attr.qkey_viol_cntr + 1);
+ spin_unlock_bh(&port->port_lock);
+}
+
+static int check_keys(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ u32 qpn, struct rfc_qp *qp)
+{
+ struct rfc_port *port = &rfc->port;
+
+ obj->pkey_index = 0;
+
+ // partition key not supported in rfc v1.0
+
+ if ((qp_type(qp) == IB_QPT_UD || qp_type(qp) == IB_QPT_GSI) &&
+ qpn != 0 && obj->mask) {
+ u32 qkey = (qpn == 1) ? GSI_QKEY : qp->attr.qkey;
+
+ if (unlikely(deth_qkey(obj) != qkey)) {
+ pr_warn("bad qkey,got 0x%x expect 0x%x for qpn 0x%x\n",
+ deth_qkey(obj), qkey, qpn);
+ set_qkey_viol_cntr(port);
+ goto err1;
+ }
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static int check_addr(struct rfc_dev *rfc, struct rfc_obj_info *obj,
+ struct rfc_qp *qp)
+{
+ if (qp_type(qp) != IB_QPT_RC && qp_type(qp) != IB_QPT_UC)
+ goto done;
+
+ if (unlikely(obj->port_num != qp->attr.port_num)) {
+ pr_warn_ratelimited("port %d != qp port %d\n",
+ obj->port_num, qp->attr.port_num);
+ goto err1;
+ }
+
+done:
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static int hdr_check(struct rfc_obj_info *obj)
+{
+ struct rfc_dev *rfc = obj->rfc;
+ struct rfc_port *port = &rfc->port;
+ struct rfc_qp *qp = NULL;
+ u32 qpn = bth_qpn(obj);
+ int index;
+ int err;
+
+ if (unlikely(bth_tver(obj) != BTH_TVER)) {
+ pr_warn_ratelimited("bad tver\n");
+ goto err1;
+ }
+
+ if (qpn != IB_MULTICAST_QPN) {
+ index = (qpn == 0) ? port->qp_smi_index :
+ ((qpn == 1) ? port->qp_gsi_index : qpn);
+ qp = rfc_pool_get_index(&rfc->qp_pool, index);
+ if (unlikely(!qp)) {
+ pr_warn_ratelimited("no qp matches qpn 0x%x\n", qpn);
+ goto err1;
+ }
+ err = check_type_state(rfc, obj, qp);
+ if (unlikely(err))
+ goto err2;
+
+ err = check_addr(rfc, obj, qp);
+ if (unlikely(err))
+ goto err2;
+
+ err = check_keys(rfc, obj, qpn, qp);
+ if (unlikely(err))
+ goto err2;
+ } else {
+ if (unlikely((obj->mask & RFC_GRH_MASK) == 0)) {
+ pr_warn_ratelimited("no grh for mcast qpn\n");
+ goto err1;
+ }
+ }
+
+ obj->qp = qp;
+ return 0;
+
+err2:
+ pr_err("%s(): line %d: error %d\n", __func__, __LINE__, err);
+ if (qp)
+ rfc_drop_ref(qp);
+err1:
+ return -EINVAL;
+}
+
+static inline u64 rfc_process_hdr(struct rfc_dev *rfc,
+ struct rfc_obj_info *obj,
+ struct rfc_recv_data *rcv,
+ struct sg_table **rsgp)
+{
+ if (obj->mask & RFC_REQ_MASK)
+ return (rfc_responder_hdr(rfc, obj->qp, rcv, rsgp));
+
+ return 0;
+}
+static inline void rfc_schedule_done(struct rfc_dev *rfc,
+ struct rfc_obj_info *obj,
+ struct rfc_recv_data *rcv, u64 *ret)
+{
+ *ret = 0;
+ if (obj->mask & RFC_REQ_MASK)
+ *ret = rfc_resp_queue_done(rfc, obj->qp, rcv);
+}
+
+u64 rfc_rcv_hdr(struct rfc_recv_data *rcv, struct sg_table **rsgp)
+{
+ int err;
+ struct rfc_obj_info *obj = &rcv->obj;
+ struct rfc_dev *rfc = obj->rfc;
+
+ obj->offset = 0;
+
+ obj->opcode = bth_opcode(obj);
+ obj->psn = bth_psn(obj);
+ obj->qp = NULL;
+ obj->mask |= rfc_opcode[obj->opcode].mask;
+
+ err = hdr_check(obj);
+ if (unlikely(err))
+ goto drop;
+
+ rfc_counter_inc(rfc, RFC_CNT_RCVD_PKTS);
+
+ return rfc_process_hdr(rfc, obj, rcv, rsgp);
+
+drop:
+ if (obj->qp)
+ rfc_drop_ref(obj->qp);
+
+ return 0;
+}
+
+
+u64 rfc_rcv_done(struct rfc_recv_data *rcv)
+{
+ int err;
+ struct rfc_obj_info *obj = &rcv->obj;
+ struct rfc_dev *rfc = obj->rfc;
+ u64 ret = 0;
+
+ obj->offset = 0;
+
+ obj->opcode = bth_opcode(obj);
+ obj->psn = bth_psn(obj);
+ obj->qp = NULL;
+ obj->mask |= rfc_opcode[obj->opcode].mask;
+
+ err = hdr_check(obj);
+ if (unlikely(err))
+ goto drop;
+
+ rfc_schedule_done(rfc, obj, rcv, &ret);
+ return ret;
+
+drop:
+ return err;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_req.c b/drivers/infiniband/sw/rfc/rfc_req.c
new file mode 100644
index 0000000..f31b0e5
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_req.c
@@ -0,0 +1,768 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <crypto/hash.h>
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+static int next_opcode(struct rfc_qp *qp, struct rfc_send_wqe *wqe,
+ u32 opcode);
+
+static inline void retry_first_read_write_send(struct rfc_qp *qp,
+ struct rfc_send_wqe *wqe,
+ unsigned int mask, int npsn)
+{
+ int i;
+
+ for (i = 0; i < npsn; i++) {
+ int to_send = (wqe->dma.resid > qp->mtu) ?
+ qp->mtu : wqe->dma.resid;
+
+ qp->req.opcode = next_opcode(qp, wqe,
+ wqe->wr.opcode);
+
+ if (wqe->wr.send_flags & IB_SEND_INLINE) {
+ wqe->dma.resid -= to_send;
+ wqe->dma.sge_offset += to_send;
+ } else {
+ advance_dma_data(&wqe->dma, to_send);
+ }
+ if (mask & WR_READ_OR_WRITE_MASK)
+ wqe->iova += qp->mtu;
+ }
+}
+
+static void req_retry_sgl(struct rfc_qp *qp)
+{
+ struct rfc_send_wqe *wqe;
+ unsigned int wqe_index;
+ unsigned int mask;
+ int npsn;
+ int first = 1;
+
+ wqe = queue_head(qp->sq.queue);
+ npsn = (qp->comp.psn - wqe->first_psn) & BTH_PSN_MASK;
+
+ qp->req.wqe_index = consumer_index(qp->sq.queue);
+ qp->req.psn = qp->comp.psn;
+ qp->req.opcode = -1;
+
+ for (wqe_index = consumer_index(qp->sq.queue);
+ wqe_index != producer_index(qp->sq.queue);
+ wqe_index = next_index(qp->sq.queue, wqe_index)) {
+ wqe = addr_from_index(qp->sq.queue, wqe_index);
+ mask = wr_opcode_mask(wqe->wr.opcode, qp);
+
+ if (wqe->state == wqe_state_posted)
+ break;
+
+ if (wqe->state == wqe_state_done)
+ continue;
+
+ wqe->iova = (mask & WR_ATOMIC_MASK) ?
+ wqe->wr.wr.atomic.remote_addr :
+ (mask & WR_READ_OR_WRITE_MASK) ?
+ wqe->wr.wr.rdma.remote_addr :
+ 0;
+
+ if (!first || (mask & WR_READ_MASK) == 0) {
+ wqe->dma.resid = wqe->dma.length;
+ wqe->dma.cur_sge = 0;
+ wqe->dma.sge_offset = 0;
+ }
+
+ if (first) {
+ first = 0;
+
+ retry_first_read_write_send(qp, wqe, mask, npsn);
+ }
+
+ wqe->state = wqe_state_posted;
+ }
+}
+
+void rnr_nak_timer(struct timer_list *t)
+{
+ struct rfc_qp *qp = from_timer(qp, t, rnr_nak_timer);
+
+ rfc_run_task(&qp->req.task, 1);
+}
+
+
+static struct rfc_send_wqe *req_next_wqe(struct rfc_qp *qp)
+{
+ struct rfc_send_wqe *wqe = queue_head(qp->sq.queue);
+ unsigned long flags;
+
+ if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
+ /* check to see if we are drained;
+ * state_lock used by requester and completer
+ */
+ spin_lock_irqsave(&qp->state_lock, flags);
+ do {
+ if (qp->req.state != QP_STATE_DRAIN) {
+ /* comp just finished */
+ spin_unlock_irqrestore(&qp->state_lock,
+ flags);
+ break;
+ }
+
+ if (wqe && ((qp->req.wqe_index !=
+ consumer_index(qp->sq.queue)) ||
+ (wqe->state != wqe_state_posted))) {
+ /* comp not done yet */
+ spin_unlock_irqrestore(&qp->state_lock,
+ flags);
+ break;
+ }
+
+ qp->req.state = QP_STATE_DRAINED;
+ spin_unlock_irqrestore(&qp->state_lock, flags);
+
+ if (qp->ibqp.event_handler) {
+ struct ib_event ev;
+
+ ev.device = qp->ibqp.device;
+ ev.element.qp = &qp->ibqp;
+ ev.event = IB_EVENT_SQ_DRAINED;
+ qp->ibqp.event_handler(&ev,
+ qp->ibqp.qp_context);
+ }
+ } while (0);
+ }
+
+ if (qp->req.wqe_index == producer_index(qp->sq.queue))
+ return NULL;
+
+ wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index);
+
+ if (unlikely((qp->req.state == QP_STATE_DRAIN ||
+ qp->req.state == QP_STATE_DRAINED) &&
+ (wqe->state != wqe_state_processing)))
+ return NULL;
+
+ if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) &&
+ (qp->req.wqe_index != consumer_index(qp->sq.queue)))) {
+ qp->req.wait_fence = 1;
+ return NULL;
+ }
+
+ wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
+ return wqe;
+}
+
+static int next_opcode_rc_sgl(struct rfc_qp *qp, u32 opcode, int fits)
+{
+ switch (opcode) {
+ case IB_WR_RDMA_WRITE:
+ return IB_OPCODE_RC_RDMA_WRITE_ONLY;
+
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ return IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE;
+
+ case IB_WR_SEND:
+ return IB_OPCODE_RC_SEND_ONLY;
+
+ case IB_WR_SEND_WITH_IMM:
+ return IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE;
+
+ case IB_WR_RDMA_READ:
+ return IB_OPCODE_RC_RDMA_READ_REQUEST;
+
+ case IB_WR_ATOMIC_CMP_AND_SWP:
+ return IB_OPCODE_RC_COMPARE_SWAP;
+
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ return IB_OPCODE_RC_FETCH_ADD;
+
+ case IB_WR_SEND_WITH_INV:
+ return IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE;
+
+ case IB_WR_REG_MR:
+ case IB_WR_LOCAL_INV:
+ return opcode;
+ }
+
+ return -EINVAL;
+}
+
+
+static int next_opcode_uc(struct rfc_qp *qp, u32 opcode, int fits)
+{
+ switch (opcode) {
+ case IB_WR_RDMA_WRITE:
+ if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
+ qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
+ return fits ?
+ IB_OPCODE_UC_RDMA_WRITE_LAST :
+ IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
+ else
+ return fits ?
+ IB_OPCODE_UC_RDMA_WRITE_ONLY :
+ IB_OPCODE_UC_RDMA_WRITE_FIRST;
+
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
+ qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
+ return fits ?
+ IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
+ IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
+ else
+ return fits ?
+ IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
+ IB_OPCODE_UC_RDMA_WRITE_FIRST;
+
+ case IB_WR_SEND:
+ if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
+ qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
+ return fits ?
+ IB_OPCODE_UC_SEND_LAST :
+ IB_OPCODE_UC_SEND_MIDDLE;
+ else
+ return fits ?
+ IB_OPCODE_UC_SEND_ONLY :
+ IB_OPCODE_UC_SEND_FIRST;
+
+ case IB_WR_SEND_WITH_IMM:
+ if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
+ qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
+ return fits ?
+ IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE :
+ IB_OPCODE_UC_SEND_MIDDLE;
+ else
+ return fits ?
+ IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE :
+ IB_OPCODE_UC_SEND_FIRST;
+ }
+
+ return -EINVAL;
+}
+
+static int next_opcode(struct rfc_qp *qp, struct rfc_send_wqe *wqe,
+ u32 opcode)
+{
+ int fits = (wqe->dma.resid <= qp->mtu);
+
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ return next_opcode_rc_sgl(qp, opcode, fits);
+
+ case IB_QPT_UC:
+ return next_opcode_uc(qp, opcode, fits);
+
+ case IB_QPT_SMI:
+ case IB_QPT_UD:
+ case IB_QPT_GSI:
+ switch (opcode) {
+ case IB_WR_SEND:
+ return IB_OPCODE_UD_SEND_ONLY;
+
+ case IB_WR_SEND_WITH_IMM:
+ return IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ return -EINVAL;
+}
+
+static inline int check_init_depth(struct rfc_qp *qp, struct rfc_send_wqe *wqe)
+{
+ int depth;
+
+ if (wqe->has_rd_atomic)
+ return 0;
+
+ qp->req.need_rd_atomic = 1;
+ depth = atomic_dec_return(&qp->req.rd_atomic);
+
+ if (depth >= 0) {
+ qp->req.need_rd_atomic = 0;
+ wqe->has_rd_atomic = 1;
+ return 0;
+ }
+
+ atomic_inc(&qp->req.rd_atomic);
+ return -EAGAIN;
+}
+
+static inline int get_mtu(struct rfc_qp *qp)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+
+ if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
+ return qp->mtu;
+
+ return rfc->port.mtu_cap;
+}
+
+static struct sg_table *init_req_object(struct rfc_qp *qp,
+ struct rfc_send_wqe *wqe,
+ int opcode, int payload,
+ struct rfc_obj_info *obj)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct rfc_port *port = &rfc->port;
+ struct sg_table *rsg;
+ struct rfc_send_wr *ibwr = &wqe->wr;
+ struct rfc_av *av;
+ int pad = 0;
+ int paylen;
+ int solicited;
+ u16 pkey;
+ u32 qp_num;
+ int ack_req;
+
+ paylen = payload;
+ pad = 0;
+
+ obj->opcode = opcode;
+ obj->qp = qp;
+ obj->psn = qp->req.psn;
+ obj->mask = rfc_opcode[opcode].mask;
+ obj->paylen = paylen;
+ obj->offset = 0;
+ obj->wqe = wqe;
+
+ /* init sgl */
+ av = rfc_get_av(obj);
+ rsg = rfc_init_object(rfc, av, paylen, obj);
+ if (unlikely(!rsg))
+ return NULL;
+
+ /* init bth */
+ solicited = (ibwr->send_flags & IB_SEND_SOLICITED) &&
+ (obj->mask & RFC_END_MASK) &&
+ ((obj->mask & (RFC_SEND_MASK)) ||
+ (obj->mask & (RFC_WRITE_MASK | RFC_IMMDT_MASK)) ==
+ (RFC_WRITE_MASK | RFC_IMMDT_MASK));
+
+ pkey = (qp_type(qp) == IB_QPT_GSI) ?
+ port->pkey_tbl[ibwr->wr.ud.pkey_index] :
+ port->pkey_tbl[qp->attr.pkey_index];
+
+ qp_num = (obj->mask & RFC_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
+ qp->attr.dest_qp_num;
+
+ ack_req = ((obj->mask & RFC_END_MASK) ||
+ (qp->req.noack_objs++ > RFC_MAX_PKT_PER_ACK));
+ if (ack_req)
+ qp->req.noack_objs = 0;
+
+ bth_init(obj, obj->opcode, solicited, 0, pad, pkey, qp_num,
+ ack_req, obj->psn);
+
+ /* init optional headers */
+ if (obj->mask & RFC_RETH_MASK) {
+ reth_set_rkey(obj, ibwr->wr.rdma.rkey);
+
+ reth_set_va(obj, wqe->iova + wqe->dma.length - wqe->dma.resid);
+
+ /*
+ * a READ/WRITE WQE may have to be fragmented into multiple
+ * NVMe READ/WRITE requests.
+ * Adjust the RETH length for READ case here.
+ */
+ reth_set_len(obj, paylen);
+ }
+
+ if (obj->mask & RFC_IMMDT_MASK)
+ immdt_set_imm(obj, ibwr->ex.imm_data);
+
+ if (obj->mask & RFC_IETH_MASK)
+ ieth_set_rkey(obj, ibwr->ex.invalidate_rkey);
+
+ if (obj->mask & RFC_ATMETH_MASK) {
+ atmeth_set_va(obj, wqe->iova);
+ if (opcode == IB_OPCODE_RC_COMPARE_SWAP ||
+ opcode == IB_OPCODE_RD_COMPARE_SWAP) {
+ atmeth_set_swap_add(obj, ibwr->wr.atomic.swap);
+ atmeth_set_comp(obj, ibwr->wr.atomic.compare_add);
+ } else {
+ atmeth_set_swap_add(obj, ibwr->wr.atomic.compare_add);
+ }
+ atmeth_set_rkey(obj, ibwr->wr.atomic.rkey);
+
+ /* write on first sgl also to tunnel the ATOMIC data */
+ memcpy(sg_virt(&rsg->sgl[1]), (struct rfc_atmeth *)
+ (obj->hdr + obj->offset +
+ rfc_opcode[obj->opcode].offset[RFC_ATMETH]),
+ sizeof(struct rfc_atmeth));
+ }
+
+ if (obj->mask & RFC_DETH_MASK) {
+ if (qp->ibqp.qp_num == 1)
+ deth_set_qkey(obj, GSI_QKEY);
+ else
+ deth_set_qkey(obj, ibwr->wr.ud.remote_qkey);
+ deth_set_sqp(obj, qp->ibqp.qp_num);
+ }
+
+ return rsg;
+}
+static void print_sgl_length(struct sg_table *rsg)
+{
+ struct scatterlist *sgl = rsg->sgl, *tmpsg;
+ int i, length = 0;
+
+ for_each_sg(sgl, tmpsg, rsg->nents, i) {
+ length += tmpsg->length;
+ }
+}
+static int fill_object(struct rfc_qp *qp, struct rfc_send_wqe *wqe,
+ struct rfc_obj_info *obj, struct sg_table *rsg,
+ int paylen)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ u32 crc = 0;
+ int err;
+
+ obj->inline_page = NULL;
+
+ if (obj->mask & RFC_WRITE_OR_SEND) {
+ if (wqe->wr.send_flags & IB_SEND_INLINE) {
+ /* Support for Inline data <= 1 page size. */
+ u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
+ int npages = paylen / PAGE_SIZE;
+ struct page *page = NULL;
+
+ npages += paylen % PAGE_SIZE?1:0;
+ page = alloc_pages(GFP_KERNEL, 0);
+ memcpy(page_address(page), tmp, paylen);
+ sg_set_page(&rsg->sgl[rsg->nents++], page, paylen, 0);
+ obj->inline_page = page;
+ wqe->dma.resid -= paylen;
+ wqe->dma.sge_offset += paylen;
+ } else {
+ err = copy_sgl(rfc, qp->pd, 0, &wqe->dma, rsg,
+ paylen, from_mem_obj, &crc);
+ if (err)
+ return err;
+ }
+ } else if (obj->mask & RFC_READ_MASK) {
+ /* Provide SGLs for READ data */
+ err = copy_sgl(rfc, qp->pd, 0, &wqe->dma, rsg, paylen,
+ to_mem_obj, NULL);
+ }
+ print_sgl_length(rsg);
+ return 0;
+}
+
+static void update_wqe_state(struct rfc_qp *qp,
+ struct rfc_send_wqe *wqe,
+ struct rfc_obj_info *obj)
+{
+ if (obj->mask & RFC_END_MASK) {
+ if (qp_type(qp) == IB_QPT_RC) {
+ /*
+ * If RDMA_READ/WRITE WQE has not been accomodated
+ * in single NVMe READ/WRITE REQUEST, keep the wqe
+ * in processing state.
+ */
+ if (wqe->dma.resid > 0) {
+ wqe->state = wqe_state_processing;
+ return;
+ }
+ wqe->state = wqe_state_pending;
+ }
+
+ } else {
+ wqe->state = wqe_state_processing;
+ }
+}
+
+static void update_wqe_psn(struct rfc_qp *qp,
+ struct rfc_send_wqe *wqe,
+ struct rfc_obj_info *obj,
+ int payload,
+ int first_obj)
+{
+ /* number of objects left to send including current one */
+ int num_obj = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
+
+ /* handle zero length object case */
+ if (num_obj == 0)
+ num_obj = 1;
+
+// if (obj->mask & RFC_START_MASK) {
+ if (first_obj) {
+ wqe->first_psn = qp->req.psn;
+ wqe->last_psn = (qp->req.psn + num_obj - 1) & BTH_PSN_MASK;
+ }
+
+ qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
+}
+
+static void save_state(struct rfc_send_wqe *wqe,
+ struct rfc_qp *qp,
+ struct rfc_send_wqe *rollback_wqe,
+ u32 *rollback_psn)
+{
+ rollback_wqe->state = wqe->state;
+ rollback_wqe->first_psn = wqe->first_psn;
+ rollback_wqe->last_psn = wqe->last_psn;
+ *rollback_psn = qp->req.psn;
+}
+
+static void rollback_state(struct rfc_send_wqe *wqe,
+ struct rfc_qp *qp,
+ struct rfc_send_wqe *rollback_wqe,
+ u32 rollback_psn)
+{
+ wqe->state = rollback_wqe->state;
+ wqe->first_psn = rollback_wqe->first_psn;
+ wqe->last_psn = rollback_wqe->last_psn;
+ qp->req.psn = rollback_psn;
+}
+
+static void update_state(struct rfc_qp *qp, struct rfc_send_wqe *wqe,
+ struct rfc_obj_info *obj, int payload)
+{
+ qp->req.opcode = obj->opcode;
+
+ if (obj->mask & RFC_END_MASK) {
+// if (!(obj->mask & RFC_ATOMIC_MASK) && wqe->dma.resid > 0) {
+ if ((obj->mask & RFC_ATOMIC_MASK) || wqe->dma.resid == 0)
+ qp->req.wqe_index = next_index(qp->sq.queue,
+ qp->req.wqe_index);
+ }
+
+ if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer)) {
+ mod_timer(&qp->retrans_timer,
+ jiffies + qp->qp_timeout_jiffies);
+ }
+}
+
+static void rfc_drain_send_queue(struct rfc_qp *qp, bool notify)
+{
+ struct rfc_send_wqe *wqe;
+ unsigned int index;
+
+ while ((wqe = queue_head(qp->sq.queue))) {
+ index = consumer_index(qp->sq.queue);
+ if (notify) {
+ wqe->status = IB_WC_WR_FLUSH_ERR;
+ rfc_comp_do_complete(qp, wqe);
+ } else {
+ advance_consumer(qp->sq.queue);
+ }
+ }
+}
+
+int rfc_requester(void *arg)
+{
+ struct rfc_qp *qp = (struct rfc_qp *)arg;
+ struct rfc_obj_info *obj = NULL;
+ struct sg_table *rsg = NULL;
+ struct rfc_send_wqe *wqe;
+ enum rfc_hdr_mask mask;
+ int payload;
+ int mtu;
+ int opcode;
+ int ret;
+ struct rfc_send_wqe rollback_wqe;
+ u32 rollback_psn;
+ int first_obj;
+
+ rfc_add_ref(qp);
+
+next_wqe:
+
+ if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR ||
+ qp->req.state == QP_STATE_RESET)) {
+ // if qp not valid or qp error
+ if (qp->req.state != QP_STATE_RESET)
+ rfc_drain_recv_queue(qp, true);
+
+ rfc_drain_send_queue(qp, qp->valid &&
+ qp->req.state == QP_STATE_ERROR);
+ goto exit;
+ }
+
+ if (unlikely(qp->req.state == QP_STATE_RESET)) {
+ qp->req.wqe_index = consumer_index(qp->sq.queue);
+ qp->req.opcode = -1;
+ qp->req.need_rd_atomic = 0;
+ qp->req.wait_psn = 0;
+ qp->req.need_retry = 0;
+ goto exit;
+ }
+
+ if (unlikely(qp->req.need_retry)) {
+ req_retry_sgl(qp);
+ qp->req.need_retry = 0;
+ }
+
+ wqe = req_next_wqe(qp);
+ if (unlikely(!wqe))
+ goto exit;
+
+ if (wqe->mask & WR_REG_MASK) {
+ if (wqe->wr.opcode == IB_WR_LOCAL_INV) {
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct rfc_mem *rmr;
+
+ rmr = rfc_pool_get_index(&rfc->mr_pool,
+ wqe->wr.ex.invalidate_rkey >> 8);
+ if (!rmr) {
+ pr_err("No mr for key %#x\n",
+ wqe->wr.ex.invalidate_rkey);
+ wqe->state = wqe_state_error;
+ wqe->status = IB_WC_MW_BIND_ERR;
+ goto exit;
+ }
+ rmr->state = RFC_MEM_STATE_FREE;
+ rfc_drop_ref(rmr);
+ wqe->state = wqe_state_done;
+ wqe->status = IB_WC_SUCCESS;
+ } else if (wqe->wr.opcode == IB_WR_REG_MR) {
+ struct rfc_mem *rmr = to_rmr(wqe->wr.wr.reg.mr);
+
+ rmr->state = RFC_MEM_STATE_VALID;
+ rmr->access = wqe->wr.wr.reg.access;
+ rmr->lkey = wqe->wr.wr.reg.key;
+ rmr->rkey = wqe->wr.wr.reg.key;
+ wqe->state = wqe_state_done;
+ wqe->status = IB_WC_SUCCESS;
+ } else {
+ goto exit;
+ }
+ qp->req.wqe_index = next_index(qp->sq.queue,
+ qp->req.wqe_index);
+ goto next_wqe;
+ }
+
+ if (unlikely(qp_type(qp) == IB_QPT_RC &&
+ qp->req.psn > (qp->comp.psn + RFC_MAX_UNACKED_PSNS))) {
+ qp->req.wait_psn = 1;
+ goto exit;
+ }
+
+ opcode = next_opcode(qp, wqe, wqe->wr.opcode);
+ if (unlikely(opcode < 0)) {
+ wqe->status = IB_WC_LOC_QP_OP_ERR;
+ goto exit;
+ }
+
+ mask = rfc_opcode[opcode].mask;
+ if (unlikely(mask & RFC_READ_OR_ATOMIC)) {
+ if (check_init_depth(qp, wqe))
+ goto exit;
+ }
+
+ mtu = ROFC_MTU;
+ payload = wqe->dma.resid;
+ first_obj = (wqe->dma.resid == wqe->dma.length);
+ if (payload > mtu) {
+ if (qp_type(qp) == IB_QPT_UD) {
+ /* C10-93.1.1: If the total sum of all the buffer
+ * lengths specified for a UD message exceeds the MTU
+ * of the port as returned by QueryHCA, the CI shall
+ * not emit any objects for this message. Further, the
+ * CI shall not generate an error due to this condition
+ */
+
+ /* fake a successful UD send */
+ wqe->first_psn = qp->req.psn;
+ wqe->last_psn = qp->req.psn;
+ qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
+ qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;
+ qp->req.wqe_index = next_index(qp->sq.queue,
+ qp->req.wqe_index);
+ wqe->state = wqe_state_done;
+ wqe->status = IB_WC_SUCCESS;
+ __rfc_do_task(&qp->comp.task);
+ rfc_drop_ref(qp);
+ return 0;
+ }
+ payload = mtu;
+ }
+
+ obj = kmalloc(sizeof(struct rfc_obj_info), GFP_KERNEL);
+
+ if (!obj)
+ goto err;
+
+ rsg = init_req_object(qp, wqe, opcode, payload, obj);
+ if (unlikely(!rsg)) {
+ pr_err("qp#%d Failed allocating rsg\n", qp_num(qp));
+ goto err;
+ }
+
+ if (fill_object(qp, wqe, obj, rsg, payload)) {
+ pr_err("qp#%d Error during fill object\n", qp_num(qp));
+ goto err;
+ }
+
+ /*
+ * To prevent a race on wqe access between requester and completer,
+ * wqe members state and psn need to be set before calling
+ * rfc_send().
+ * Otherwise, completer might initiate an unjustified retry flow.
+ */
+ save_state(wqe, qp, &rollback_wqe, &rollback_psn);
+ update_wqe_state(qp, wqe, obj);
+ update_wqe_psn(qp, wqe, obj, payload, first_obj);
+ ret = rfc_xmit_object(to_rdev(qp->ibqp.device), qp, obj, rsg);
+ if (ret) {
+ rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
+
+ if (ret == -EAGAIN) {
+ pr_err("rfc_xmit_object error\n");
+ rfc_run_task(&qp->req.task, 1);
+ goto exit;
+ }
+
+ goto err;
+ }
+
+ update_state(qp, wqe, obj, payload);
+
+ goto next_wqe;
+
+err:
+ pr_err("rfc request error. qp#%d\n", qp_num(qp));
+ wqe->status = IB_WC_LOC_PROT_ERR;
+ wqe->state = wqe_state_error;
+
+exit:
+ rfc_drop_ref(qp);
+ return -EAGAIN;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_resp.c b/drivers/infiniband/sw/rfc/rfc_resp.c
new file mode 100644
index 0000000..a383105
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_resp.c
@@ -0,0 +1,1444 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+enum resp_states {
+ RESPST_NONE,
+ RESPST_GET_REQ,
+ RESPST_CHK_PSN,
+ RESPST_CHK_OP_SEQ,
+ RESPST_CHK_OP_VALID,
+ RESPST_CHK_RESOURCE,
+ RESPST_CHK_LENGTH,
+ RESPST_CHK_RKEY,
+ RESPST_EXECUTE,
+ RESPST_READ_REPLY,
+ RESPST_COMPLETE,
+ RESPST_ACKNOWLEDGE,
+ RESPST_CLEANUP,
+ RESPST_DUPLICATE_REQUEST,
+ RESPST_ERR_MALFORMED_WQE,
+ RESPST_ERR_UNSUPPORTED_OPCODE,
+ RESPST_ERR_MISALIGNED_ATOMIC,
+ RESPST_ERR_PSN_OUT_OF_SEQ,
+ RESPST_ERR_MISSING_OPCODE_FIRST,
+ RESPST_ERR_MISSING_OPCODE_LAST_C,
+ RESPST_ERR_MISSING_OPCODE_LAST_D1E,
+ RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
+ RESPST_ERR_RNR,
+ RESPST_ERR_RKEY_VIOLATION,
+ RESPST_ERR_LENGTH,
+ RESPST_ERR_CQ_OVERFLOW,
+ RESPST_ERROR,
+ RESPST_RESET,
+ RESPST_DONE,
+ RESPST_EXIT,
+};
+
+static char *resp_state_name[] = {
+ [RESPST_NONE] = "NONE",
+ [RESPST_GET_REQ] = "GET_REQ",
+ [RESPST_CHK_PSN] = "CHK_PSN",
+ [RESPST_CHK_OP_SEQ] = "CHK_OP_SEQ",
+ [RESPST_CHK_OP_VALID] = "CHK_OP_VALID",
+ [RESPST_CHK_RESOURCE] = "CHK_RESOURCE",
+ [RESPST_CHK_LENGTH] = "CHK_LENGTH",
+ [RESPST_CHK_RKEY] = "CHK_RKEY",
+ [RESPST_EXECUTE] = "EXECUTE",
+ [RESPST_READ_REPLY] = "READ_REPLY",
+ [RESPST_COMPLETE] = "COMPLETE",
+ [RESPST_ACKNOWLEDGE] = "ACKNOWLEDGE",
+ [RESPST_CLEANUP] = "CLEANUP",
+ [RESPST_DUPLICATE_REQUEST] = "DUPLICATE_REQUEST",
+ [RESPST_ERR_MALFORMED_WQE] = "ERR_MALFORMED_WQE",
+ [RESPST_ERR_UNSUPPORTED_OPCODE] = "ERR_UNSUPPORTED_OPCODE",
+ [RESPST_ERR_MISALIGNED_ATOMIC] = "ERR_MISALIGNED_ATOMIC",
+ [RESPST_ERR_PSN_OUT_OF_SEQ] = "ERR_PSN_OUT_OF_SEQ",
+ [RESPST_ERR_MISSING_OPCODE_FIRST] = "ERR_MISSING_OPCODE_FIRST",
+ [RESPST_ERR_MISSING_OPCODE_LAST_C] = "ERR_MISSING_OPCODE_LAST_C",
+ [RESPST_ERR_MISSING_OPCODE_LAST_D1E] = "ERR_MISSING_OPCODE_LAST_D1E",
+ [RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ",
+ [RESPST_ERR_RNR] = "ERR_RNR",
+ [RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION",
+ [RESPST_ERR_LENGTH] = "ERR_LENGTH",
+ [RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",
+ [RESPST_ERROR] = "ERROR",
+ [RESPST_RESET] = "RESET",
+ [RESPST_DONE] = "DONE",
+ [RESPST_EXIT] = "EXIT",
+};
+
+u64 rfc_resp_queue_done(struct rfc_dev *rfc, struct rfc_qp *qp,
+ struct rfc_recv_data *rcv)
+{
+ struct rfc_obj_info *obj = &rcv->obj;
+ u64 ret = 0, resp = 0;
+
+ ret = rfc_responder_done_direct(qp, obj, &resp);
+ return resp;
+}
+
+static inline enum resp_states get_req(struct rfc_qp *qp,
+ struct rfc_obj_info **obj_p)
+{
+ struct rfc_recv_data *rcv_datap;
+
+ *obj_p = NULL;
+ if (!list_empty(&qp->req_objs)) {
+ rcv_datap = list_first_entry(&qp->req_objs,
+ struct rfc_recv_data, rcv_obj);
+ *obj_p = &rcv_datap->obj;
+ return (qp->resp.res) ? RESPST_READ_REPLY : RESPST_CHK_PSN;
+ }
+ return RESPST_EXIT;
+}
+
+static enum resp_states check_psn(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ int diff = psn_compare(obj->psn, qp->resp.psn);
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ if (diff > 0) {
+ if (qp->resp.sent_psn_nak)
+ return RESPST_CLEANUP;
+
+ qp->resp.sent_psn_nak = 1;
+ rfc_counter_inc(rfc, RFC_CNT_OUT_OF_SEQ_REQ);
+ return RESPST_ERR_PSN_OUT_OF_SEQ;
+
+ } else if (diff < 0) {
+ rfc_counter_inc(rfc, RFC_CNT_DUP_REQ);
+ return RESPST_DUPLICATE_REQUEST;
+ }
+
+ if (qp->resp.sent_psn_nak)
+ qp->resp.sent_psn_nak = 0;
+
+ break;
+
+ case IB_QPT_UC:
+ if (qp->resp.drop_msg || diff != 0) {
+ if (obj->mask & RFC_START_MASK) {
+ qp->resp.drop_msg = 0;
+ return RESPST_CHK_OP_SEQ;
+ }
+
+ qp->resp.drop_msg = 1;
+ return RESPST_CLEANUP;
+ }
+ break;
+ default:
+ break;
+ }
+
+ return RESPST_CHK_OP_SEQ;
+}
+
+static enum resp_states check_op_seq(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ switch (qp->resp.opcode) {
+ case IB_OPCODE_RC_SEND_FIRST:
+ case IB_OPCODE_RC_SEND_MIDDLE:
+ switch (obj->opcode) {
+ case IB_OPCODE_RC_SEND_MIDDLE:
+ case IB_OPCODE_RC_SEND_LAST:
+ case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
+ case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_C;
+ }
+
+ case IB_OPCODE_RC_RDMA_WRITE_FIRST:
+ case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+ switch (obj->opcode) {
+ case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_RC_RDMA_WRITE_LAST:
+ case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_C;
+ }
+
+ default:
+ switch (obj->opcode) {
+ case IB_OPCODE_RC_SEND_MIDDLE:
+ case IB_OPCODE_RC_SEND_LAST:
+ case IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE:
+ case IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE:
+ case IB_OPCODE_RC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_RC_RDMA_WRITE_LAST:
+ case IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ return RESPST_ERR_MISSING_OPCODE_FIRST;
+ default:
+ return RESPST_CHK_OP_VALID;
+ }
+ }
+ break;
+
+ case IB_QPT_UC:
+ switch (qp->resp.opcode) {
+ case IB_OPCODE_UC_SEND_FIRST:
+ case IB_OPCODE_UC_SEND_MIDDLE:
+ switch (obj->opcode) {
+ case IB_OPCODE_UC_SEND_MIDDLE:
+ case IB_OPCODE_UC_SEND_LAST:
+ case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
+ }
+
+ case IB_OPCODE_UC_RDMA_WRITE_FIRST:
+ case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+ switch (obj->opcode) {
+ case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_UC_RDMA_WRITE_LAST:
+ case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_D1E;
+ }
+
+ default:
+ switch (obj->opcode) {
+ case IB_OPCODE_UC_SEND_MIDDLE:
+ case IB_OPCODE_UC_SEND_LAST:
+ case IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE:
+ case IB_OPCODE_UC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_UC_RDMA_WRITE_LAST:
+ case IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ qp->resp.drop_msg = 1;
+ return RESPST_CLEANUP;
+ default:
+ return RESPST_CHK_OP_VALID;
+ }
+ }
+ break;
+
+ default:
+ return RESPST_CHK_OP_VALID;
+ }
+}
+
+static enum resp_states check_op_valid(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ if (((obj->mask & RFC_READ_MASK) &&
+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) ||
+ ((obj->mask & RFC_WRITE_MASK) &&
+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) ||
+ ((obj->mask & RFC_ATOMIC_MASK) &&
+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) {
+ return RESPST_ERR_UNSUPPORTED_OPCODE;
+ }
+
+ break;
+
+ case IB_QPT_UC:
+ if ((obj->mask & RFC_WRITE_MASK) &&
+ !(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_WRITE)) {
+ qp->resp.drop_msg = 1;
+ return RESPST_CLEANUP;
+ }
+
+ break;
+
+ case IB_QPT_UD:
+ case IB_QPT_SMI:
+ case IB_QPT_GSI:
+ break;
+
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
+ return RESPST_CHK_RESOURCE;
+}
+
+static enum resp_states get_srq_wqe(struct rfc_qp *qp)
+{
+ struct rfc_srq *srq = qp->srq;
+ struct rfc_queue *q = srq->rq.queue;
+ struct rfc_recv_wqe *wqe;
+ struct ib_event ev;
+
+ if (srq->error)
+ return RESPST_ERR_RNR;
+
+ spin_lock_bh(&srq->rq.consumer_lock);
+
+ wqe = queue_head(q);
+ if (!wqe) {
+ spin_unlock_bh(&srq->rq.consumer_lock);
+ return RESPST_ERR_RNR;
+ }
+
+ /* note kernel and user space recv wqes have same size */
+ memcpy(&qp->resp.srq_wqe, wqe, sizeof(qp->resp.srq_wqe));
+
+ qp->resp.wqe = &qp->resp.srq_wqe.wqe;
+ advance_consumer(q);
+
+ if (srq->limit && srq->ibsrq.event_handler &&
+ (queue_count(q) < srq->limit)) {
+ srq->limit = 0;
+ goto event;
+ }
+
+ spin_unlock_bh(&srq->rq.consumer_lock);
+ return RESPST_CHK_LENGTH;
+
+event:
+ spin_unlock_bh(&srq->rq.consumer_lock);
+ ev.device = qp->ibqp.device;
+ ev.element.srq = qp->ibqp.srq;
+ ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
+ srq->ibsrq.event_handler(&ev, srq->ibsrq.srq_context);
+ return RESPST_CHK_LENGTH;
+}
+
+static enum resp_states check_resource(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ struct rfc_srq *srq = qp->srq;
+
+ if (qp->resp.state == QP_STATE_ERROR) {
+ if (qp->resp.wqe) {
+ qp->resp.status = IB_WC_WR_FLUSH_ERR;
+ return RESPST_COMPLETE;
+ } else if (!srq) {
+ qp->resp.wqe = queue_head(qp->rq.queue);
+ if (qp->resp.wqe) {
+ qp->resp.status = IB_WC_WR_FLUSH_ERR;
+ return RESPST_COMPLETE;
+ } else {
+ return RESPST_EXIT;
+ }
+ } else {
+ return RESPST_EXIT;
+ }
+ }
+
+ if (obj->mask & RFC_READ_OR_ATOMIC) {
+ /* it is the requesters job to not send
+ * too many read/atomic ops, we just
+ * recycle the responder resource queue
+ */
+ if (likely(qp->attr.max_dest_rd_atomic > 0))
+ return RESPST_CHK_LENGTH;
+ else
+ return RESPST_ERR_TOO_MANY_RDMA_ATM_REQ;
+ }
+
+ if (obj->mask & RFC_RWR_MASK) {
+ if (srq)
+ return get_srq_wqe(qp);
+
+ qp->resp.wqe = queue_head(qp->rq.queue);
+ return (qp->resp.wqe) ? RESPST_CHK_LENGTH : RESPST_ERR_RNR;
+ }
+
+ return RESPST_CHK_LENGTH;
+}
+
+static enum resp_states check_length(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ switch (qp_type(qp)) {
+ case IB_QPT_RC:
+ return RESPST_CHK_RKEY;
+
+ case IB_QPT_UC:
+ return RESPST_CHK_RKEY;
+
+ default:
+ return RESPST_CHK_RKEY;
+ }
+}
+
+static enum resp_states check_rkey(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ struct rfc_mem *mem = NULL;
+ u64 va;
+ u32 rkey;
+ u32 resid;
+ u32 objlen;
+ int mtu = ROFC_MTU;
+ enum resp_states state;
+ int access;
+
+ if (obj->mask & (RFC_READ_MASK | RFC_WRITE_MASK)) {
+ if (obj->mask & RFC_RETH_MASK) {
+ qp->resp.va = reth_va(obj);
+ qp->resp.rkey = reth_rkey(obj);
+ qp->resp.resid = reth_len(obj);
+ }
+ access = (obj->mask & RFC_READ_MASK) ? IB_ACCESS_REMOTE_READ
+ : IB_ACCESS_REMOTE_WRITE;
+ } else if (obj->mask & RFC_ATOMIC_MASK) {
+ /* For ATOMIC data, check_rkey will be called in done context.
+ * So, take the values from the allocated extended header page.
+ */
+ qp->resp.va = atmeth_va(obj);
+ qp->resp.rkey = atmeth_rkey(obj);
+ qp->resp.resid = sizeof(u64);
+ access = IB_ACCESS_REMOTE_ATOMIC;
+ } else {
+ return RESPST_EXECUTE;
+ }
+
+ /* A zero-byte op is not required to set an addr or rkey. */
+ if ((obj->mask & (RFC_READ_MASK | RFC_WRITE_OR_SEND)) &&
+ (obj->mask & RFC_RETH_MASK) &&
+ reth_len(obj) == 0) {
+ return RESPST_EXECUTE;
+ }
+
+ va = qp->resp.va;
+ rkey = qp->resp.rkey;
+ resid = qp->resp.resid;
+ objlen = payload_size(obj);
+
+ mem = lookup_mem(qp->pd, access, rkey, lookup_remote);
+ if (!mem) {
+ state = RESPST_ERR_RKEY_VIOLATION;
+ goto err;
+ }
+
+ if (unlikely(mem->state == RFC_MEM_STATE_FREE)) {
+ state = RESPST_ERR_RKEY_VIOLATION;
+ goto err;
+ }
+
+ if (mem_check_range(mem, va, resid)) {
+ state = RESPST_ERR_RKEY_VIOLATION;
+ goto err;
+ }
+
+ if (obj->mask & RFC_WRITE_MASK) {
+ if (resid > mtu) {
+ if (objlen != mtu || bth_pad(obj)) {
+ state = RESPST_ERR_LENGTH;
+ goto err;
+ }
+ } else {
+ if (objlen != resid) {
+ state = RESPST_ERR_LENGTH;
+ goto err;
+ }
+ }
+ }
+
+
+ qp->resp.mr = mem;
+ return RESPST_EXECUTE;
+
+err:
+ if (mem)
+ rfc_drop_ref(mem);
+ return state;
+}
+
+static enum resp_states send_data_in(struct rfc_qp *qp, void *data_addr,
+ int data_len)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+
+ err = copy_data(rfc, qp->pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
+ data_addr, data_len, to_mem_obj, NULL);
+ if (unlikely(err))
+ return (err == -ENOSPC) ? RESPST_ERR_LENGTH
+ : RESPST_ERR_MALFORMED_WQE;
+
+ return RESPST_NONE;
+}
+
+/* Guarantee atomicity of atomic operations at the machine level. */
+static DEFINE_SPINLOCK(atomic_ops_lock);
+
+static enum resp_states process_atomic(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+
+ u64 iova = atmeth_va(obj);
+ u64 *vaddr;
+ enum resp_states ret;
+ struct rfc_mem *mr = qp->resp.mr;
+
+ if (mr->state != RFC_MEM_STATE_VALID) {
+ ret = RESPST_ERR_RKEY_VIOLATION;
+ goto out;
+ }
+
+ vaddr = iova_to_vaddr(mr, iova, sizeof(u64));
+
+ /* check vaddr is 8 bytes aligned. */
+ if (!vaddr || (uintptr_t)vaddr & 7) {
+ ret = RESPST_ERR_MISALIGNED_ATOMIC;
+ goto out;
+ }
+
+ spin_lock_bh(&atomic_ops_lock);
+
+ qp->resp.atomic_orig = *vaddr;
+
+ if (obj->opcode == IB_OPCODE_RC_COMPARE_SWAP ||
+ obj->opcode == IB_OPCODE_RD_COMPARE_SWAP) {
+ if (*vaddr == atmeth_comp(obj))
+ *vaddr = atmeth_swap_add(obj);
+ } else {
+ *vaddr += atmeth_swap_add(obj);
+ }
+
+ spin_unlock_bh(&atomic_ops_lock);
+
+ ret = RESPST_NONE;
+out:
+ return ret;
+}
+
+static struct sg_table *prepare_ack_object(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct rfc_obj_info *ack,
+ int opcode,
+ int payload,
+ u32 psn,
+ u8 syndrome,
+ u32 *crcp)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct sg_table *rsg = NULL;
+ u32 crc = 0;
+ int paylen;
+ int pad = 0;
+ int err;
+
+ /*
+ * allocate object
+ */
+ paylen = payload;
+
+ ack->opcode = opcode;
+ ack->qp = qp;
+ ack->mask = rfc_opcode[opcode].mask;
+ ack->offset = obj->offset;
+ ack->paylen = paylen;
+ rsg = rfc_init_object(rfc, &qp->pri_av, paylen, ack);
+ if (!rsg)
+ return NULL;
+
+ /* fill in bth using the request object headers */
+ memcpy(ack->hdr, obj->hdr, obj->offset + RFC_BTH_BYTES);
+
+ bth_set_opcode(ack, opcode);
+ bth_set_qpn(ack, qp->attr.dest_qp_num);
+ bth_set_pad(ack, pad);
+ bth_set_se(ack, 0);
+ bth_set_psn(ack, psn);
+ bth_set_ack(ack, 0);
+ ack->psn = psn;
+
+ if (ack->mask & RFC_AETH_MASK) {
+ aeth_set_syn(ack, syndrome);
+ aeth_set_msn(ack, qp->resp.msn);
+ }
+
+ if (ack->mask & RFC_ATMACK_MASK)
+ atmack_set_orig(ack, qp->resp.atomic_orig);
+
+ err = rfc_prepare(rfc, ack, rsg, &crc);
+
+ if (err)
+ return NULL;
+
+ if (crcp) {
+ /* CRC computation will be continued by the caller */
+ *crcp = crc;
+ }
+
+ return rsg;
+}
+
+
+// Send Read reply as NVMe READ DATA
+static enum resp_states read_reply_sgl(struct rfc_qp *qp,
+ struct rfc_obj_info *req_obj,
+ struct sg_table *rsg)
+{
+ int mtu = ROFC_MTU;
+ enum resp_states state;
+ int payload;
+ int err;
+ struct resp_res *res = qp->resp.res;
+
+ if (!res) {
+ /* This is the first time we process that request. Get a
+ * resource
+ */
+ res = &qp->resp.resources[qp->resp.res_head];
+
+ free_rd_atomic_resource(qp, res);
+ rfc_advance_resp_resource(qp);
+
+ res->type = RFC_READ_MASK;
+
+ res->read.va = qp->resp.va;
+ res->read.va_org = qp->resp.va;
+
+ res->first_psn = req_obj->psn;
+
+ if (reth_len(req_obj)) {
+ res->last_psn = (req_obj->psn +
+ (reth_len(req_obj) + mtu - 1) /
+ mtu - 1) & BTH_PSN_MASK;
+ } else {
+ res->last_psn = res->first_psn;
+ }
+ res->cur_psn = req_obj->psn;
+
+ res->read.resid = qp->resp.resid;
+ res->read.length = qp->resp.resid;
+ res->read.rkey = qp->resp.rkey;
+
+ /* note res inherits the reference to mr from qp */
+ res->read.mr = qp->resp.mr;
+ qp->resp.mr = NULL;
+
+ qp->resp.res = res;
+ res->state = rdatm_res_state_new;
+ }
+
+ payload = min_t(int, res->read.resid, mtu);
+
+ err = rfc_mem_copy_sgl(res->read.mr, res->read.va, rsg,
+ payload, from_mem_obj, NULL);
+ if (err)
+ pr_err("Failed copying memory\n");
+
+ res->read.va += payload;
+ res->read.resid -= payload;
+ res->cur_psn = (res->cur_psn + 1) & BTH_PSN_MASK;
+
+ if (res->read.resid > 0) {
+ qp->resp.res = NULL;
+ qp->resp.opcode = -1;
+ if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
+ qp->resp.psn = res->cur_psn;
+ state = RESPST_CLEANUP;
+ } else {
+ qp->resp.res = NULL;
+ qp->resp.opcode = -1;
+ if (psn_compare(res->cur_psn, qp->resp.psn) >= 0)
+ qp->resp.psn = res->cur_psn;
+ state = RESPST_CLEANUP;
+ }
+
+ return state;
+}
+static void build_rdma_network_hdr(union rdma_network_hdr *hdr,
+ struct rfc_obj_info *obj)
+{
+ struct iphdr *iph = &hdr->roce4grh;
+
+ memset(hdr, 0, sizeof(*hdr));
+
+ iph->version = IPVERSION;
+ iph->ihl = sizeof(struct iphdr) >> 2;
+ iph->frag_off = 0;
+ iph->protocol = IPPROTO_UDP;
+ iph->tos = 0;
+ iph->daddr = obj->daddr;
+ iph->saddr = obj->saddr;
+ iph->ttl = 50;
+}
+
+struct page *ackpage;
+
+static enum resp_states rfc_resp_get_sgl(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ struct sg_table **rsg)
+{
+ enum resp_states err;
+ uint32_t npages = 0;
+ struct sg_table *temprsg = NULL;
+ struct rfc_recv_data *rdata = container_of(obj, struct rfc_recv_data,
+ obj);
+
+ temprsg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
+ if (unlikely(!temprsg)) {
+ pr_err("%s(): line %d\n", __func__, __LINE__);
+ return -1;
+ }
+ npages = SG_MAX_SINGLE_ALLOC;
+ if (sg_alloc_table(temprsg, npages, GFP_KERNEL)) {
+ kfree(temprsg);
+ return -1;
+ }
+ temprsg->orig_nents = SG_MAX_SINGLE_ALLOC;
+ temprsg->nents = 0;
+ *rsg = temprsg;
+ if (obj->mask & RFC_SEND_MASK) {
+ if (qp_type(qp) == IB_QPT_UD ||
+ qp_type(qp) == IB_QPT_SMI ||
+ qp_type(qp) == IB_QPT_GSI) {
+
+ /* Allocate the first page for MAD data. THe contents
+ * will be copied later along with rdma_network_hdr.
+ */
+ struct page *page = alloc_page(GFP_KERNEL);
+
+ sg_set_page(temprsg->sgl, page, payload_size(obj), 0);
+ temprsg->nents++;
+ rdata->eth_page = page;
+ } else {
+ err = copy_sgl(to_rdev(qp->ibqp.device), qp->pd,
+ IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
+ temprsg, payload_size(obj), to_mem_obj, NULL);
+ }
+ } else if (obj->mask & RFC_WRITE_MASK) {
+ err = rfc_mem_copy_sgl(qp->resp.mr, qp->resp.va, temprsg,
+ payload_size(obj), to_mem_obj, NULL);
+ } else if (obj->mask & RFC_READ_MASK) {
+ err = read_reply_sgl(qp, obj, temprsg);
+ return RESPST_READ_REPLY;
+ } else if (obj->mask & RFC_ATOMIC_MASK) {
+ // Allocate page for receiving ATOMIC data
+ struct page *page = alloc_page(GFP_KERNEL);
+
+ sg_set_page(temprsg->sgl, page, sizeof(struct rfc_atmeth), 0);
+ temprsg->nents = 1;
+ rdata->eth_page = page;
+
+ } else {
+ /* Unreachable */
+ WARN_ON_ONCE(1);
+ }
+
+
+ if (obj->mask & RFC_COMP_MASK) {
+ /* We successfully processed this new request. */
+ return RESPST_COMPLETE;
+ } else if (qp_type(qp) == IB_QPT_RC)
+ return RESPST_ACKNOWLEDGE;
+ else
+ return RESPST_CLEANUP;
+}
+
+static enum resp_states find_resp_done_state(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ int err;
+ struct rfc_recv_data *rdata = container_of(obj, struct rfc_recv_data,
+ obj);
+
+ if (obj->mask & RFC_SEND_MASK) {
+ if (qp_type(qp) == IB_QPT_UD ||
+ qp_type(qp) == IB_QPT_SMI ||
+ qp_type(qp) == IB_QPT_GSI) {
+ union rdma_network_hdr hdr;
+
+ build_rdma_network_hdr(&hdr, obj);
+
+ err = send_data_in(qp, &hdr, sizeof(hdr));
+
+ /*
+ * length of last 2 words containing src and dest ip
+ * has already been subtracted
+ */
+ err = send_data_in(qp, page_address(rdata->eth_page),
+ payload_size(obj));
+ __free_pages(rdata->eth_page, 0);
+ rdata->eth_page = NULL;
+ }
+ } else if (obj->mask & RFC_WRITE_MASK) {
+ qp->resp.va += payload_size(obj);
+ qp->resp.resid -= payload_size(obj);
+ } else if (obj->mask & RFC_READ_MASK) {
+ /* For RDMA Read we can increment the msn now. See C9-148. */
+ qp->resp.msn++;
+ goto find_state;
+ } else if (obj->mask & RFC_ATOMIC_MASK) {
+ err = process_atomic(qp, obj);
+ } else {
+ /* Unreachable */
+ WARN_ON_ONCE(1);
+ }
+
+ /* next expected psn, read handles this separately */
+ qp->resp.psn = (obj->psn + 1) & BTH_PSN_MASK;
+
+ qp->resp.opcode = obj->opcode;
+ qp->resp.status = IB_WC_SUCCESS;
+
+
+find_state:
+ if (obj->mask & RFC_COMP_MASK) {
+ /* We successfully processed this new request. */
+ qp->resp.msn++;
+ return RESPST_COMPLETE;
+ } else if (qp_type(qp) == IB_QPT_RC)
+ return RESPST_ACKNOWLEDGE;
+ else
+ return RESPST_CLEANUP;
+}
+
+static enum resp_states do_complete(struct rfc_qp *qp,
+ struct rfc_obj_info *obj)
+{
+ struct rfc_cqe cqe;
+ struct ib_wc *wc = &cqe.ibwc;
+ struct ib_uverbs_wc *uwc = &cqe.uibwc;
+ struct rfc_recv_wqe *wqe = qp->resp.wqe;
+ struct rfc_recv_data *rdata =
+ container_of(obj, struct rfc_recv_data, obj);
+
+ if (unlikely(!wqe))
+ return RESPST_CLEANUP;
+
+ memset(&cqe, 0, sizeof(cqe));
+
+ wc->wr_id = wqe->wr_id;
+ wc->status = qp->resp.status;
+ wc->qp = &qp->ibqp;
+
+ /* fields after status are not required for errors */
+ if (wc->status == IB_WC_SUCCESS) {
+ wc->opcode = (obj->mask & RFC_IMMDT_MASK &&
+ obj->mask & RFC_WRITE_MASK) ?
+ IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
+ wc->vendor_err = 0;
+ wc->byte_len = wqe->dma.length - wqe->dma.resid;
+
+ /* fields after byte_len are different between kernel and user
+ * space
+ */
+ if (qp->rcq->is_user) {
+ uwc->wc_flags = IB_WC_GRH;
+
+ if (obj->mask & RFC_IMMDT_MASK) {
+ uwc->wc_flags |= IB_WC_WITH_IMM;
+ uwc->ex.imm_data = immdt_imm(obj);
+ }
+
+ if (obj->mask & RFC_IETH_MASK) {
+ uwc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ uwc->ex.invalidate_rkey = ieth_rkey(obj);
+ }
+
+ uwc->qp_num = qp->ibqp.qp_num;
+
+ if (obj->mask & RFC_DETH_MASK)
+ uwc->src_qp = deth_sqp(obj);
+
+ uwc->port_num = qp->attr.port_num;
+ } else {
+
+ wc->wc_flags = IB_WC_GRH | IB_WC_WITH_NETWORK_HDR_TYPE;
+ if (rdata->protocol == htons(ETH_P_IP))
+ wc->network_hdr_type = RDMA_NETWORK_IPV4;
+ else
+ wc->network_hdr_type = RDMA_NETWORK_IPV6;
+
+ if (obj->mask & RFC_IMMDT_MASK) {
+ wc->wc_flags |= IB_WC_WITH_IMM;
+ wc->ex.imm_data = immdt_imm(obj);
+ }
+
+ if (obj->mask & RFC_IETH_MASK) {
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ struct rfc_mem *rmr;
+
+ wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+ wc->ex.invalidate_rkey = ieth_rkey(obj);
+
+ rmr = rfc_pool_get_index(&rfc->mr_pool,
+ wc->ex.invalidate_rkey >> 8);
+ if (unlikely(!rmr)) {
+ pr_err("Bad rkey %#x invalidation\n",
+ wc->ex.invalidate_rkey);
+ return RESPST_ERROR;
+ }
+ rmr->state = RFC_MEM_STATE_FREE;
+ rfc_drop_ref(rmr);
+ }
+
+ wc->qp = &qp->ibqp;
+
+ if (obj->mask & RFC_DETH_MASK)
+ wc->src_qp = deth_sqp(obj);
+
+ wc->port_num = qp->attr.port_num;
+ }
+ }
+
+ /* have copy for srq and reference for !srq */
+ if (!qp->srq)
+ advance_consumer(qp->rq.queue);
+
+ qp->resp.wqe = NULL;
+
+ if (rfc_cq_post(qp->rcq, &cqe, obj ? bth_se(obj) : 1))
+ return RESPST_ERR_CQ_OVERFLOW;
+
+ if (qp->resp.state == QP_STATE_ERROR)
+ return RESPST_CHK_RESOURCE;
+
+ if (!obj)
+ return RESPST_DONE;
+ else if (qp_type(qp) == IB_QPT_RC)
+ return RESPST_ACKNOWLEDGE;
+ else
+ return RESPST_CLEANUP;
+}
+
+static int send_ack(struct rfc_qp *qp, struct rfc_obj_info *obj,
+ u8 syndrome, u32 psn)
+{
+ int err = 0;
+ struct rfc_obj_info *ack_obj;
+ struct sg_table *rsg = NULL;
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+
+ ack_obj = kmalloc(sizeof(struct rfc_obj_info), GFP_KERNEL);
+ rsg = prepare_ack_object(qp, obj, ack_obj, IB_OPCODE_RC_ACKNOWLEDGE,
+ 0, psn, syndrome, NULL);
+ if (!rsg) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ err = rfc_xmit_object(rfc, qp, ack_obj, rsg);
+
+ if (err)
+ pr_err_ratelimited("Failed sending ack\n");
+
+err1:
+ return err;
+}
+
+/* Process a class A or C. Both are treated the same in this implementation. */
+static void do_class_ac_error(struct rfc_qp *qp, u8 syndrome,
+ enum ib_wc_status status)
+{
+ qp->resp.aeth_syndrome = syndrome;
+ qp->resp.status = status;
+
+ /* indicate that we should go through the ERROR state */
+ qp->resp.goto_error = 1;
+}
+
+static enum resp_states do_class_d1e_error(struct rfc_qp *qp)
+{
+ /* UC */
+ if (qp->srq) {
+ /* Class E */
+ qp->resp.drop_msg = 1;
+ if (qp->resp.wqe) {
+ qp->resp.status = IB_WC_REM_INV_REQ_ERR;
+ return RESPST_COMPLETE;
+ } else {
+ return RESPST_CLEANUP;
+ }
+ } else {
+ /* Class D1. This object may be the start of a
+ * new message and could be valid. The previous
+ * message is invalid and ignored. reset the
+ * recv wr to its original state
+ */
+ if (qp->resp.wqe) {
+ qp->resp.wqe->dma.resid = qp->resp.wqe->dma.length;
+ qp->resp.wqe->dma.cur_sge = 0;
+ qp->resp.wqe->dma.sge_offset = 0;
+ qp->resp.opcode = -1;
+ }
+
+ if (qp->resp.mr) {
+ rfc_drop_ref(qp->resp.mr);
+ qp->resp.mr = NULL;
+ }
+
+ return RESPST_CLEANUP;
+ }
+}
+void rfc_drain_recv_queue(struct rfc_qp *qp, bool notify)
+{
+ if (notify)
+ return;
+
+ while (!qp->srq && qp->rq.queue && queue_head(qp->rq.queue))
+ advance_consumer(qp->rq.queue);
+}
+
+u64 rfc_responder_hdr(struct rfc_dev *blkrfc,
+ struct rfc_qp *qp,
+ struct rfc_recv_data *rcv,
+ struct sg_table **rsgp)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ enum resp_states state;
+ struct rfc_obj_info *obj = &rcv->obj;
+ struct sg_table *rsg = NULL;
+ u64 ret = 0;
+ u8 syn = 0;
+ u32 msn = 0;
+
+ *rsgp = NULL;
+ rfc_add_ref(qp);
+
+ qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
+
+ if (!qp->valid) {
+ ret = -EINVAL;
+ goto done_hdr;
+ }
+
+ switch (qp->resp.state) {
+ case QP_STATE_RESET:
+ state = RESPST_RESET;
+ break;
+
+ default:
+ state = RESPST_GET_REQ;
+ break;
+ }
+ while (1) {
+ pr_debug("%s(): qp#%d state = %s\n", __func__, qp_num(qp),
+ resp_state_name[state]);
+ switch (state) {
+ case RESPST_GET_REQ:
+ state = RESPST_CHK_PSN;
+ break;
+ case RESPST_CHK_PSN:
+ state = check_psn(qp, obj);
+ break;
+ case RESPST_CHK_OP_SEQ:
+ state = check_op_seq(qp, obj);
+ break;
+ case RESPST_CHK_OP_VALID:
+ state = check_op_valid(qp, obj);
+ break;
+ case RESPST_CHK_RESOURCE:
+ state = check_resource(qp, obj);
+ break;
+ case RESPST_CHK_LENGTH:
+ state = check_length(qp, obj);
+ break;
+ case RESPST_CHK_RKEY:
+ if (obj->mask & RFC_ATOMIC_MASK) {
+ /* Defer rkey check to done context */
+ state = RESPST_EXECUTE;
+ } else {
+ state = check_rkey(qp, obj);
+ }
+
+ break;
+ case RESPST_EXECUTE:
+ state = rfc_resp_get_sgl(qp, obj, &rsg);
+ *rsgp = rsg;
+ rcv->data_valid = 1;
+ ret = 0;
+ goto done_hdr;
+
+ case RESPST_DUPLICATE_REQUEST:
+ ret = 0;
+ goto exit_hdr;
+ case RESPST_ERR_PSN_OUT_OF_SEQ:
+ /* RC only - Class B. Drop object. */
+ ret = (AETH_NAK_PSN_SEQ_ERROR << 24) |
+ (qp->resp.psn & AETH_MSN_MASK);
+ syn = AETH_NAK_PSN_SEQ_ERROR;
+ msn = qp->resp.psn & AETH_MSN_MASK;
+ goto exit_hdr;
+
+ case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ:
+ case RESPST_ERR_MISSING_OPCODE_FIRST:
+ case RESPST_ERR_MISSING_OPCODE_LAST_C:
+ case RESPST_ERR_UNSUPPORTED_OPCODE:
+ case RESPST_ERR_MISALIGNED_ATOMIC:
+ /* RC Only - Class C. */
+ do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
+ IB_WC_REM_INV_REQ_ERR);
+ ret = (AETH_NAK_INVALID_REQ << 24) |
+ (qp->resp.psn & AETH_MSN_MASK);
+ syn = AETH_NAK_INVALID_REQ;
+ msn = qp->resp.psn & AETH_MSN_MASK;
+ goto exit_hdr;
+
+ case RESPST_ERR_MISSING_OPCODE_LAST_D1E:
+ state = do_class_d1e_error(qp);
+ ret = 0;
+ goto exit_hdr;
+ case RESPST_ERR_RNR:
+ if (qp_type(qp) == IB_QPT_RC) {
+ rfc_counter_inc(rfc, RFC_CNT_SND_RNR);
+ /* RC - class B */
+ ret = (AETH_RNR_NAK << 24) |
+ (AETH_MSN_MASK & qp->attr.min_rnr_timer);
+ syn = AETH_RNR_NAK;
+ msn = AETH_MSN_MASK & qp->attr.min_rnr_timer;
+ } else {
+ /* UD/UC - class D */
+ qp->resp.drop_msg = 1;
+ }
+ goto exit_hdr;
+
+ case RESPST_ERR_RKEY_VIOLATION:
+ if (qp_type(qp) == IB_QPT_RC) {
+ /* Class C */
+ do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
+ IB_WC_REM_ACCESS_ERR);
+ ret = (AETH_NAK_REM_ACC_ERR << 24) |
+ (qp->resp.psn & AETH_MSN_MASK);
+ syn = AETH_NAK_REM_ACC_ERR;
+ msn = qp->resp.psn & AETH_MSN_MASK;
+ } else {
+ qp->resp.drop_msg = 1;
+ if (qp->srq) {
+ /* UC/SRQ Class D */
+ qp->resp.status = IB_WC_REM_ACCESS_ERR;
+ } else {
+ /* UC/non-SRQ Class E. */
+ }
+ }
+ state = RESPST_ERROR;
+
+ case RESPST_ERR_LENGTH:
+ if (qp_type(qp) == IB_QPT_RC) {
+ /* Class C */
+ do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
+ IB_WC_REM_INV_REQ_ERR);
+ ret = (AETH_NAK_INVALID_REQ << 24) |
+ (qp->resp.psn & AETH_MSN_MASK);
+ syn = AETH_NAK_INVALID_REQ;
+ msn = qp->resp.psn & AETH_MSN_MASK;
+ } else if (qp->srq) {
+ /* UC/UD - class E */
+ qp->resp.status = IB_WC_REM_INV_REQ_ERR;
+ } else {
+ /* UC/UD - class D */
+ qp->resp.drop_msg = 1;
+ }
+ state = RESPST_ERROR;
+
+ case RESPST_ERR_MALFORMED_WQE:
+ /* All, Class A. */
+ do_class_ac_error(qp, AETH_NAK_REM_OP_ERR,
+ IB_WC_LOC_QP_OP_ERR);
+ ret = (AETH_NAK_REM_OP_ERR << 24) |
+ (qp->resp.psn & AETH_MSN_MASK);
+ syn = AETH_NAK_REM_OP_ERR;
+ msn = qp->resp.psn & AETH_MSN_MASK;
+ state = RESPST_ERROR;
+
+ case RESPST_ERR_CQ_OVERFLOW:
+ /* All - Class G */
+ state = RESPST_ERROR;
+ break;
+
+ case RESPST_DONE:
+ if (qp->resp.goto_error) {
+ state = RESPST_ERROR;
+ break;
+ }
+ goto exit_hdr;
+
+ case RESPST_EXIT:
+ if (qp->resp.goto_error) {
+ state = RESPST_ERROR;
+ break;
+ }
+ goto exit_hdr;
+
+ case RESPST_RESET:
+ qp->resp.wqe = NULL;
+ goto exit_hdr;
+
+ case RESPST_ERROR:
+ qp->resp.goto_error = 0;
+ pr_warn("qp#%d moved to error state\n", qp_num(qp));
+ rfc_qp_error(qp);
+ goto exit_hdr;
+ default:
+ goto exit_hdr;
+ }
+ }
+exit_hdr:
+ if (rsg) {
+ sg_free_table(rsg);
+ kfree(rsg);
+ }
+ rsg = NULL;
+done_hdr:
+ rfc_drop_ref(qp);
+ return ret;
+}
+
+
+// Returns u64 value which would contain atomic_orig in case of ATOMIC REQ
+u64 rfc_responder_done_direct(struct rfc_qp *qp,
+ struct rfc_obj_info *obj,
+ u64 *resp)
+{
+ struct rfc_dev *rfc = to_rdev(qp->ibqp.device);
+ enum resp_states state;
+ struct rfc_recv_data *rcv = NULL;
+ u64 ret = 0;
+
+ rfc_add_ref(qp);
+
+ qp->resp.aeth_syndrome = AETH_ACK_UNLIMITED;
+
+ if (!qp->valid)
+ goto done;
+
+ switch (qp->resp.state) {
+ case QP_STATE_RESET:
+ state = RESPST_RESET;
+ break;
+
+ default:
+ state = RESPST_GET_REQ;
+ break;
+ }
+
+ while (1) {
+ pr_debug("%s(): qp#%d state = %s\n", __func__, qp_num(qp),
+ resp_state_name[state]);
+ switch (state) {
+ case RESPST_GET_REQ:
+ rcv = container_of(obj, struct rfc_recv_data,
+ obj);
+ if (obj->mask & RFC_ATOMIC_MASK) {
+ /*
+ * For ATOMIC data, rkey check and subsequent
+ * fetching of rfc_mem is skipped during
+ * responder_hdr. Do the check now.
+ */
+ state = RESPST_CHK_RKEY;
+ break;
+ }
+ /*
+ * recv_data is not valid if
+ * cmdiu->xfer_len = 0
+ * as, hdr check is skipped.
+ */
+ if (rcv->data_valid)
+ state = RESPST_EXECUTE;
+ else
+ state = RESPST_CHK_PSN;
+ break;
+ case RESPST_CHK_PSN:
+ state = check_psn(qp, obj);
+ break;
+ case RESPST_CHK_OP_SEQ:
+ state = check_op_seq(qp, obj);
+ break;
+ case RESPST_CHK_OP_VALID:
+ state = check_op_valid(qp, obj);
+ break;
+ case RESPST_CHK_RESOURCE:
+ state = check_resource(qp, obj);
+ break;
+ case RESPST_CHK_LENGTH:
+ state = check_length(qp, obj);
+ break;
+ case RESPST_CHK_RKEY:
+ state = check_rkey(qp, obj);
+ break;
+ case RESPST_EXECUTE:
+ state = find_resp_done_state(qp, obj);
+ if (obj->mask & RFC_ATOMIC_MASK) {
+ ret = qp->resp.atomic_orig;
+ *resp = qp->resp.atomic_orig;
+ } else
+ ret = 0;
+ if (rcv)
+ rcv->data_valid = 1;
+ break;
+ case RESPST_COMPLETE:
+ state = do_complete(qp, obj);
+ break;
+ case RESPST_READ_REPLY:
+ break;
+ case RESPST_ACKNOWLEDGE:
+ state = RESPST_CLEANUP;
+ break;
+
+ case RESPST_CLEANUP:
+ if (qp->resp.mr) {
+ rfc_drop_ref(qp->resp.mr);
+ qp->resp.mr = NULL;
+ }
+
+ state = RESPST_DONE;
+ break;
+
+ case RESPST_DUPLICATE_REQUEST:
+ state = RESPST_CLEANUP;
+ break;
+
+ case RESPST_ERR_PSN_OUT_OF_SEQ:
+ /* RC only - Class B. Drop object. */
+ send_ack(qp, obj, AETH_NAK_PSN_SEQ_ERROR, qp->resp.psn);
+ state = RESPST_CLEANUP;
+ break;
+
+ case RESPST_ERR_TOO_MANY_RDMA_ATM_REQ:
+ case RESPST_ERR_MISSING_OPCODE_FIRST:
+ case RESPST_ERR_MISSING_OPCODE_LAST_C:
+ case RESPST_ERR_UNSUPPORTED_OPCODE:
+ case RESPST_ERR_MISALIGNED_ATOMIC:
+ /* RC Only - Class C. */
+ do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
+ IB_WC_REM_INV_REQ_ERR);
+ state = RESPST_COMPLETE;
+ break;
+
+ case RESPST_ERR_MISSING_OPCODE_LAST_D1E:
+ state = do_class_d1e_error(qp);
+ break;
+ case RESPST_ERR_RNR:
+ if (qp_type(qp) == IB_QPT_RC) {
+ rfc_counter_inc(rfc, RFC_CNT_SND_RNR);
+ /* RC - class B */
+ send_ack(qp, obj, AETH_RNR_NAK |
+ (~AETH_TYPE_MASK &
+ qp->attr.min_rnr_timer),
+ obj->psn);
+ } else {
+ /* UD/UC - class D */
+ qp->resp.drop_msg = 1;
+ }
+ state = RESPST_CLEANUP;
+ break;
+
+ case RESPST_ERR_RKEY_VIOLATION:
+ if (qp_type(qp) == IB_QPT_RC) {
+ /* Class C */
+ do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
+ IB_WC_REM_ACCESS_ERR);
+ state = RESPST_COMPLETE;
+ } else {
+ qp->resp.drop_msg = 1;
+ if (qp->srq) {
+ /* UC/SRQ Class D */
+ qp->resp.status = IB_WC_REM_ACCESS_ERR;
+ state = RESPST_COMPLETE;
+ } else {
+ /* UC/non-SRQ Class E. */
+ state = RESPST_CLEANUP;
+ }
+ }
+ break;
+
+ case RESPST_ERR_LENGTH:
+ if (qp_type(qp) == IB_QPT_RC) {
+ /* Class C */
+ do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
+ IB_WC_REM_INV_REQ_ERR);
+ state = RESPST_COMPLETE;
+ } else if (qp->srq) {
+ /* UC/UD - class E */
+ qp->resp.status = IB_WC_REM_INV_REQ_ERR;
+ state = RESPST_COMPLETE;
+ } else {
+ /* UC/UD - class D */
+ qp->resp.drop_msg = 1;
+ state = RESPST_CLEANUP;
+ }
+ break;
+
+ case RESPST_ERR_MALFORMED_WQE:
+ /* All, Class A. */
+ do_class_ac_error(qp, AETH_NAK_REM_OP_ERR,
+ IB_WC_LOC_QP_OP_ERR);
+ state = RESPST_COMPLETE;
+ break;
+
+ case RESPST_ERR_CQ_OVERFLOW:
+ /* All - Class G */
+ state = RESPST_ERROR;
+ break;
+
+ case RESPST_DONE:
+ if (qp->resp.goto_error) {
+ state = RESPST_ERROR;
+ break;
+ }
+
+ goto done;
+
+ case RESPST_EXIT:
+ if (qp->resp.goto_error) {
+ state = RESPST_ERROR;
+ break;
+ }
+
+ goto done;
+
+ case RESPST_RESET:
+ qp->resp.wqe = NULL;
+ goto done;
+
+ case RESPST_ERROR:
+ qp->resp.goto_error = 0;
+ pr_warn("qp#%d moved to error state\n", qp_num(qp));
+ rfc_qp_error(qp);
+ goto done;
+
+ default:
+ WARN_ON_ONCE(1);
+ }
+ }
+
+done:
+ rfc_drop_ref(qp);
+ return ret;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_srq.c b/drivers/infiniband/sw/rfc/rfc_srq.c
new file mode 100644
index 0000000..27df589
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_srq.c
@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+
+int rfc_srq_chk_attr(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_attr *attr, enum ib_srq_attr_mask mask)
+{
+ if (srq && srq->error) {
+ pr_warn("srq in error state\n");
+ goto err1;
+ }
+
+ if (mask & IB_SRQ_MAX_WR) {
+ if (attr->max_wr > rfc->attr.max_srq_wr) {
+ pr_warn("max_wr(%d) > max_srq_wr(%d)\n",
+ attr->max_wr, rfc->attr.max_srq_wr);
+ goto err1;
+ }
+
+ if (attr->max_wr <= 0) {
+ pr_warn("max_wr(%d) <= 0\n", attr->max_wr);
+ goto err1;
+ }
+
+ if (srq && srq->limit && (attr->max_wr < srq->limit)) {
+ pr_warn("max_wr (%d) < srq->limit (%d)\n",
+ attr->max_wr, srq->limit);
+ goto err1;
+ }
+
+ if (attr->max_wr < RFC_MIN_SRQ_WR)
+ attr->max_wr = RFC_MIN_SRQ_WR;
+ }
+
+ if (mask & IB_SRQ_LIMIT) {
+ if (attr->srq_limit > rfc->attr.max_srq_wr) {
+ pr_warn("srq_limit(%d) > max_srq_wr(%d)\n",
+ attr->srq_limit, rfc->attr.max_srq_wr);
+ goto err1;
+ }
+
+ if (srq && (attr->srq_limit > srq->rq.queue->buf->index_mask)) {
+ pr_warn("srq_limit (%d) > cur limit(%d)\n",
+ attr->srq_limit,
+ srq->rq.queue->buf->index_mask);
+ goto err1;
+ }
+ }
+
+ if (mask == IB_SRQ_INIT_MASK) {
+ if (attr->max_sge > rfc->attr.max_srq_sge) {
+ pr_warn("max_sge(%d) > max_srq_sge(%d)\n",
+ attr->max_sge, rfc->attr.max_srq_sge);
+ goto err1;
+ }
+
+ if (attr->max_sge < RFC_MIN_SRQ_SGE)
+ attr->max_sge = RFC_MIN_SRQ_SGE;
+ }
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+int rfc_srq_from_init(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_init_attr *init,
+ struct ib_ucontext *context,
+ struct rfc_create_srq_resp __user *uresp)
+{
+ int err;
+ int srq_wqe_size;
+ struct rfc_queue *q;
+
+ srq->ibsrq.event_handler = init->event_handler;
+ srq->ibsrq.srq_context = init->srq_context;
+ srq->limit = init->attr.srq_limit;
+ srq->srq_num = srq->pelem.index;
+ srq->rq.max_wr = init->attr.max_wr;
+ srq->rq.max_sge = init->attr.max_sge;
+
+ srq_wqe_size = rcv_wqe_size(srq->rq.max_sge);
+
+ spin_lock_init(&srq->rq.producer_lock);
+ spin_lock_init(&srq->rq.consumer_lock);
+
+ q = rfc_queue_init(rfc, &srq->rq.max_wr,
+ srq_wqe_size);
+ if (!q) {
+ pr_warn("unable to allocate queue for srq\n");
+ return -ENOMEM;
+ }
+
+ srq->rq.queue = q;
+
+ err = do_mmap_info(rfc, uresp ? &uresp->mi : NULL, context, q->buf,
+ q->buf_size, &q->ip);
+ if (err)
+ return err;
+
+ if (uresp) {
+ if (copy_to_user(&uresp->srq_num, &srq->srq_num,
+ sizeof(uresp->srq_num)))
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+int rfc_srq_from_attr(struct rfc_dev *rfc, struct rfc_srq *srq,
+ struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
+ struct rfc_modify_srq_cmd *ucmd)
+{
+ int err;
+ struct rfc_queue *q = srq->rq.queue;
+ struct mminfo __user *mi = NULL;
+
+ if (mask & IB_SRQ_MAX_WR) {
+ /*
+ * This is completely screwed up, the response is supposed to
+ * be in the outbuf not like this.
+ */
+ mi = u64_to_user_ptr(ucmd->mmap_info_addr);
+
+ err = rfc_queue_resize(q, &attr->max_wr,
+ rcv_wqe_size(srq->rq.max_sge),
+ srq->rq.queue->ip ?
+ srq->rq.queue->ip->context :
+ NULL,
+ mi, &srq->rq.producer_lock,
+ &srq->rq.consumer_lock);
+ if (err)
+ goto err2;
+ }
+
+ if (mask & IB_SRQ_LIMIT)
+ srq->limit = attr->srq_limit;
+
+ return 0;
+
+err2:
+ rfc_queue_cleanup(q);
+ srq->rq.queue = NULL;
+ return err;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_sysfs.c b/drivers/infiniband/sw/rfc/rfc_sysfs.c
new file mode 100644
index 0000000..5f70cb2
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_sysfs.c
@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "rfc.h"
+#include "rfc_net.h"
+
+/* Copy argument and remove trailing CR. Return the new length. */
+static int sanitize_arg(const char *val, char *intf, int intf_len)
+{
+ int len;
+
+ if (!val)
+ return 0;
+
+ /* Remove newline. */
+ for (len = 0; len < intf_len - 1 && val[len] && val[len] != '\n'; len++)
+ intf[len] = val[len];
+ intf[len] = 0;
+
+ if (len == 0 || (val[len] != 0 && val[len] != '\n'))
+ return 0;
+
+ return len;
+}
+
+static void rfc_set_port_state(struct net_device *ndev)
+{
+ struct rfc_dev *rfc = net_to_rfc(ndev);
+ bool is_up = netif_running(ndev) && netif_carrier_ok(ndev);
+
+ if (!rfc)
+ goto out;
+
+ if (is_up)
+ rfc_port_up(rfc);
+ else
+ rfc_port_down(rfc); /* down for unknown state */
+out:
+ return;
+}
+
+static int rfc_param_set_add(const char *val, const struct kernel_param *kp)
+{
+ int len;
+ int err = 0;
+ char intf[32];
+ struct net_device *ndev = NULL;
+ struct rfc_dev *rfc;
+
+ len = sanitize_arg(val, intf, sizeof(intf));
+ if (!len) {
+ pr_err("add: invalid interface name\n");
+ err = -EINVAL;
+ goto err;
+ }
+
+ ndev = dev_get_by_name(&init_net, intf);
+ if (!ndev) {
+ pr_err("interface %s not found\n", intf);
+ err = -EINVAL;
+ goto err;
+ }
+
+ if (net_to_rfc(ndev)) {
+ pr_err("already configured on %s\n", intf);
+ err = -EINVAL;
+ goto err;
+ }
+
+ if (strcmp(ndev->name, "rfcnet0")) {
+ pr_err("cannot add non rfc net devices %s\n", ndev->name);
+ err = -EINVAL;
+ goto err;
+ }
+
+ rfc = rfc_net_add(ndev);
+ if (!rfc) {
+ pr_err("failed to add %s\n", intf);
+ err = -EINVAL;
+ goto err;
+ }
+
+ rfc_set_port_state(ndev);
+ pr_info("added %s to %s\n", rfc->ib_dev.name, intf);
+err:
+ if (ndev)
+ dev_put(ndev);
+ return err;
+}
+
+static int rfc_param_set_remove(const char *val, const struct kernel_param *kp)
+{
+ int len;
+ char intf[32];
+ struct rfc_dev *rfc;
+
+ len = sanitize_arg(val, intf, sizeof(intf));
+ if (!len) {
+ pr_err("add: invalid interface name\n");
+ return -EINVAL;
+ }
+
+ if (strncmp("all", intf, len) == 0) {
+ pr_info("rfc_sys: remove all");
+ rfc_remove_all();
+ return 0;
+ }
+
+ rfc = get_rfc_by_name(intf);
+
+ if (!rfc) {
+ pr_err("not configured on %s\n", intf);
+ return -EINVAL;
+ }
+
+ list_del(&rfc->list);
+ rfc_remove(rfc);
+
+ return 0;
+}
+
+static const struct kernel_param_ops rfc_add_ops = {
+ .set = rfc_param_set_add,
+};
+
+static const struct kernel_param_ops rfc_remove_ops = {
+ .set = rfc_param_set_remove,
+};
+
+module_param_cb(add, &rfc_add_ops, NULL, 0200);
+MODULE_PARM_DESC(add, "Create RFC device over network interface");
+module_param_cb(remove, &rfc_remove_ops, NULL, 0200);
+MODULE_PARM_DESC(remove, "Remove RFC device over network interface");
diff --git a/drivers/infiniband/sw/rfc/rfc_task.c b/drivers/infiniband/sw/rfc/rfc_task.c
new file mode 100644
index 0000000..49fe62d
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_task.c
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/hardirq.h>
+
+#include "rfc_task.h"
+#include "rfc.h"
+#include "rfc_loc.h"
+
+int __rfc_do_task(struct rfc_task *task)
+
+{
+ int ret;
+
+ while ((ret = task->func(task->arg)) == 0)
+ ;
+
+ task->ret = ret;
+
+ return ret;
+}
+
+/*
+ * this locking is due to a potential race where
+ * a second caller finds the task already running
+ * but looks just after the last call to func
+ */
+void rfc_do_task(unsigned long data)
+{
+ int cont;
+ int ret;
+ unsigned long flags;
+ struct rfc_task *task = (struct rfc_task *)data;
+
+ spin_lock_irqsave(&task->state_lock, flags);
+ switch (task->state) {
+ case TASK_STATE_START:
+ task->state = TASK_STATE_BUSY;
+ spin_unlock_irqrestore(&task->state_lock, flags);
+ break;
+
+ case TASK_STATE_BUSY:
+ task->state = TASK_STATE_ARMED;
+ /* fall through */
+ case TASK_STATE_ARMED:
+ spin_unlock_irqrestore(&task->state_lock, flags);
+ return;
+
+ default:
+ spin_unlock_irqrestore(&task->state_lock, flags);
+ pr_warn("%s failed with bad state %d\n", __func__, task->state);
+ return;
+ }
+
+ do {
+ cont = 0;
+ ret = task->func(task->arg);
+
+ spin_lock_irqsave(&task->state_lock, flags);
+ switch (task->state) {
+ case TASK_STATE_BUSY:
+ if (ret)
+ task->state = TASK_STATE_START;
+ else
+ cont = 1;
+ break;
+
+ /* soneone tried to run the task since the last time we called
+ * func, so we will call one more time regardless of the
+ * return value
+ */
+ case TASK_STATE_ARMED:
+ task->state = TASK_STATE_BUSY;
+ cont = 1;
+ break;
+
+ default:
+ pr_warn("%s failed with bad state %d\n", __func__,
+ task->state);
+ }
+ spin_unlock_irqrestore(&task->state_lock, flags);
+ } while (cont);
+
+ task->ret = ret;
+}
+
+int rfc_init_task(void *obj, struct rfc_task *task,
+ void *arg, int (*func)(void *), char *name)
+{
+ task->obj = obj;
+ task->arg = arg;
+ task->func = func;
+ snprintf(task->name, sizeof(task->name), "%s", name);
+ task->destroyed = false;
+
+ tasklet_init(&task->tasklet, rfc_do_task, (unsigned long)task);
+
+ task->state = TASK_STATE_START;
+ spin_lock_init(&task->state_lock);
+
+ return 0;
+}
+
+void rfc_cleanup_task(struct rfc_task *task)
+{
+ unsigned long flags;
+ bool idle;
+
+ /*
+ * Mark the task, then wait for it to finish. It might be
+ * running in a non-tasklet (direct call) context.
+ */
+ task->destroyed = true;
+
+ do {
+ spin_lock_irqsave(&task->state_lock, flags);
+ idle = (task->state == TASK_STATE_START);
+ spin_unlock_irqrestore(&task->state_lock, flags);
+ } while (!idle);
+
+ if (!strncmp(task->name, "req", strlen("req"))) {
+ pr_debug("%s(): Kill requester work for qp #%d\n",
+ __func__, qp_num((struct rfc_qp *)task->arg));
+ flush_workqueue(task->queue);
+ destroy_workqueue(task->queue);
+ return;
+ }
+
+ tasklet_kill(&task->tasklet);
+}
+
+void rfc_run_task(struct rfc_task *task, int sched)
+{
+ if (task->destroyed)
+ return;
+
+ if (!strncmp(task->name, "req", strlen("req"))) {
+ queue_work(task->queue, &task->work);
+ return;
+ }
+
+ if (sched)
+ tasklet_schedule(&task->tasklet);
+ else
+ rfc_do_task((unsigned long)task);
+}
+
+void rfc_disable_task(struct rfc_task *task)
+{
+ tasklet_disable(&task->tasklet);
+}
+
+void rfc_enable_task(struct rfc_task *task)
+{
+ tasklet_enable(&task->tasklet);
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_task.h b/drivers/infiniband/sw/rfc/rfc_task.h
new file mode 100644
index 0000000..a75bd30
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_task.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_TASK_H
+#define RFC_TASK_H
+
+enum {
+ TASK_STATE_START = 0,
+ TASK_STATE_BUSY = 1,
+ TASK_STATE_ARMED = 2,
+};
+
+/*
+ * data structure to describe a 'task' which is a short
+ * function that returns 0 as long as it needs to be
+ * called again.
+ */
+struct rfc_task {
+ void *obj;
+ struct tasklet_struct tasklet;
+ struct work_struct work;
+ struct workqueue_struct *queue;
+ int state;
+ spinlock_t state_lock; /* spinlock for task state */
+ void *arg;
+ int (*func)(void *arg);
+ int ret;
+ char name[16];
+ bool destroyed;
+};
+
+/*
+ * init rfc_task structure
+ * arg => parameter to pass to fcn
+ * fcn => function to call until it returns != 0
+ */
+int rfc_init_task(void *obj, struct rfc_task *task,
+ void *arg, int (*func)(void *), char *name);
+
+/* cleanup task */
+void rfc_cleanup_task(struct rfc_task *task);
+
+/*
+ * raw call to func in loop without any checking
+ * can call when tasklets are disabled
+ */
+int __rfc_do_task(struct rfc_task *task);
+
+/*
+ * common function called by any of the main tasklets
+ * If there is any chance that there is additional
+ * work to do someone must reschedule the task before
+ * leaving
+ */
+void rfc_do_task(unsigned long data);
+
+/* run a task, else schedule it to run as a tasklet, The decision
+ * to run or schedule tasklet is based on the parameter sched.
+ */
+void rfc_run_task(struct rfc_task *task, int sched);
+
+/* keep a task from scheduling */
+void rfc_disable_task(struct rfc_task *task);
+
+/* allow task to run */
+void rfc_enable_task(struct rfc_task *task);
+
+#endif /* RFC_TASK_H */
diff --git a/drivers/infiniband/sw/rfc/rfc_tb.c b/drivers/infiniband/sw/rfc/rfc_tb.c
new file mode 100644
index 0000000..ba11a33
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_tb.c
@@ -0,0 +1,795 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+
+#include <linux/types.h>
+#include <linux/string.h>
+/* Disk on RAM Driver */
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/file.h>
+#include <linux/stat.h>
+#include <linux/errno.h>
+#include <linux/major.h>
+#include <linux/blkpg.h>
+#include <linux/types.h>
+#include <linux/genhd.h>
+#include <linux/blkdev.h>
+#include <linux/errno.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/scatterlist.h>
+#include "rfc.h"
+#include "rfc_obj.h"
+#include "../../../drivers/nvme/host/nvme.h"
+#include "../../../drivers/nvme/host/fabrics.h"
+#include <linux/nvme.h>
+
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/time.h>
+#include <linux/kernel_stat.h>
+
+#define MBR_SIZE SECTOR_SIZE
+#define RB_SECTOR_SIZE 512
+#define RB_DEVICE_SIZE 1024 //2048 /* sectors */
+#define RB_FIRST_MINOR 0
+#define RB_MINOR_CNT 16
+
+static DEFINE_SPINLOCK(rfc_req_lock);
+#define LOCK(x, y) spin_lock_irqsave((x), (y))
+#define UNLOCK(x, y) spin_unlock_irqrestore((x), (y))
+#define LOCK_INIT(x) spin_lock_init((x))
+
+#define RFC_SKB_SECTOR 0xa5
+static u8 *dev_data;
+struct work_struct rfcwork;
+
+struct rfc_data {
+ struct sg_table *sg_head;
+ __be32 daddr;
+ int len;
+ int op;
+ void *data;
+ struct list_head active_objs;
+ struct rdma_hdr_common *rdma_hdr;
+};
+
+struct list_head rfc_obj_list;
+
+static struct workqueue_struct *rfcblk_work;
+
+
+static u_int rfcblk_major;
+
+/*
+ * The internal structure representation of our Device
+ */
+static struct rfcblk_device
+{
+ /* Size is the size of the device (in sectors) */
+ unsigned int size;
+ /* For exclusive access to our request queue */
+ spinlock_t lock;
+ /* Our request queue */
+ struct request_queue *rfcblk_queue;
+ /* This is kernel's representation of an individual disk device */
+ struct gendisk *rfcblk_disk;
+} rfcblk_dev;
+
+struct list_head rdma_dev_list;
+struct list_rdma_ip_dev {
+ char node[264];
+ u32 ipaddr;
+ struct list_head rdma_devs;
+
+};
+
+static int get_ip_addr_from_name(char *dev_name)
+{
+ struct net_device *device = NULL;
+ struct in_device *in_dev = NULL;
+ struct in_ifaddr *if_info = NULL;
+ int ipaddr = 0;
+
+ device = dev_get_by_name(&init_net, dev_name);
+ if (!device) {
+ pr_err("\n%sunabe to get device\n", __func__);
+ return 0;
+ }
+
+ in_dev = (struct in_device *)device->ip_ptr;
+ if (!in_dev) {
+ pr_err("\n%sunabe to get in device\n", __func__);
+ return 0;
+ }
+ if_info = in_dev->ifa_list;
+ if (!if_info) {
+ pr_err("\n%sunabe to get info\n", __func__);
+ return 0;
+ }
+
+ for (; if_info; if_info = if_info->ifa_next) {
+ if (!(strcmp(if_info->ifa_label, dev_name))) {
+ ipaddr = if_info->ifa_address;
+ break;
+ }
+ }
+ return ipaddr;
+}
+
+int rfc_update_ip_addr(char *dev_name)
+{
+ int ipaddr = 0;
+ char *data = NULL;
+
+ data = kmalloc(OBJ_SIZE, GFP_ATOMIC);
+ if (data) {
+ ipaddr = get_ip_addr_from_name(dev_name);
+ sprintf(data, "RDMA_DEVICE:%d", ipaddr);
+ memcpy(dev_data, data, OBJ_SIZE);
+ kfree(data);
+ } else
+ pr_err("\nrfc_update_ip_addr failed\n");
+ return 0;
+}
+EXPORT_SYMBOL(rfc_update_ip_addr);
+
+static void block_complete(struct bio *bio)
+{
+ complete((struct completion *)bio->bi_private);
+}
+
+/*if the nvme device is an rdma device append the inof to the list*/
+static int is_rdma_dev_append_list(char *dev_name)
+{
+ char path[264];
+ struct block_device *bdev1 = NULL;
+ struct page *page = alloc_pages(GFP_ATOMIC, 0);
+ struct completion event;
+ struct bio *bio;
+ char *p;
+ u32 ipaddr;
+ u8 *buffer;
+ struct list_rdma_ip_dev *rdma_d;
+ int ret = 0;
+
+ buffer = page_address(page);
+ memset(buffer, 0x0, 1024);
+ strcpy(path, "/dev/");
+ strcat(path, dev_name);
+ bdev1 = blkdev_get_by_path(path, FMODE_READ|FMODE_WRITE, NULL);
+ if (!bdev1) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return 0;
+ }
+ bio = bio_alloc(GFP_NOIO, 4);
+ bio_set_dev(bio, bdev1);
+ bio->bi_iter.bi_sector = 0x0;
+ bio_add_page(bio, page, OBJ_SIZE, 0);
+ init_completion(&event);
+ bio->bi_private = &event;
+ bio->bi_end_io = block_complete;
+ bio->bi_opf = REQ_OP_READ;
+ submit_bio(bio);
+
+ wait_for_completion(&event);
+ bio_put(bio);
+ if (strstr(buffer, "RDMA")) {
+ rdma_d = kmalloc(sizeof(struct list_rdma_ip_dev), GFP_ATOMIC);
+
+ p = buffer + sizeof("RDMA_DEVICE");
+ ret = kstrtou32(p, 0, &ipaddr);
+ rdma_d->ipaddr = ipaddr;
+ strcpy(rdma_d->node, path);
+ list_add_tail(&rdma_d->rdma_devs, &rdma_dev_list);
+ }
+ blkdev_put(bdev1, FMODE_READ|FMODE_WRITE);
+ return 0;
+
+}
+
+static char *get_devinfo_rdma_list(u32 ipaddr)
+{
+ struct list_head *ptr;
+ struct list_rdma_ip_dev *rdma_dev;
+ struct block_device *bdev1 = NULL;
+
+ list_for_each(ptr, &rdma_dev_list) {
+ rdma_dev = list_entry(ptr, struct list_rdma_ip_dev, rdma_devs);
+
+ bdev1 = blkdev_get_by_path(rdma_dev->node,
+ FMODE_READ|FMODE_WRITE, NULL);
+ if (!bdev1) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return 0;
+ }
+ blkdev_put(bdev1, FMODE_READ|FMODE_WRITE);
+ if (ipaddr == rdma_dev->ipaddr)
+ return rdma_dev->node;
+ }
+
+ return NULL;
+
+}
+
+char *dev_path_get_by_list(u32 ipaddr);
+char *dev_path_get_by_list(u32 ipaddr)
+{
+ int skip = 100;
+ struct device *dev;
+
+ struct class_dev_iter *iter;
+ struct block_device *bdev = NULL;
+ struct device *ddev = NULL;
+ struct gendisk *disk;
+
+ if (!list_empty(&rdma_dev_list))
+ return get_devinfo_rdma_list(ipaddr);
+
+ bdev = blkdev_get_by_path("/dev/rfcblk", FMODE_READ|FMODE_WRITE, NULL);
+ if (!bdev) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return 0;
+ }
+
+ ddev = (&(bdev->bd_disk)->part0.__dev);
+ if (!ddev) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return 0;
+ }
+
+ iter = kmalloc(sizeof(*iter), GFP_KERNEL);
+ if (!iter)
+ return ERR_PTR(-ENOMEM);
+
+ class_dev_iter_init(iter, ddev->class, NULL, NULL);
+
+ do {
+
+ dev = class_dev_iter_next(iter);
+ if (!dev)
+ goto out;
+ disk = dev_to_disk(dev);
+ if (strstr(disk->disk_name, "nvme"))
+ is_rdma_dev_append_list(disk->disk_name);
+
+ } while (skip--);
+
+
+out:
+ blkdev_put(bdev, FMODE_READ|FMODE_WRITE);
+ return get_devinfo_rdma_list(ipaddr);
+}
+EXPORT_SYMBOL(dev_path_get_by_list);
+
+static int rfcblk_memory_init(void)
+{
+ dev_data = vmalloc(RB_DEVICE_SIZE * RB_SECTOR_SIZE);
+ if (dev_data == NULL)
+ return -ENOMEM;
+ /* Setup its partition table */
+ memset(dev_data, 0xff, RB_DEVICE_SIZE * RB_SECTOR_SIZE);
+ return RB_DEVICE_SIZE;
+}
+
+static void rfcblk_memory_cleanup(void)
+{
+ vfree(dev_data);
+}
+
+static void rfcblk_memory_write(sector_t sector_off, u8 *buffer,
+ unsigned int sectors)
+{
+ memcpy(dev_data + sector_off * RB_SECTOR_SIZE, buffer,
+ sectors * RB_SECTOR_SIZE);
+}
+
+static void rfcblk_memory_read(sector_t sector_off, u8 *buffer,
+ unsigned int sectors)
+{
+ memcpy(buffer, dev_data + sector_off * RB_SECTOR_SIZE,
+ sectors * RB_SECTOR_SIZE);
+}
+
+static int rfcblk_open(struct block_device *bdev, fmode_t mode)
+{
+ unsigned int unit = iminor(bdev->bd_inode);
+
+ if (unit > RB_MINOR_CNT)
+ return -ENODEV;
+ return 0;
+}
+
+static void rfcblk_close(struct gendisk *disk, fmode_t mode)
+{
+ pr_info("\n%s\n", __func__);
+}
+
+static int rfcblk_transfer(struct request *req)
+{
+
+
+ int dir = rq_data_dir(req);
+ sector_t start_sector = blk_rq_pos(req);
+ unsigned int sector_cnt = blk_rq_sectors(req);
+
+ struct bio_vec bv;
+ struct req_iterator iter;
+
+ sector_t sector_offset;
+ unsigned int sectors;
+ u8 *buffer;
+
+ int ret = 0;
+ struct bio *bio = NULL;
+
+ sector_offset = 0;
+ bio = req->bio;
+
+ rq_for_each_segment(bv, req, iter) {
+
+ buffer = page_address(bv.bv_page) + bv.bv_offset;
+ if (bv.bv_len % RB_SECTOR_SIZE != 0) {
+ pr_err("biosize %d is not a multiple of secto size %d\n"
+ , bv.bv_len, RB_SECTOR_SIZE);
+ ret = -EIO;
+ }
+ sectors = bv.bv_len / RB_SECTOR_SIZE;
+
+ if (dir == WRITE) {
+ if (start_sector == RFC_SKB_SECTOR) {
+ push_data_blk_skb(buffer);
+ return ret;
+ }
+
+ rfcblk_memory_write(start_sector + sector_offset,
+ buffer, sectors);
+ } else /* Read from the device */ {
+ if (start_sector == 0x0)
+ rfcblk_memory_read(start_sector + sector_offset,
+ buffer, sectors);
+ }
+
+ sector_offset += sectors;
+ }
+
+ if (sector_offset != sector_cnt) {
+ pr_err("\n bio info doesn't match with the request info");
+ ret = -EIO;
+ }
+ return ret;
+}
+
+/*
+ * Represents a block I/O request for us to execute
+ */
+static void rfcblk_request(struct request_queue *q)
+{
+ struct request *req;
+ int ret;
+ /* Gets the current request from the dispatch queue */
+ while ((req = blk_fetch_request(q)) != NULL) {
+ ret = rfcblk_transfer(req);
+ __blk_end_request_all(req, ret);
+
+ }
+}
+
+/*
+ * These are the file operations that performed on the ram block device
+ */
+static const struct block_device_operations rfcblk_fops = {
+ .owner = THIS_MODULE,
+ .open = rfcblk_open,
+ .release = rfcblk_close,
+};
+static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
+ struct nvme_ns_head **head, int *srcu_idx)
+{
+#ifdef CONFIG_NVME_MULTIPATH
+ if (disk->fops == &nvme_ns_head_ops) {
+ *head = disk->private_data;
+ *srcu_idx = srcu_read_lock(&(*head)->srcu);
+ return nvme_find_path(*head);
+ }
+#endif
+ *head = NULL;
+ *srcu_idx = -1;
+ return disk->private_data;
+}
+
+static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
+{
+ if (head)
+ srcu_read_unlock(&head->srcu, idx);
+}
+
+static u32 nvme_known_admin_effects(u8 opcode)
+{
+ switch (opcode) {
+ case nvme_admin_format_nvm:
+ return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
+ NVME_CMD_EFFECTS_CSE_MASK;
+ case nvme_admin_sanitize_nvm:
+ return NVME_CMD_EFFECTS_CSE_MASK;
+ default:
+ break;
+ }
+ return 0;
+}
+
+static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
+ u8 opcode)
+{
+ u32 effects = 0;
+
+ if (ns) {
+ if (ctrl->effects)
+ effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
+ if (effects & ~NVME_CMD_EFFECTS_CSUPP)
+ dev_warn(ctrl->device,
+ "IO command:%02x has unhandled effects:%08x\n",
+ opcode, effects);
+ return 0;
+ }
+
+ if (ctrl->effects)
+ effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
+ else
+ effects = nvme_known_admin_effects(opcode);
+
+ /*
+ * For simplicity, IO to all namespaces is quiesced even if the command
+ * effects say only one namespace is affected.
+ */
+ if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
+ nvme_start_freeze(ctrl);
+ nvme_wait_freeze(ctrl);
+ }
+ return effects;
+}
+
+static void bio_map_kern_endio(struct bio *bio)
+{
+ bio_put(bio);
+}
+
+
+unsigned long nvme_send_object(struct sg_table *sg_head,
+ struct rdma_hdr_common *rdma_hdr, __be32 ipaddr,
+ int op, int *status);
+unsigned long nvme_send_object(struct sg_table *sg_head,
+ struct rdma_hdr_common *rdma_hdr, __be32 ipaddr,
+ int op, int *status)
+{
+ struct block_device *bdev = NULL;
+ struct nvme_ns_head *head = NULL;
+ struct nvme_ns *ns;
+ int srcu_idx, ret;
+ struct nvme_command cmd;
+ u32 effects;
+ struct request_queue *q;
+ struct gendisk *disk;
+ bool write;
+ struct request *req;
+ struct bio *bio = NULL;
+ struct rdma_hdr_common *rdmacmd;
+ int i = 0;
+ struct scatterlist *sg, *sg_list_start;
+ char *dev_path = NULL;
+ unsigned long result = 0;
+
+ *status = NVME_SC_DNR;
+
+ rdmacmd = (struct rdma_hdr_common *)&cmd.common.cdw10[0];
+ dev_path = dev_path_get_by_list(ipaddr);
+ if (!dev_path) {
+ pr_err("\n%s dev_path_get_by_list failed %d\n",
+ __func__, ipaddr);
+ return -EINVAL;
+ }
+ bdev = blkdev_get_by_path(dev_path, FMODE_READ|FMODE_WRITE, NULL);
+ if (!bdev) {
+ pr_err("\n%sunabe to open device\n", __func__);
+ return -EINVAL;
+ }
+
+ ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
+ if (unlikely(!ns)) {
+ ret = -EWOULDBLOCK;
+ return ret;
+ } else {
+ memset(&cmd, 0, sizeof(cmd));
+ cmd.common.opcode = nvme_cmd_write;
+ if (op == nvme_cmd_write)
+ cmd.common.opcode = nvme_cmd_write;
+ else
+ cmd.common.opcode = nvme_cmd_read;
+
+ cmd.common.nsid = 1;
+ memcpy(rdmacmd, rdma_hdr, sizeof(struct rdma_hdr_common));
+
+ effects = nvme_passthru_start(ns->ctrl, ns, 0x1);
+ q = ns ? ns->queue : ns->ctrl->admin_q;
+
+ write = nvme_is_write(&cmd);
+ ns = q->queuedata;
+ disk = ns ? ns->disk : NULL;
+
+
+ req = nvme_alloc_request(q, &cmd, 0, NVME_QID_ANY);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+ if (sg_head->nents > 1) {
+ bio = bio_kmalloc(GFP_KERNEL, (sg_head->nents-1));
+ if (!bio)
+ return -ENOMEM;
+ sg_list_start = sg_head->sgl;
+ ret = sg_head->nents;
+ for_each_sg(sg_list_start, sg, ret, i) {
+ if (i == 0)
+ continue;
+ if (bio_add_pc_page(q,
+ bio, sg_page(sg), sg->length,
+ sg->offset) < sg->length) {
+ /* we don't upport partial mappingss */
+ bio_put(bio);
+ return (-EINVAL);
+ }
+
+ }
+
+ bio->bi_end_io = bio_map_kern_endio;
+
+ bio->bi_opf &= ~REQ_OP_MASK;
+ bio->bi_opf |= req_op(req);
+ ret = blk_rq_append_bio(req, &bio);
+ if (unlikely(ret)) {
+ /* request is too big */
+ bio_put(bio);
+ return ret;
+ }
+
+ } else {
+
+ pr_err("\n %s got only one SGL\n", __func__);
+
+ bio = bio_kmalloc(GFP_KERNEL, 0);
+ if (!bio)
+ return (-ENOMEM);//ERR_PTR(-ENOMEM);
+ bio->bi_end_io = bio_map_kern_endio;
+
+ bio->bi_opf &= ~REQ_OP_MASK;
+ bio->bi_opf |= req_op(req);
+ ret = blk_rq_append_bio(req, &bio);
+ if (unlikely(ret)) {
+ /* request is too big */
+ bio_put(bio);
+ return ret;
+ }
+
+ }
+
+ blk_execute_rq(req->q, disk, req, 0);
+ result = le64_to_cpu(nvme_req(req)->result.u64);
+ if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
+ ret = -EINTR;
+ else
+ ret = nvme_req(req)->status;
+ *status = nvme_req(req)->status;
+
+ if (req)
+ blk_mq_free_request(req);
+ }
+
+ nvme_put_ns_from_disk(head, srcu_idx);
+ blkdev_put(bdev, FMODE_READ|FMODE_WRITE);
+
+ return result;
+}
+EXPORT_SYMBOL(nvme_send_object);
+
+static int sendobject(struct rfc_data *r_data)
+{
+ int status = 0;
+
+ u64 resp = nvme_send_object(r_data->sg_head,
+ r_data->rdma_hdr, r_data->daddr,
+ r_data->op, &status);
+ rfc_send_data_done(r_data->data, r_data->sg_head, resp, status);
+ return status;
+}
+
+static void rfc_process_object(struct work_struct *work)
+{
+ struct rfc_data *rq_node;
+ int ret = 0;
+ unsigned long flags = 0;
+
+ LOCK(&rfc_req_lock, flags);
+ while (!list_empty(&rfc_obj_list)) {
+ rq_node = list_first_entry(&rfc_obj_list,
+ struct rfc_data, active_objs);
+ UNLOCK(&rfc_req_lock, flags);
+ ret = sendobject(rq_node);
+ if (ret) {
+ /* IO ERROR: cancel all subsequent requests
+ * and flush the workqueue
+ */
+
+ }
+ LOCK(&rfc_req_lock, flags);
+ list_del(&rq_node->active_objs);
+ UNLOCK(&rfc_req_lock, flags);
+ kfree(rq_node);
+ LOCK(&rfc_req_lock, flags);
+ }
+ UNLOCK(&rfc_req_lock, flags);
+}
+
+int rfc_sendobject(struct rdma_hdr_common *rdma_hdr, struct sg_table *sg_head,
+ int len, __be32 daddr, void *data, int op)
+{
+ int nents = 0;
+ struct scatterlist *sgl;
+ unsigned long flags = 0;
+ struct rfc_data *rfcp = kmalloc(sizeof(struct rfc_data) +
+ sizeof(struct rdma_hdr_common),
+ GFP_KERNEL);
+ rfcp->data = data;
+ rfcp->op = op;
+ nents = sg_head->nents;
+ sgl = sg_head->sgl;
+ rfcp->len = len;
+ rfcp->sg_head = sg_head;
+ rfcp->daddr = daddr;
+ rfcp->rdma_hdr = (struct rdma_hdr_common *)(rfcp + 1);
+ memcpy(rfcp->rdma_hdr, rdma_hdr, sizeof(struct rdma_hdr_common));
+ LOCK(&rfc_req_lock, flags);
+ list_add_tail(&rfcp->active_objs, &rfc_obj_list);
+ UNLOCK(&rfc_req_lock, flags);
+ queue_work(rfcblk_work, &rfcwork);
+ return 0;
+}
+EXPORT_SYMBOL(rfc_sendobject);
+
+static int rfcblk_stat_proc_show(struct seq_file *m, void *v)
+{
+ return 0;
+}
+
+static int rfcblk_stat_proc_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, rfcblk_stat_proc_show, NULL);
+}
+
+static const struct file_operations rfcblk_stat_proc_fops = {
+ .open = rfcblk_stat_proc_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int proc_rfcblk_stat_init(void)
+{
+ proc_create("rfcblk_stats", 0, NULL, &rfcblk_stat_proc_fops);
+ return 0;
+}
+/*
+ * This is the registration and initialization section of the ram blockdevice
+ * driver
+ */
+int rfcblk_init(void)
+{
+ int ret;
+
+ ret = rfcblk_memory_init();
+ /* Set up our RAM Device */
+ if (ret < 0)
+ return ret;
+
+ rfcblk_dev.size = RB_DEVICE_SIZE;//(1024*1024);
+
+ rfcblk_major = register_blkdev(rfcblk_major, "rfcblk");
+ if (rfcblk_major <= 0) {
+ pr_err("rfcblk: Unable to get Major Number\n");
+ rfcblk_memory_cleanup();
+ return -EBUSY;
+ }
+ spin_lock_init(&rfcblk_dev.lock);
+ rfcblk_dev.rfcblk_queue = blk_init_queue(rfcblk_request,
+ &rfcblk_dev.lock);
+ if (rfcblk_dev.rfcblk_queue == NULL) {
+ pr_err("rfcblk: blk_init_queue failure\n");
+ unregister_blkdev(rfcblk_major, "rfcblk");
+ rfcblk_memory_cleanup();
+ return -ENOMEM;
+ }
+
+ rfcblk_dev.rfcblk_disk = alloc_disk(RB_MINOR_CNT);
+ if (!rfcblk_dev.rfcblk_disk) {
+ pr_err("rfcblk: alloc_disk failure\n");
+ blk_cleanup_queue(rfcblk_dev.rfcblk_queue);
+ unregister_blkdev(rfcblk_major, "rfcblk");
+ rfcblk_memory_cleanup();
+ return -ENOMEM;
+ }
+
+ /* Setting the major number */
+ rfcblk_dev.rfcblk_disk->major = rfcblk_major;
+ /* Setting the first mior number */
+ rfcblk_dev.rfcblk_disk->first_minor = RB_FIRST_MINOR;
+ /* Initializing the device operations */
+ rfcblk_dev.rfcblk_disk->fops = &rfcblk_fops;
+ /* Driver-specific own internal data */
+ rfcblk_dev.rfcblk_disk->private_data = &rfcblk_dev;
+ rfcblk_dev.rfcblk_disk->queue = rfcblk_dev.rfcblk_queue;
+
+ sprintf(rfcblk_dev.rfcblk_disk->disk_name, "rfcblk");
+ /* Setting the capacity of the device in its gendisk structure */
+ set_capacity(rfcblk_dev.rfcblk_disk, rfcblk_dev.size);
+
+ /* Adding the disk to the system */
+ add_disk(rfcblk_dev.rfcblk_disk);
+ /* Now the disk is "live" */
+ INIT_LIST_HEAD(&rfc_obj_list);
+ INIT_LIST_HEAD(&rdma_dev_list);
+ rfcblk_work = create_workqueue("rfcblk_work");
+
+ INIT_WORK(&rfcwork, rfc_process_object);
+ proc_rfcblk_stat_init();
+ return 0;
+}
+EXPORT_SYMBOL(rfcblk_init);
+
+/*
+ * This is the unregistration and uninitialization section of the ram block
+ * device driver
+ */
+void rfcblk_cleanup(void)
+{
+ del_gendisk(rfcblk_dev.rfcblk_disk);
+ put_disk(rfcblk_dev.rfcblk_disk);
+ blk_cleanup_queue(rfcblk_dev.rfcblk_queue);
+ unregister_blkdev(rfcblk_major, "rfcblk");
+ rfcblk_memory_cleanup();
+ destroy_workqueue(rfcblk_work);
+}
+EXPORT_SYMBOL(rfcblk_cleanup);
+
diff --git a/drivers/infiniband/sw/rfc/rfc_verbs.c b/drivers/infiniband/sw/rfc/rfc_verbs.c
new file mode 100644
index 0000000..3d8c4dd
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_verbs.c
@@ -0,0 +1,1323 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/dma-mapping.h>
+#include <net/addrconf.h>
+#include "rfc.h"
+#include "rfc_loc.h"
+#include "rfc_queue.h"
+#include "rfc_hw_counters.h"
+
+static int rfc_query_device(struct ib_device *dev,
+ struct ib_device_attr *attr,
+ struct ib_udata *uhw)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+
+ if (uhw->inlen || uhw->outlen)
+ return -EINVAL;
+
+ *attr = rfc->attr;
+ return 0;
+}
+
+static int rfc_query_port(struct ib_device *dev,
+ u8 port_num, struct ib_port_attr *attr)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+ struct rfc_port *port;
+ int rc = -EINVAL;
+
+ if (unlikely(port_num != 1)) {
+ pr_warn("invalid port_number %d\n", port_num);
+ goto out;
+ }
+
+ port = &rfc->port;
+
+ /* *attr being zeroed by the caller, avoid zeroing it here */
+ *attr = port->attr;
+
+ mutex_lock(&rfc->usdev_lock);
+ rc = ib_get_eth_speed(dev, port_num, &attr->active_speed,
+ &attr->active_width);
+ mutex_unlock(&rfc->usdev_lock);
+
+out:
+ return rc;
+}
+
+static struct net_device *rfc_get_netdev(struct ib_device *device,
+ u8 port_num)
+{
+ struct rfc_dev *rfc = to_rdev(device);
+
+ if (rfc->ndev) {
+ dev_hold(rfc->ndev);
+ return rfc->ndev;
+ }
+
+ return NULL;
+}
+
+static int rfc_query_pkey(struct ib_device *device,
+ u8 port_num, u16 index, u16 *pkey)
+{
+ struct rfc_dev *rfc = to_rdev(device);
+ struct rfc_port *port;
+
+ if (unlikely(port_num != 1)) {
+ dev_warn(device->dev.parent, "invalid port_num = %d\n",
+ port_num);
+ goto err1;
+ }
+
+ port = &rfc->port;
+
+ if (unlikely(index >= port->attr.pkey_tbl_len)) {
+ dev_warn(device->dev.parent, "invalid index = %d\n",
+ index);
+ goto err1;
+ }
+
+ *pkey = port->pkey_tbl[index];
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static int rfc_modify_device(struct ib_device *dev,
+ int mask, struct ib_device_modify *attr)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+
+ if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID)
+ rfc->attr.sys_image_guid = cpu_to_be64(attr->sys_image_guid);
+
+ if (mask & IB_DEVICE_MODIFY_NODE_DESC) {
+ memcpy(rfc->ib_dev.node_desc,
+ attr->node_desc, sizeof(rfc->ib_dev.node_desc));
+ }
+
+ return 0;
+}
+
+static int rfc_modify_port(struct ib_device *dev,
+ u8 port_num, int mask, struct ib_port_modify *attr)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+ struct rfc_port *port;
+
+ if (unlikely(port_num != 1)) {
+ pr_warn("invalid port_num = %d\n", port_num);
+ goto err1;
+ }
+
+ port = &rfc->port;
+
+ port->attr.port_cap_flags |= attr->set_port_cap_mask;
+ port->attr.port_cap_flags &= ~attr->clr_port_cap_mask;
+
+ if (mask & IB_PORT_RESET_QKEY_CNTR)
+ port->attr.qkey_viol_cntr = 0;
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static enum rdma_link_layer rfc_get_link_layer(struct ib_device *dev,
+ u8 port_num)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+
+ return rfc_link_layer(rfc, port_num);
+}
+
+static struct ib_ucontext *rfc_alloc_ucontext(struct ib_device *dev,
+ struct ib_udata *udata)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+ struct rfc_ucontext *uc;
+
+ uc = rfc_alloc(&rfc->uc_pool);
+ return uc ? &uc->ibuc : ERR_PTR(-ENOMEM);
+}
+
+static int rfc_dealloc_ucontext(struct ib_ucontext *ibuc)
+{
+ struct rfc_ucontext *uc = to_ruc(ibuc);
+
+ rfc_drop_ref(uc);
+ return 0;
+}
+
+static int rfc_port_immutable(struct ib_device *dev, u8 port_num,
+ struct ib_port_immutable *immutable)
+{
+ int err;
+ struct ib_port_attr attr;
+
+ immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+
+ err = ib_query_port(dev, port_num, &attr);
+ if (err)
+ return err;
+
+ immutable->pkey_tbl_len = attr.pkey_tbl_len;
+ immutable->gid_tbl_len = attr.gid_tbl_len;
+ immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+
+ return 0;
+}
+
+static struct ib_pd *rfc_alloc_pd(struct ib_device *dev,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ struct rfc_dev *rfc = to_rdev(dev);
+ struct rfc_pd *pd;
+
+ pd = rfc_alloc(&rfc->pd_pool);
+ return pd ? &pd->ibpd : ERR_PTR(-ENOMEM);
+}
+
+static int rfc_dealloc_pd(struct ib_pd *ibpd)
+{
+ struct rfc_pd *pd = to_rpd(ibpd);
+
+ rfc_drop_ref(pd);
+ return 0;
+}
+
+static int rfc_init_av(struct rfc_dev *rfc, struct rdma_ah_attr *attr,
+ struct rfc_av *av)
+{
+ int err;
+ union ib_gid sgid;
+ struct ib_gid_attr sgid_attr;
+
+ err = ib_get_cached_gid(&rfc->ib_dev, rdma_ah_get_port_num(attr),
+ rdma_ah_read_grh(attr)->sgid_index, &sgid,
+ &sgid_attr);
+ if (err) {
+ pr_err("Failed to query sgid. err = %d\n", err);
+ return err;
+ }
+
+ rfc_av_from_attr(rdma_ah_get_port_num(attr), av, attr);
+ rfc_av_fill_ip_info(av, attr, &sgid_attr, &sgid);
+ dev_put(sgid_attr.ndev);
+ return 0;
+}
+
+static struct ib_ah *rfc_create_ah(struct ib_pd *ibpd,
+ struct rdma_ah_attr *attr,
+ struct ib_udata *udata)
+
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_ah *ah;
+
+ err = rfc_av_chk_attr(rfc, attr);
+ if (err)
+ goto err1;
+
+ ah = rfc_alloc(&rfc->ah_pool);
+ if (!ah) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ rfc_add_ref(pd);
+ ah->pd = pd;
+
+ err = rfc_init_av(rfc, attr, &ah->av);
+ if (err)
+ goto err2;
+
+ return &ah->ibah;
+
+err2:
+ rfc_drop_ref(pd);
+ rfc_drop_ref(ah);
+err1:
+ return ERR_PTR(err);
+}
+
+static int rfc_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibah->device);
+ struct rfc_ah *ah = to_rah(ibah);
+
+ err = rfc_av_chk_attr(rfc, attr);
+ if (err)
+ return err;
+
+ err = rfc_init_av(rfc, attr, &ah->av);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+static int rfc_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr)
+{
+ struct rfc_ah *ah = to_rah(ibah);
+
+ memset(attr, 0, sizeof(*attr));
+ attr->type = ibah->type;
+ rfc_av_to_attr(&ah->av, attr);
+ return 0;
+}
+
+static int rfc_destroy_ah(struct ib_ah *ibah)
+{
+ struct rfc_ah *ah = to_rah(ibah);
+
+ rfc_drop_ref(ah->pd);
+ rfc_drop_ref(ah);
+ return 0;
+}
+
+static int post_one_recv(struct rfc_rq *rq, struct ib_recv_wr *ibwr)
+{
+ int err;
+ int i;
+ u32 length;
+ struct rfc_recv_wqe *recv_wqe;
+ int num_sge = ibwr->num_sge;
+
+ if (unlikely(queue_full(rq->queue))) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ if (unlikely(num_sge > rq->max_sge)) {
+ err = -EINVAL;
+ goto err1;
+ }
+
+ length = 0;
+ for (i = 0; i < num_sge; i++)
+ length += ibwr->sg_list[i].length;
+
+ recv_wqe = producer_addr(rq->queue);
+ recv_wqe->wr_id = ibwr->wr_id;
+ recv_wqe->num_sge = num_sge;
+
+ memcpy(recv_wqe->dma.sge, ibwr->sg_list,
+ num_sge * sizeof(struct ib_sge));
+
+ recv_wqe->dma.length = length;
+ recv_wqe->dma.resid = length;
+ recv_wqe->dma.num_sge = num_sge;
+ recv_wqe->dma.cur_sge = 0;
+ recv_wqe->dma.sge_offset = 0;
+
+ /* make sure all changes to the work queue are written before we
+ * update the producer pointer
+ */
+ smp_wmb();
+
+ advance_producer(rq->queue);
+ return 0;
+
+err1:
+ return err;
+}
+
+static struct ib_srq *rfc_create_srq(struct ib_pd *ibpd,
+ struct ib_srq_init_attr *init,
+ struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_srq *srq;
+ struct ib_ucontext *context = udata ? ibpd->uobject->context : NULL;
+ struct rfc_create_srq_resp __user *uresp = NULL;
+
+ if (udata) {
+ if (udata->outlen < sizeof(*uresp))
+ return ERR_PTR(-EINVAL);
+ uresp = udata->outbuf;
+ }
+
+ err = rfc_srq_chk_attr(rfc, NULL, &init->attr, IB_SRQ_INIT_MASK);
+ if (err)
+ goto err1;
+
+ srq = rfc_alloc(&rfc->srq_pool);
+ if (!srq) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ rfc_add_index(srq);
+ rfc_add_ref(pd);
+ srq->pd = pd;
+
+ err = rfc_srq_from_init(rfc, srq, init, context, uresp);
+ if (err)
+ goto err2;
+
+ return &srq->ibsrq;
+
+err2:
+ rfc_drop_ref(pd);
+ rfc_drop_index(srq);
+ rfc_drop_ref(srq);
+err1:
+ return ERR_PTR(err);
+}
+
+static int rfc_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
+ enum ib_srq_attr_mask mask,
+ struct ib_udata *udata)
+{
+ int err;
+ struct rfc_srq *srq = to_rsrq(ibsrq);
+ struct rfc_dev *rfc = to_rdev(ibsrq->device);
+ struct rfc_modify_srq_cmd ucmd = {};
+
+ if (udata) {
+ if (udata->inlen < sizeof(ucmd))
+ return -EINVAL;
+
+ err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
+ if (err)
+ return err;
+ }
+
+ err = rfc_srq_chk_attr(rfc, srq, attr, mask);
+ if (err)
+ goto err1;
+
+ err = rfc_srq_from_attr(rfc, srq, attr, mask, &ucmd);
+ if (err)
+ goto err1;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+static int rfc_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
+{
+ struct rfc_srq *srq = to_rsrq(ibsrq);
+
+ if (srq->error)
+ return -EINVAL;
+
+ attr->max_wr = srq->rq.queue->buf->index_mask;
+ attr->max_sge = srq->rq.max_sge;
+ attr->srq_limit = srq->limit;
+ return 0;
+}
+
+static int rfc_destroy_srq(struct ib_srq *ibsrq)
+{
+ struct rfc_srq *srq = to_rsrq(ibsrq);
+
+ if (srq->rq.queue)
+ rfc_queue_cleanup(srq->rq.queue);
+
+ rfc_drop_ref(srq->pd);
+ rfc_drop_index(srq);
+ rfc_drop_ref(srq);
+
+ return 0;
+}
+
+static int rfc_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr)
+{
+ int err = 0;
+ unsigned long flags;
+ struct rfc_srq *srq = to_rsrq(ibsrq);
+
+ spin_lock_irqsave(&srq->rq.producer_lock, flags);
+
+ while (wr) {
+ err = post_one_recv(&srq->rq, wr);
+ if (unlikely(err))
+ break;
+ wr = wr->next;
+ }
+
+ spin_unlock_irqrestore(&srq->rq.producer_lock, flags);
+
+ if (err)
+ *bad_wr = wr;
+
+ return err;
+}
+
+static struct ib_qp *rfc_create_qp(struct ib_pd *ibpd,
+ struct ib_qp_init_attr *init,
+ struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_qp *qp;
+ struct rfc_create_qp_resp __user *uresp = NULL;
+
+ if (udata) {
+ if (udata->outlen < sizeof(*uresp))
+ return ERR_PTR(-EINVAL);
+ uresp = udata->outbuf;
+ }
+
+ err = rfc_qp_chk_init(rfc, init);
+ if (err)
+ goto err1;
+
+ qp = rfc_alloc(&rfc->qp_pool);
+ if (!qp) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ if (udata) {
+ if (udata->inlen) {
+ err = -EINVAL;
+ goto err2;
+ }
+ qp->is_user = 1;
+ }
+
+ rfc_add_index(qp);
+
+ err = rfc_qp_from_init(rfc, qp, pd, init, uresp, ibpd);
+ if (err)
+ goto err3;
+
+ return &qp->ibqp;
+
+err3:
+ rfc_drop_index(qp);
+err2:
+ rfc_drop_ref(qp);
+err1:
+ return ERR_PTR(err);
+}
+
+static int rfc_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+ int mask, struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibqp->device);
+ struct rfc_qp *qp = to_rqp(ibqp);
+
+ err = rfc_qp_chk_attr(rfc, qp, attr, mask);
+ if (err)
+ goto err1;
+
+ err = rfc_qp_from_attr(qp, attr, mask, udata);
+ if (err)
+ goto err1;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+static int rfc_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+ int mask, struct ib_qp_init_attr *init)
+{
+ struct rfc_qp *qp = to_rqp(ibqp);
+
+ rfc_qp_to_init(qp, init);
+ rfc_qp_to_attr(qp, attr, mask);
+
+ return 0;
+}
+
+static int rfc_destroy_qp(struct ib_qp *ibqp)
+{
+ struct rfc_qp *qp = to_rqp(ibqp);
+
+ rfc_qp_destroy(qp);
+ rfc_drop_index(qp);
+ rfc_drop_ref(qp);
+ return 0;
+}
+
+static int validate_send_wr(struct rfc_qp *qp, struct ib_send_wr *ibwr,
+ unsigned int mask, unsigned int length)
+{
+ int num_sge = ibwr->num_sge;
+ struct rfc_sq *sq = &qp->sq;
+
+ if (unlikely(num_sge > sq->max_sge))
+ goto err1;
+
+ if (unlikely(mask & WR_ATOMIC_MASK)) {
+ if (length < 8)
+ goto err1;
+
+ if (atomic_wr(ibwr)->remote_addr & 0x7)
+ goto err1;
+ }
+
+ if (unlikely((ibwr->send_flags & IB_SEND_INLINE) &&
+ (length > sq->max_inline)))
+ goto err1;
+
+ return 0;
+
+err1:
+ return -EINVAL;
+}
+
+static void init_send_wr(struct rfc_qp *qp, struct rfc_send_wr *wr,
+ struct ib_send_wr *ibwr)
+{
+ wr->wr_id = ibwr->wr_id;
+ wr->num_sge = ibwr->num_sge;
+ wr->opcode = ibwr->opcode;
+ wr->send_flags = ibwr->send_flags;
+
+ if (qp_type(qp) == IB_QPT_UD ||
+ qp_type(qp) == IB_QPT_SMI ||
+ qp_type(qp) == IB_QPT_GSI) {
+ wr->wr.ud.remote_qpn = ud_wr(ibwr)->remote_qpn;
+ wr->wr.ud.remote_qkey = ud_wr(ibwr)->remote_qkey;
+ if (qp_type(qp) == IB_QPT_GSI)
+ wr->wr.ud.pkey_index = ud_wr(ibwr)->pkey_index;
+ if (wr->opcode == IB_WR_SEND_WITH_IMM)
+ wr->ex.imm_data = ibwr->ex.imm_data;
+ } else {
+ switch (wr->opcode) {
+ case IB_WR_RDMA_WRITE_WITH_IMM:
+ wr->ex.imm_data = ibwr->ex.imm_data;
+ /* fall through */
+ case IB_WR_RDMA_READ:
+ case IB_WR_RDMA_WRITE:
+ wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr;
+ wr->wr.rdma.rkey = rdma_wr(ibwr)->rkey;
+ break;
+ case IB_WR_SEND_WITH_IMM:
+ wr->ex.imm_data = ibwr->ex.imm_data;
+ break;
+ case IB_WR_SEND_WITH_INV:
+ wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
+ break;
+ case IB_WR_ATOMIC_CMP_AND_SWP:
+ case IB_WR_ATOMIC_FETCH_AND_ADD:
+ wr->wr.atomic.remote_addr =
+ atomic_wr(ibwr)->remote_addr;
+ wr->wr.atomic.compare_add =
+ atomic_wr(ibwr)->compare_add;
+ wr->wr.atomic.swap = atomic_wr(ibwr)->swap;
+ wr->wr.atomic.rkey = atomic_wr(ibwr)->rkey;
+ break;
+ case IB_WR_LOCAL_INV:
+ wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey;
+ break;
+ case IB_WR_REG_MR:
+ wr->wr.reg.mr = reg_wr(ibwr)->mr;
+ wr->wr.reg.key = reg_wr(ibwr)->key;
+ wr->wr.reg.access = reg_wr(ibwr)->access;
+ break;
+ default:
+ break;
+ }
+ }
+}
+
+static int init_send_wqe(struct rfc_qp *qp, struct ib_send_wr *ibwr,
+ unsigned int mask, unsigned int length,
+ struct rfc_send_wqe *wqe)
+{
+ int num_sge = ibwr->num_sge;
+ struct ib_sge *sge;
+ int i;
+ u8 *p;
+
+ init_send_wr(qp, &wqe->wr, ibwr);
+
+ if (qp_type(qp) == IB_QPT_UD ||
+ qp_type(qp) == IB_QPT_SMI ||
+ qp_type(qp) == IB_QPT_GSI)
+ memcpy(&wqe->av, &to_rah(ud_wr(ibwr)->ah)->av, sizeof(wqe->av));
+
+ if (unlikely(ibwr->send_flags & IB_SEND_INLINE)) {
+ p = wqe->dma.inline_data;
+
+ sge = ibwr->sg_list;
+ for (i = 0; i < num_sge; i++, sge++) {
+ memcpy(p, (void *)(uintptr_t)sge->addr,
+ sge->length);
+
+ p += sge->length;
+ }
+ } else if (mask & WR_REG_MASK) {
+ wqe->mask = mask;
+ wqe->state = wqe_state_posted;
+ return 0;
+ } else
+ memcpy(wqe->dma.sge, ibwr->sg_list,
+ num_sge * sizeof(struct ib_sge));
+
+ wqe->iova = mask & WR_ATOMIC_MASK ? atomic_wr(ibwr)->remote_addr :
+ mask & WR_READ_OR_WRITE_MASK ? rdma_wr(ibwr)->remote_addr : 0;
+ wqe->mask = mask;
+ wqe->dma.length = length;
+ wqe->dma.resid = length;
+ wqe->dma.num_sge = num_sge;
+ wqe->dma.cur_sge = 0;
+ wqe->dma.sge_offset = 0;
+ wqe->state = wqe_state_posted;
+ wqe->ssn = atomic_add_return(1, &qp->ssn);
+
+ return 0;
+}
+
+static int post_one_send(struct rfc_qp *qp, struct ib_send_wr *ibwr,
+ unsigned int mask, u32 length)
+{
+ int err;
+ struct rfc_sq *sq = &qp->sq;
+ struct rfc_send_wqe *send_wqe;
+ unsigned long flags;
+
+ err = validate_send_wr(qp, ibwr, mask, length);
+ if (err)
+ return err;
+
+ spin_lock_irqsave(&qp->sq.sq_lock, flags);
+
+ if (unlikely(queue_full(sq->queue))) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ send_wqe = producer_addr(sq->queue);
+
+ err = init_send_wqe(qp, ibwr, mask, length, send_wqe);
+ if (unlikely(err))
+ goto err1;
+
+ /*
+ * make sure all changes to the work queue are
+ * written before we update the producer pointer
+ */
+ smp_wmb();
+
+ advance_producer(sq->queue);
+ spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
+
+ return 0;
+
+err1:
+ spin_unlock_irqrestore(&qp->sq.sq_lock, flags);
+ return err;
+}
+
+static int rfc_post_send_kernel(struct rfc_qp *qp, struct ib_send_wr *wr,
+ struct ib_send_wr **bad_wr)
+{
+ int err = 0;
+ unsigned int mask;
+ unsigned int length = 0;
+ int i;
+ int must_sched;
+
+ while (wr) {
+ mask = wr_opcode_mask(wr->opcode, qp);
+ if (unlikely(!mask)) {
+ err = -EINVAL;
+ *bad_wr = wr;
+ break;
+ }
+
+ if (unlikely((wr->send_flags & IB_SEND_INLINE) &&
+ !(mask & WR_INLINE_MASK))) {
+ err = -EINVAL;
+ *bad_wr = wr;
+ break;
+ }
+
+ length = 0;
+ for (i = 0; i < wr->num_sge; i++)
+ length += wr->sg_list[i].length;
+
+ err = post_one_send(qp, wr, mask, length);
+
+ if (err) {
+ *bad_wr = wr;
+ break;
+ }
+ wr = wr->next;
+ }
+
+ /*
+ * Must sched in case of GSI QP because ib_send_mad() hold irq lock,
+ * and the requester call ip_local_out_sk() that takes spin_lock_bh.
+ */
+ must_sched = (qp_type(qp) == IB_QPT_GSI) ||
+ (queue_count(qp->sq.queue) > 1);
+
+ rfc_run_task(&qp->req.task, must_sched);
+
+ return err;
+}
+
+static int rfc_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
+ struct ib_send_wr **bad_wr)
+{
+ struct rfc_qp *qp = to_rqp(ibqp);
+
+ if (unlikely(!qp->valid)) {
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ if (unlikely(qp->req.state < QP_STATE_READY)) {
+ *bad_wr = wr;
+ return -EINVAL;
+ }
+
+ if (qp->is_user) {
+ /* Utilize process context to do protocol processing */
+ rfc_run_task(&qp->req.task, 0);
+ return 0;
+ } else
+ return rfc_post_send_kernel(qp, wr, bad_wr);
+}
+
+static int rfc_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
+ struct ib_recv_wr **bad_wr)
+{
+ int err = 0;
+ struct rfc_qp *qp = to_rqp(ibqp);
+ struct rfc_rq *rq = &qp->rq;
+ unsigned long flags;
+
+ if (unlikely((qp_state(qp) < IB_QPS_INIT) || !qp->valid)) {
+ *bad_wr = wr;
+ err = -EINVAL;
+ goto err1;
+ }
+
+ if (unlikely(qp->srq)) {
+ *bad_wr = wr;
+ err = -EINVAL;
+ goto err1;
+ }
+
+ spin_lock_irqsave(&rq->producer_lock, flags);
+
+ while (wr) {
+ err = post_one_recv(rq, wr);
+ if (unlikely(err)) {
+ *bad_wr = wr;
+ break;
+ }
+ wr = wr->next;
+ }
+
+ spin_unlock_irqrestore(&rq->producer_lock, flags);
+
+ if (qp->resp.state == QP_STATE_ERROR)
+ rfc_run_task(&qp->resp.task, 1);
+
+err1:
+ return err;
+}
+
+static struct ib_cq *rfc_create_cq(struct ib_device *dev,
+ const struct ib_cq_init_attr *attr,
+ struct ib_ucontext *context,
+ struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(dev);
+ struct rfc_cq *cq;
+ struct rfc_create_cq_resp __user *uresp = NULL;
+
+ if (udata) {
+ if (udata->outlen < sizeof(*uresp))
+ return ERR_PTR(-EINVAL);
+ uresp = udata->outbuf;
+ }
+
+ if (attr->flags)
+ return ERR_PTR(-EINVAL);
+
+ err = rfc_cq_chk_attr(rfc, NULL, attr->cqe, attr->comp_vector);
+ if (err)
+ goto err1;
+
+ cq = rfc_alloc(&rfc->cq_pool);
+ if (!cq) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ err = rfc_cq_from_init(rfc, cq, attr->cqe, attr->comp_vector,
+ context, uresp);
+ if (err)
+ goto err2;
+
+ return &cq->ibcq;
+
+err2:
+ rfc_drop_ref(cq);
+err1:
+ return ERR_PTR(err);
+}
+
+static int rfc_destroy_cq(struct ib_cq *ibcq)
+{
+ struct rfc_cq *cq = to_rcq(ibcq);
+
+ rfc_cq_disable(cq);
+
+ rfc_drop_ref(cq);
+ return 0;
+}
+
+static int rfc_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
+{
+ int err;
+ struct rfc_cq *cq = to_rcq(ibcq);
+ struct rfc_dev *rfc = to_rdev(ibcq->device);
+ struct rfc_resize_cq_resp __user *uresp = NULL;
+
+ if (udata) {
+ if (udata->outlen < sizeof(*uresp))
+ return -EINVAL;
+ uresp = udata->outbuf;
+ }
+
+ err = rfc_cq_chk_attr(rfc, cq, cqe, 0);
+ if (err)
+ goto err1;
+
+ err = rfc_cq_resize_queue(cq, cqe, uresp);
+ if (err)
+ goto err1;
+
+ return 0;
+
+err1:
+ return err;
+}
+
+static int rfc_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
+{
+ int i;
+ struct rfc_cq *cq = to_rcq(ibcq);
+ struct rfc_cqe *cqe;
+ unsigned long flags;
+
+ spin_lock_irqsave(&cq->cq_lock, flags);
+ for (i = 0; i < num_entries; i++) {
+ cqe = queue_head(cq->queue);
+ if (!cqe)
+ break;
+
+ memcpy(wc++, &cqe->ibwc, sizeof(*wc));
+ advance_consumer(cq->queue);
+ }
+ spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+ return i;
+}
+
+static int rfc_peek_cq(struct ib_cq *ibcq, int wc_cnt)
+{
+ struct rfc_cq *cq = to_rcq(ibcq);
+ int count = queue_count(cq->queue);
+
+ return (count > wc_cnt) ? wc_cnt : count;
+}
+
+static int rfc_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
+{
+ struct rfc_cq *cq = to_rcq(ibcq);
+ unsigned long irq_flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&cq->cq_lock, irq_flags);
+ if (cq->notify != IB_CQ_NEXT_COMP)
+ cq->notify = flags & IB_CQ_SOLICITED_MASK;
+
+ if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !queue_empty(cq->queue))
+ ret = 1;
+
+ spin_unlock_irqrestore(&cq->cq_lock, irq_flags);
+
+ return ret;
+}
+
+static struct ib_mr *rfc_get_dma_mr(struct ib_pd *ibpd, int access)
+{
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_mem *mr;
+ int err;
+
+ mr = rfc_alloc(&rfc->mr_pool);
+ if (!mr) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ rfc_add_index(mr);
+
+ rfc_add_ref(pd);
+
+ err = rfc_mem_init_dma(rfc, pd, access, mr);
+ if (err)
+ goto err2;
+
+ return &mr->ibmr;
+
+err2:
+ rfc_drop_ref(pd);
+ rfc_drop_index(mr);
+ rfc_drop_ref(mr);
+err1:
+ return ERR_PTR(err);
+}
+
+static struct ib_mr *rfc_reg_user_mr(struct ib_pd *ibpd,
+ u64 start,
+ u64 length,
+ u64 iova,
+ int access, struct ib_udata *udata)
+{
+ int err;
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_mem *mr;
+
+ mr = rfc_alloc(&rfc->mr_pool);
+ if (!mr) {
+ err = -ENOMEM;
+ goto err2;
+ }
+
+ rfc_add_index(mr);
+
+ rfc_add_ref(pd);
+
+ err = rfc_mem_init_user(rfc, pd, start, length, iova,
+ access, udata, mr);
+ if (err)
+ goto err3;
+
+ return &mr->ibmr;
+
+err3:
+ rfc_drop_ref(pd);
+ rfc_drop_index(mr);
+ rfc_drop_ref(mr);
+err2:
+ return ERR_PTR(err);
+}
+
+static int rfc_dereg_mr(struct ib_mr *ibmr)
+{
+ struct rfc_mem *mr = to_rmr(ibmr);
+
+ mr->state = RFC_MEM_STATE_ZOMBIE;
+ rfc_drop_ref(mr->pd);
+ rfc_drop_index(mr);
+ rfc_drop_ref(mr);
+ return 0;
+}
+
+static struct ib_mr *rfc_alloc_mr(struct ib_pd *ibpd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ struct rfc_dev *rfc = to_rdev(ibpd->device);
+ struct rfc_pd *pd = to_rpd(ibpd);
+ struct rfc_mem *mr;
+ int err;
+
+ if (mr_type != IB_MR_TYPE_MEM_REG)
+ return ERR_PTR(-EINVAL);
+
+ mr = rfc_alloc(&rfc->mr_pool);
+ if (!mr) {
+ err = -ENOMEM;
+ goto err1;
+ }
+
+ rfc_add_index(mr);
+
+ rfc_add_ref(pd);
+
+ err = rfc_mem_init_fast(rfc, pd, max_num_sg, mr);
+ if (err)
+ goto err2;
+
+ return &mr->ibmr;
+
+err2:
+ rfc_drop_ref(pd);
+ rfc_drop_index(mr);
+ rfc_drop_ref(mr);
+err1:
+ return ERR_PTR(err);
+}
+
+static int rfc_set_page(struct ib_mr *ibmr, u64 addr)
+{
+ struct rfc_mem *mr = to_rmr(ibmr);
+ struct rfc_map *map;
+ struct rfc_phys_buf *buf;
+
+ if (unlikely(mr->nbuf == mr->num_buf))
+ return -ENOMEM;
+
+ map = mr->map[mr->nbuf / RFC_BUF_PER_MAP];
+ buf = &map->buf[mr->nbuf % RFC_BUF_PER_MAP];
+
+ buf->addr = addr;
+ buf->size = ibmr->page_size;
+ mr->nbuf++;
+
+ return 0;
+}
+
+static int rfc_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
+ int sg_nents, unsigned int *sg_offset)
+{
+ struct rfc_mem *mr = to_rmr(ibmr);
+ int n;
+
+ mr->nbuf = 0;
+
+ n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, rfc_set_page);
+
+ mr->va = ibmr->iova;
+ mr->iova = ibmr->iova;
+ mr->length = ibmr->length;
+ mr->page_shift = ilog2(ibmr->page_size);
+ mr->page_mask = ibmr->page_size - 1;
+ mr->offset = mr->iova & mr->page_mask;
+
+ return n;
+}
+
+static int rfc_attach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
+{
+ int err = -1;
+
+ // multicast not supported in rfc v1.0
+ return err;
+}
+
+static int rfc_detach_mcast(struct ib_qp *ibqp, union ib_gid *mgid, u16 mlid)
+{
+ return -1;
+}
+
+static ssize_t parent_show(struct device *device,
+ struct device_attribute *attr, char *buf)
+{
+ struct rfc_dev *rfc = container_of(device, struct rfc_dev,
+ ib_dev.dev);
+
+ return snprintf(buf, 16, "%s\n", rfc_parent_name(rfc, 1));
+}
+
+static DEVICE_ATTR_RO(parent);
+
+static struct device_attribute *rfc_dev_attributes[] = {
+ &dev_attr_parent,
+};
+
+int rfc_register_device(struct rfc_dev *rfc)
+{
+ int err;
+ int i;
+ struct ib_device *dev = &rfc->ib_dev;
+ struct crypto_shash *tfm;
+
+ strlcpy(dev->name, "rfc%d", IB_DEVICE_NAME_MAX);
+ strlcpy(dev->node_desc, "rfc", sizeof(dev->node_desc));
+
+ dev->owner = THIS_MODULE;
+ dev->node_type = RDMA_NODE_IB_CA;
+ dev->phys_port_cnt = 1;
+ dev->num_comp_vectors = num_possible_cpus();
+ dev->dev.parent = rfc_dma_device(rfc);
+ dev->local_dma_lkey = 0;
+ addrconf_addr_eui48((unsigned char *)&dev->node_guid,
+ rfc->ndev->dev_addr);
+ dev->dev.dma_ops = &dma_virt_ops;
+ dma_coerce_mask_and_coherent(&dev->dev,
+ dma_get_required_mask(&dev->dev));
+
+ dev->uverbs_abi_ver = RFC_UVERBS_ABI_VERSION;
+ dev->uverbs_cmd_mask = BIT_ULL(IB_USER_VERBS_CMD_GET_CONTEXT)
+ | BIT_ULL(IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)
+ | BIT_ULL(IB_USER_VERBS_CMD_QUERY_DEVICE)
+ | BIT_ULL(IB_USER_VERBS_CMD_QUERY_PORT)
+ | BIT_ULL(IB_USER_VERBS_CMD_ALLOC_PD)
+ | BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_PD)
+ | BIT_ULL(IB_USER_VERBS_CMD_CREATE_SRQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_SRQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_QUERY_SRQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_SRQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_POST_SRQ_RECV)
+ | BIT_ULL(IB_USER_VERBS_CMD_CREATE_QP)
+ | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_QP)
+ | BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP)
+ | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP)
+ | BIT_ULL(IB_USER_VERBS_CMD_POST_SEND)
+ | BIT_ULL(IB_USER_VERBS_CMD_POST_RECV)
+ | BIT_ULL(IB_USER_VERBS_CMD_CREATE_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_RESIZE_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_PEEK_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ)
+ | BIT_ULL(IB_USER_VERBS_CMD_REG_MR)
+ | BIT_ULL(IB_USER_VERBS_CMD_DEREG_MR)
+ | BIT_ULL(IB_USER_VERBS_CMD_CREATE_AH)
+ | BIT_ULL(IB_USER_VERBS_CMD_MODIFY_AH)
+ | BIT_ULL(IB_USER_VERBS_CMD_QUERY_AH)
+ | BIT_ULL(IB_USER_VERBS_CMD_DESTROY_AH)
+ | BIT_ULL(IB_USER_VERBS_CMD_ATTACH_MCAST)
+ | BIT_ULL(IB_USER_VERBS_CMD_DETACH_MCAST)
+ ;
+
+ dev->query_device = rfc_query_device;
+ dev->modify_device = rfc_modify_device;
+ dev->query_port = rfc_query_port;
+ dev->modify_port = rfc_modify_port;
+ dev->get_link_layer = rfc_get_link_layer;
+ dev->get_netdev = rfc_get_netdev;
+ dev->query_pkey = rfc_query_pkey;
+ dev->alloc_ucontext = rfc_alloc_ucontext;
+ dev->dealloc_ucontext = rfc_dealloc_ucontext;
+ dev->mmap = rfc_mmap;
+ dev->get_port_immutable = rfc_port_immutable;
+ dev->alloc_pd = rfc_alloc_pd;
+ dev->dealloc_pd = rfc_dealloc_pd;
+ dev->create_ah = rfc_create_ah;
+ dev->modify_ah = rfc_modify_ah;
+ dev->query_ah = rfc_query_ah;
+ dev->destroy_ah = rfc_destroy_ah;
+ dev->create_srq = rfc_create_srq;
+ dev->modify_srq = rfc_modify_srq;
+ dev->query_srq = rfc_query_srq;
+ dev->destroy_srq = rfc_destroy_srq;
+ dev->post_srq_recv = rfc_post_srq_recv;
+ dev->create_qp = rfc_create_qp;
+ dev->modify_qp = rfc_modify_qp;
+ dev->query_qp = rfc_query_qp;
+ dev->destroy_qp = rfc_destroy_qp;
+ dev->post_send = rfc_post_send;
+ dev->post_recv = rfc_post_recv;
+ dev->create_cq = rfc_create_cq;
+ dev->destroy_cq = rfc_destroy_cq;
+ dev->resize_cq = rfc_resize_cq;
+ dev->poll_cq = rfc_poll_cq;
+ dev->peek_cq = rfc_peek_cq;
+ dev->req_notify_cq = rfc_req_notify_cq;
+ dev->get_dma_mr = rfc_get_dma_mr;
+ dev->reg_user_mr = rfc_reg_user_mr;
+ dev->dereg_mr = rfc_dereg_mr;
+ dev->alloc_mr = rfc_alloc_mr;
+ dev->map_mr_sg = rfc_map_mr_sg;
+ dev->attach_mcast = rfc_attach_mcast;
+ dev->detach_mcast = rfc_detach_mcast;
+ dev->get_hw_stats = rfc_ib_get_hw_stats;
+ dev->alloc_hw_stats = rfc_ib_alloc_hw_stats;
+
+ tfm = crypto_alloc_shash("crc32", 0, 0);
+ if (IS_ERR(tfm)) {
+ pr_err("failed to allocate crc algorithm err:%ld\n",
+ PTR_ERR(tfm));
+ return PTR_ERR(tfm);
+ }
+ rfc->tfm = tfm;
+
+ dev->driver_id = RDMA_DRIVER_RFC;
+ err = ib_register_device(dev, NULL);
+ if (err) {
+ pr_warn("%s failed with error %d\n", __func__, err);
+ goto err1;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(rfc_dev_attributes); ++i) {
+ err = device_create_file(&dev->dev, rfc_dev_attributes[i]);
+ if (err) {
+ pr_warn("%s failed with error %d for attr number %d\n",
+ __func__, err, i);
+ goto err2;
+ }
+ }
+
+ return 0;
+
+err2:
+ ib_unregister_device(dev);
+err1:
+ crypto_free_shash(rfc->tfm);
+
+ return err;
+}
+
+int rfc_unregister_device(struct rfc_dev *rfc)
+{
+ int i;
+ struct ib_device *dev = &rfc->ib_dev;
+
+ for (i = 0; i < ARRAY_SIZE(rfc_dev_attributes); ++i)
+ device_remove_file(&dev->dev, rfc_dev_attributes[i]);
+
+ ib_unregister_device(dev);
+
+ return 0;
+}
diff --git a/drivers/infiniband/sw/rfc/rfc_verbs.h b/drivers/infiniband/sw/rfc/rfc_verbs.h
new file mode 100644
index 0000000..ea5cffd
--- /dev/null
+++ b/drivers/infiniband/sw/rfc/rfc_verbs.h
@@ -0,0 +1,465 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2018 Broadcom INC. All rights reserved.
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RFC_VERBS_H
+#define RFC_VERBS_H
+
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <rdma/rdma_user_rfc.h>
+#include "rfc_pool.h"
+#include "rfc_task.h"
+#include "rfc_hw_counters.h"
+#include "rfc_hdr.h"
+
+static inline int pkey_match(u16 key1, u16 key2)
+{
+ return (((key1 & 0x7fff) != 0) &&
+ ((key1 & 0x7fff) == (key2 & 0x7fff)) &&
+ ((key1 & 0x8000) || (key2 & 0x8000))) ? 1 : 0;
+}
+
+/* Return >0 if psn_a > psn_b
+ * 0 if psn_a == psn_b
+ * <0 if psn_a < psn_b
+ */
+static inline int psn_compare(u32 psn_a, u32 psn_b)
+{
+ s32 diff;
+
+ diff = (psn_a - psn_b) << 8;
+ return diff;
+}
+
+struct rfc_ucontext {
+ struct rfc_pool_entry pelem;
+ struct ib_ucontext ibuc;
+};
+
+struct rfc_pd {
+ struct rfc_pool_entry pelem;
+ struct ib_pd ibpd;
+};
+
+struct rfc_ah {
+ struct rfc_pool_entry pelem;
+ struct ib_ah ibah;
+ struct rfc_pd *pd;
+ struct rfc_av av;
+};
+
+struct rfc_cqe {
+ union {
+ struct ib_wc ibwc;
+ struct ib_uverbs_wc uibwc;
+ };
+};
+
+struct rfc_cq {
+ struct rfc_pool_entry pelem;
+ struct ib_cq ibcq;
+ struct rfc_queue *queue;
+ spinlock_t cq_lock;
+ u8 notify;
+ bool is_dying;
+ int is_user;
+ struct tasklet_struct comp_task;
+};
+
+enum wqe_state {
+ wqe_state_posted,
+ wqe_state_processing,
+ wqe_state_pending,
+ wqe_state_done,
+ wqe_state_error,
+};
+
+struct rfc_sq {
+ int max_wr;
+ int max_sge;
+ int max_inline;
+ spinlock_t sq_lock; /* guard queue */
+ struct rfc_queue *queue;
+};
+
+struct rfc_rq {
+ int max_wr;
+ int max_sge;
+ spinlock_t producer_lock; /* guard queue producer */
+ spinlock_t consumer_lock; /* guard queue consumer */
+ struct rfc_queue *queue;
+};
+
+struct rfc_srq {
+ struct rfc_pool_entry pelem;
+ struct ib_srq ibsrq;
+ struct rfc_pd *pd;
+ struct rfc_rq rq;
+ u32 srq_num;
+
+ int limit;
+ int error;
+};
+
+enum rfc_qp_state {
+ QP_STATE_RESET,
+ QP_STATE_INIT,
+ QP_STATE_READY,
+ QP_STATE_DRAIN, /* req only */
+ QP_STATE_DRAINED, /* req only */
+ QP_STATE_ERROR
+};
+
+struct rfc_req_info {
+ enum rfc_qp_state state;
+ int wqe_index;
+ u32 psn;
+ int opcode;
+ atomic_t rd_atomic;
+ int wait_fence;
+ int need_rd_atomic;
+ int wait_psn;
+ int need_retry;
+ int noack_objs;
+ struct rfc_task task;
+};
+
+struct rfc_comp_info {
+ u32 psn;
+ int opcode;
+ int timeout;
+ int timeout_retry;
+ u32 retry_cnt;
+ u32 rnr_retry;
+ struct rfc_task task;
+};
+
+enum rdatm_res_state {
+ rdatm_res_state_next,
+ rdatm_res_state_new,
+ rdatm_res_state_replay,
+};
+
+struct resp_res {
+ int type;
+ u32 first_psn;
+ u32 last_psn;
+ u32 cur_psn;
+ enum rdatm_res_state state;
+
+ union {
+ struct {
+ struct rfc_mem *mr;
+ u64 va_org;
+ u32 rkey;
+ u32 length;
+ u64 va;
+ u32 resid;
+ } read;
+ };
+};
+
+struct rfc_resp_info {
+ enum rfc_qp_state state;
+ u32 msn;
+ u32 psn;
+ int opcode;
+ int drop_msg;
+ int goto_error;
+ int sent_psn_nak;
+ enum ib_wc_status status;
+ u8 aeth_syndrome;
+
+ /* Receive only */
+ struct rfc_recv_wqe *wqe;
+
+ /* RDMA read / atomic only */
+ u64 va;
+ struct rfc_mem *mr;
+ u32 resid;
+ u32 rkey;
+ u64 atomic_orig;
+
+ /* SRQ only */
+ struct {
+ struct rfc_recv_wqe wqe;
+ struct ib_sge sge[RFC_MAX_SGE];
+ } srq_wqe;
+
+ /* Responder resources. It's a circular list where the oldest
+ * resource is dropped first.
+ */
+ struct resp_res *resources;
+ unsigned int res_head;
+ unsigned int res_tail;
+ struct resp_res *res;
+ struct rfc_task task;
+};
+
+struct rfc_qp {
+ struct rfc_pool_entry pelem;
+ struct ib_qp ibqp;
+ struct ib_qp_attr attr;
+ unsigned int valid;
+ unsigned int mtu;
+ int is_user;
+
+ struct rfc_pd *pd;
+ struct rfc_srq *srq;
+ struct rfc_cq *scq;
+ struct rfc_cq *rcq;
+
+ enum ib_sig_type sq_sig_type;
+
+ struct rfc_sq sq;
+ struct rfc_rq rq;
+
+ struct socket *sk;
+ u32 dst_cookie;
+
+ struct rfc_av pri_av;
+ struct rfc_av alt_av;
+
+ /* list of mcast groups qp has joined (for cleanup) */
+ struct list_head grp_list;
+ spinlock_t grp_lock; /* guard grp_list */
+ struct list_head req_objs;
+ struct list_head resp_objs;
+ struct list_head send_objs;
+ struct rfc_req_info req;
+ struct rfc_comp_info comp;
+ struct rfc_resp_info resp;
+
+ atomic_t ssn;
+
+ /* Timer for retranmitting object when ACKs have been lost. RC
+ * only. The requester sets it when it is not already
+ * started. The responder resets it whenever an ack is
+ * received.
+ */
+ struct timer_list retrans_timer;
+ u64 qp_timeout_jiffies;
+
+ /* Timer for handling RNR NAKS. */
+ struct timer_list rnr_nak_timer;
+
+ spinlock_t state_lock; /* guard requester and completer */
+ struct rfc_recv_data rcv_data; /* per qp receive data buffer */
+ struct execute_work cleanup_work;
+};
+
+enum rfc_mem_state {
+ RFC_MEM_STATE_ZOMBIE,
+ RFC_MEM_STATE_INVALID,
+ RFC_MEM_STATE_FREE,
+ RFC_MEM_STATE_VALID,
+};
+
+enum rfc_mem_type {
+ RFC_MEM_TYPE_NONE,
+ RFC_MEM_TYPE_DMA,
+ RFC_MEM_TYPE_MR,
+ RFC_MEM_TYPE_FMR,
+ RFC_MEM_TYPE_MW,
+};
+
+#define RFC_BUF_PER_MAP (PAGE_SIZE / sizeof(struct rfc_phys_buf))
+
+struct rfc_phys_buf {
+ u64 addr;
+ u64 size;
+};
+
+struct rfc_map {
+ struct rfc_phys_buf buf[RFC_BUF_PER_MAP];
+};
+
+struct rfc_mem {
+ struct rfc_pool_entry pelem;
+ union {
+ struct ib_mr ibmr;
+ struct ib_mw ibmw;
+ };
+
+ struct rfc_pd *pd;
+ struct ib_umem *umem;
+
+ u32 lkey;
+ u32 rkey;
+
+ enum rfc_mem_state state;
+ enum rfc_mem_type type;
+ u64 va;
+ u64 iova;
+ size_t length;
+ u32 offset;
+ int access;
+
+ int page_shift;
+ int page_mask;
+ int map_shift;
+ int map_mask;
+
+ u32 num_buf;
+ u32 nbuf;
+
+ u32 max_buf;
+ u32 num_map;
+
+ struct rfc_map **map;
+};
+
+struct rfc_mc_grp {
+ struct rfc_pool_entry pelem;
+ spinlock_t mcg_lock; /* guard group */
+ struct rfc_dev *rfc;
+ struct list_head qp_list;
+ union ib_gid mgid;
+ int num_qp;
+ u32 qkey;
+ u16 pkey;
+};
+
+struct rfc_mc_elem {
+ struct rfc_pool_entry pelem;
+ struct list_head qp_list;
+ struct list_head grp_list;
+ struct rfc_qp *qp;
+ struct rfc_mc_grp *grp;
+};
+
+struct rfc_port {
+ struct ib_port_attr attr;
+ u16 *pkey_tbl;
+ __be64 port_guid;
+ __be64 subnet_prefix;
+ spinlock_t port_lock; /* guard port */
+ unsigned int mtu_cap;
+ /* special QPs */
+ u32 qp_smi_index;
+ u32 qp_gsi_index;
+};
+
+struct rfc_dev {
+ struct ib_device ib_dev;
+ struct ib_device_attr attr;
+ int max_ucontext;
+ int max_inline_data;
+ struct kref ref_cnt;
+ struct mutex usdev_lock;
+
+ struct net_device *ndev;
+
+ int xmit_errors;
+
+ struct rfc_pool uc_pool;
+ struct rfc_pool pd_pool;
+ struct rfc_pool ah_pool;
+ struct rfc_pool srq_pool;
+ struct rfc_pool qp_pool;
+ struct rfc_pool cq_pool;
+ struct rfc_pool mr_pool;
+ struct rfc_pool mw_pool;
+ struct rfc_pool mc_grp_pool;
+ struct rfc_pool mc_elem_pool;
+
+ spinlock_t pending_lock; /* guard pending_mmaps */
+ struct list_head pending_mmaps;
+
+ spinlock_t mmap_offset_lock; /* guard mmap_offset */
+ int mmap_offset;
+
+ u64 stats_counters[RFC_NUM_OF_COUNTERS];
+
+ struct rfc_port port;
+ struct list_head list;
+ struct crypto_shash *tfm;
+ char objname[64];
+};
+
+static inline void rfc_counter_inc(struct rfc_dev *rfc, enum rfc_counters cnt)
+{
+ rfc->stats_counters[cnt]++;
+}
+
+static inline struct rfc_dev *to_rdev(struct ib_device *dev)
+{
+ return dev ? container_of(dev, struct rfc_dev, ib_dev) : NULL;
+}
+
+static inline struct rfc_ucontext *to_ruc(struct ib_ucontext *uc)
+{
+ return uc ? container_of(uc, struct rfc_ucontext, ibuc) : NULL;
+}
+
+static inline struct rfc_pd *to_rpd(struct ib_pd *pd)
+{
+ return pd ? container_of(pd, struct rfc_pd, ibpd) : NULL;
+}
+
+static inline struct rfc_ah *to_rah(struct ib_ah *ah)
+{
+ return ah ? container_of(ah, struct rfc_ah, ibah) : NULL;
+}
+
+static inline struct rfc_srq *to_rsrq(struct ib_srq *srq)
+{
+ return srq ? container_of(srq, struct rfc_srq, ibsrq) : NULL;
+}
+
+static inline struct rfc_qp *to_rqp(struct ib_qp *qp)
+{
+ return qp ? container_of(qp, struct rfc_qp, ibqp) : NULL;
+}
+
+static inline struct rfc_cq *to_rcq(struct ib_cq *cq)
+{
+ return cq ? container_of(cq, struct rfc_cq, ibcq) : NULL;
+}
+
+static inline struct rfc_mem *to_rmr(struct ib_mr *mr)
+{
+ return mr ? container_of(mr, struct rfc_mem, ibmr) : NULL;
+}
+
+static inline struct rfc_mem *to_rmw(struct ib_mw *mw)
+{
+ return mw ? container_of(mw, struct rfc_mem, ibmw) : NULL;
+}
+
+int rfc_register_device(struct rfc_dev *rfc);
+int rfc_unregister_device(struct rfc_dev *rfc);
+
+#endif /* RFC_VERBS_H */
diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
index 33ee8d3..f053c8c 100644
--- a/drivers/nvme/target/fc.c
+++ b/drivers/nvme/target/fc.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2016 Avago Technologies. All rights reserved.
*
@@ -1705,23 +1706,88 @@ enum {
* Start of FCP handling
* **********************
*/
+//extern struct sg_table *rxe_get_target_sgl(void *buf, int len);
+
+u64 (*get_rdma_sgl_fn)(void *buf, int len, struct sg_table **rsgp,
+ char *name) = NULL;
+/*int fc_register_rdma_alloc_tgt_pgs(u64 (*fc_rdma_sgl_fn)(void *buf, int len,
+ struct sg_table **rsgp, char *name));*/
+
+int fc_register_rdma_alloc_tgt_pgs(u64 (*fc_rdma_sgl_fn)(void *buf, int len,
+ struct sg_table **rsgp, char *name))
+{
+ get_rdma_sgl_fn = fc_rdma_sgl_fn;
+ return 0;
+}
+EXPORT_SYMBOL(fc_register_rdma_alloc_tgt_pgs);
+
+static int is_target_rdma_device(struct nvmet_fc_fcp_iod *fod)
+{
+ if (fod) {
+ if (fod->cmdiubuf.sqe.common.cdw10[5] != 0x0) {
+ if (!(strcmp(fod->req.ns->bdev->bd_disk->disk_name,
+ "rfcblk")))
+ return 1;
+ else
+ return 0;
+ }
+ }
+ return 0;
+}
+EXPORT_SYMBOL(is_target_rdma_device);
static int
nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
{
struct scatterlist *sg;
unsigned int nent;
+ int i = 0;
+ struct scatterlist *sg1;
+ struct scatterlist *sg_list_start;
+
+ if (is_target_rdma_device(fod)) {
+ struct sg_table *sg_head = NULL;
+ u64 resp;
+
+ if (!get_rdma_sgl_fn)
+ goto alloc_pgs;
+
+ resp = get_rdma_sgl_fn(&fod->cmdiubuf.sqe.common.cdw10[0],
+ fod->req.transfer_len, &sg_head,
+ fod->req.ns->bdev->bd_disk->disk_name);
+
+ if (!sg_head) {
+ fod->req.rsp->result.u64 = cpu_to_le64(resp);
+ pr_err("%s(): returning NULL SGL : resp = 0x%llx\n",
+ __func__, resp);
+ return NVME_SC_INTERNAL;
+ }
+ nent = sg_head->nents;
+ sg = kmalloc_array(nent, sizeof(struct scatterlist),
+ GFP_KERNEL);
+ if (!sg)
+ goto out;
+ sg_init_table(sg, sg_head->nents);
+ sg_list_start = sg_head->sgl;
+ for_each_sg(sg_list_start, sg1, sg_head->nents, i) {
+ sg_set_page(&sg[i], sg_page(sg1), sg1->length,
+ sg1->offset);
+ }
+ } else {
+
+alloc_pgs:
- sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
- if (!sg)
- goto out;
+ sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
+ if (!sg)
+ goto out;
+ }
fod->data_sg = sg;
fod->data_sg_cnt = nent;
fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
- ((fod->io_dir == NVMET_FCP_WRITE) ?
- DMA_FROM_DEVICE : DMA_TO_DEVICE));
- /* note: write from initiator perspective */
+ ((fod->io_dir == NVMET_FCP_WRITE) ?
+ DMA_FROM_DEVICE : DMA_TO_DEVICE));
+ /* note: write from initiator perspective */
return 0;
@@ -1738,7 +1804,11 @@ enum {
fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
((fod->io_dir == NVMET_FCP_WRITE) ?
DMA_FROM_DEVICE : DMA_TO_DEVICE));
- sgl_free(fod->data_sg);
+ if (!is_target_rdma_device(fod))
+ sgl_free(fod->data_sg);
+ else
+ kfree(fod->data_sg);
+
fod->data_sg = NULL;
fod->data_sg_cnt = 0;
}
@@ -1884,12 +1954,15 @@ enum {
fcpreq->sg = &fod->data_sg[fod->offset / PAGE_SIZE];
fcpreq->sg_cnt = DIV_ROUND_UP(tlen, PAGE_SIZE);
+ if (is_target_rdma_device(fod))
+ fcpreq->sg_cnt = fod->data_sg_cnt;
+
/*
* If the last READDATA request: check if LLDD supports
* combined xfr with response.
*/
if ((op == NVMET_FCOP_READDATA) &&
- ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
+ ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
(tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
fcpreq->op = NVMET_FCOP_READDATA_RSP;
nvmet_fc_prep_fcp_rsp(tgtport, fod);
@@ -2177,13 +2250,18 @@ enum {
}
fod->req.transfer_len = xfrlen;
+ if (is_target_rdma_device(fod)) {
+ if (fod->req.transfer_len != fod->req.data_len)
+ fod->req.data_len = fod->req.transfer_len;
+ }
/* keep a running counter of tail position */
atomic_inc(&fod->queue->sqtail);
if (fod->req.transfer_len) {
ret = nvmet_fc_alloc_tgt_pgs(fod);
if (ret) {
+ pr_err("\n%s: %d ret %d\n", __func__, __LINE__, ret);
nvmet_req_complete(&fod->req, ret);
return;
}
diff --git a/drivers/nvme/target/io-cmd.c b/drivers/nvme/target/io-cmd.c
index cd23441..9067bab 100644
--- a/drivers/nvme/target/io-cmd.c
+++ b/drivers/nvme/target/io-cmd.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* NVMe I/O command implementation.
* Copyright (c) 2015-2016 HGST, a Western Digital Company.
@@ -15,6 +16,9 @@
#include <linux/blkdev.h>
#include <linux/module.h>
#include "nvmet.h"
+#include <uapi/scsi/fc/fc_fs.h>
+#include <uapi/scsi/fc/fc_els.h>
+#include <linux/nvme-fc-driver.h>
static void nvmet_bio_done(struct bio *bio)
{
@@ -32,6 +36,29 @@ static inline u32 nvmet_rw_len(struct nvmet_req *req)
return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) <<
req->ns->blksize_shift;
}
+u64 (*set_fc_rdma_done_fn)(void *buf, int len, char *name) = NULL;
+
+int fc_register_rdma_dma_done(u64 (*fc_rdma_dma_done_fn)(void *buf,
+ int len, char *name))
+{
+ set_fc_rdma_done_fn = fc_rdma_dma_done_fn;
+ return 0;
+}
+EXPORT_SYMBOL(fc_register_rdma_dma_done);
+
+static int is_execute_rdma_device(struct nvmet_req *req)
+{
+ if (req) {
+ if (req->cmd->common.cdw10[5] != 0x0) {
+ if (!(strcmp(req->ns->bdev->bd_disk->disk_name,
+ "rfcblk")))
+ return 1;
+ else
+ return 0;
+ }
+ }
+ return 0;
+}
static void nvmet_execute_rw(struct nvmet_req *req)
{
@@ -41,7 +68,22 @@ static void nvmet_execute_rw(struct nvmet_req *req)
sector_t sector;
blk_qc_t cookie;
int op, op_flags = 0, i;
-
+ u64 resp = 0;
+
+ if (is_execute_rdma_device(req)) {
+ if (!set_fc_rdma_done_fn) {
+ pr_err("\n%s: %d set_fc_rdma_done_fn nott registered:\n",
+ __func__, __LINE__);
+ nvmet_req_complete(req, 0);
+ return;
+ }
+ resp = set_fc_rdma_done_fn(&req->cmd->common.cdw10[0],
+ req->transfer_len, req->ns->bdev->bd_disk->disk_name);
+ // Get the result from the done function.
+ req->rsp->result.u64 = cpu_to_le64(resp);
+ nvmet_req_complete(req, 0);
+ return;
+ }
if (!req->sg_cnt) {
nvmet_req_complete(req, 0);
return;
diff --git a/include/linux/nvme-fc-driver.h b/include/linux/nvme-fc-driver.h
index 496ff75..763fb4d 100644
--- a/include/linux/nvme-fc-driver.h
+++ b/include/linux/nvme-fc-driver.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2016, Avago Technologies
*
@@ -893,4 +894,9 @@ int nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *tgtport,
void nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *tgtport,
struct nvmefc_tgt_fcp_req *fcpreq);
+int fc_register_rdma_alloc_tgt_pgs(u64 (*fc_rdma_sgl_fn)(void *buf, int len,
+ struct sg_table **rsgp, char *name));
+int fc_register_rdma_dma_done(u64 (*fc_rdma_dma_done_fn)(void *buf,
+ int len, char *name));
+
#endif /* _NVME_FC_DRIVER_H */
diff --git a/include/uapi/rdma/rdma_user_ioctl_cmds.h b/include/uapi/rdma/rdma_user_ioctl_cmds.h
index 1da5a1e..955bb45 100644
--- a/include/uapi/rdma/rdma_user_ioctl_cmds.h
+++ b/include/uapi/rdma/rdma_user_ioctl_cmds.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
@@ -92,6 +93,7 @@ enum rdma_driver_id {
RDMA_DRIVER_HNS,
RDMA_DRIVER_USNIC,
RDMA_DRIVER_RXE,
+ RDMA_DRIVER_RFC,
RDMA_DRIVER_HFI1,
RDMA_DRIVER_QIB,
};
diff --git a/include/uapi/rdma/rdma_user_rfc.h b/include/uapi/rdma/rdma_user_rfc.h
new file mode 100644
index 0000000..1c2daeb
--- /dev/null
+++ b/include/uapi/rdma/rdma_user_rfc.h
@@ -0,0 +1,179 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef RDMA_USER_RFC_H
+#define RDMA_USER_RFC_H
+
+#include <linux/types.h>
+#include <linux/socket.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+
+union rfc_gid {
+ __u8 raw[16];
+ struct {
+ __be64 subnet_prefix;
+ __be64 interface_id;
+ } global;
+};
+
+struct rfc_global_route {
+ union rfc_gid dgid;
+ __u32 flow_label;
+ __u8 sgid_index;
+ __u8 hop_limit;
+ __u8 traffic_class;
+};
+
+struct rfc_av {
+ __u8 port_num;
+ __u8 network_type;
+ __u16 reserved1;
+ __u32 reserved2;
+ struct rfc_global_route grh;
+ union {
+ struct sockaddr_in _sockaddr_in;
+ struct sockaddr_in6 _sockaddr_in6;
+ } sgid_addr, dgid_addr;
+};
+
+struct rfc_send_wr {
+ __aligned_u64 wr_id;
+ __u32 num_sge;
+ __u32 opcode;
+ __u32 send_flags;
+ union {
+ __be32 imm_data;
+ __u32 invalidate_rkey;
+ } ex;
+ union {
+ struct {
+ __aligned_u64 remote_addr;
+ __u32 rkey;
+ __u32 reserved;
+ } rdma;
+ struct {
+ __aligned_u64 remote_addr;
+ __aligned_u64 compare_add;
+ __aligned_u64 swap;
+ __u32 rkey;
+ __u32 reserved;
+ } atomic;
+ struct {
+ __u32 remote_qpn;
+ __u32 remote_qkey;
+ __u16 pkey_index;
+ } ud;
+ /* reg is only used by the kernel and is not part of the uapi */
+ struct {
+ union {
+ struct ib_mr *mr;
+ __aligned_u64 reserved;
+ };
+ __u32 key;
+ __u32 access;
+ } reg;
+ } wr;
+};
+
+struct rfc_sge {
+ __aligned_u64 addr;
+ __u32 length;
+ __u32 lkey;
+};
+
+struct mminfo {
+ __aligned_u64 offset;
+ __u32 size;
+ __u32 pad;
+};
+
+struct rfc_dma_info {
+ __u32 length;
+ __u32 resid;
+ __u32 cur_sge;
+ __u32 num_sge;
+ __u32 sge_offset;
+ __u32 reserved;
+ union {
+ __u8 inline_data[0];
+ struct rfc_sge sge[0];
+ };
+};
+
+struct rfc_send_wqe {
+ struct rfc_send_wr wr;
+ struct rfc_av av;
+ __u32 status;
+ __u32 state;
+ __aligned_u64 iova;
+ __u32 mask;
+ __u32 first_psn;
+ __u32 last_psn;
+ __u32 ack_length;
+ __u32 ssn;
+ __u32 has_rd_atomic;
+ struct rfc_dma_info dma;
+};
+
+struct rfc_recv_wqe {
+ __aligned_u64 wr_id;
+ __u32 num_sge;
+ __u32 padding;
+ struct rfc_dma_info dma;
+};
+
+struct rfc_create_cq_resp {
+ struct mminfo mi;
+};
+
+struct rfc_resize_cq_resp {
+ struct mminfo mi;
+};
+
+struct rfc_create_qp_resp {
+ struct mminfo rq_mi;
+ struct mminfo sq_mi;
+};
+
+struct rfc_create_srq_resp {
+ struct mminfo mi;
+ __u32 srq_num;
+ __u32 reserved;
+};
+
+struct rfc_modify_srq_cmd {
+ __aligned_u64 mmap_info_addr;
+};
+
+#endif /* RDMA_USER_RFC_H */
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
