Signed-off-by: Alexander Nezhinsky <alexandern@xxxxxxxxxxxx>
---
usr/iscsi/iser_ib.c | 3103 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 3103 insertions(+), 0 deletions(-)
create mode 100644 usr/iscsi/iser_ib.c
diff --git a/usr/iscsi/iser_ib.c b/usr/iscsi/iser_ib.c
new file mode 100644
index 0000000..e1a1f15
--- /dev/null
+++ b/usr/iscsi/iser_ib.c
@@ -0,0 +1,3103 @@
+/*
+ * iSCSI extensions for RDMA (iSER) data path
+ *
+ * Copyright (C) 2007 Dennis Dalessandro (dennis@xxxxxxx)
+ * Copyright (C) 2007 Ananth Devulapalli (ananth@xxxxxxx)
+ * Copyright (C) 2007 Pete Wyckoff (pw@xxxxxxx)
+ * Copyright (C) 2010 Voltaire, Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation, version 2 of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301 USA
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <assert.h>
+#include <netdb.h>
+#include <sys/epoll.h>
+#include <infiniband/verbs.h>
+#include <rdma/rdma_cma.h>
+
+#include "util.h"
+#include "iscsid.h"
+#include "target.h"
+#include "iser.h"
+#include "driver.h"
+#include "scsi.h"
+
+#if defined(HAVE_VALGRIND) && !defined(NDEBUG)
+#include <valgrind/memcheck.h>
+#else
+#define VALGRIND_MAKE_MEM_DEFINED(addr, len)
+#endif
+
+/* global, across all devices */
+static struct rdma_event_channel *rdma_evt_channel;
+static struct rdma_cm_id *cma_listen_id;
+
+/* accepted at RDMA layer, but not yet established */
+static struct list_head temp_conn;
+
+/* all devices */
+static struct list_head iser_dev_list;
+
+/* all iser connections */
+static struct list_head iser_conn_list;
+
+#define uint64_from_ptr(p) (uint64_t)(uintptr_t)(p)
+#define ptr_from_int64(p) (void *)(unsigned long)(p)
+
+#define ISER_LISTEN_PORT 3260
+
+short int iser_listen_port = ISER_LISTEN_PORT;
+char *iser_portal_addr;
+
+/*
+ * Crazy hard-coded linux iser settings need 128 * 8 slots + slop, plus
+ * room for our rdmas and send requests.
+ */
+#define MAX_WQE 1800
+
+#define RDMA_TRANSFER_SIZE (512 * 1024)
+
+#define MAX_POLL_WC 32
+
+#define ISER_MAX_QUEUE_CMD 128 // iSCSI cmd window size
+
+struct iscsi_sense_data {
+ uint16_t length;
+ uint8_t data[0];
+} __packed;
+
+/*
+ * Number of allocatable data buffers, each of this size. Do at least 128
+ * for linux iser. The membuf size is rounded up at initialization time
+ * to the hardware page size so that allocations for direct IO devices are
+ * aligned.
+ */
+static int membuf_num = 16 * ISER_MAX_QUEUE_CMD;
+static size_t membuf_size = RDMA_TRANSFER_SIZE;
+
+static int iser_conn_get(struct iser_conn *conn);
+static void iser_conn_put(struct iser_conn *conn);
+
+static void iser_free_login_resources(struct iser_conn *conn);
+static void iser_free_ff_resources(struct iser_conn *conn);
+
+static void iser_cqe_handler(int fd __attribute__ ((unused)),
+ int events __attribute__ ((unused)),
+ void *data);
+
+static void iser_sched_poll_cq(struct event_data *tev);
+static void iser_sched_consume_cq(struct event_data *tev);
+static void iser_sched_tx(struct event_data *evt);
+static void iser_sched_post_recv(struct event_data *evt);
+static void iser_sched_buf_alloc(struct event_data *evt);
+static void iser_sched_iosubmit(struct event_data *evt);
+static void iser_sched_rdma_rd(struct event_data *evt);
+
+
+static inline struct iser_conn *ISER_CONN(struct iscsi_connection *iscsi_conn)
+{
+ return container_of(iscsi_conn, struct iser_conn, h);
+}
+
+static void iser_pdu_init(struct iser_pdu *pdu, void *buf)
+{
+ pdu->iser_hdr = (struct iser_hdr *) buf;
+ buf += sizeof(struct iser_hdr);
+ pdu->bhs = (struct iscsi_hdr *) buf;
+ buf += sizeof(struct iscsi_hdr);
+ pdu->ahs = pdu->data.buf = buf;
+ pdu->ahssize = pdu->data.size = 0;
+}
+
+static void iser_send_pdu_init(struct iser_pdu *pdu, void *buf)
+{
+ iser_pdu_init(pdu, buf);
+ memset(pdu->iser_hdr, 0, sizeof(struct iser_hdr));
+ pdu->iser_hdr->flags = ISCSI_CTRL;
+}
+
+static void iser_rxd_init(struct iser_work_req *rxd,
+ struct iser_task *task,
+ void *buf, unsigned size,
+ struct ibv_mr *srmr)
+{
+ rxd->recv = 1;
+ rxd->task = task;
+
+ rxd->sge.addr = uint64_from_ptr(buf);
+ rxd->sge.length = size;
+ rxd->sge.lkey = srmr->lkey;
+
+ rxd->recv_wr.wr_id = uint64_from_ptr(rxd);
+ rxd->recv_wr.sg_list = &rxd->sge;
+ rxd->recv_wr.num_sge = 1;
+ rxd->recv_wr.next = NULL;
+}
+
+static void iser_txd_init(struct iser_work_req *txd,
+ struct iser_task *task,
+ void *buf, unsigned size,
+ struct ibv_mr *srmr)
+{
+ txd->recv = 0;
+ txd->task = task;
+
+ txd->sge.addr = uint64_from_ptr(buf);
+ txd->sge.length = size;
+ txd->sge.lkey = srmr->lkey;
+
+ txd->send_wr.wr_id = uint64_from_ptr(txd);
+ txd->send_wr.next = NULL;
+ txd->send_wr.sg_list = &txd->sge;
+ txd->send_wr.num_sge = 1;
+ txd->send_wr.opcode = IBV_WR_SEND;
+ txd->send_wr.send_flags = IBV_SEND_SIGNALED;
+
+ INIT_LIST_HEAD(&txd->wr_list);
+}
+
+static void iser_rdmad_init(struct iser_work_req *rdmad,
+ struct iser_task *task,
+ struct ibv_mr *srmr)
+{
+ rdmad->recv = 0;
+ rdmad->task = task;
+
+ // txd->sge.addr, txd->sge.length to be set before posting
+ rdmad->sge.lkey = srmr->lkey;
+
+ rdmad->send_wr.wr_id = uint64_from_ptr(rdmad);
+ rdmad->send_wr.sg_list = &rdmad->sge;
+ rdmad->send_wr.num_sge = 1;
+ rdmad->send_wr.next = NULL;
+ // txd->send_wr.opcode to be set before posting
+ rdmad->send_wr.send_flags = IBV_SEND_SIGNALED;
+ // rdmad->send_wr.wr.rdma.(remote_addr,rkey) to be set before post
+
+ INIT_LIST_HEAD(&rdmad->wr_list);
+}
+
+static void iser_task_init(struct iser_task *task,
+ struct iser_conn *conn,
+ void *pdu_data_buf,
+ struct ibv_mr *srmr)
+{
+ task->conn = conn;
+ task->unsolicited = 0;
+
+ iser_rxd_init(&task->rxd, task, pdu_data_buf, conn->rsize, srmr);
+ iser_pdu_init(&task->req, pdu_data_buf);
+ pdu_data_buf += conn->rsize;
+
+ iser_txd_init(&task->txd, task, pdu_data_buf, conn->ssize, srmr);
+ iser_send_pdu_init(&task->rsp, pdu_data_buf);
+
+ INIT_LIST_HEAD(&task->in_buf_list);
+ INIT_LIST_HEAD(&task->out_buf_list);
+ iser_rdmad_init(&task->rdmad, task, conn->dev->membuf_mr);
+
+ INIT_LIST_HEAD(&task->exec_list);
+ INIT_LIST_HEAD(&task->rdma_list);
+ INIT_LIST_HEAD(&task->tx_list);
+ INIT_LIST_HEAD(&task->recv_list);
+
+ INIT_LIST_HEAD(&task->session_list);
+ INIT_LIST_HEAD(&task->dout_task_list);
+}
+
+static void iser_unsolicited_task_init(struct iser_task *task,
+ struct iser_conn *conn,
+ void *pdu_data_buf,
+ struct ibv_mr *srmr)
+{
+ task->conn = conn;
+ task->unsolicited = 1;
+
+ iser_txd_init(&task->txd, task, pdu_data_buf, conn->ssize, srmr);
+ iser_send_pdu_init(&task->rsp, pdu_data_buf);
+}
+
+static void iser_task_add_out_pdu_buf(struct iser_task *task,
+ struct membuf *buf,
+ unsigned int offset)
+{
+ buf->offset = offset;
+ task->out_buf_num ++;
+ if (!list_empty(&task->out_buf_list)) {
+ struct membuf *cur_buf;
+ list_for_each_entry(cur_buf, &task->out_buf_list, task_list) {
+ if (offset < cur_buf->offset) {
+ dprintf("task:%p offset:%d size:%d buf:%p add before:%p\n",
+ task, offset, buf->size, buf->buf, cur_buf->buf);
+ list_add_tail(&buf->task_list, &cur_buf->task_list);
+ return;
+ }
+ }
+ }
+ dprintf("task:%p offset:%d size:%d buf:%p add last\n",
+ task, offset, buf->size, buf->buf);
+ list_add_tail(&buf->task_list, &task->out_buf_list);
+}
+
+static void iser_task_add_out_rdma_buf(struct iser_task *task,
+ struct membuf *buf,
+ unsigned int offset)
+{
+ buf->offset = offset;
+ task->out_buf_num ++;
+ dprintf("task:%p offset:%d size:%d buf:%p add last\n",
+ task, offset, buf->size, buf->buf);
+ list_add_tail(&buf->task_list, &task->out_buf_list);
+}
+
+static void iser_task_del_out_buf(struct iser_task *task,
+ struct membuf *buf)
+{
+ dprintf("task:%p offset:%d size:%d buf:%p\n",
+ task, buf->offset, buf->size, buf->buf);
+ list_del(&buf->task_list);
+ task->out_buf_num --;
+}
+
+static void iser_task_add_in_rdma_buf(struct iser_task *task,
+ struct membuf *buf,
+ unsigned int offset)
+{
+ dprintf("task:%p offset:0x%d size:%d buf:%p add last\n",
+ task, offset, buf->size, buf->buf);
+ buf->offset = offset;
+ task->in_buf_num ++;
+ list_add_tail(&buf->task_list, &task->in_buf_list);
+}
+
+static void iser_task_del_in_buf(struct iser_task *task,
+ struct membuf *buf)
+{
+ dprintf("task:%p offset:%d size:%d buf:%p\n",
+ task, buf->offset, buf->size, buf->buf);
+ list_del(&buf->task_list);
+ task->in_buf_num --;
+}
+
+static int iser_post_recv(struct iser_conn *conn, struct iser_task *task,
+ int num_recv_bufs)
+{
+ struct ibv_recv_wr *bad_wr;
+ int err, nr_posted;
+
+ err = ibv_post_recv(conn->qp_hndl, &task->rxd.recv_wr, &bad_wr);
+ if (!err)
+ nr_posted = num_recv_bufs;
+ else {
+ nr_posted = 0;
+ eprintf("failed to post rx buf %p", &task->rxd.recv_wr);
+ }
+ //conn->recvl_posted += nr_posted;
+
+ dprintf("chained:%d last task:%p wr_id:0x%lx sge_sz:%u\n",
+ nr_posted, task, (unsigned long)task->rxd.recv_wr.wr_id,
+ task->rxd.sge.length);
+
+ while (nr_posted--)
+ iser_conn_get(conn);
+
+ return err;
+}
+
+static void iser_prep_resp_send_req(struct iser_task *task,
+ struct iser_work_req *next_wr,
+ int signaled)
+{
+ struct iser_pdu *rsp = &task->rsp;
+ struct iscsi_hdr *bhs = rsp->bhs;
+ struct iser_work_req *txd = &task->txd;
+
+ bhs->hlength = rsp->ahssize / 4;
+ hton24(bhs->dlength, rsp->data.size);
+
+ txd->sge.length = sizeof(struct iscsi_hdr) + sizeof(struct iser_hdr);
+ txd->sge.length += rsp->ahssize;
+ txd->sge.length += rsp->data.size;
+
+ txd->send_wr.next = (next_wr ? &next_wr->send_wr : NULL);
+ txd->send_wr.send_flags = (signaled ? IBV_SEND_SIGNALED : 0);
+
+ dprintf("task:%p wr_id:0x%lx tag:0x%04lx "
+ "dtbuf:%p dtsz:%u ahs_sz:%u stat:0x%x statsn:0x%x expcmdsn:0x%x\n",
+ task, (unsigned long)txd->send_wr.wr_id,
+ (unsigned long)task->tag,
+ rsp->data.buf, rsp->data.size, rsp->ahssize,
+ bhs->rsvd2[1], ntohl(bhs->statsn), ntohl(bhs->exp_statsn));
+}
+
+static void iser_prep_rdma_wr_send_req(struct iser_task *task,
+ struct iser_work_req *next_wr,
+ int signaled)
+{
+ struct iser_work_req *rdmad = &task->rdmad;
+ struct membuf *rdma_buf;
+ uint64_t offset = 0; // ToDo: multiple RDMA-Write buffers
+
+ /* RDMA-Write buffer is the only one on the list */
+ // ToDo: multiple RDMA-Write buffers, use rdma_buf->offset
+ rdma_buf = list_first_entry(&task->in_buf_list, struct membuf, task_list);
+ rdmad->sge.addr = uint64_from_ptr(rdma_buf->buf);
+
+ if (task->rdma_wr_remains > rdma_buf->size)
+ rdmad->sge.length = rdma_buf->size;
+ else
+ rdmad->sge.length = task->rdma_wr_remains;
+ task->rdma_wr_remains -= rdmad->sge.length;
+
+ rdmad->send_wr.next = (next_wr ? &next_wr->send_wr : NULL);
+ rdmad->send_wr.opcode = IBV_WR_RDMA_WRITE;
+ rdmad->send_wr.send_flags = (signaled ? IBV_SEND_SIGNALED : 0);
+
+ rdmad->send_wr.wr.rdma.remote_addr = task->rem_read_va + offset;
+ // ToDo: multiple RDMA-Write buffers
+ // offset += rdmad->sge.length
+ rdmad->send_wr.wr.rdma.rkey = task->rem_read_stag;
+
+ dprintf(" task:%p wr_id:0x%lx tag:0x%04lx daddr:0x%lx dsz:%u "
+ "bufsz:%u rdma:%d lkey:%x raddr:%lx rkey:%x rems:%u\n",
+ task, (unsigned long)rdmad->send_wr.wr_id,
+ (unsigned long)task->tag, (unsigned long)rdmad->sge.addr,
+ rdmad->sge.length, rdma_buf->size, rdma_buf->rdma,
+ rdmad->sge.lkey, (unsigned long)rdmad->send_wr.wr.rdma.remote_addr,
+ rdmad->send_wr.wr.rdma.rkey, task->rdma_wr_remains);
+}
+
+static void iser_prep_rdma_rd_send_req(struct iser_task *task,
+ struct iser_work_req *next_wr,
+ int signaled)
+{
+ struct iser_work_req *rdmad = &task->rdmad;
+ struct membuf *rdma_buf;
+ uint64_t offset;
+ int cur_req_sz;
+
+ /* RDMA-Read buffer is always put at the list's tail */
+ // ToDo: multiple RDMA-Write buffers
+ rdma_buf = container_of(task->out_buf_list.prev, struct membuf, task_list);
+ offset = rdma_buf->offset;
+ rdmad->sge.addr = uint64_from_ptr(rdma_buf->buf) + offset;
+
+ if (task->rdma_rd_sz <= rdma_buf->size)
+ cur_req_sz = task->rdma_rd_sz;
+ else
+ cur_req_sz = rdma_buf->size;
+
+ // ToDo: multiple RDMA-Write buffers
+ // task->rdma_rd_offset += cur_req_sz;
+
+ rdmad->sge.length = cur_req_sz;
+
+ rdmad->send_wr.next = (next_wr ? &next_wr->send_wr : NULL);
+ rdmad->send_wr.opcode = IBV_WR_RDMA_READ;
+ rdmad->send_wr.send_flags = (signaled ? IBV_SEND_SIGNALED : 0);
+ rdmad->send_wr.wr.rdma.remote_addr =
+ (uint64_t) task->rem_write_va;
+ // ToDo: multiple RDMA-Write buffers: + (offset - task->unsol_sz);
+ rdmad->send_wr.wr.rdma.rkey =
+ (uint32_t) task->rem_write_stag;
+
+ dprintf("task:%p wr_id:0x%lx tag:0x%04lx daddr:0x%lx dsz:%u "
+ "bufsz:%u rdma:%d lkey:%x raddr:%lx rkey:%x rems:%u\n",
+ task, (unsigned long)rdmad->send_wr.wr_id,
+ (unsigned long)task->tag, (unsigned long)rdmad->sge.addr,
+ rdmad->sge.length, rdma_buf->size, rdma_buf->rdma,
+ rdmad->sge.lkey, (unsigned long)rdmad->send_wr.wr.rdma.remote_addr,
+ rdmad->send_wr.wr.rdma.rkey, task->rdma_rd_sz);
+}
+
+static int iser_post_send(struct iser_conn *conn, struct iser_work_req *iser_send)
+{
+ struct ibv_send_wr *bad_wr;
+ int err;
+
+ err = ibv_post_send(conn->qp_hndl, &iser_send->send_wr, &bad_wr);
+ if (err)
+ eprintf("ibv_post_send err %d\n", err);
+ return err;
+}
+
+static inline void iser_set_rsp_cmd_sn(struct iscsi_session *session,
+ struct iscsi_hdr *rsp)
+{
+ if (session) {
+ rsp->exp_statsn = cpu_to_be32(session->exp_cmd_sn);
+ rsp->max_statsn = cpu_to_be32(session->exp_cmd_sn + ISER_MAX_QUEUE_CMD);
+ }
+}
+
+/*
+ * One pool of registered memory per device (per PD that is).
+ */
+static int iser_init_rdma_buf_pool(struct iser_device *dev)
+{
+ uint8_t *regbuf, *task_buf;
+ size_t pool_size, list_size;
+ struct membuf *mp;
+ int i;
+
+ membuf_size = roundup(membuf_size, pagesize);
+ pool_size = membuf_num * membuf_size;
+ regbuf = valloc(pool_size);
+ if (!regbuf) {
+ eprintf("malloc regbuf sz:%zu\n", pool_size);
+ return -ENOMEM;
+ }
+
+ list_size = membuf_num * sizeof(*mp);
+ task_buf = malloc(list_size);
+ if (!task_buf) {
+ eprintf("malloc task_buf sz:%zu\n", list_size);
+ free(regbuf);
+ return -ENOMEM;
+ }
+
+ dev->membuf_mr = ibv_reg_mr(dev->pd, regbuf, pool_size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!dev->membuf_mr) {
+ eprintf("register regbuf\n");
+ free(regbuf);
+ free(task_buf);
+ return -1;
+ }
+ dprintf("pool buf:%p list:%p mr:%p lkey:0x%x\n",
+ regbuf, task_buf, dev->membuf_mr, dev->membuf_mr->lkey);
+
+ dev->membuf_regbuf = regbuf;
+ dev->membuf_listbuf = task_buf;
+ INIT_LIST_HEAD(&dev->membuf_free);
+ INIT_LIST_HEAD(&dev->membuf_alloc);
+
+ for (i = 0; i < membuf_num; i++) {
+ mp = (void *) task_buf;
+ task_buf += sizeof(*mp);
+
+ list_add_tail(&mp->pool_list, &dev->membuf_free);
+ INIT_LIST_HEAD(&mp->task_list);
+ mp->buf = regbuf;
+ mp->size = membuf_size;
+ mp->rdma = 1;
+
+ regbuf += membuf_size;
+ }
+
+ return 0;
+}
+
+static struct membuf *iser_alloc_rdma_buf(struct iser_conn *conn,
+ size_t sz)
+{
+ struct iser_device *dev = conn->dev;
+ struct list_head *buf_list;
+ struct membuf *mem;
+
+ if (list_empty(&dev->membuf_free)) {
+ // eprintf("free list empty\n");
+ dev->waiting_for_mem = 1;
+ return NULL;
+ }
+ if (sz > membuf_size) {
+ eprintf("size %zu too big\n", sz);
+ exit(1);
+ }
+
+ buf_list = dev->membuf_free.next;
+ list_del(buf_list);
+ list_add_tail(buf_list, &dev->membuf_alloc);
+ mem = list_entry(buf_list, struct membuf, pool_list);
+
+ dprintf("alloc:%p sz:%zu\n", mem, sz);
+ return mem;
+}
+
+static void iser_free_rdma_buf(struct iser_conn *conn, struct membuf *mem)
+{
+ struct iser_device *dev;
+
+ if (!mem || !mem->rdma)
+ return;
+
+ dprintf("free %p\n", mem);
+ list_del(&mem->pool_list);
+ // add to the free list head to reuse recently used buffers first
+ dev = conn->dev;
+ list_add(&mem->pool_list, &dev->membuf_free);
+ if (dev->waiting_for_mem) {
+ dev->waiting_for_mem = 0;
+ tgt_add_sched_event(&conn->sched_buf_alloc);
+ }
+}
+
+static void iser_task_free_out_bufs(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct membuf *membuf, *mbnext;
+
+ list_for_each_entry_safe(membuf, mbnext, &task->out_buf_list, task_list) {
+ iser_task_del_out_buf(task, membuf);
+ if (membuf->rdma)
+ iser_free_rdma_buf(conn, membuf);
+ }
+ assert(task->out_buf_num == 0);
+}
+
+static void iser_task_free_in_bufs(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct membuf *membuf, *mbnext;
+
+ list_for_each_entry_safe(membuf, mbnext, &task->in_buf_list, task_list) {
+ iser_task_del_in_buf(task, membuf);
+ iser_free_rdma_buf(conn, membuf);
+ }
+ assert(task->in_buf_num == 0);
+}
+
+static void iser_complete_task(struct iser_task *task)
+{
+ if (task->opcode == ISCSI_OP_SCSI_CMD) {
+ target_cmd_done(&task->scmd);
+ list_del(&task->session_list);
+ if (task->is_read)
+ iser_task_free_in_bufs(task);
+ if (task->is_write) {
+ iser_task_free_out_bufs(task);
+ // ToDo: multiple out buffers, no copy
+ // iser_task_free_dout_tasks(task);
+ }
+ }
+ dprintf("task:%p\n", task);
+ //iser_conn_put(task->conn);
+}
+
+/*
+ * Called at accept time, builds resources just for login phase.
+ */
+
+#define NUM_LOGIN_TASKS 2 /* one posted for req rx, one for reply tx */
+
+static int iser_alloc_login_resources(struct iser_conn *conn)
+{
+ unsigned long size;
+ struct iser_task *login_task[NUM_LOGIN_TASKS];
+ uint8_t *pdu_data_buf, *task_buf;
+ unsigned int i;
+ int err = 0;
+
+ dprintf("login tasks num:%u, rx/tx sz %u/%u\n",
+ NUM_LOGIN_TASKS, conn->ssize, conn->rsize);
+
+ size = NUM_LOGIN_TASKS * (conn->rsize + conn->ssize);
+ conn->login_data_pool = malloc(size);
+ if (!conn->login_data_pool) {
+ eprintf("malloc pdu_data_buf %lu\n", size);
+ err = -1;
+ goto out;
+ }
+
+ conn->login_data_mr =
+ ibv_reg_mr(conn->dev->pd, conn->login_data_pool, size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!conn->login_data_mr) {
+ eprintf("ibv_reg_mr pdu_data_buf failed\n");
+ free(conn->login_data_pool);
+ conn->login_data_pool = NULL;
+ err = -1;
+ goto out;
+ }
+
+ size = NUM_LOGIN_TASKS * sizeof(struct iser_task);
+ conn->login_task_pool = malloc(size);
+ if (!conn->login_task_pool) {
+ eprintf("malloc login_task_buf %lu\n", size);
+ ibv_dereg_mr(conn->login_data_mr);
+ conn->login_data_mr = NULL;
+ free(conn->login_data_pool);
+ conn->login_data_pool = NULL;
+ goto out;
+ }
+ memset(conn->login_task_pool, 0, size);
+
+ conn->login_res_alloc = 1;
+ pdu_data_buf = conn->login_data_pool;
+ task_buf = conn->login_task_pool;
+ for (i = 0; i < NUM_LOGIN_TASKS; i++) {
+ login_task[i] = (struct iser_task *) task_buf;
+ task_buf += sizeof(struct iser_task);
+
+ iser_task_init(login_task[i], conn, pdu_data_buf,
+ conn->login_data_mr);
+ pdu_data_buf += conn->rsize + conn->ssize;
+ }
+
+ dprintf("post_recv login rx task:%p\n", login_task[0]);
+ err = iser_post_recv(conn, login_task[0], 1);
+ if (err) {
+ eprintf("post_recv login rx task failed, %m\n");
+ iser_free_login_resources(conn);
+ goto out;
+ }
+ dprintf("saved login tx task:%p\n", login_task[1]);
+ conn->login_tx_task = login_task[1];
+out:
+ return err;
+}
+
+/*
+ * When ready for full-feature mode, free login-phase resources.
+ */
+static void iser_free_login_resources(struct iser_conn *conn)
+{
+ int err;
+
+ if (!conn->login_res_alloc)
+ return;
+
+ dprintf("freeing, login phase %d\n", conn->login_phase);
+
+ /* release mr and free the lists */
+ if (conn->login_data_mr) {
+ err = ibv_dereg_mr(conn->login_data_mr);
+ if (err)
+ eprintf("ibv_dereg_mr\n");
+ }
+ if (conn->login_data_pool)
+ free(conn->login_data_pool);
+ if (conn->login_task_pool)
+ free(conn->login_task_pool);
+ conn->login_res_alloc = 0;
+}
+
+/*
+ * Called when ready for full feature, builds resources.
+ */
+static int iser_alloc_ff_resources(struct iser_conn *conn)
+{
+ unsigned long size;
+ uint8_t *pdu_data_buf, *task_buf;
+ struct iser_task *task;
+ unsigned int i;
+ int err = 0;
+
+ dprintf("conn ssize:%u rsize:%u max_outst:%u\n",
+ conn->ssize, conn->rsize, conn->max_outst_pdu);
+
+ size = (conn->rsize + conn->ssize) * conn->max_outst_pdu;
+ conn->pdu_data_pool = malloc(size);
+ if (!conn->pdu_data_pool) {
+ eprintf("malloc pdu_data_buf %lu\n", size);
+ err = -1;
+ goto out;
+ }
+
+ conn->pdu_data_mr = ibv_reg_mr(conn->dev->pd, conn->pdu_data_pool, size,
+ IBV_ACCESS_LOCAL_WRITE);
+ if (!conn->pdu_data_mr) {
+ eprintf("register pdu_data_buf\n");
+ free(conn->pdu_data_pool);
+ conn->pdu_data_pool = NULL;
+ err = -1;
+ goto out;
+ }
+
+ size = conn->max_outst_pdu * sizeof(struct iser_task);
+ conn->task_pool = malloc(size);
+ if (!conn->task_pool) {
+ eprintf("malloc task_buf %lu\n", size);
+ ibv_dereg_mr(conn->pdu_data_mr);
+ conn->pdu_data_mr = NULL;
+ free(conn->pdu_data_pool);
+ conn->pdu_data_pool = NULL;
+ goto out;
+ }
+ memset(conn->task_pool, 0, size);
+
+ conn->ff_res_alloc = 1;
+ pdu_data_buf = conn->pdu_data_pool;
+ task_buf = conn->task_pool;
+ for (i = 1; i < conn->max_outst_pdu; i++) {
+ task = (void *) task_buf;
+ task_buf += sizeof(*task);
+
+ iser_task_init(task, conn, pdu_data_buf, conn->pdu_data_mr);
+ pdu_data_buf += conn->rsize + conn->ssize;
+
+ err = iser_post_recv(conn, task, 1);
+ if (err) {
+ eprintf("post_recv (%d/%d): %m\n",
+ i, conn->max_outst_pdu);
+ iser_free_ff_resources(conn);
+ break;
+ }
+ }
+ task = (void *) task_buf;
+ iser_unsolicited_task_init(task, conn, pdu_data_buf, conn->pdu_data_mr);
+ conn->nop_in_task = task;
+
+out:
+ return err;
+}
+
+/*
+ * On connection shutdown.
+ */
+static void iser_free_ff_resources(struct iser_conn *conn)
+{
+ int err;
+
+ if (!conn->ff_res_alloc)
+ return;
+
+ dprintf("freeing conn %p\n", conn);
+
+ /* release mr and free the lists */
+ dprintf("dereg mr %p\n", conn->pdu_data_mr);
+ if (conn->pdu_data_mr) {
+ err = ibv_dereg_mr(conn->pdu_data_mr);
+ if (err)
+ eprintf("ibv_dereg_mr\n");
+ }
+ if (conn->pdu_data_pool)
+ free(conn->pdu_data_pool);
+ if (conn->task_pool)
+ free(conn->task_pool);
+ conn->ff_res_alloc = 0;
+ dprintf("exit\n");
+}
+
+/*
+ * Allocate resources for this new connection. Called after login, when
+ * final negotiated transfer parameters are known.
+ */
+int iser_login_complete(struct iscsi_connection *iscsi_conn)
+{
+ struct iser_conn *conn = ISER_CONN(iscsi_conn);
+ unsigned int irdsl, trdsl, outst_pdu, hdrsz;
+ int err = -1;
+
+ dprintf("entry\n");
+
+ /*
+ * Build full feature connection structures, but leave around the
+ * login ones until the final send finishes.
+ */
+ conn->login_phase = LOGIN_PHASE_LAST_SEND; /* one more send, then done */
+ irdsl = iscsi_conn->session_param[ISCSI_PARAM_INITIATOR_RDSL].val;
+ trdsl = iscsi_conn->session_param[ISCSI_PARAM_TARGET_RDSL].val;
+ outst_pdu =
+ iscsi_conn->session_param[ISCSI_PARAM_MAX_OUTST_PDU].val; // ToDo: outstanding pdus num
+
+ /* hack, ib/ulp/iser does not have this param, but reading the code
+ * shows their formula for max tx dtos outstanding
+ * = cmds_max * (1 + dataouts) + rx_misc + tx_misc
+ */
+#define ISER_INFLIGHT_DATAOUTS 0
+#define ISER_MAX_RX_MISC_PDUS 4
+#define ISER_MAX_TX_MISC_PDUS 6
+
+ // ToDo: outstanding pdus num
+// if (outst_pdu == 0)
+ outst_pdu =
+ 3 * ISER_MAX_QUEUE_CMD * (1 + ISER_INFLIGHT_DATAOUTS) +
+ ISER_MAX_RX_MISC_PDUS + ISER_MAX_TX_MISC_PDUS;
+
+ /* RDSLs do not include headers. */
+ hdrsz = sizeof(struct iser_hdr) +
+ sizeof(struct iscsi_hdr) +
+ sizeof(struct iscsi_ecdb_ahdr) +
+ sizeof(struct iscsi_rlength_ahdr);
+
+ conn->ssize = hdrsz + irdsl;
+ conn->rsize = hdrsz + trdsl;
+ conn->max_outst_pdu = outst_pdu;
+ err = iser_alloc_ff_resources(conn);
+ if (err) {
+ eprintf("iser_alloc_resources failed\n");
+ goto out;
+ }
+ // How much data to grab in an RDMA operation, read or write.
+ //iscsi_conn->data_inout_max_length = RDMA_TRANSFER_SIZE;
+ iscsi_conn->session_param[ISCSI_PARAM_MAX_XMIT_DLENGTH].val =
+ RDMA_TRANSFER_SIZE;
+ out:
+ return err;
+}
+
+/*
+ * First time a new connection is received on an RDMA device, record
+ * it and build a PD and static memory.
+ */
+static int iser_device_init(struct iser_device *dev)
+{
+ int cqe_num;
+ int err = -1;
+
+ dprintf("dev %p\n", dev);
+ dev->pd = ibv_alloc_pd(dev->ibv_hndl);
+ if (dev->pd == NULL) {
+ eprintf("ibv_alloc_pd failed\n");
+ goto out;
+ }
+
+ err = iser_init_rdma_buf_pool(dev);
+ if (err) {
+ eprintf("iser_init_rdma_buf_pool failed\n");
+ goto out;
+ }
+
+ err = ibv_query_device(dev->ibv_hndl, &dev->device_attr);
+ if (err < 0) {
+ eprintf("ibv_query_device: %m\n");
+ goto out;
+ }
+ cqe_num = dev->device_attr.max_cqe;
+ dprintf("max %d CQEs\n", cqe_num);
+
+ err = -1;
+ dev->cq_channel = ibv_create_comp_channel(dev->ibv_hndl);
+ if (dev->cq_channel == NULL) {
+ eprintf("ibv_create_comp_channel failed: %m\n");
+ goto out;
+ }
+
+ dev->cq = ibv_create_cq(dev->ibv_hndl, cqe_num, NULL,
+ dev->cq_channel, 0);
+ if (dev->cq == NULL) {
+ eprintf("ibv_create_cq failed: %m\n");
+ goto out;
+ }
+
+ tgt_init_sched_event(&dev->poll_sched, iser_sched_poll_cq, dev);
+
+ err = ibv_req_notify_cq(dev->cq, 0);
+ if (err) {
+ eprintf("ibv_req_notify failed: %s\n", strerror(err));
+ goto out;
+ }
+
+ err = tgt_event_add(dev->cq_channel->fd, EPOLLIN, iser_cqe_handler,
+ dev);
+ if (err) {
+ eprintf("tgt_event_add failed: %m\n");
+ goto out;
+
+ }
+
+ list_add_tail(&dev->list, &iser_dev_list);
+
+out:
+ return err;
+}
+
+static int iser_ib_clear_iosubmit_list(struct iser_conn *conn)
+{
+ struct iser_task *task;
+
+ dprintf("start\n");
+ while (!list_empty(&conn->iosubmit_list)) {
+ task = list_first_entry(&conn->iosubmit_list, struct iser_task, exec_list);
+ list_del(&task->exec_list);
+ iser_complete_task(task);
+ }
+ return 0;
+}
+
+static int iser_ib_clear_rdma_rd_list(struct iser_conn *conn)
+{
+ struct iser_task *task;
+
+ dprintf("start\n");
+ while (!list_empty(&conn->rdma_rd_list)) {
+ task = list_first_entry(&conn->rdma_rd_list, struct iser_task, rdma_list);
+ list_del(&task->rdma_list);
+ }
+ return 0;
+}
+
+static int iser_ib_clear_tx_list(struct iser_conn *conn)
+{
+ struct iser_task *task;
+
+ dprintf("start\n");
+ while (!list_empty(&conn->resp_tx_list)) {
+ task = list_first_entry(&conn->resp_tx_list, struct iser_task, tx_list);
+ list_del(&task->tx_list);
+ }
+ return 0;
+}
+
+static int iser_ib_clear_sent_list(struct iser_conn *conn)
+{
+ struct iser_task *task;
+
+ dprintf("start\n");
+ while (!list_empty(&conn->sent_list)) {
+ task = list_first_entry(&conn->sent_list, struct iser_task, tx_list);
+ list_del(&task->tx_list);
+ }
+ return 0;
+}
+
+static int iser_ib_clear_post_recv_list(struct iser_conn *conn)
+{
+ struct iser_task *task;
+
+ dprintf("start\n");
+ while (!list_empty(&conn->post_recv_list)) {
+ task = list_first_entry(&conn->post_recv_list, struct iser_task, recv_list);
+ list_del(&task->recv_list);
+ }
+ return 0;
+}
+
+int iser_conn_init(struct iser_conn *conn)
+{
+ conn->h.refcount = 0;
+ conn->h.state = STATE_INIT;
+ param_set_defaults(conn->h.session_param, session_keys);
+
+ INIT_LIST_HEAD(&conn->h.clist);
+
+ INIT_LIST_HEAD(&conn->buf_alloc_list);
+ INIT_LIST_HEAD(&conn->rdma_rd_list);
+ /* INIT_LIST_HEAD(&conn->rdma_wr_list); */
+ INIT_LIST_HEAD(&conn->iosubmit_list);
+ INIT_LIST_HEAD(&conn->resp_tx_list);
+ INIT_LIST_HEAD(&conn->sent_list);
+ INIT_LIST_HEAD(&conn->post_recv_list);
+
+ return 0;
+}
+
+/*
+ * Start closing connection. Transfer IB QP to error state.
+ * This will be followed by WC error and buffers flush events.
+ * We also should expect DISCONNECTED and TIMEWAIT_EXIT events.
+ * Only after the draining is over we are sure to have reclaimed
+ * all buffers (and tasks). After the RDMA CM events are collected,
+ * the connection QP may be destroyed, and its number may be recycled.
+ */
+void iser_conn_close(struct iser_conn *conn)
+{
+ int err;
+
+ if (conn->h.state == STATE_CLOSE)
+ return;
+ conn->h.state = STATE_CLOSE;
+
+ dprintf("rdma_disconnect conn:%p\n", &conn->h);
+ err = rdma_disconnect(conn->cm_id);
+ if (err)
+ eprintf("rdma_disconnect: %s\n", strerror(-err));
+
+ list_del(&conn->conn_list);
+
+ tgt_remove_sched_event(&conn->sched_buf_alloc);
+ tgt_remove_sched_event(&conn->sched_rdma_rd);
+ tgt_remove_sched_event(&conn->sched_iosubmit);
+ tgt_remove_sched_event(&conn->sched_tx);
+ tgt_remove_sched_event(&conn->sched_post_recv);
+
+ if (conn->login_phase != LOGIN_PHASE_FF) {
+ iser_ib_clear_iosubmit_list(conn);
+ iser_ib_clear_rdma_rd_list(conn);
+ }
+ iser_ib_clear_tx_list(conn);
+ iser_ib_clear_sent_list(conn);
+ iser_ib_clear_post_recv_list(conn);
+
+ eprintf("connection:%p closed, refcnt:%d\n", conn, conn->h.refcount);
+}
+
+/*
+ * Called when the connection is freed, from iscsi, but won't do anything until
+ * all posted WRs have gone away. So also called again from RX progress when
+ * it notices this happens.
+ */
+void iser_conn_free(struct iser_conn *conn)
+{
+ int err;
+
+ dprintf("conn:%p rx.posted:%d refcnt:%d\n", conn, conn->recvl_posted,
+ conn->h.refcount);
+ assert(conn->h.refcount == 0);
+
+ /* try to free unconditionally, resources freed only if necessary */
+ iser_free_login_resources(conn);
+ iser_free_ff_resources(conn);
+
+ if (conn->qp_hndl) {
+ err = ibv_destroy_qp(conn->qp_hndl);
+ if (err)
+ eprintf("ibv_destroy_qp: %s\n", strerror(-err));
+ else
+ dprintf("destroyed qp:%p\n", conn->qp_hndl);
+ }
+ if (conn->cm_id) {
+ err = rdma_destroy_id(conn->cm_id);
+ if (err)
+ eprintf("rdma_destroy_id conn:%p cm_id:%p, %s\n",
+ &conn->h, conn->cm_id, strerror(-err));
+ else
+ dprintf("destroyed cm_id:%p\n", conn->cm_id);
+ }
+
+ /* delete from session; was put there by conn_add_to_session() */
+ list_del(&conn->h.clist);
+
+ if (conn->h.initiator)
+ free(conn->h.initiator);
+
+ if (conn->h.session)
+ session_put(conn->h.session);
+
+ conn->h.state = STATE_INIT;
+ free(conn);
+ dprintf("conn:%p freed\n", conn);
+}
+
+static void iser_sched_conn_free(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+
+ iser_conn_free(conn);
+}
+
+int iser_conn_get(struct iser_conn *conn)
+{
+ /* TODO: check state */
+ conn->h.refcount++;
+ dprintf("refcnt:%d\n", conn->h.refcount);
+ return 0;
+}
+
+void iser_conn_put(struct iser_conn *conn)
+{
+ conn->h.refcount--;
+ dprintf("refcnt:%d\n", conn->h.refcount);
+ if (!conn->h.refcount) {
+ assert(conn->h.state == STATE_CLOSE);
+ tgt_add_sched_event(&conn->sched_conn_free);
+ }
+}
+
+static int iser_show(struct iscsi_connection *iscsi_conn, char *buf,
+ int rest)
+{
+ struct iser_conn *conn = ISER_CONN(iscsi_conn);
+ int err;
+ char host[NI_MAXHOST];
+
+ err = getnameinfo((struct sockaddr *) &conn->peer_addr,
+ sizeof(conn->peer_addr), host, sizeof(host),
+ NULL, 0, NI_NUMERICHOST);
+ if (err) {
+ eprintf("getnameinfo: %m\n");
+ return 0;
+ }
+ return snprintf(buf, rest, "RDMA IP Address: %s", host);
+}
+
+static int iser_getsockname(struct iscsi_connection *iscsi_conn,
+ struct sockaddr *sa, socklen_t * len)
+{
+ struct iser_conn *conn = ISER_CONN(iscsi_conn);
+
+ if (*len > sizeof(conn->self_addr))
+ *len = sizeof(conn->self_addr);
+ memcpy(sa, &conn->self_addr, *len);
+ return 0;
+}
+
+static int iser_getpeername(struct iscsi_connection *iscsi_conn,
+ struct sockaddr *sa, socklen_t * len)
+{
+ struct iser_conn *conn = ISER_CONN(iscsi_conn);
+
+ if (*len > sizeof(conn->peer_addr))
+ *len = sizeof(conn->peer_addr);
+ memcpy(sa, &conn->peer_addr, *len);
+ return 0;
+}
+
+static void iser_cm_connect_request(struct rdma_cm_event *ev)
+{
+ struct rdma_cm_id *cm_id = ev->id;
+ struct ibv_qp_init_attr qp_init_attr;
+ struct iser_conn *conn;
+ struct iser_device *dev;
+ unsigned int hdrsz;
+ int err, dev_found;
+
+ struct rdma_conn_param conn_param = {
+ .responder_resources = 1,
+ .initiator_depth = 1,
+ .retry_count = 5,
+ };
+
+ /* find device */
+ dev_found = 0;
+ list_for_each_entry(dev, &iser_dev_list, list) {
+ if (dev->ibv_hndl == cm_id->verbs) {
+ dev_found = 1;
+ break;
+ }
+ }
+ if (!dev_found) {
+ dev = malloc(sizeof(*dev));
+ if (dev == NULL) {
+ eprintf("unable to allocate dev\n");
+ goto reject;
+ }
+ dev->ibv_hndl = cm_id->verbs;
+ err = iser_device_init(dev);
+ if (err) {
+ free(dev);
+ goto reject;
+ }
+ }
+
+ /* build a new connection structure */
+ conn = zalloc(sizeof(*conn));
+ if (!conn) {
+ eprintf("unable to allocate conn\n");
+ goto reject;
+ }
+
+ err = iser_conn_init(conn);
+ if (err) {
+ free(conn);
+ goto reject;
+ }
+ dprintf("Created conn:%p cm_id:%p\n", &conn->h, cm_id);
+ list_add(&conn->conn_list, &iser_conn_list);
+
+ /* ToDo: finalize these callbacks after tcp/iser code merge */
+ conn->h.op.conn_show = iser_show;
+ conn->h.op.conn_getsockname = iser_getsockname;
+ conn->h.op.conn_getpeername = iser_getpeername;
+
+ /* relate iser and rdma connections */
+ conn->cm_id = cm_id;
+ cm_id->context = conn;
+
+ conn->dev = dev;
+ conn->login_phase = LOGIN_PHASE_START;
+
+ tgt_init_sched_event(&conn->sched_buf_alloc, iser_sched_buf_alloc,
+ conn);
+ tgt_init_sched_event(&conn->sched_iosubmit, iser_sched_iosubmit,
+ conn);
+ tgt_init_sched_event(&conn->sched_rdma_rd, iser_sched_rdma_rd,
+ conn);
+ tgt_init_sched_event(&conn->sched_tx, iser_sched_tx,
+ conn);
+ tgt_init_sched_event(&conn->sched_post_recv, iser_sched_post_recv,
+ conn);
+ tgt_init_sched_event(&conn->sched_conn_free, iser_sched_conn_free,
+ conn);
+
+ /* initiator is dst, target is src */
+ memcpy(&conn->peer_addr, &cm_id->route.addr.dst_addr,
+ sizeof(conn->peer_addr));
+ memcpy(&conn->self_addr, &cm_id->route.addr.src_addr,
+ sizeof(conn->self_addr));
+#ifndef NDEBUG
+ {
+ char str[256];
+
+ iser_show(&conn->h, str, sizeof(str));
+ str[sizeof(str) - 1] = 0;
+ dprintf("new conn %p from %s\n", conn, str);
+ }
+#endif
+
+ /* create qp next */
+ memset(&qp_init_attr, 0, sizeof(qp_init_attr));
+ /* wire both send and recv to the same CQ */
+ qp_init_attr.send_cq = dev->cq;
+ qp_init_attr.recv_cq = dev->cq;
+ qp_init_attr.cap.max_send_wr = MAX_WQE;
+ qp_init_attr.cap.max_recv_wr = MAX_WQE;
+ qp_init_attr.cap.max_send_sge = 1; /* scatter/gather entries */
+ qp_init_attr.cap.max_recv_sge = 1;
+ qp_init_attr.qp_type = IBV_QPT_RC;
+ /* only generate completion queue entries if requested */
+ qp_init_attr.sq_sig_all = 0;
+
+ err = rdma_create_qp(cm_id, dev->pd, &qp_init_attr);
+ if (err) {
+ eprintf("create qp failed\n");
+ goto free_conn;
+ }
+ conn->qp_hndl = cm_id->qp;
+ VALGRIND_MAKE_MEM_DEFINED(conn->qp_hndl, sizeof(*conn->qp_hndl));
+
+ /*
+ * Post buffers for the login phase, only.
+ */
+ hdrsz = sizeof(struct iser_hdr) +
+ sizeof(struct iscsi_hdr) +
+ sizeof(struct iscsi_ecdb_ahdr) +
+ sizeof(struct iscsi_rlength_ahdr);
+ conn->ssize = hdrsz + 8192;
+ conn->rsize = hdrsz + 8192;
+ err = iser_alloc_login_resources(conn);
+ if (err)
+ goto free_conn;
+
+ conn_param.initiator_depth = dev->device_attr.max_qp_init_rd_atom;
+ if (conn_param.initiator_depth > ev->param.conn.initiator_depth)
+ conn_param.initiator_depth = ev->param.conn.initiator_depth;
+
+ /* Increment reference count to be able to wait for TIMEWAIT_EXIT
+ when finalizing the disconnect process */
+ iser_conn_get(conn);
+
+ /* now we can actually accept the connection */
+ err = rdma_accept(conn->cm_id, &conn_param);
+ if (err) {
+ eprintf("rdma_accept failed conn:%p cm_id:%p\n",
+ &conn->h, cm_id);
+ goto free_conn;
+ }
+
+ conn->h.state = STATE_START;
+ dprintf("accepted conn:%p cm_id:%p\n", &conn->h, cm_id);
+
+ return;
+
+free_conn:
+ iser_conn_free(conn);
+
+reject:
+ err = rdma_reject(cm_id, NULL, 0);
+ if (err)
+ eprintf("rdma_reject failed: %s\n", strerror(-err));
+}
+
+/*
+ * Finish putting the connection together, now that the other side
+ * has ACKed our acceptance. Moves it from the temp_conn to the
+ * iser_conn_list.
+ *
+ * Release the temporary conn_info and glue it into iser_conn_list.
+ */
+static void iser_cm_conn_established(struct rdma_cm_event *ev)
+{
+ struct rdma_cm_id *cm_id = ev->id;
+ struct iser_conn *conn = cm_id->context;
+
+ eprintf("conn:%p cm_id:%p\n", &conn->h, cm_id);
+ assert(conn->h.state == STATE_START);
+ conn->h.state = STATE_READY;
+}
+
+/*
+ * Handle RDMA_CM_EVENT_DISCONNECTED or an equivalent event.
+ * Start closing the target's side connection.
+*/
+static void iser_cm_disconnected(struct rdma_cm_event *ev)
+{
+ struct rdma_cm_id *cm_id = ev->id;
+ struct iser_conn *conn = cm_id->context;
+ enum rdma_cm_event_type ev_type = ev->event;
+
+ eprintf("conn:%p cm_id:%p event:%d, %s\n", &conn->h, cm_id,
+ ev_type, rdma_event_str(ev_type));
+ iser_conn_close(conn);
+}
+
+/*
+ * Handle RDMA_CM_EVENT_TIMEWAIT_EXIT which is expected to be the last
+ * event during the lifecycle of a connection, when it had been shut down
+ * and the network has cleared from the remaining in-flight messages.
+*/
+static void iser_cm_timewait_exit(struct rdma_cm_event *ev)
+{
+ struct rdma_cm_id *cm_id = ev->id;
+ struct iser_conn *conn = cm_id->context;
+
+ eprintf("conn:%p cm_id:%p\n", &conn->h, cm_id);
+
+ /* Refcount was incremented just before accepting the connection,
+ typically this is the last decrement and the connection will be
+ released instantly */
+ iser_conn_put(conn);
+}
+
+/*
+ * Handle RDMA CM events.
+ */
+static void iser_handle_rdmacm(int fd __attribute__ ((unused)),
+ int events __attribute__ ((unused)),
+ void *data __attribute__ ((unused)))
+{
+ struct rdma_cm_event *ev;
+ enum rdma_cm_event_type ev_type;
+ int err;
+
+ err = rdma_get_cm_event(rdma_evt_channel, &ev);
+ if (err) {
+ eprintf("rdma_get_cm_event failed\n");
+ return;
+ }
+
+ VALGRIND_MAKE_MEM_DEFINED(ev, sizeof(*ev));
+
+ ev_type = ev->event;
+ switch (ev_type) {
+ case RDMA_CM_EVENT_CONNECT_REQUEST:
+ iser_cm_connect_request(ev);
+ break;
+
+ case RDMA_CM_EVENT_ESTABLISHED:
+ iser_cm_conn_established(ev);
+ break;
+
+ case RDMA_CM_EVENT_CONNECT_ERROR:
+ case RDMA_CM_EVENT_REJECTED:
+ case RDMA_CM_EVENT_ADDR_CHANGE:
+ case RDMA_CM_EVENT_DISCONNECTED:
+ iser_cm_disconnected(ev);
+ break;
+
+ case RDMA_CM_EVENT_TIMEWAIT_EXIT:
+ iser_cm_timewait_exit(ev);
+ break;
+
+ case RDMA_CM_EVENT_MULTICAST_JOIN:
+ case RDMA_CM_EVENT_MULTICAST_ERROR:
+ eprintf("UD-related event:%d, %s - ignored\n", ev_type,
+ rdma_event_str(ev_type));
+ break;
+
+ case RDMA_CM_EVENT_DEVICE_REMOVAL:
+ eprintf("Unsupported event:%d, %s - ignored\n", ev_type,
+ rdma_event_str(ev_type));
+ break;
+
+ case RDMA_CM_EVENT_ADDR_RESOLVED:
+ case RDMA_CM_EVENT_ADDR_ERROR:
+ case RDMA_CM_EVENT_ROUTE_RESOLVED:
+ case RDMA_CM_EVENT_ROUTE_ERROR:
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ case RDMA_CM_EVENT_UNREACHABLE:
+ eprintf("Active side event:%d, %s - ignored\n", ev_type,
+ rdma_event_str(ev_type));
+ break;
+
+ default:
+ eprintf("Illegal event:%d - ignored\n", ev_type);
+ break;
+ }
+
+ err = rdma_ack_cm_event(ev);
+ if (err) {
+ eprintf("ack cm event failed, %s\n",
+ rdma_event_str(ev_type));
+ }
+}
+
+static inline void schedule_task_iosubmit(struct iser_task *task,
+ struct iser_conn *conn)
+{
+ struct iscsi_cmd *req_bhs = (struct iscsi_cmd *) task->req.bhs;
+
+ list_add_tail(&task->exec_list, &conn->iosubmit_list);
+ tgt_add_sched_event(&conn->sched_iosubmit);
+ dprintf("task:%p tag:0x%04lx cmdsn:0x%x\n",
+ task, (unsigned long)task->tag,
+ be32_to_cpu(req_bhs->cmdsn));
+}
+
+static inline void schedule_rdma_read(struct iser_task *task,
+ struct iser_conn *conn)
+{
+ list_add_tail(&task->rdma_list, &conn->rdma_rd_list);
+ tgt_add_sched_event(&conn->sched_rdma_rd);
+ dprintf("task:%p tag:0x%04lx\n", task, (unsigned long)task->tag);
+}
+
+static inline void schedule_resp_tx(struct iser_task *task,
+ struct iser_conn *conn)
+{
+ dprintf("task:%p tag:0x%04lx\n", task, (unsigned long)task->tag);
+
+ list_add_tail(&task->tx_list, &conn->resp_tx_list);
+ tgt_add_sched_event(&conn->sched_tx);
+}
+
+static inline void schedule_post_recv(struct iser_task *task,
+ struct iser_conn *conn)
+{
+ list_add_tail(&task->recv_list, &conn->post_recv_list);
+ tgt_add_sched_event(&conn->sched_post_recv);
+ dprintf("task:%p tag:0x%04lx\n", task, (unsigned long)task->tag);
+}
+
+static int iser_logout_exec(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_logout *req = (struct iscsi_logout *) task->req.bhs;
+ uint32_t cmd_sn = be32_to_cpu(req->cmdsn);
+ int is_immediate = req->opcode & ISCSI_OP_IMMEDIATE;
+ struct iscsi_logout_rsp *rsp =
+ (struct iscsi_logout_rsp *) task->rsp.bhs;
+
+ memset(rsp, 0, BHS_SIZE);
+ rsp->opcode = ISCSI_OP_LOGOUT_RSP;
+ rsp->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp->itt = req->itt;
+ rsp->statsn = cpu_to_be32(conn->h.stat_sn++);
+ if (session->exp_cmd_sn == cmd_sn && !is_immediate)
+ session->exp_cmd_sn++;
+ iser_set_rsp_cmd_sn(session, task->rsp.bhs);
+
+ task->rsp.ahssize = 0;
+ task->rsp.data.size = 0;
+
+ schedule_resp_tx(task, conn);
+ return 0;
+}
+
+static int iser_nop_out_exec(struct iser_task *task)
+{
+ struct iscsi_nopin *rsp_bhs =
+ (struct iscsi_nopin *) task->rsp.bhs;
+ struct iser_conn *conn = task->conn;
+
+ rsp_bhs->opcode = ISCSI_OP_NOOP_IN;
+ rsp_bhs->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp_bhs->rsvd2 = 0;
+ rsp_bhs->rsvd3 = 0;
+ memset(rsp_bhs->lun, 0, sizeof(rsp_bhs->lun));
+ rsp_bhs->itt = task->req.bhs->itt;
+ rsp_bhs->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
+ rsp_bhs->statsn = cpu_to_be32(conn->h.stat_sn);
+ if (task->req.bhs->itt != cpu_to_be32(ISCSI_RESERVED_TAG))
+ conn->h.stat_sn++;
+
+ iser_set_rsp_cmd_sn(conn->h.session, task->rsp.bhs);
+
+ memset(rsp_bhs->rsvd4, 0, sizeof(rsp_bhs->rsvd4));
+ task->rsp.ahssize = 0;
+
+ if (task->out_len > 0)
+ task->rsp.data.buf = task->req.data.buf;
+ task->rsp.data.size = task->out_len;
+
+ schedule_resp_tx(task, conn);
+ return 0;
+}
+
+static int iser_send_ping_nop_in(struct iser_task *task)
+{
+ struct iscsi_nopin *rsp_bhs =
+ (struct iscsi_nopin *) task->rsp.bhs;
+ struct iser_conn *conn = task->conn;
+
+ task->opcode = ISCSI_OP_NOOP_IN;
+ task->tag = 0xffffffff;
+
+ rsp_bhs->opcode = ISCSI_OP_NOOP_IN;
+ rsp_bhs->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp_bhs->rsvd2 = 0;
+ rsp_bhs->rsvd3 = 0;
+ memset(rsp_bhs->lun, 0, sizeof(rsp_bhs->lun));
+ rsp_bhs->itt = 0xffffffff;
+ rsp_bhs->ttt = 0xffffffff;
+ rsp_bhs->statsn = cpu_to_be32(conn->h.stat_sn);
+ iser_set_rsp_cmd_sn(conn->h.session, task->rsp.bhs);
+
+ memset(rsp_bhs->rsvd4, 0, sizeof(rsp_bhs->rsvd4));
+ task->rsp.ahssize = 0;
+ task->rsp.data.size = 0;
+
+ dprintf("task:%p conn:%p\n", task, &conn->h);
+
+ schedule_resp_tx(task, conn);
+ return 0;
+}
+
+/*
+static int iser_send_reject(struct iser_task *task, uint8 reason)
+{
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_hdr *req = (struct iscsi_hdr *) task->req.bhs;
+ struct iscsi_reject *rsp = (struct iscsi_reject *) task->rsp.bhs;
+
+ memset(rsp, 0, BHS_SIZE);
+ rsp->opcode = ISCSI_OP_REJECT;
+ rsp->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp->reason = reason;
+ hton24(rsp->dlength, sizeof(struct iser_hdr));
+ rsp->ffffffff = 0xffffffff;
+ rsp->itt = req->itt;
+ rsp->statsn = cpu_to_be32(conn->h.stat_sn++);
+ iser_set_rsp_cmd_sn(session, task->rsp.bhs);
+
+ task->rsp.ahssize = 0;
+ task->rsp.data.buf = task->req.bhs;
+ task->rsp.data.size = sizeof(struct iser_hdr);
+
+ schedule_resp_tx(task, conn);
+ return 0;
+}
+*/
+
+static int iser_tm_exec(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_tm *req_bhs = (struct iscsi_tm *) task->req.bhs;
+ int fn = 0;
+ int err = 0;
+
+ eprintf("TM itt:0x%x cmdsn:0x%x "
+ "ref.itt:0x%x ref.cmdsn:0x%x "
+ "exp.cmdsn:0x%x "
+ "lun:0x%02x%02x%02x%02x%02x%02x%02x%02x\n",
+ be32_to_cpu(req_bhs->itt), be32_to_cpu(req_bhs->cmdsn),
+ be32_to_cpu(req_bhs->rtt), be32_to_cpu(req_bhs->refcmdsn),
+ be32_to_cpu(req_bhs->exp_statsn),
+ req_bhs->lun[0], req_bhs->lun[1], req_bhs->lun[2], req_bhs->lun[3],
+ req_bhs->lun[4], req_bhs->lun[5], req_bhs->lun[6], req_bhs->lun[7]);
+
+ switch (req_bhs->flags & ISCSI_FLAG_TM_FUNC_MASK) {
+ case ISCSI_TM_FUNC_ABORT_TASK:
+ fn = ABORT_TASK;
+ break;
+ case ISCSI_TM_FUNC_ABORT_TASK_SET:
+ fn = ABORT_TASK_SET;
+ break;
+ case ISCSI_TM_FUNC_CLEAR_ACA:
+ fn = CLEAR_TASK_SET;
+ break;
+ case ISCSI_TM_FUNC_CLEAR_TASK_SET:
+ fn = CLEAR_ACA;
+ break;
+ case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
+ fn = LOGICAL_UNIT_RESET;
+ break;
+ case ISCSI_TM_FUNC_TARGET_WARM_RESET:
+ case ISCSI_TM_FUNC_TARGET_COLD_RESET:
+ case ISCSI_TM_FUNC_TASK_REASSIGN:
+ err = ISCSI_TMF_RSP_NOT_SUPPORTED;
+ eprintf("unsupported TMF %d\n",
+ req_bhs->flags & ISCSI_FLAG_TM_FUNC_MASK);
+ break;
+ default:
+ err = ISCSI_TMF_RSP_REJECTED;
+ eprintf("unknown TMF %d\n",
+ req_bhs->flags & ISCSI_FLAG_TM_FUNC_MASK);
+ }
+
+ if (err)
+ task->result = err;
+ else
+ target_mgmt_request(session->target->tid, session->tsih,
+ (unsigned long) task, fn, req_bhs->lun,
+ req_bhs->rtt, 0);
+ return err;
+}
+
+static int iser_tm_done(struct mgmt_req *mreq)
+{
+ struct iser_task *task =
+ (struct iser_task *) (unsigned long) mreq->mid;
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_tm_rsp *rsp_bhs =
+ (struct iscsi_tm_rsp *) task->rsp.bhs;
+
+ memset(rsp_bhs, 0, sizeof(*rsp_bhs));
+ rsp_bhs->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
+ rsp_bhs->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp_bhs->itt = task->req.bhs->itt;
+ switch (mreq->result) {
+ case 0:
+ rsp_bhs->response = ISCSI_TMF_RSP_COMPLETE;
+ break;
+ case -EINVAL:
+ rsp_bhs->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
+ break;
+ case -EEXIST:
+ /*
+ * the command completed or we could not find it so
+ * we retrun no task here
+ */
+ rsp_bhs->response = ISCSI_TMF_RSP_NO_TASK;
+ break;
+ default:
+ rsp_bhs->response = ISCSI_TMF_RSP_REJECTED;
+ break;
+ }
+
+ rsp_bhs->statsn = cpu_to_be32(conn->h.stat_sn++);
+ iser_set_rsp_cmd_sn(session, task->rsp.bhs);
+
+ // Must be zero according to 10.6.3
+ task->rsp.ahssize = 0;
+ task->rsp.data.size = 0;
+
+ schedule_resp_tx(task, conn);
+
+ return 0;
+}
+
+static int iser_scsi_cmd_buf_alloc(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct membuf *mem;
+
+ // when a buffer needs to be allocated and it fails, leave task on
+ // the list waiting until a new buffer becomes available
+ if (task->is_write) {
+ if (task->out_len > 0) {
+ if (task->rdma_rd_sz > 0) {
+ // ToDo: multiple RDMA-Read buffers
+ mem = iser_alloc_rdma_buf(conn, task->rdma_rd_sz);
+ if (mem == NULL) {
+ eprintf("free list empty, task:%p tag:0x%04lx\n",
+ task, (unsigned long)task->tag);
+ return -ENOMEM;
+ }
+ dprintf("out-buf:%p sz:%u rdma:%d\n",
+ mem->buf, mem->size, mem->rdma);
+ // ToDo: multiple RDMA-Read buffers
+ iser_task_add_out_rdma_buf(task, mem, task->unsol_sz);
+ schedule_rdma_read(task, conn);
+ return 0;
+ }
+ if (task->unsol_remains > 0)
+ return 0;
+ }
+ } // ToDo: bi-dir
+ else if (task->is_read) {
+ if (task->in_len > 0) {
+ mem = iser_alloc_rdma_buf(conn, task->rdma_wr_sz);
+ if (mem == NULL) {
+ eprintf("free list empty, task:%p tag:0x%04lx\n",
+ task, (unsigned long)task->tag);
+ return -ENOMEM;
+ }
+ dprintf("in-buf:%p sz:%u rdma:%d\n",
+ mem->buf, mem->size, mem->rdma);
+ // ToDo: multiple rdma write bufs
+ iser_task_add_in_rdma_buf(task, mem, 0);
+ }
+ }
+ schedule_task_iosubmit(task, conn);
+ return 0;
+}
+
+static int scsi_cmd_attr(unsigned int flags)
+{
+ int attr;
+
+ switch (flags & ISCSI_FLAG_CMD_ATTR_MASK) {
+ case ISCSI_ATTR_UNTAGGED:
+ case ISCSI_ATTR_SIMPLE:
+ attr = MSG_SIMPLE_TAG;
+ break;
+ case ISCSI_ATTR_HEAD_OF_QUEUE:
+ attr = MSG_HEAD_TAG;
+ break;
+ case ISCSI_ATTR_ORDERED:
+ default:
+ attr = MSG_ORDERED_TAG;
+ }
+ return attr;
+}
+
+static void iser_task_free_dout_tasks(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct iser_task *dout_task, *tnext;
+
+ list_for_each_entry_safe(dout_task, tnext, &task->dout_task_list, dout_task_list) {
+ list_del(&dout_task->dout_task_list);
+ iser_task_del_out_buf(task, &dout_task->req.data);
+ schedule_post_recv(dout_task, conn);
+ }
+}
+
+static void iser_scsi_cmd_iosubmit(struct iser_task *task)
+{
+ struct iscsi_cmd *req_bhs = (struct iscsi_cmd *) task->req.bhs;
+ struct scsi_cmd *scmd = &task->scmd;
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct membuf *data;
+
+ // ToDo: implement iser_task_handle_ahs(itask);
+
+ scmd->state = 0;
+ scsi_set_in_length(scmd, 0);
+ scsi_set_out_length(scmd, 0);
+
+ if (task->is_write) {
+ scsi_set_out_resid(scmd, 0);
+ /* It's either the last buffer, which is RDMA, or the only buffer */
+ data = list_entry(task->out_buf_list.prev, struct membuf, task_list);
+ // ToDo: multiple RDMA-Write buffers
+ if (task->out_buf_num > 1) {
+ unsigned char *ptr = data->buf;
+ struct membuf *cur_buf;
+
+ list_for_each_entry(cur_buf, &task->out_buf_list, task_list) {
+ if (cur_buf->rdma)
+ break;
+ memcpy(ptr, cur_buf->buf, cur_buf->size);
+ ptr += cur_buf->size;
+ }
+ iser_task_free_dout_tasks(task);
+ }
+
+ scsi_set_out_buffer(scmd, data->buf);
+ scsi_set_out_length(scmd, task->out_len);
+ }
+ if (task->is_read) {
+ scsi_set_in_resid(scmd, 0);
+ // ToDo: multiple RDMA-Read buffers
+ data = list_entry(task->in_buf_list.next, struct membuf, task_list);
+ scsi_set_in_buffer(scmd, data->buf);
+ scsi_set_in_length(scmd, task->in_len);
+ }
+
+ scmd->cmd_itn_id = session->tsih;
+ scmd->scb = req_bhs->cdb;
+ scmd->scb_len = sizeof(req_bhs->cdb);
+ memcpy(scmd->lun, req_bhs->lun, sizeof(scmd->lun));
+ scmd->attribute = scsi_cmd_attr(req_bhs->flags);
+ scmd->tag = task->tag; //req_bhs->itt;
+ scmd->result = 0;
+ scmd->mreq = NULL;
+ scmd->sense_len = 0;
+
+ dprintf("task:%p tag:0x%04lx\n", task, (unsigned long)task->tag);
+
+ // ToDo: handle bidi ahs and allocations
+ // the problem is that there is only one buffer
+ // in scsi command
+
+ target_cmd_queue(session->target->tid, scmd);
+}
+
+static void iser_rsp_set_read_resid(struct iscsi_cmd_rsp *rsp_bhs,
+ int in_resid)
+{
+ if (in_resid > 0) {
+ rsp_bhs->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
+ rsp_bhs->residual_count = cpu_to_be32((uint32_t)in_resid);
+ }
+ else {
+ rsp_bhs->flags |= ISCSI_FLAG_CMD_OVERFLOW;
+ rsp_bhs->residual_count = cpu_to_be32(((uint32_t)-in_resid));
+ }
+ rsp_bhs->bi_residual_count = 0;
+}
+
+static void iser_rsp_set_bidir_resid(struct iscsi_cmd_rsp *rsp_bhs,
+ int in_resid)
+{
+ if (in_resid > 0) {
+ rsp_bhs->flags |= ISCSI_FLAG_CMD_BIDI_UNDERFLOW;
+ rsp_bhs->bi_residual_count = cpu_to_be32((uint32_t)in_resid);
+ }
+ else {
+ rsp_bhs->flags |= ISCSI_FLAG_CMD_BIDI_OVERFLOW;
+ rsp_bhs->bi_residual_count = cpu_to_be32(((uint32_t)-in_resid));
+ }
+ rsp_bhs->residual_count = 0;
+}
+
+static int iser_scsi_cmd_done(uint64_t nid, int result,
+ struct scsi_cmd *scmd)
+{
+ struct iser_task *task = container_of(scmd, struct iser_task, scmd);
+ struct iscsi_cmd_rsp *rsp_bhs = (struct iscsi_cmd_rsp *) task->rsp.bhs;
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ unsigned char sense_len = scmd->sense_len;
+ int in_resid;
+
+ if (nid != scmd->cmd_itn_id) {
+ eprintf("unexpected itn_id in scsi command\n");
+ exit(1);
+ }
+
+ if (conn->h.state == STATE_CLOSE) {
+ /* Connection is closed, but its resources are not released
+ to allow receiving completion of such late tasks.
+ When all tasks are released, and connection refcnt
+ drops to zero, then all the resources can be freed. */
+ iser_complete_task(task);
+ return 0;
+ }
+
+ rsp_bhs->opcode = ISCSI_OP_SCSI_CMD_RSP;
+ rsp_bhs->flags = ISCSI_FLAG_CMD_FINAL;
+ rsp_bhs->response = ISCSI_STATUS_CMD_COMPLETED;
+ rsp_bhs->cmd_status = scsi_get_result(scmd);
+ *((uint64_t *) rsp_bhs->rsvd) = (uint64_t) 0;
+ rsp_bhs->itt = task->tag;
+ rsp_bhs->rsvd1 = 0;
+ rsp_bhs->statsn = cpu_to_be32(conn->h.stat_sn++);
+ iser_set_rsp_cmd_sn(session, task->rsp.bhs);
+ rsp_bhs->exp_datasn = 0;
+
+ if (task->is_read)
+ {
+ in_resid = scsi_get_in_resid(scmd);
+ if (in_resid != 0) {
+ if (!task->is_write)
+ iser_rsp_set_read_resid(rsp_bhs, in_resid);
+ else
+ iser_rsp_set_bidir_resid(rsp_bhs, in_resid);
+ if (in_resid > 0)
+ task->rdma_wr_remains = task->in_len - in_resid;
+ }
+ //schedule_rdma_write(task, conn);
+ }
+ else {
+ rsp_bhs->bi_residual_count = 0;
+ rsp_bhs->residual_count = 0;
+ }
+ task->rsp.ahssize = 0;
+ task->rsp.data.size = 0;
+
+ if (sense_len) {
+ struct iscsi_sense_data *sense =
+ (struct iscsi_sense_data *) task->rsp.data.buf;
+ sense->length = cpu_to_be16(sense_len);
+ memcpy(sense->data, scmd->sense_buffer, sense_len);
+ task->rsp.data.size = sense_len + sizeof(*sense);
+ }
+ dprintf("task:%p tag:0x%04lx status:%x statsn:%d flags:0x%x "
+ "in_len:%d out_len:%d rsd:%d birsd:%d\n",
+ task, (unsigned long)task->tag, rsp_bhs->cmd_status,
+ conn->h.stat_sn-1, rsp_bhs->flags,
+ scsi_get_in_length(scmd), scsi_get_out_length(scmd),
+ ntohl(rsp_bhs->residual_count),
+ ntohl(rsp_bhs->bi_residual_count));
+
+ schedule_resp_tx(task, conn);
+
+ return 0;
+}
+
+/*
+static int iser_task_handle_ahs(struct iser_task *task)
+{
+ struct scsi_cmd *scmd = &task->scmd;
+ struct iscsi_connection *conn = task->conn;
+ struct iscsi_cmd *req_bhs = (struct iscsi_cmd *) &task->req.;
+ uint32_t data_len;
+ uint8_t *ahs;
+ int ahslen;
+ enum data_direction dir = scsi_get_data_dir(scmd);
+
+ ahs = task->ahs;
+ ahslen = req_bhs->hlength * 4;
+ if (ahslen >= 4) {
+ struct iscsi_ecdb_ahdr *ahs_extcdb = (void *) ahs;
+
+ if (ahs_extcdb->ahstype == ISCSI_AHSTYPE_CDB) {
+ int extcdb_len = ntohs(ahs_extcdb->ahslength) - 1;
+ unsigned char *p = (void *)task->extdata;
+
+ if (4 + extcdb_len > ahslen) {
+ eprintf("AHS len:%d too short for extcdb %d\n",
+ ahslen, extcdb_len);
+ return -EINVAL;
+ }
+ if (extcdb_len + sizeof(req_bhs->cdb) > 260) {
+ eprintf("invalid extcdb len:%d\n", extcdb_len);
+
+ return -EINVAL;
+ }
+
+ memcpy(p, req_bhs->cdb, sizeof(req_bhs->cdb));
+ memmove(p + sizeof(req_bhs->cdb), ahs_extcdb->ecdb,
+ extcdb_len);
+
+ scmd->scb = p;
+ scmd->scb_len = sizeof(req_bhs->cdb) + extcdb_len;
+
+ ahs += 4 + extcdb_len;
+ ahslen -= 4 + extcdb_len;
+ }
+ }
+
+ if (dir == DATA_BIDIRECTIONAL && ahslen >= 8) {
+ struct iscsi_rlength_ahdr *ahs_bidi = (void *) ahs;
+ if (ahs_bidi->ahstype == ISCSI_AHSTYPE_RLENGTH) {
+ uint32_t in_length = ntohl(ahs_bidi->read_length);
+
+ dprintf("bidi read len %u\n", in_length);
+
+ if (in_length) {
+ uint32_t len;
+ void *buf;
+
+ len = roundup(in_length,
+ conn->tp->data_padding);
+ buf = conn->tp->alloc_data_buf(conn, len);
+ if (!buf)
+ return -ENOMEM;
+
+ scsi_set_in_buffer(scmd, buf);
+ scsi_set_in_length(scmd, in_length);
+ }
+ }
+ }
+}
+*/
+
+static void iser_sched_buf_alloc(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+ struct iser_task *task;
+ int err;
+
+ while (!list_empty(&conn->buf_alloc_list)) {
+ task = list_first_entry(&conn->buf_alloc_list,
+ struct iser_task,
+ exec_list);
+ err = iser_scsi_cmd_buf_alloc(task);
+ if (err == -ENOMEM)
+ break;
+ list_del(&task->exec_list);
+ }
+}
+
+static inline int task_batch_type(struct iser_task *task)
+{
+ struct iscsi_cmd *req_bhs = (struct iscsi_cmd *) task->req.bhs;
+
+ switch (req_bhs->cdb[0]) {
+ case SYNCHRONIZE_CACHE:
+ case SYNCHRONIZE_CACHE_16:
+ case WRITE_6:
+ case WRITE_10:
+ case WRITE_12:
+ case WRITE_16:
+ case READ_6:
+ case READ_10:
+ case READ_12:
+ case READ_16:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+static void iser_sched_iosubmit(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+ struct iser_task *task = NULL;
+
+ while (!list_empty(&conn->iosubmit_list)) {
+ task = list_first_entry(&conn->iosubmit_list,
+ struct iser_task, exec_list);
+ list_del(&task->exec_list);
+ iser_scsi_cmd_iosubmit(task);
+ }
+}
+
+static void iser_sched_rdma_rd(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+ struct iser_work_req *first_wr = NULL;
+ struct iser_task *prev_task = NULL;
+ struct iser_task *task = NULL;
+ int err;
+
+ if (conn->h.state == STATE_CLOSE) {
+ dprintf("ignoring rdma_rd for closed conn\n");
+ // ToDo: free all tasks and buffers
+ return;
+ }
+
+ while (!list_empty(&conn->rdma_rd_list)) {
+ task = list_first_entry(&conn->rdma_rd_list,
+ struct iser_task, rdma_list);
+ list_del(&task->rdma_list);
+
+ iser_prep_rdma_rd_send_req(task, NULL, 1);
+ if (first_wr == NULL)
+ first_wr = &task->rdmad;
+ else
+ prev_task->rdmad.send_wr.next = &task->rdmad.send_wr;
+ prev_task = task;
+ }
+ if (prev_task) {
+ prev_task->rdmad.send_wr.next = NULL;
+ // submit the chain of rdma-rd requests, start from the first
+ err = iser_post_send(conn, first_wr);
+ }
+ // ToDo: error handling
+}
+
+static void iser_sched_tx(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+ struct iser_work_req *first_wr = NULL;
+ struct iser_task *prev_task = NULL;
+ struct iser_task *task;
+ struct iser_work_req *cur_send_wr;
+ int err;
+
+ if (conn->h.state == STATE_CLOSE) {
+ dprintf("ignoring tx for closed conn\n");
+ return;
+ }
+
+ while (!list_empty(&conn->resp_tx_list)) {
+ task = list_first_entry(&conn->resp_tx_list,
+ struct iser_task,
+ tx_list);
+ list_del(&task->tx_list);
+ list_add_tail(&task->tx_list, &conn->sent_list);
+
+ if (task->is_read) {
+ iser_prep_rdma_wr_send_req(task, &task->txd, 0);
+ cur_send_wr = &task->rdmad;
+ }
+ else
+ cur_send_wr = &task->txd;
+
+ if (prev_task == NULL)
+ first_wr = cur_send_wr;
+ else
+ prev_task->txd.send_wr.next = &cur_send_wr->send_wr;
+
+ iser_prep_resp_send_req(task, NULL, 1);
+ prev_task = task;
+ }
+ if (prev_task) {
+ prev_task->txd.send_wr.next = NULL;
+ // submit the chain of rdma-wr & tx reqs, start from the first
+ err = iser_post_send(conn, first_wr);
+ // ToDo: error handling
+ }
+}
+
+static void iser_sched_post_recv(struct event_data *evt)
+{
+ struct iser_conn *conn = (struct iser_conn *) evt->data;
+ struct iser_task *first_task = NULL;
+ struct iser_task *prev_task = NULL;
+ struct iser_task *task;
+ int num_recv_bufs = 0;
+ int err;
+
+ if (conn->h.state == STATE_CLOSE) {
+ dprintf("ignoring post recv for closed conn\n");
+ return;
+ }
+
+ while (!list_empty(&conn->post_recv_list)) {
+ task = list_first_entry(&conn->post_recv_list,
+ struct iser_task,
+ recv_list);
+ list_del(&task->recv_list);
+
+ if (prev_task == NULL)
+ first_task = task;
+ else
+ prev_task->rxd.recv_wr.next = &task->rxd.recv_wr;
+
+ prev_task = task;
+ num_recv_bufs ++;
+ }
+ if (prev_task) {
+ prev_task->rxd.recv_wr.next = NULL;
+ // post the chain of recv buffers, start from the first
+ err = iser_post_recv(conn, first_task, num_recv_bufs);
+ // ToDo: error handling
+ }
+}
+
+static int iser_scsi_cmd_rx(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_cmd *req_bhs = (struct iscsi_cmd *) task->req.bhs;
+ unsigned int flags = req_bhs->flags;
+ uint32_t imm_data_sz = ntoh24(req_bhs->dlength);
+ uint32_t xfer_sz = ntohl(req_bhs->data_length);
+
+ task->is_read = flags & ISCSI_FLAG_CMD_READ;
+ task->is_write = flags & ISCSI_FLAG_CMD_WRITE;
+
+ if (task->is_write) {
+ task->out_len = xfer_sz;
+
+ /* add immediate data to the task */
+
+
+ if (!conn->h.session_param[ISCSI_PARAM_INITIAL_R2T_EN].val) {
+ task->unsol_sz = conn->h.session_param[ISCSI_PARAM_FIRST_BURST].val;
+ if (task->out_len < task->unsol_sz)
+ task->unsol_sz = task->out_len;
+ }
+ else
+ task->unsol_sz = 0;
+
+ if (conn->h.session_param[ISCSI_PARAM_IMM_DATA_EN].val) {
+ if (imm_data_sz > 0) {
+ if (task->unsol_sz == 0)
+ task->unsol_sz = imm_data_sz;
+ iser_task_add_out_pdu_buf(task, &task->req.data, 0);
+ }
+ }
+ else if (imm_data_sz > 0) {
+ eprintf("ImmediateData disabled but received\n");
+ return -EINVAL;
+ }
+
+ /* immediate data is the first chunk of the unsolicited data */
+ task->unsol_remains = task->unsol_sz - imm_data_sz;
+ /* rdma-reads cover the entire solicited data */
+ task->rdma_rd_sz = task->rdma_rd_remains =
+ task->out_len - task->unsol_sz;
+ // ToDo: multiple RDMA-Write buffers
+ // task->rdma_rd_offset = task->unsol_sz;
+
+ task->in_len = task->rdma_wr_sz = task->rdma_wr_remains = 0;
+
+ if (!task->is_read)
+ scsi_set_data_dir(&task->scmd, DATA_WRITE);
+ else
+ scsi_set_data_dir(&task->scmd, DATA_BIDIRECTIONAL);
+ } else {
+ task->in_len = task->rdma_wr_sz =
+ task->rdma_wr_remains = xfer_sz;
+ task->out_len = task->unsol_sz = task->unsol_remains =
+ task->rdma_rd_sz = task->rdma_rd_remains = 0;
+
+ if (task->is_read)
+ scsi_set_data_dir(&task->scmd, DATA_READ);
+ else
+ scsi_set_data_dir(&task->scmd, DATA_NONE);
+ }
+
+ list_add_tail(&task->session_list, &session->cmd_list);
+
+ dprintf("task:%p tag:0x%04lx scsi_op:0x%x %s%s in_len:%d out_len:%d "
+ "imm_sz:%d unsol_sz:%d cmdsn:0x%x expcmdsn:0x%x\n",
+ task, (long unsigned)task->tag, req_bhs->cdb[0],
+ task->is_read ? "rd" : "", task->is_write ? "wr" : "",
+ task->in_len, task->out_len, imm_data_sz, task->unsol_sz,
+ ntohl(req_bhs->cmdsn), session->exp_cmd_sn);
+
+ return 0;
+}
+
+static int iser_data_out_rx(struct iser_task *dout_task)
+{
+ struct iser_conn *conn = dout_task->conn;
+ struct iscsi_session *session = conn->h.session;
+ struct iscsi_data *req_bhs =
+ (struct iscsi_data *) dout_task->req.bhs;
+ struct iser_task *task;
+ int err = 0;
+
+ list_for_each_entry(task, &session->cmd_list, session_list) {
+ if (task->tag == req_bhs->itt)
+ goto found;
+ }
+ return -EINVAL;
+
+found:
+ iser_task_add_out_pdu_buf(task, &dout_task->req.data,
+ be32_to_cpu(req_bhs->offset));
+ list_add_tail(&dout_task->dout_task_list, &task->dout_task_list);
+
+ // ToDo: BUG!!! add an indication that it's data-out task so that
+ // it can be released when the buffer is released
+
+ task->unsol_remains -= ntoh24(req_bhs->dlength);
+
+ dprintf("task:%p tag:0x%04lx, dout taskptr:%p out_len:%d "
+ "uns_rem:%d rdma_rd_rem:%d expcmdsn:0x%x\n",
+ task, (long unsigned)task->tag, dout_task, task->out_len,
+ task->unsol_remains, task->rdma_rd_remains,
+ session->exp_cmd_sn);
+
+ // ToDo: look at the task counters !!!
+ if (req_bhs->ttt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
+ if (req_bhs->flags & ISCSI_FLAG_CMD_FINAL) {
+ if (!task_pending(task)) {
+ // ToDo: this condition is weird ...
+ if (task->rdma_rd_remains == 0 && task->unsol_remains == 0)
+ schedule_task_iosubmit(task,conn);
+ }
+ }
+ } else {
+ if (!(req_bhs->flags & ISCSI_FLAG_CMD_FINAL))
+ return err;
+
+ if (task->rdma_rd_remains == 0 && task->unsol_remains == 0)
+ schedule_task_iosubmit(task,conn);
+ }
+ return err;
+}
+
+static void iser_pdu_copy(struct iser_pdu *dst_pdu, struct iser_pdu *src_pdu)
+{
+ memcpy(dst_pdu->iser_hdr, src_pdu->iser_hdr, sizeof(*src_pdu->iser_hdr));
+ memcpy(dst_pdu->bhs, src_pdu->bhs, sizeof(*src_pdu->bhs));
+ dst_pdu->ahssize = src_pdu->ahssize;
+ // ToDo ahs alloc and copy
+
+ dst_pdu->data.size = src_pdu->data.size;
+ memcpy(dst_pdu->data.buf, src_pdu->data.buf, src_pdu->data.size);
+}
+
+static void iser_login_rx(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+ struct iser_task *tx_task = conn->login_tx_task;
+
+ iser_login_exec(task);
+ iser_pdu_copy(&tx_task->rsp, &task->rsp);
+
+ if (conn->login_phase != LOGIN_PHASE_LAST_SEND)
+ iser_post_recv(conn, task, 1);
+ else {
+ dprintf("transitioning to full-feature, no repost\n");
+ }
+ iser_prep_resp_send_req(tx_task, NULL, 1);
+ iser_post_send(conn, &tx_task->txd);
+}
+
+static int iser_nop_out_rx(struct iser_task *task)
+{
+ struct iser_conn *conn = task->conn;
+// struct iscsi_session *session = conn->h.session;
+ struct iscsi_nopout *req_bhs = (struct iscsi_nopout *) task->req.bhs;
+ int err = 0;
+
+ if (req_bhs->ttt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
+ /*
+ * We don't request a NOP-Out by sending a NOP-In.
+ * See 10.18.2 in the draft 20.
+ */
+ eprintf("initiator bug\n");
+ err = -ISCSI_REASON_PROTOCOL_ERROR;
+ goto reject;
+ }
+ if (req_bhs->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
+ if (req_bhs->opcode & ISCSI_OP_IMMEDIATE) {
+ dprintf("No response to Nop-Out\n");
+ iser_post_recv(conn, task, 1);
+ return -EAGAIN; // ToDo: fix the ret codes
+ } else {
+ eprintf("initiator bug\n");
+ err = -ISCSI_REASON_PROTOCOL_ERROR;
+ goto reject;
+ }
+ }
+
+ task->out_len = ntoh24(req_bhs->dlength);
+ dprintf("nop-out task:%p cmdsn:0x%x data_sz:%d\n",
+ task, ntohl(req_bhs->cmdsn), task->out_len);
+
+ return 0;
+
+reject: // ToDo: prepare and send reject pdu
+ /* iser_send_reject(task, ISCSI_REASON_INVALID_PDU_FIELD); */
+ return err;
+}
+
+static int iser_task_delivery(struct iser_task *task)
+{
+ int err;
+
+ switch (task->opcode) {
+ case ISCSI_OP_SCSI_CMD:
+ err = iser_scsi_cmd_buf_alloc(task);
+ if (err == -ENOMEM)
+ list_add_tail(&task->exec_list,
+ &task->conn->buf_alloc_list);
+ break;
+ case ISCSI_OP_NOOP_OUT:
+ err = iser_nop_out_exec(task);
+ break;
+ case ISCSI_OP_LOGOUT:
+ err = iser_logout_exec(task);
+ break;
+ case ISCSI_OP_SCSI_TMFUNC:
+ err = iser_tm_exec(task);
+ break;
+ case ISCSI_OP_TEXT:
+ err = iser_text_exec(task);
+ break;
+ default:
+ eprintf("Internal error: Unexpected op:0x%x\n",task->opcode);
+ err = -EINVAL;
+ break;
+ }
+ return err;
+}
+
+static inline int cmdsn_cmp(uint32_t sn1, uint32_t sn2)
+{
+ if (sn1 == sn2)
+ return 0;
+
+ return ((int32_t)sn1 - (int32_t)sn2) > 0 ? 1 : -1;
+}
+
+// queues the task according to cmd-sn, no exec here
+static int iser_queue_task(struct iscsi_session *session,
+ struct iser_task *task)
+{
+ struct iscsi_hdr *req_bhs = (struct iscsi_hdr *) task->req.bhs;
+ int is_immediate = req_bhs->opcode & ISCSI_OP_IMMEDIATE;
+ uint32_t cmd_sn = be32_to_cpu(req_bhs->statsn);
+ struct list_head *cmp_entry;
+ int err;
+
+ if (is_immediate) {
+ dprintf("exec imm task task:%p cmd_sn:0x%x\n", task, cmd_sn);
+ err = iser_task_delivery(task);
+ if (!err || err == -ENOMEM)
+ return 0;
+ else
+ return err;
+ }
+
+ /* if the current command is the expected one, exec it
+ and all others possibly acumulated on the queue */
+ while (session->exp_cmd_sn == cmd_sn) {
+ session->exp_cmd_sn++;
+ dprintf("exec task:%p cmd_sn:0x%x\n", task, cmd_sn);
+ err = iser_task_delivery(task);
+ if (err && err != -ENOMEM) {
+ // when no free buffers remain, the task will wait
+ // on queue, so it is not a real error, but we should
+ // not attempt to start other tasks until more
+ // memory becomes available
+ return err; // ToDo: what if there are more tasks in case of error
+ }
+
+ if (list_empty(&session->pending_cmd_list))
+ return 0;
+
+ task = list_first_entry(&session->pending_cmd_list,
+ struct iser_task, exec_list);
+ list_del(&task->exec_list);
+ clear_task_pending(task);
+ cmd_sn = be32_to_cpu(task->req.bhs->statsn);
+ }
+
+ /* cmd_sn > (exp_cmd_sn+ISER_MAX_QUEUE_CMD), i.e. beyond allowed window */
+ if (cmdsn_cmp(cmd_sn, session->exp_cmd_sn+ISER_MAX_QUEUE_CMD) == 1) {
+ eprintf("unexpected cmd_sn:0x%x, max:0x%x\n",
+ cmd_sn, session->exp_cmd_sn+ISER_MAX_QUEUE_CMD);
+ return -EINVAL;
+ }
+
+ /* insert the current task, ordered by cmd_sn */
+ list_for_each_prev(cmp_entry, &session->pending_cmd_list) {
+ struct iser_task *cmp_task;
+ uint32_t cmp_cmd_sn;
+ int cmp_res;
+
+ cmp_task = list_entry(cmp_entry, struct iser_task, exec_list);
+ cmp_cmd_sn = be32_to_cpu(cmp_task->req.bhs->statsn);
+ cmp_res = cmdsn_cmp(cmd_sn, cmp_cmd_sn);
+ if (cmp_res == 1) { /* cmd_sn > cmp_cmd_sn */
+ dprintf("inserted cmdsn:0x%x after cmdsn:0x%x\n",
+ cmd_sn, cmp_cmd_sn);
+ break;
+ }
+ else if (cmp_res == -1) { /* cmd_sn < cmp_cmd_sn */
+ dprintf("inserting cmdsn:0x%x skip cmdsn:0x%x\n",
+ cmd_sn, cmp_cmd_sn);
+ continue;
+ }
+ else { /* cmd_sn == cmp_cmd_sn */
+ eprintf("duplicate cmd_sn:0x%x, exp:%u\n",
+ cmd_sn, session->exp_cmd_sn);
+ return -EINVAL;
+ }
+ }
+ list_add(&task->exec_list, cmp_entry);
+ set_task_pending(task);
+ return 0;
+}
+
+static int iser_parse_req_headers(struct iser_task *task)
+{
+ struct iser_hdr *iser_hdr = task->req.iser_hdr;
+ struct iscsi_hdr *iscsi_hdr = task->req.bhs;
+ int err = -1;
+
+ switch (iser_hdr->flags & 0xF0) {
+ case ISCSI_CTRL:
+ if (iser_hdr->flags & ISER_RSV) {
+ task->rem_read_stag =
+ be32_to_cpu(iser_hdr->read_stag);
+ task->rem_read_va = be64_to_cpu(iser_hdr->read_va);
+ dprintf("task:%p rstag:0x%x va:0x%lx\n", task,
+ task->rem_read_stag, task->rem_read_va);
+ }
+ if (iser_hdr->flags & ISER_WSV) {
+ task->rem_write_stag =
+ be32_to_cpu(iser_hdr->write_stag);
+ task->rem_write_va =
+ be64_to_cpu(iser_hdr->write_va);
+ dprintf("task:%p wstag:0x%x va:0x%lx\n", task,
+ task->rem_write_stag, task->rem_write_va);
+ }
+ err = 0;
+ break;
+ case ISER_HELLO:
+ dprintf("iSER Hello message??\n");
+ break;
+ default:
+ eprintf("malformed iser iser_hdr, flags 0x%02x\n",
+ iser_hdr->flags);
+ break;
+ }
+
+ task->opcode = iscsi_hdr->opcode & ISCSI_OPCODE_MASK;
+ task->req.ahssize = iscsi_hdr->hlength * 4;
+ task->req.data.buf += task->req.ahssize;
+ task->req.data.size = ntoh24(iscsi_hdr->dlength);
+ task->req.data.rdma = 0;
+
+ return err;
+}
+
+static void iser_rx_handler(struct iser_work_req *rxd)
+{
+ struct iser_task *task = rxd->task;
+ struct iser_conn *conn = task->conn;
+ int queue_task = 1;
+ int err = 0;
+
+ //--conn->recvl_posted;
+ iser_conn_put(conn);
+
+ if (conn->h.state == STATE_CLOSE)
+ return;
+
+ conn->h.exp_stat_sn = be32_to_cpu(task->req.bhs->exp_statsn);
+ task->tag = task->req.bhs->itt;
+ task->is_read = task->is_write = 0;
+
+ err = iser_parse_req_headers(task);
+ if (err)
+ goto out;
+
+ INIT_LIST_HEAD(&task->in_buf_list);
+ task->in_buf_num = 0;
+
+ INIT_LIST_HEAD(&task->out_buf_list);
+ task->out_buf_num = 0;
+
+ switch (task->opcode) {
+ case ISCSI_OP_LOGIN:
+ dprintf("login rx\n");
+ iser_login_rx(task);
+ queue_task = 0;
+ break;
+ case ISCSI_OP_SCSI_CMD:
+ err = iser_scsi_cmd_rx(task);
+ break;
+ case ISCSI_OP_SCSI_DATA_OUT:
+ err = iser_data_out_rx(task);
+ queue_task = 0;
+ break;
+ case ISCSI_OP_NOOP_OUT:
+ err = iser_nop_out_rx(task);
+ break;
+ case ISCSI_OP_LOGOUT:
+ dprintf("logout rx\n");
+ break;
+ case ISCSI_OP_SCSI_TMFUNC:
+ dprintf("tmfunc rx\n");
+ break;
+ case ISCSI_OP_TEXT:
+ dprintf("text rx\n");
+ break;
+ case ISCSI_OP_SNACK:
+ eprintf("Cannot handle SNACK yet\n");
+ err = -EINVAL;
+ break;
+ default:
+ eprintf("Unknown op 0x%x\n",task->opcode);
+ err = -EINVAL;
+ break;
+ }
+ if (!err && queue_task)
+ err = iser_queue_task(conn->h.session, task);
+out:
+ if (err) {
+ eprintf("task:%p conn:%p err:%d\n", task, &conn->h, err);
+ iser_conn_close(conn);
+ }
+}
+
+/*
+static void iser_send_chain_complete(struct iser_work_req *txd)
+{
+ struct iser_task *task = txd->task;
+
+ for (;;) {
+ if (txd->send_wr.opcode == IBV_WR_SEND) {
+ iser_complete_task(task);
+ iser_post_recv(task->conn, task);
+ }
+ else if (txd->send_wr.opcode != IBV_WR_RDMA_WRITE) {
+ eprintf("Unexpected send_wr.opcode:0x%x wr:%p\n",
+ txd->send_wr.opcode, &txd->send_wr);
+ exit(1);
+ }
+ if (!txd->send_wr.next)
+ break;
+
+ txd = container_of(txd->send_wr.next, struct iser_work_req,
+ send_wr);
+ task = txd->task;
+ }
+}
+*/
+
+static void iser_tx_complete_handler(struct iser_work_req *txd)
+{
+ struct iser_task *task = txd->task;
+ struct iser_conn *conn = task->conn;
+
+ dprintf("task:%p tag:0x%04lx\n", task, task->tag);
+
+ list_del(&task->tx_list); /* remove from conn->sent_list */
+ if (!task->unsolicited)
+ iser_complete_task(task);
+ else {
+ conn->nop_in_task = task;
+ return;
+ }
+
+ if (conn->h.state == STATE_CLOSE)
+ return;
+
+ if (conn->login_phase == LOGIN_PHASE_FF)
+ schedule_post_recv(task, conn);
+ else if (conn->login_phase == LOGIN_PHASE_LAST_SEND) {
+ dprintf("last login send completed, release, to ff\n");
+ iser_free_login_resources(conn);
+ conn->login_phase = LOGIN_PHASE_FF;
+ conn->h.state = STATE_FULL;
+ }
+}
+
+static void iser_rdma_wr_complete_handler(struct iser_work_req *rdmad)
+{
+ struct iser_task *task = rdmad->task;
+ struct iser_conn *conn = task->conn;
+
+ eprintf("task:%p tag:0x%04lx\n", task, task->tag);
+ exit(1);
+
+ if (conn->h.state == STATE_CLOSE)
+ return;
+
+ //iser_send_chain_complete(rdmad);
+}
+
+/*
+ * Called from CQ processing. Hands completed write data to iscsi.
+ */
+static void iser_rdma_rd_complete_handler(struct iser_work_req *rdmad)
+{
+ struct iser_task *task = rdmad->task;
+ struct iser_conn *conn = task->conn;
+
+ // ToDo: need iser_complete_task?
+
+ if (conn->h.state == STATE_CLOSE)
+ return;
+
+ task->rdma_rd_remains -= rdmad->sge.length;
+ dprintf("task:%p tag:0x%04lx, rems rdma:%d unsol:%d\n",
+ task, task->tag, task->rdma_rd_remains,
+ task->unsol_remains);
+ if (task->rdma_rd_remains == 0 && task->unsol_remains == 0)
+ schedule_task_iosubmit(task,conn);
+}
+
+/*
+ * Deal with just one work completion.
+ */
+static void handle_wc(struct ibv_wc *wc)
+{
+ void *wr_id = ptr_from_int64(wc->wr_id);
+
+ switch (wc->opcode) {
+ case IBV_WC_SEND:
+ dprintf("tx complete, wr_id:%p len:%d\n", wr_id, wc->byte_len);
+ iser_tx_complete_handler(wr_id);
+ break;
+ case IBV_WC_RECV:
+ dprintf("rx cmd, wr_id:%p len:%d\n", wr_id, wc->byte_len);
+ iser_rx_handler(wr_id);
+ break;
+ case IBV_WC_RDMA_WRITE:
+ dprintf("rdma write done, wr_id:%p len:%d\n", wr_id, wc->byte_len);
+ iser_rdma_wr_complete_handler(wr_id);
+ break;
+ case IBV_WC_RDMA_READ:
+ dprintf("rdma read done, wr_id:%p len:%d\n", wr_id, wc->byte_len);
+ iser_rdma_rd_complete_handler(wr_id);
+ break;
+ default:
+ eprintf("unexpected opcode %d\n", wc->opcode);
+ exit(1);
+ }
+}
+
+static void handle_wc_flush(struct ibv_wc *wc)
+{
+ struct iser_work_req *wr = ptr_from_int64(wc->wr_id);
+ struct iser_task *task = wr->task;
+ struct iser_conn *conn = task->conn;
+
+ dprintf("conn:%p wr_id:0x%llx task:%p\n",
+ &conn->h, (unsigned long long) wc->wr_id, task);
+
+ if (conn->h.state == STATE_CLOSE) {
+ //--conn->recvl_posted;
+ iser_conn_put(conn);
+ } else {
+ eprintf("conn %p wr flush err\n", conn);
+ /* call disconnect now? */
+ }
+}
+
+static void handle_wc_error(struct ibv_wc *wc)
+{
+ struct iser_work_req *wr = ptr_from_int64(wc->wr_id);
+ struct iser_task *task = wr->task;
+ struct iser_conn *conn = task->conn;
+ int complete = 1;
+ char *op_str;
+
+ if (wr->recv) {
+ op_str = "recv";
+ complete = 0;
+ } else {
+ switch (wr->send_wr.opcode) {
+ case IBV_WC_SEND:
+ op_str = "send";
+ break;
+ case IBV_WC_RDMA_WRITE:
+ op_str = "rdma_wr";
+ break;
+ case IBV_WC_RDMA_READ:
+ op_str = "rdma_rd";
+ break;
+ default:
+ op_str = "Unknown";
+ complete = 0;
+ break;
+ }
+ }
+ eprintf("conn:%p task:%p tag:0x%04lx op:%s WC status:%d wr_id:0x%llx\n",
+ &conn->h, task, task->tag, op_str, wc->status,
+ (unsigned long long) wc->wr_id);
+
+ if (complete)
+ iser_complete_task(task);
+ iser_conn_close(conn);
+}
+
+/*
+ * Could read as many entries as possible without blocking, but
+ * that just fills up a list of tasks. Instead pop out of here
+ * so that tx progress, like issuing rdma reads and writes, can
+ * happen periodically.
+ */
+static int iser_poll_cq(struct iser_device *dev, int max_wc)
+{
+ int err = 0, numwc = 0;
+ struct ibv_wc wc;
+
+ for (;;) {
+ err = ibv_poll_cq(dev->cq, 1, &wc);
+ if (err < 0) {
+ eprintf("ibv_poll_cq %d\n", err);
+ break;
+ } else if (err == 0) {
+ break;
+ }
+
+ VALGRIND_MAKE_MEM_DEFINED(&wc, sizeof(wc));
+ if (wc.status == IBV_WC_SUCCESS) {
+ handle_wc(&wc);
+ if (++numwc == max_wc) {
+ err = 1;
+ break;
+ }
+ } else if (wc.status == IBV_WC_WR_FLUSH_ERR)
+ handle_wc_flush(&wc);
+ else
+ handle_wc_error(&wc);
+ }
+ return err;
+}
+
+static int num_delayed_arm = 0;
+#define MAX_NUM_DELAYED_ARM 8
+
+static void iser_poll_cq_armable(struct iser_device *dev)
+{
+ int err;
+
+ err = iser_poll_cq(dev, MAX_POLL_WC);
+ if (err < 0)
+ exit(1);
+
+ if (err == 0) {
+ if (++num_delayed_arm == MAX_NUM_DELAYED_ARM) {
+ num_delayed_arm = 0;
+ /* no more completions on cq, arm the completion interrupts */
+ err = ibv_req_notify_cq(dev->cq, 0);
+ if (err) {
+ eprintf("ibv_req_notify_cq: %s\n", strerror(err));
+ exit(1);
+ }
+ dev->poll_sched.sched_handler = iser_sched_consume_cq;
+ }
+ else
+ dev->poll_sched.sched_handler = iser_sched_poll_cq;
+ } else
+ dev->poll_sched.sched_handler = iser_sched_poll_cq;
+
+ tgt_add_sched_event(&dev->poll_sched);
+}
+
+/* Scheduled to poll cq after a completion event has been
+ received and acknowledged, if no more completions are found
+ the interrupts are re-armed */
+static void iser_sched_poll_cq(struct event_data *tev)
+{
+ struct iser_device *dev = tev->data;
+ iser_poll_cq_armable(dev);
+}
+
+/* Scheduled to consume completion events that could arrive
+ after the cq had been seen empty but just before
+ the notification interrupts were re-armed.
+ Intended to consume those remaining completions only,
+ this function does not re-arm interrupts. */
+static void iser_sched_consume_cq(struct event_data *tev)
+{
+ struct iser_device *dev = tev->data;
+ int err;
+
+ err = iser_poll_cq(dev, MAX_POLL_WC);
+ if (err < 0)
+ exit(1);
+}
+
+/*
+ * Called from main event loop when a CQ notification is available.
+ */
+static void iser_cqe_handler(int fd __attribute__ ((unused)),
+ int events __attribute__ ((unused)),
+ void *data)
+{
+ struct iser_device *dev = data;
+ void *cq_context;
+ int err;
+
+ err = ibv_get_cq_event(dev->cq_channel, &dev->cq, &cq_context);
+ if (err != 0) {
+ eprintf("notification, but no CQ event\n");
+ exit(1);
+ }
+
+ ibv_ack_cq_events(dev->cq, 1);
+
+ /* if a poll was previosuly scheduled, remove it,
+ as it will be scheduled when necessary */
+ if (dev->poll_sched.scheduled)
+ tgt_remove_sched_event(&dev->poll_sched);
+
+ iser_poll_cq_armable(dev);
+}
+
+/*
+ * Init entire iscsi transport. Begin listening for connections.
+ */
+static int iser_ib_init(void)
+{
+ int err;
+ struct sockaddr_in sock_addr;
+ short int port = iser_listen_port;
+
+ rdma_evt_channel = rdma_create_event_channel();
+ if (!rdma_evt_channel) {
+ eprintf("Failed to initialize RDMA; load kernel modules?\n");
+ return -1;
+ }
+
+ err = rdma_create_id(rdma_evt_channel, &cma_listen_id, NULL,
+ RDMA_PS_TCP);
+ if (err) {
+ eprintf("rdma_create_id: %s\n", strerror(err));
+ return -1;
+ }
+
+ memset(&sock_addr, 0, sizeof(sock_addr));
+ sock_addr.sin_family = AF_INET;
+ sock_addr.sin_port = htons(port);
+ sock_addr.sin_addr.s_addr = INADDR_ANY;
+ err =
+ rdma_bind_addr(cma_listen_id, (struct sockaddr *) &sock_addr);
+ if (err) {
+ if (err == -1)
+ eprintf("rdma_bind_addr -1: %m\n");
+ else
+ eprintf("rdma_bind_addr: %s\n", strerror(-err));
+ return -1;
+ }
+
+ /* 0 == maximum backlog */
+ err = rdma_listen(cma_listen_id, 0);
+ if (err) {
+ if (err == -1)
+ eprintf("rdma_listen -1: %m\n");
+ else
+ eprintf("rdma_listen: %s\n", strerror(-err));
+ return -1;
+ }
+
+ dprintf("listening for iser connections on port %d\n", port);
+ err = tgt_event_add(cma_listen_id->channel->fd, EPOLLIN,
+ iser_handle_rdmacm, NULL);
+ if (err)
+ return err;
+
+ INIT_LIST_HEAD(&iser_dev_list);
+ INIT_LIST_HEAD(&iser_conn_list);
+ INIT_LIST_HEAD(&temp_conn);
+
+ return err;
+}
+
+#include <signal.h>
+#include <sys/time.h>
+
+#define ISER_TIMER_INT_SEC 5
+static struct itimerval iser_timer =
+ {{ISER_TIMER_INT_SEC,0},
+ {ISER_TIMER_INT_SEC,0}};
+static int timer_fd[2] = {0,0};
+static unsigned int ntimer = 0;
+
+static void iser_timer_sig_handler(int data)
+{
+ int err;
+
+ // ntimer++;
+ err = write(timer_fd[1], &ntimer, sizeof(ntimer));
+ if (err < 0) {
+ eprintf("Failed to write to pipe, %m\n");
+ return;
+ }
+}
+
+static void iser_timer_evt_handler(int fd, int events, void *data)
+{
+ struct iser_conn *conn;
+ struct iser_task *task;
+ unsigned int n;
+ int err;
+
+ err = read(timer_fd[0], &n, sizeof(n));
+ if (err < 0) {
+ eprintf("Failed to write to pipe, %m\n");
+ return;
+ }
+
+ list_for_each_entry(conn, &iser_conn_list, conn_list) {
+ if (conn->h.state != STATE_FULL)
+ continue;
+ task = conn->nop_in_task;
+ if (!task)
+ continue;
+ conn->nop_in_task = NULL;
+ iser_send_ping_nop_in(task);
+ }
+}
+
+static int iser_start_timer(void)
+{
+ struct sigaction s;
+ int err;
+
+ sigemptyset(&s.sa_mask);
+ sigaddset(&s.sa_mask, SIGALRM);
+ s.sa_flags = 0;
+ s.sa_handler = iser_timer_sig_handler;
+ err = sigaction(SIGALRM, &s, NULL);
+ if (err) {
+ eprintf("Failed to setup timer handler\n");
+ return err;
+ }
+
+ err = setitimer(ITIMER_REAL, &iser_timer, 0);
+ if (err) {
+ eprintf("Failed to set timer\n");
+ return err;
+ }
+
+ err = pipe(timer_fd);
+ if (err) {
+ eprintf("Failed to open timer pipe\n");
+ return err;
+ }
+
+ err = tgt_event_add(timer_fd[0], EPOLLIN, iser_timer_evt_handler, NULL);
+ if (err) {
+ eprintf("failed to add iser timer epoll event\n");
+ return err;
+ }
+
+ return 0;
+}
+
+static int iser_stop_timer(void)
+{
+ int err;
+
+ tgt_event_del(timer_fd[0]);
+
+ if (timer_fd[0] > 0)
+ close(timer_fd[0]);
+ if (timer_fd[1] > 0)
+ close(timer_fd[1]);
+
+ err = setitimer(ITIMER_REAL, 0, 0);
+ if (err)
+ eprintf("Failed to stop timer\n");
+
+ return err;
+}
+
+static int iser_init(int index, char *args)
+{
+ int err;
+
+ err = iser_ib_init();
+ if (err)
+ return err;
+
+ /* err = iser_start_timer();
+ if (err)
+ return err;
+ */
+
+ return 0;
+}
+
+static void iser_exit(void)
+{
+ iser_stop_timer();
+}
+
+static int iser_target_create(struct target *t)
+{
+ struct iscsi_target *target;
+ int err;
+
+ err = iscsi_target_create(t);
+ if (err)
+ return err;
+
+ target = target_find_by_id(t->tid);
+ assert(target != NULL);
+
+ target->rdma = 1;
+ target->session_param[ISCSI_PARAM_INITIAL_R2T_EN].val = 1;
+ target->session_param[ISCSI_PARAM_IMM_DATA_EN].val = 0;
+
+ return 0;
+}
+
+static const char *lld_param_port = "port";
+
+static int iser_param_parser(char *p)
+{
+ char *q;
+
+ while (*p) {
+ if (!strncmp(p, lld_param_port, strlen(lld_param_port))) {
+ q = p + strlen(lld_param_port) + 1;
+ iser_listen_port = atoi(q);
+ eprintf("iser listen port:%d\n", iser_listen_port);
+ }
+
+ p += strcspn(p, ",");
+ if (*p == ',')
+ ++p;
+ }
+ return 0;
+}
+
+static struct tgt_driver iser = {
+ .name = "iser",
+ .use_kernel = 0,
+ .init = iser_init,
+ .exit = iser_exit,
+ .target_create = iser_target_create,
+ .target_destroy = iscsi_target_destroy,
+
+ .update = iscsi_target_update,
+ .show = iscsi_target_show,
+ .cmd_end_notify = iser_scsi_cmd_done,
+ .mgmt_end_notify = iser_tm_done,
+ .transportid = iscsi_transportid,
+ .default_bst = "rdwr",
+};
+
+__attribute__ ((constructor))
+static void iser_driver_constructor(void)
+{
+ register_driver(&iser);
+
+ setup_param("iser", iser_param_parser);
+}
+
--
1.6.5
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
