[PATCH 2/3] cxgb4i v5: iscsi driver From: Karen Xie <kxie@xxxxxxxxxxx> Add cxgb4i iSCSI driver. This patch implements the cxgb4i iscsi connection acceleration for the open-iscsi initiator. The cxgb4i driver offers the iscsi PDU based offload: - digest insertion and verification - payload direct-placement into host memory buffer. Signed-off-by: Karen Xie <kxie@xxxxxxxxxxx> --- drivers/scsi/cxgbi/Kconfig | 7 drivers/scsi/cxgbi/Makefile | 1 drivers/scsi/cxgbi/cxgb4i/Kbuild | 3 drivers/scsi/cxgbi/cxgb4i/cxgb4i.c | 1551 ++++++++++++++++++++++++++++++++++++ drivers/scsi/cxgbi/cxgb4i/cxgb4i.h | 42 + 5 files changed, 1604 insertions(+), 0 deletions(-) create mode 100644 drivers/scsi/cxgbi/Kconfig create mode 100644 drivers/scsi/cxgbi/Makefile create mode 100644 drivers/scsi/cxgbi/cxgb4i/Kbuild create mode 100644 drivers/scsi/cxgbi/cxgb4i/cxgb4i.c create mode 100644 drivers/scsi/cxgbi/cxgb4i/cxgb4i.h diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig index 158284f..704622d 100644 --- a/drivers/scsi/Kconfig +++ b/drivers/scsi/Kconfig @@ -371,6 +371,7 @@ config ISCSI_TCP http://open-iscsi.org source "drivers/scsi/cxgb3i/Kconfig" +source "drivers/scsi/cxgbi/Kconfig" source "drivers/scsi/bnx2i/Kconfig" source "drivers/scsi/be2iscsi/Kconfig" diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile index 2a3fca2..1694ab7 100644 --- a/drivers/scsi/Makefile +++ b/drivers/scsi/Makefile @@ -133,6 +133,7 @@ obj-$(CONFIG_SCSI_STEX) += stex.o obj-$(CONFIG_SCSI_MVSAS) += mvsas/ obj-$(CONFIG_PS3_ROM) += ps3rom.o obj-$(CONFIG_SCSI_CXGB3_ISCSI) += libiscsi.o libiscsi_tcp.o cxgb3i/ +obj-$(CONFIG_SCSI_CXGB4_ISCSI) += libiscsi.o libiscsi_tcp.o cxgbi/ obj-$(CONFIG_SCSI_BNX2_ISCSI) += libiscsi.o bnx2i/ obj-$(CONFIG_BE2ISCSI) += libiscsi.o be2iscsi/ obj-$(CONFIG_SCSI_PMCRAID) += pmcraid.o diff --git a/drivers/scsi/cxgbi/Kconfig b/drivers/scsi/cxgbi/Kconfig new file mode 100644 index 0000000..96c887a --- /dev/null +++ b/drivers/scsi/cxgbi/Kconfig @@ -0,0 +1,7 @@ +config SCSI_CXGB4_ISCSI + tristate "Chelsio T4 iSCSI support" + depends on CHELSIO_T4_DEPENDS + select CHELSIO_T4 + select SCSI_ISCSI_ATTRS + ---help--- + This driver supports iSCSI offload for the Chelsio T4 devices. diff --git a/drivers/scsi/cxgbi/Makefile b/drivers/scsi/cxgbi/Makefile new file mode 100644 index 0000000..9e8f604 --- /dev/null +++ b/drivers/scsi/cxgbi/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_SCSI_CXGB4_ISCSI) += libcxgbi.o cxgb4i/ diff --git a/drivers/scsi/cxgbi/cxgb4i/Kbuild b/drivers/scsi/cxgbi/cxgb4i/Kbuild new file mode 100644 index 0000000..b9f4af7 --- /dev/null +++ b/drivers/scsi/cxgbi/cxgb4i/Kbuild @@ -0,0 +1,3 @@ +EXTRA_CFLAGS += -I$(srctree)/drivers/net/cxgb4 + +obj-$(CONFIG_SCSI_CXGB4_ISCSI) += cxgb4i.o diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c new file mode 100644 index 0000000..6391d69 --- /dev/null +++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c @@ -0,0 +1,1551 @@ +/* + * cxgb4i.c: Chelsio T4 iSCSI driver. + * + * Copyright (c) 2010 Chelsio Communications, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@xxxxxxxxxxx) + * Rakesh Ranjan (rranjan@xxxxxxxxxxx) + */ + +#include <linux/version.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <scsi/scsi_host.h> +#include <net/tcp.h> +#include <net/dst.h> +#include <linux/netdevice.h> + +#include "t4_msg.h" +#include "cxgb4.h" +#include "cxgb4_uld.h" +#include "t4fw_api.h" +#include "l2t.h" +#include "cxgb4i.h" + +#define DRV_MODULE_NAME "cxgb4i" +#define DRV_MODULE_DESC "Chelsio T4 iSCSI Driver" +#define DRV_MODULE_VERSION "0.9.0" +#define DRV_MODULE_RELDATE "May 2010" + +static char version[] = + DRV_MODULE_DESC " " DRV_MODULE_NAME + " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; + +MODULE_AUTHOR("Chelsio Communications, Inc."); +MODULE_DESCRIPTION(DRV_MODULE_DESC); +MODULE_VERSION(DRV_MODULE_VERSION); +MODULE_LICENSE("GPL"); + +unsigned int dbg_level; +module_param(dbg_level, uint, 0644); +MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)"); + +static int cxgb4i_rcv_win = 256 * 1024; +module_param(cxgb4i_rcv_win, int, 0644); +MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes"); + +static int cxgb4i_snd_win = 128 * 1024; +module_param(cxgb4i_snd_win, int, 0644); +MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes"); + +static int cxgb4i_rx_credit_thres = 10 * 1024; +module_param(cxgb4i_rx_credit_thres, int, 0644); +MODULE_PARM_DESC(cxgb4i_rx_credit_thres, + "RX credits return threshold in bytes (default=10KB)"); + +static unsigned int cxgb4i_max_connect = (8 * 1024); +module_param(cxgb4i_max_connect, uint, 0644); +MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections"); + +static unsigned short cxgb4i_sport_base = 20000; +module_param(cxgb4i_sport_base, ushort, 0644); +MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)"); + +#define cxgb4i_log_error cxgbi_dbg_error +#define cxgb4i_log_warn cxgbi_dbg_warn +#define cxgb4i_log_info cxgbi_dbg_info +#define cxgb4i_log_debug(level, fmt, args...) \ + cxgbi_dbg_print(level, fmt, ## args) + +static void *t4_uld_add(const struct cxgb4_lld_info *); +static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *); +static int t4_uld_state_change(void *, enum cxgb4_state state); + +static const struct cxgb4_uld_info cxgb4i_uld_info = { + .name = DRV_MODULE_NAME, + .add = t4_uld_add, + .rx_handler = t4_uld_rx_handler, + .state_change = t4_uld_state_change, +}; + +static struct scsi_host_template cxgb4i_host_template = { + .module = THIS_MODULE, + .name = DRV_MODULE_NAME, + .proc_name = DRV_MODULE_NAME, + .can_queue = CXGB4I_SCSI_HOST_QDEPTH, +}; + +static struct iscsi_transport cxgb4i_iscsi_transport = { + .owner = THIS_MODULE, + .name = DRV_MODULE_NAME, +}; + +/* + * CPL (Chelsio Protocol Language) defines a message passing interface between + * the host driver and Chelsio asic. + * The section below implments CPLs that related to iscsi tcp connection + * open/close/abort and data send/receive. + */ +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define RCV_BUFSIZ_MASK 0x3FFU +#define MAX_IMM_TX_PKT_LEN 128 + +static inline void set_queue(struct sk_buff *skb, unsigned int queue, + const struct cxgbi_sock *csk) +{ + skb->queue_mapping = queue; +} + +static int push_tx_frames(struct cxgbi_sock *, int); + +/* + * is_ofld_imm - check whether a packet can be sent as immediate data + * @skb: the packet + * + * Returns true if a packet can be sent as an offload WR with immediate + * data. We currently use the same limit as for Ethernet packets. + */ +static inline int is_ofld_imm(const struct sk_buff *skb) +{ + return skb->len <= (MAX_IMM_TX_PKT_LEN - + sizeof(struct fw_ofld_tx_data_wr)); +} + +static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, + struct l2t_entry *e) +{ + struct cpl_act_open_req *req; + int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); + unsigned long long opt0; + unsigned int opt2; + unsigned int qid_atid = ((unsigned int)csk->atid) | + (((unsigned int)csk->rss_qid) << 14); + + opt0 = KEEP_ALIVE(1) | + WND_SCALE(wscale) | + MSS_IDX(csk->mss_idx) | + L2T_IDX(((struct l2t_entry *)csk->l2t)->idx) | + TX_CHAN(csk->tx_chan) | + SMAC_SEL(csk->smac_idx) | + ULP_MODE(ULP_MODE_ISCSI) | + RCV_BUFSIZ(cxgb4i_rcv_win >> 10); + opt2 = RX_CHANNEL(0) | + RSS_QUEUE_VALID | + (1 << 20) | (1 << 22) | + RSS_QUEUE(csk->rss_qid); + + set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); + req = (struct cpl_act_open_req *)skb->head; + + INIT_TP_WR(req, 0); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, + qid_atid)); + req->local_port = csk->saddr.sin_port; + req->peer_port = csk->daddr.sin_port; + req->local_ip = csk->saddr.sin_addr.s_addr; + req->peer_ip = csk->daddr.sin_addr.s_addr; + req->opt0 = cpu_to_be64(opt0); + req->params = 0; + req->opt2 = cpu_to_be32(opt2); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", + csk, &req->local_ip, ntohs(req->local_port), + &req->peer_ip, ntohs(req->peer_port), + csk->atid, csk->rss_qid); + + cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); +} + +static void send_close_req(struct cxgbi_sock *csk) +{ + struct sk_buff *skb = csk->cpl_close; + struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; + unsigned int tid = csk->tid; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u.\n", + csk, csk->state, csk->flags, csk->tid); + + csk->cpl_close = NULL; + set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); + INIT_TP_WR(req, tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); + req->rsvd = 0; + + cxgbi_sock_skb_entail(csk, skb); + if (csk->state >= CTP_ESTABLISHED) + push_tx_frames(csk, 1); +} + +static void abort_arp_failure(void *handle, struct sk_buff *skb) +{ + struct cxgbi_sock *csk = (struct cxgbi_sock *)handle; + struct cpl_abort_req *req; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx, tid %u, abort.\n", + csk, csk->state, csk->flags, csk->tid); + req = (struct cpl_abort_req *)skb->data; + req->cmd = CPL_ABORT_NO_RST; + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); +} + +static void send_abort_req(struct cxgbi_sock *csk) +{ + struct cpl_abort_req *req; + struct sk_buff *skb = csk->cpl_abort_req; + + if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev) + return; + cxgbi_sock_set_state(csk, CTP_ABORTING); + cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); + cxgbi_sock_purge_write_queue(csk); + + csk->cpl_abort_req = NULL; + req = (struct cpl_abort_req *)skb->head; + set_queue(skb, CPL_PRIORITY_DATA, csk); + req->cmd = CPL_ABORT_SEND_RST; + t4_set_arp_err_handler(skb, csk, abort_arp_failure); + INIT_TP_WR(req, csk->tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); + req->rsvd0 = htonl(csk->snd_nxt); + req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", + csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, + req->rsvd1); + + cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); +} + +static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) +{ + struct sk_buff *skb = csk->cpl_abort_rpl; + struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, status %d.\n", + csk, csk->state, csk->flags, csk->tid, rst_status); + + csk->cpl_abort_rpl = NULL; + set_queue(skb, CPL_PRIORITY_DATA, csk); + INIT_TP_WR(rpl, csk->tid); + OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); + rpl->cmd = rst_status; + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); +} + +/* + * CPL connection rx data ack: host -> + * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of + * credits sent. + */ +static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) +{ + struct sk_buff *skb; + struct cpl_rx_data_ack *req; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, credit %u.\n", + csk, csk->state, csk->flags, csk->tid, credits); + + skb = alloc_cpl(sizeof(*req), 0, GFP_ATOMIC); + if (!skb) { + cxgb4i_log_info("csk 0x%p, credit %u, OOM.\n", csk, credits); + return 0; + } + req = (struct cpl_rx_data_ack *)skb->head; + + set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); + INIT_TP_WR(req, csk->tid); + OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, + csk->tid)); + req->credit_dack = cpu_to_be32(RX_CREDITS(credits) | RX_FORCE_ACK(1)); + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); + return credits; +} + +/* + * sgl_len - calculates the size of an SGL of the given capacity + * @n: the number of SGL entries + * Calculates the number of flits needed for a scatter/gather list that + * can hold the given number of entries. + */ +static inline unsigned int sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +/* + * calc_tx_flits_ofld - calculate # of flits for an offload packet + * @skb: the packet + * + * Returns the number of flits needed for the given offload packet. + * These packets are already fully constructed and no additional headers + * will be added. + */ +static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) +{ + unsigned int flits, cnt; + + if (is_ofld_imm(skb)) + return DIV_ROUND_UP(skb->len, 8); + flits = skb_transport_offset(skb) / 8; + cnt = skb_shinfo(skb)->nr_frags; + if (skb->tail != skb->transport_header) + cnt++; + return flits + sgl_len(cnt); +} + +static inline void send_tx_flowc_wr(struct cxgbi_sock *csk) +{ + struct sk_buff *skb; + struct fw_flowc_wr *flowc; + int flowclen, i; + + flowclen = 80; + skb = alloc_cpl(flowclen, 0, GFP_ATOMIC); + flowc = (struct fw_flowc_wr *)skb->head; + flowc->op_to_nparams = + htonl(FW_WR_OP(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS(8)); + flowc->flowid_len16 = + htonl(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 16)) | + FW_WR_FLOWID(csk->tid)); + flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; + flowc->mnemval[0].val = htonl(0); + flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; + flowc->mnemval[1].val = htonl(csk->tx_chan); + flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; + flowc->mnemval[2].val = htonl(csk->tx_chan); + flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; + flowc->mnemval[3].val = htonl(csk->rss_qid); + flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; + flowc->mnemval[4].val = htonl(csk->snd_nxt); + flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; + flowc->mnemval[5].val = htonl(csk->rcv_nxt); + flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; + flowc->mnemval[6].val = htonl(cxgb4i_snd_win); + flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; + flowc->mnemval[7].val = htonl(csk->advmss); + flowc->mnemval[8].mnemonic = 0; + flowc->mnemval[8].val = 0; + for (i = 0; i < 9; i++) { + flowc->mnemval[i].r4[0] = 0; + flowc->mnemval[i].r4[1] = 0; + flowc->mnemval[i].r4[2] = 0; + } + set_queue(skb, CPL_PRIORITY_DATA, csk); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n", + csk, csk->tid, 0, csk->tx_chan, csk->rss_qid, + csk->snd_nxt, csk->rcv_nxt, cxgb4i_snd_win, + csk->advmss); + + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); +} + +static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, + int dlen, int len, u32 credits, int compl) +{ + struct fw_ofld_tx_data_wr *req; + unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3; + unsigned int wr_ulp_mode = 0; + + req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req)); + + if (is_ofld_imm(skb)) { + req->op_to_immdlen = htonl(FW_WR_OP(FW_OFLD_TX_DATA_WR) | + FW_WR_COMPL(compl) | + FW_WR_IMMDLEN(dlen)); + req->flowid_len16 = htonl(FW_WR_FLOWID(csk->tid) | + FW_WR_LEN16(credits)); + } else { + req->op_to_immdlen = + cpu_to_be32(FW_WR_OP(FW_OFLD_TX_DATA_WR) | + FW_WR_COMPL(compl) | + FW_WR_IMMDLEN(0)); + req->flowid_len16 = + cpu_to_be32(FW_WR_FLOWID(csk->tid) | + FW_WR_LEN16(credits)); + } + if (submode) + wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE(ULP2_MODE_ISCSI) | + FW_OFLD_TX_DATA_WR_ULPSUBMODE(submode); + req->tunnel_to_proxy = htonl(wr_ulp_mode) | + FW_OFLD_TX_DATA_WR_SHOVE(skb_peek(&csk->write_queue) ? 0 : 1); + req->plen = htonl(len); + if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) + cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, tid 0x%x, skb 0x%p.\n", + csk, csk->tid, skb); +} + +static void arp_failure_skb_discard(void *handle, struct sk_buff *skb) +{ + kfree_skb(skb); +} + +static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) +{ + int total_size = 0; + struct sk_buff *skb; + + if (unlikely(csk->state < CTP_ESTABLISHED || + csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, in closing state.\n", + csk, csk->state, csk->flags, csk->tid); + return 0; + } + + while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { + int dlen = skb->len; + int len = skb->len; + unsigned int credits_needed; + + skb_reset_transport_header(skb); + if (is_ofld_imm(skb)) + credits_needed = DIV_ROUND_UP(dlen + + sizeof(struct fw_ofld_tx_data_wr), 16); + else + credits_needed = DIV_ROUND_UP(8*calc_tx_flits_ofld(skb) + + sizeof(struct fw_ofld_tx_data_wr), + 16); + + if (csk->wr_cred < credits_needed) { + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_TX, + "csk 0x%p, skb %u/%u, wr %d < %u.\n", + csk, skb->len, skb->data_len, + credits_needed, csk->wr_cred); + break; + } + __skb_unlink(skb, &csk->write_queue); + set_queue(skb, CPL_PRIORITY_DATA, csk); + skb->csum = credits_needed; + csk->wr_cred -= credits_needed; + csk->wr_una_cred += credits_needed; + cxgbi_sock_enqueue_wr(csk, skb); + + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_TX, + "csk 0x%p, enqueue, skb len %u/%u, " + "wr %d, left %u, unack %u.\n", + csk, skb->len, skb->data_len, + credits_needed, csk->wr_cred, + csk->wr_una_cred); + + if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { + if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { + send_tx_flowc_wr(csk); + skb->csum += 5; + csk->wr_cred -= 5; + csk->wr_una_cred += 5; + } + if ((req_completion && + csk->wr_una_cred == credits_needed) || + csk->wr_una_cred >= csk->wr_max_cred >> 1) { + req_completion = 1; + csk->wr_una_cred = 0; + } + len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); + make_tx_data_wr(csk, skb, dlen, len, credits_needed, + req_completion); + csk->snd_nxt += len; + cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); + } + total_size += skb->truesize; + t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard); + + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_TX, + "csk 0x%p, tid 0x%x, send skb 0x%p.\n", + csk, csk->tid, skb); + + cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); + } + return total_size; +} + +static inline void free_atid(struct cxgbi_sock *csk) +{ + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); + + if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { + cxgb4_free_atid(lldi->tids, csk->atid); + cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); + cxgbi_sock_put(csk); + } +} + +static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data; + unsigned short tcp_opt = ntohs(req->tcp_opt); + unsigned int tid = GET_TID(req); + unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + u32 rcv_isn = be32_to_cpu(req->rcv_isn); + + csk = lookup_atid(t, atid); + if (unlikely(!csk)) { + cxgb4i_log_error("NO connection for atid %u, cdev 0x%p\n", + atid, cdev); + goto rel_skb; + } + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx, tid %u, atid %u, rseq %u.\n", + csk, csk->state, csk->flags, tid, atid, rcv_isn); + + cxgbi_sock_get(csk); + csk->tid = tid; + cxgb4_insert_tid(lldi->tids, csk, tid); + cxgbi_sock_set_flag(csk, CTPF_HAS_TID); + + free_atid(csk); + + spin_lock_bh(&csk->lock); + if (unlikely(csk->state != CTP_ACTIVE_OPEN)) + cxgb4i_log_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", + csk, csk->state, csk->flags, csk->tid); + + if (csk->retry_timer.function) { + del_timer(&csk->retry_timer); + csk->retry_timer.function = NULL; + } + + csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; + /* + * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't + * pass through opt0. + */ + if (cxgb4i_rcv_win > (RCV_BUFSIZ_MASK << 10)) + csk->rcv_wup -= cxgb4i_rcv_win - (RCV_BUFSIZ_MASK << 10); + + csk->advmss = lldi->mtus[GET_TCPOPT_MSS(tcp_opt)] - 40; + if (GET_TCPOPT_TSTAMP(tcp_opt)) + csk->advmss -= 12; + if (csk->advmss < 128) + csk->advmss = 128; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, mss_idx %u, advmss %u.\n", + csk, GET_TCPOPT_MSS(tcp_opt), csk->advmss); + + cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); + + if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) + send_abort_req(csk); + else { + if (skb_queue_len(&csk->write_queue)) + push_tx_frames(csk, 0); + cxgbi_conn_tx_open(csk); + } + spin_unlock_bh(&csk->lock); + +rel_skb: + __kfree_skb(skb); +} + +static int act_open_rpl_status_to_errno(int status) +{ + switch (status) { + case CPL_ERR_CONN_RESET: + return -ECONNREFUSED; + case CPL_ERR_ARP_MISS: + return -EHOSTUNREACH; + case CPL_ERR_CONN_TIMEDOUT: + return -ETIMEDOUT; + case CPL_ERR_TCAM_FULL: + return -ENOMEM; + case CPL_ERR_CONN_EXIST: + return -EADDRINUSE; + default: + return -EIO; + } +} + +static void csk_act_open_retry_timer(unsigned long data) +{ + struct sk_buff *skb; + struct cxgbi_sock *csk = (struct cxgbi_sock *)data; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u.\n", + csk, csk->state, csk->flags, csk->tid); + + cxgbi_sock_get(csk); + spin_lock_bh(&csk->lock); + skb = alloc_cpl(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC); + if (!skb) + cxgbi_sock_fail_act_open(csk, -ENOMEM); + else { + skb->sk = (struct sock *)csk; + t4_set_arp_err_handler(skb, csk, + cxgbi_sock_act_open_req_arp_failure); + send_act_open_req(csk, skb, csk->l2t); + } + spin_unlock_bh(&csk->lock); + cxgbi_sock_put(csk); +} + +static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data; + unsigned int tid = GET_TID(rpl); + unsigned int atid = + GET_TID_TID(GET_AOPEN_ATID(be32_to_cpu(rpl->atid_status))); + unsigned int status = GET_AOPEN_STATUS(be32_to_cpu(rpl->atid_status)); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + + csk = lookup_atid(t, atid); + if (unlikely(!csk)) { + cxgb4i_log_error("NO matching conn. atid %u, tid %u.\n", + atid, tid); + goto rel_skb; + } + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx, status %u, atid %u, tid %u.\n", + csk, csk->state, csk->flags, status, atid, tid); + + if (status && status != CPL_ERR_TCAM_FULL && + status != CPL_ERR_CONN_EXIST && + status != CPL_ERR_ARP_MISS) + cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl)); + + cxgbi_sock_get(csk); + spin_lock_bh(&csk->lock); + + if (status == CPL_ERR_CONN_EXIST && + csk->retry_timer.function != csk_act_open_retry_timer) { + csk->retry_timer.function = csk_act_open_retry_timer; + mod_timer(&csk->retry_timer, jiffies + HZ / 2); + } else + cxgbi_sock_fail_act_open(csk, + act_open_rpl_status_to_errno(status)); + + spin_unlock_bh(&csk->lock); + cxgbi_sock_put(csk); +rel_skb: + __kfree_skb(skb); +} + +static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data; + unsigned int tid = GET_TID(req); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) { + cxgb4i_log_error("can't find connection for tid %u\n", tid); + goto rel_skb; + } + cxgbi_sock_rcv_peer_close(csk); +rel_skb: + __kfree_skb(skb); +} + +static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data; + unsigned int tid = GET_TID(rpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) { + cxgb4i_log_error("can't find connection for tid %u\n", tid); + goto rel_skb; + } + cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); +rel_skb: + __kfree_skb(skb); +} + +static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, + int *need_rst) +{ + switch (abort_reason) { + case CPL_ERR_BAD_SYN: /* fall through */ + case CPL_ERR_CONN_RESET: + return csk->state > CTP_ESTABLISHED ? + -EPIPE : -ECONNRESET; + case CPL_ERR_XMIT_TIMEDOUT: + case CPL_ERR_PERSIST_TIMEDOUT: + case CPL_ERR_FINWAIT2_TIMEDOUT: + case CPL_ERR_KEEPALIVE_TIMEDOUT: + return -ETIMEDOUT; + default: + return -EIO; + } +} + +static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data; + unsigned int tid = GET_TID(req); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + int rst_status = CPL_ABORT_NO_RST; + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) { + cxgb4i_log_error("can't find connection for tid %u\n", tid); + goto rel_skb; + } + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx, tid %u, status 0x%x.\n", + csk, csk->state, csk->flags, csk->tid, req->status); + + if (req->status == CPL_ERR_RTX_NEG_ADVICE || + req->status == CPL_ERR_PERSIST_NEG_ADVICE) + goto rel_skb; + + cxgbi_sock_get(csk); + spin_lock_bh(&csk->lock); + + if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) { + cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); + cxgbi_sock_set_state(csk, CTP_ABORTING); + goto done; + } + + cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); + send_abort_rpl(csk, rst_status); + + if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { + csk->err = abort_status_to_errno(csk, req->status, &rst_status); + cxgbi_sock_closed(csk); + } +done: + spin_unlock_bh(&csk->lock); + cxgbi_sock_put(csk); +rel_skb: + __kfree_skb(skb); +} + +static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data; + unsigned int tid = GET_TID(rpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + + csk = lookup_tid(t, tid); + if (!csk) + goto rel_skb; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "status 0x%x, csk 0x%p, s %u, 0x%lx.\n", + rpl->status, csk, csk ? csk->state : 0, + csk ? csk->flags : 0UL); + + if (rpl->status == CPL_ERR_ABORT_FAILED) + goto rel_skb; + + cxgbi_sock_rcv_abort_rpl(csk); +rel_skb: + __kfree_skb(skb); +} + +static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; + unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); + unsigned int tid = GET_TID(cpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + struct sk_buff *lskb; + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) { + cxgb4i_log_error("can't find conn. for tid %u\n", tid); + goto rel_skb; + } + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, + "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", + csk, csk->state, csk->flags, csk->tid, skb, skb->len, + pdu_len_ddp); + + spin_lock_bh(&csk->lock); + + if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, bad state.\n", + csk, csk->state, csk->flags, csk->tid); + if (csk->state != CTP_ABORTING) + goto abort_conn; + else + goto discard; + } + + cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq); + skb_reset_transport_header(skb); + __skb_pull(skb, sizeof(*cpl)); + __pskb_trim(skb, ntohs(cpl->len)); + + if (!csk->skb_ulp_lhdr) { + unsigned char *bhs; + unsigned int hlen, dlen; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, + "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n", + csk, csk->state, csk->flags, csk->tid, skb); + csk->skb_ulp_lhdr = skb; + lskb = csk->skb_ulp_lhdr; + cxgbi_skcb_set_flag(lskb, SKCBF_RX_HDR); + + if (cxgbi_skcb_tcp_seq(lskb) != csk->rcv_nxt) { + cxgb4i_log_info("tid 0x%x, CPL_ISCSI_HDR, bad seq got " + "0x%x, exp 0x%x\n", + csk->tid, cxgbi_skcb_tcp_seq(lskb), + csk->rcv_nxt); + goto abort_conn; + } + + bhs = lskb->data; + hlen = ntohs(cpl->len); + dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF; + + if ((hlen + dlen) != ISCSI_PDU_LEN(pdu_len_ddp) - 40) { + cxgb4i_log_info("tid 0x%x, CPL_ISCSI_HDR, pdu len " + "mismatch %u != %u + %u, seq 0x%x\n", + csk->tid, + ISCSI_PDU_LEN(pdu_len_ddp) - 40, + hlen, dlen, cxgbi_skcb_tcp_seq(skb)); + goto abort_conn; + } + + cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3); + if (dlen) + cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len; + csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n", + csk, skb, *bhs, hlen, dlen, + ntohl(*((unsigned int *)(bhs + 16))), + ntohl(*((unsigned int *)(bhs + 24)))); + + } else { + lskb = csk->skb_ulp_lhdr; + cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, + "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", + csk, csk->state, csk->flags, skb, lskb); + } + + __skb_queue_tail(&csk->receive_queue, skb); + spin_unlock_bh(&csk->lock); + return; + +abort_conn: + send_abort_req(csk); +discard: + spin_unlock_bh(&csk->lock); +rel_skb: + __kfree_skb(skb); +} + +static void do_rx_data_ddp(struct cxgbi_device *cdev, + struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct sk_buff *lskb; + struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data; + unsigned int tid = GET_TID(rpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + unsigned int status = ntohl(rpl->ddpvld); + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) { + cxgb4i_log_error("can't find connection for tid %u\n", tid); + goto rel_skb; + } + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, + "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n", + csk, csk->state, csk->flags, skb, status, + csk->skb_ulp_lhdr); + + spin_lock_bh(&csk->lock); + + if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u, bad state.\n", + csk, csk->state, csk->flags, csk->tid); + if (csk->state != CTP_ABORTING) + goto abort_conn; + else + goto discard; + } + + if (!csk->skb_ulp_lhdr) { + cxgb4i_log_error("tid 0x%x, rcv RX_DATA_DDP w/o pdu header\n", + csk->tid); + goto abort_conn; + } + + lskb = csk->skb_ulp_lhdr; + csk->skb_ulp_lhdr = NULL; + + cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS); + cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc); + + if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb)) { + cxgb4i_log_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n", + csk->tid, ntohs(rpl->len), + cxgbi_skcb_rx_pdulen(lskb)); + } + + if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) { + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad.\n", + csk, lskb, status); + cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR); + } + if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) { + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad.\n", + csk, lskb, status); + cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR); + } + if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) { + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n", + csk, lskb, status); + cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR); + } + if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) && + !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) { + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n", + csk, lskb, status); + cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD); + } + cxgb4i_log_debug(1 << CXGBI_DBG_PDU_RX, + "csk 0x%p, lskb 0x%p, f 0x%lx.\n", + csk, lskb, cxgbi_skcb_flags(lskb)); + + cxgbi_conn_pdu_ready(csk); + spin_unlock_bh(&csk->lock); + goto rel_skb; + +abort_conn: + send_abort_req(csk); +discard: + spin_unlock_bh(&csk->lock); +rel_skb: + __kfree_skb(skb); +} + +static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cxgbi_sock *csk; + struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data; + unsigned int tid = GET_TID(rpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + + csk = lookup_tid(t, tid); + if (unlikely(!csk)) + cxgb4i_log_error("can't find connection for tid %u\n", tid); + else + cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una), + rpl->seq_vld); + __kfree_skb(skb); +} + +static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) +{ + struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; + unsigned int tid = GET_TID(rpl); + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct tid_info *t = lldi->tids; + struct cxgbi_sock *csk; + + csk = lookup_tid(t, tid); + if (!csk) + cxgb4i_log_error("can't find conn. for tid %u\n", tid); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, tid 0x%x, status 0x%x.\n", + csk, tid, rpl->status); + + if (rpl->status != CPL_ERR_NONE) + cxgb4i_log_error("csk 0x%p,%u, SET_TCB_RPL status %u.\n", + csk, tid, rpl->status); + + __kfree_skb(skb); +} + +static int alloc_cpls(struct cxgbi_sock *csk) +{ + csk->cpl_close = alloc_cpl(sizeof(struct cpl_close_con_req), + 0, GFP_NOIO); + if (!csk->cpl_close) + return -ENOMEM; + + csk->cpl_abort_req = alloc_cpl(sizeof(struct cpl_abort_req), + 0, GFP_NOIO); + if (!csk->cpl_abort_req) + goto free_cpls; + + csk->cpl_abort_rpl = alloc_cpl(sizeof(struct cpl_abort_rpl), + 0, GFP_NOIO); + if (!csk->cpl_abort_rpl) + goto free_cpls; + return 0; + +free_cpls: + cxgbi_sock_free_cpl_skbs(csk); + return -ENOMEM; +} + +static inline void l2t_put(struct cxgbi_sock *csk) +{ + if (csk->l2t) { + cxgb4_l2t_release(csk->l2t); + csk->l2t = NULL; + cxgbi_sock_put(csk); + } +} + +static void release_offload_resources(struct cxgbi_sock *csk) +{ + struct cxgb4_lld_info *lldi; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u.\n", + csk, csk->state, csk->flags, csk->tid); + + cxgbi_sock_free_cpl_skbs(csk); + if (csk->wr_cred != csk->wr_max_cred) { + cxgbi_sock_purge_wr_queue(csk); + cxgbi_sock_reset_wr_list(csk); + } + + l2t_put(csk); + if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) + free_atid(csk); + else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { + lldi = cxgbi_cdev_priv(csk->cdev); + cxgb4_remove_tid(lldi->tids, 0, csk->tid); + cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); + cxgbi_sock_put(csk); + } + csk->dst = NULL; + csk->cdev = NULL; +} + +static int init_act_open(struct cxgbi_sock *csk) +{ + struct cxgbi_device *cdev = csk->cdev; + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct net_device *ndev = cdev->ports[csk->port_id]; + struct port_info *pi = netdev_priv(ndev); + struct sk_buff *skb = NULL; + unsigned int step; + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,%u,0x%lx,%u.\n", + csk, csk->state, csk->flags, csk->tid); + + csk->atid = cxgb4_alloc_atid(lldi->tids, csk); + if (csk->atid < 0) { + cxgb4i_log_error("%s, NO atid available.\n", ndev->name); + return -EINVAL; + } + cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); + cxgbi_sock_get(csk); + + csk->l2t = cxgb4_l2t_get(lldi->l2t, csk->dst->neighbour, ndev, 0); + if (!csk->l2t) { + cxgb4i_log_error("%s, cannot alloc l2t.\n", ndev->name); + goto rel_resource; + } + cxgbi_sock_get(csk); + + skb = alloc_cpl(sizeof(struct cpl_act_open_req), 0, GFP_NOIO); + if (!skb) + goto rel_resource; + skb->sk = (struct sock *)csk; + t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure); + + if (!csk->mtu) + csk->mtu = dst_mtu(csk->dst); + cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx); + csk->tx_chan = cxgb4_port_chan(ndev); + /* SMT two entries per row */ + csk->smac_idx = ((cxgb4_port_viid(ndev) & 0x7F)) << 1; + step = lldi->ntxq / lldi->nchan; + csk->txq_idx = cxgb4_port_idx(ndev) * step; + step = lldi->nrxq / lldi->nchan; + csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step]; + csk->wr_max_cred = csk->wr_cred = lldi->wr_cred; + csk->wr_una_cred = 0; + cxgbi_sock_reset_wr_list(csk); + csk->err = 0; + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p,p%d,%s, %u,%u,%u, mss %u,%u, smac %u.\n", + csk, pi->port_id, ndev->name, csk->tx_chan, + csk->txq_idx, csk->rss_qid, csk->mtu, csk->mss_idx, + csk->smac_idx); + + cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); + send_act_open_req(csk, skb, csk->l2t); + return 0; + +rel_resource: + if (skb) + __kfree_skb(skb); + return -EINVAL; +} + +cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = { + [CPL_ACT_ESTABLISH] = do_act_establish, + [CPL_ACT_OPEN_RPL] = do_act_open_rpl, + [CPL_PEER_CLOSE] = do_peer_close, + [CPL_ABORT_REQ_RSS] = do_abort_req_rss, + [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss, + [CPL_CLOSE_CON_RPL] = do_close_con_rpl, + [CPL_FW4_ACK] = do_fw4_ack, + [CPL_ISCSI_HDR] = do_rx_iscsi_hdr, + [CPL_SET_TCB_RPL] = do_set_tcb_rpl, + [CPL_RX_DATA_DDP] = do_rx_data_ddp, +}; + +int cxgb4i_ofld_init(struct cxgbi_device *cdev) +{ + int rc; + + if (cxgb4i_max_connect > CXGB4I_MAX_CONN) + cxgb4i_max_connect = CXGB4I_MAX_CONN; + + rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base, + cxgb4i_max_connect); + if (rc < 0) + return rc; + + cdev->csk_release_offload_resources = release_offload_resources; + cdev->csk_push_tx_frames = push_tx_frames; + cdev->csk_send_abort_req = send_abort_req; + cdev->csk_send_close_req = send_close_req; + cdev->csk_send_rx_credits = send_rx_credits; + cdev->csk_alloc_cpls = alloc_cpls; + cdev->csk_init_act_open = init_act_open; + + cxgb4i_log_info("cdev 0x%p, offload up, added.\n", cdev); + return 0; +} + +/* + * functions to program the pagepod in h/w + */ +static inline void ulp_mem_io_set_hdr(struct ulp_mem_io *req, + unsigned int dlen, unsigned int pm_addr) +{ + struct ulptx_sgl *sgl; + unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + + sizeof(struct ulptx_sgl), 16); + + INIT_ULPTX_WR(req, wr_len, 0, 0); + req->cmd = htonl(ULPTX_CMD(ULP_TX_MEM_WRITE)); + req->dlen = htonl(ULP_MEMIO_DATA_LEN(dlen >> 5)); + req->lock_addr = htonl(ULP_MEMIO_ADDR(pm_addr >> 5)); + req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); + sgl = (struct ulptx_sgl *)(req + 1); + sgl->cmd_nsge = htonl(ULPTX_CMD(ULP_TX_SC_DSGL) | ULPTX_NSGE(1)); + sgl->len0 = htonl(dlen); +} + +static int ddp_ppod_write_sgl(struct cxgbi_device *cdev, unsigned int port_id, + struct cxgbi_pagepod_hdr *hdr, unsigned int idx, + unsigned int npods, + struct cxgbi_gather_list *gl, + unsigned int gl_pidx) +{ + struct cxgbi_ddp_info *ddp = cdev->ddp; + unsigned int dlen, pm_addr; + struct sk_buff *skb; + struct ulp_mem_io *req; + struct ulptx_sgl *sgl; + struct cxgbi_pagepod *ppod; + unsigned int i; + + dlen = PPOD_SIZE * npods; + pm_addr = idx * PPOD_SIZE + ddp->llimit; + + skb = alloc_cpl(sizeof(*req) + sizeof(*sgl), dlen, GFP_ATOMIC); + if (!skb) { + cxgb4i_log_error("cdev 0x%p, idx %u, npods %u, OOM\n", + cdev, idx, npods); + return -ENOMEM; + } + req = (struct ulp_mem_io *)skb->head; + set_queue(skb, CPL_PRIORITY_CONTROL, NULL); + + ulp_mem_io_set_hdr(req, dlen, pm_addr); + sgl = (struct ulptx_sgl *)(req + 1); + ppod = (struct cxgbi_pagepod *)(sgl + 1); + sgl->addr0 = cpu_to_be64(virt_to_phys(ppod)); + + for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES_MAX) { + if (!hdr && !gl) + cxgbi_ddp_ppod_clear(ppod); + else + cxgbi_ddp_ppod_set(ppod, hdr, gl, gl_pidx); + } + + cxgb4_ofld_send(cdev->ports[port_id], skb); + return 0; +} + +static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr, + unsigned int idx, unsigned int npods, + struct cxgbi_gather_list *gl) +{ + unsigned int i, cnt; + int err = 0; + + for (i = 0; i < npods; i += cnt, idx += cnt) { + cnt = npods - i; + if (cnt > ULPMEM_DSGL_MAX_NPPODS) + cnt = ULPMEM_DSGL_MAX_NPPODS; + err = ddp_ppod_write_sgl(csk->cdev, csk->port_id, hdr, + idx, cnt, gl, 4 * i); + if (err < 0) + break; + } + return err; +} + +static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag, + unsigned int idx, unsigned int npods) +{ + unsigned int i, cnt; + int err; + + for (i = 0; i < npods; i += cnt, idx += cnt) { + cnt = npods - i; + if (cnt > ULPMEM_DSGL_MAX_NPPODS) + cnt = ULPMEM_DSGL_MAX_NPPODS; + err = ddp_ppod_write_sgl(chba->cdev, chba->port_id, NULL, + idx, cnt, NULL, 0); + if (err < 0) + break; + } +} + +static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid, + int pg_idx, bool reply) +{ + struct sk_buff *skb; + struct cpl_set_tcb_field *req; + u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0; + + if (!pg_idx) + return 0; + + skb = alloc_cpl(sizeof(*req), 0, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + /* set up ulp submode and page size */ + val = (val & 0x03) << 2; + val |= TCB_ULP_TYPE(ULP2_MODE_ISCSI); + + req = (struct cpl_set_tcb_field *)skb->head; + INIT_TP_WR(req, csk->tid); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); + req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid)); + req->word_cookie = htons(TCB_WORD(W_TCB_ULP_RAW)); + req->mask = cpu_to_be64(TCB_ULP_TYPE(TCB_ULP_TYPE_MASK)); + req->val = cpu_to_be64(val); + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, tid 0x%x, pg_idx %u.\n", + csk, csk->tid, pg_idx); + + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); + return 0; +} + +static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, + int hcrc, int dcrc, int reply) +{ + struct sk_buff *skb; + struct cpl_set_tcb_field *req; + u64 val = (hcrc ? ULP_CRC_HEADER : 0) | (dcrc ? ULP_CRC_DATA : 0); + + val = TCB_ULP_RAW(val); + val |= TCB_ULP_TYPE(ULP2_MODE_ISCSI); + + skb = alloc_cpl(sizeof(*req), 0, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + csk->hcrc_len = (hcrc ? 4 : 0); + csk->dcrc_len = (dcrc ? 4 : 0); + /* set up ulp submode and page size */ + req = (struct cpl_set_tcb_field *)skb->head; + INIT_TP_WR(req, tid); + OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid)); + req->word_cookie = htons(TCB_WORD(W_TCB_ULP_RAW)); + req->mask = cpu_to_be64(TCB_ULP_RAW(TCB_ULP_RAW_MASK)); + req->val = cpu_to_be64(val); + set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); + + cxgb4i_log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, + "csk 0x%p, tid 0x%x, crc %d,%d.\n", + csk, csk->tid, hcrc, dcrc); + + cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); + return 0; +} + +static int cxgb4i_ddp_init(struct cxgbi_device *cdev) +{ + struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); + struct cxgbi_ddp_info *ddp = cdev->ddp; + unsigned int tagmask, pgsz_factor[4]; + int err; + + if (ddp) { + kref_get(&ddp->refcnt); + cxgb4i_log_warn("cdev 0x%p, ddp 0x%p already set up\n", + cdev, cdev->ddp); + return -EALREADY; + } + + err = cxgbi_ddp_init(cdev, lldi->vr->iscsi.start, + lldi->vr->iscsi.start + lldi->vr->iscsi.size - 1, + lldi->iscsi_iolen, lldi->iscsi_iolen); + if (err < 0) + return err; + + ddp = cdev->ddp; + + tagmask = ddp->idx_mask << PPOD_IDX_SHIFT; + cxgbi_ddp_page_size_factor(pgsz_factor); + cxgb4_iscsi_init(lldi->ports[0], tagmask, pgsz_factor); + + cdev->csk_ddp_free_gl_skb = NULL; + cdev->csk_ddp_alloc_gl_skb = NULL; + cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; + cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; + cdev->csk_ddp_set = ddp_set_map; + cdev->csk_ddp_clear = ddp_clear_map; + + cxgb4i_log_info("cxgb4i 0x%p tag: sw %u, rsvd %u,%u, mask 0x%x.\n", + cdev, cdev->tag_format.sw_bits, + cdev->tag_format.rsvd_bits, + cdev->tag_format.rsvd_shift, + cdev->tag_format.rsvd_mask); + cxgb4i_log_info("cxgb4i 0x%p, nppods %u, bits %u, mask 0x%x,0x%x " + "pkt %u/%u, %u/%u\n", + cdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask, + ddp->rsvd_tag_mask, ddp->max_txsz, lldi->iscsi_iolen, + ddp->max_rxsz, lldi->iscsi_iolen); + cxgb4i_log_info("cxgb4i 0x%p max payload size: %u/%u, %u/%u.\n", + cdev, cdev->tx_max_size, ddp->max_txsz, + cdev->rx_max_size, ddp->max_rxsz); + return 0; +} + +static void *t4_uld_add(const struct cxgb4_lld_info *lldi) +{ + struct cxgbi_device *cdev; + struct port_info *pi; + int i, rc; + + cxgb4i_log_info("%s", version); + + cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports); + if (!cdev) { + cxgb4i_log_info("t4 device 0x%p, register failed.\n", lldi); + return NULL; + } + cxgb4i_log_debug(1 << CXGBI_DBG_DEV, + "t4 0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n", + cdev, lldi->adapter_type, lldi->nports, + lldi->ports[0]->name, lldi->nchan, lldi->ntxq, + lldi->nrxq, lldi->wr_cred); + for (i = 0; i < lldi->nrxq; i++) + cxgb4i_log_debug(1 << CXGBI_DBG_DEV, + "t4 0x%p, rxq id #%d: %u.\n", + cdev, i, lldi->rxq_ids[i]); + + memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi)); + cdev->flags = CXGBI_FLAG_DEV_T4; + cdev->pdev = lldi->pdev; + cdev->ports = lldi->ports; + cdev->nports = lldi->nports; + cdev->mtus = lldi->mtus; + cdev->nmtus = NMTUS; + cdev->snd_win = cxgb4i_snd_win; + cdev->rcv_win = cxgb4i_rcv_win; + cdev->rx_credit_thres = cxgb4i_rx_credit_thres; + cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN; + cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr); + cdev->itp = &cxgb4i_iscsi_transport; + + rc = cxgbi_iscsi_init(cdev); + if (rc) { + cxgb4i_log_info("t4 0x%p iscsi init failed\n", cdev); + goto err_out; + } + rc = cxgb4i_ddp_init(cdev); + if (rc) { + cxgb4i_log_info("t4 0x%p ddp init failed\n", cdev); + goto err_out; + } + rc = cxgb4i_ofld_init(cdev); + if (rc) { + cxgb4i_log_info("t4 0x%p ofld init failed\n", cdev); + goto err_out; + } + + rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN, + &cxgb4i_host_template); + if (rc) + goto err_out; + + for (i = 0; i < cdev->nports; i++) { + pi = netdev_priv(lldi->ports[i]); + cdev->hbas[i]->port_id = pi->port_id; + } + cxgb4i_log_info("cdev 0x%p, f 0x%x, open, err %d.\n", + cdev, cdev->flags, rc); + return cdev; + +err_out: + cxgbi_device_unregister(cdev); + return ERR_PTR(-ENOMEM); +} + +#define RX_PULL_LEN 128 +static int t4_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *pgl) +{ + const struct cpl_act_establish *rpl; + struct sk_buff *skb; + unsigned int opc; + struct cxgbi_device *cdev = handle; + + if (pgl == NULL) { + unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; + + skb = alloc_cpl(len, 0, GFP_ATOMIC); + if (!skb) + goto nomem; + skb_copy_to_linear_data(skb, &rsp[1], len); + } else { + if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) { + cxgb4i_log_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", + pgl->va, be64_to_cpu(*rsp), + be64_to_cpu(*(u64 *)pgl->va), + pgl->tot_len); + return 0; + } + skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN); + if (unlikely(!skb)) + goto nomem; + } + + rpl = (struct cpl_act_establish *)skb->data; + opc = rpl->ot.opcode; + cxgb4i_log_debug(1 << CXGBI_DBG_TOE, + "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p\n", + cdev, opc, rpl->ot.opcode_tid, + ntohl(rpl->ot.opcode_tid), skb); + if (cxgb4i_cplhandlers[opc]) + cxgb4i_cplhandlers[opc](cdev, skb); + else { + cxgb4i_log_error("No handler for opcode 0x%x\n", opc); + __kfree_skb(skb); + } + return 0; +nomem: + cxgb4i_log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out\n"); + return 1; +} + +static int t4_uld_state_change(void *handle, enum cxgb4_state state) +{ + struct cxgbi_device *cdev = handle; + + switch (state) { + case CXGB4_STATE_UP: + cxgb4i_log_info("cdev 0x%p, UP.\n", cdev); + /* re-initialize */ + break; + case CXGB4_STATE_START_RECOVERY: + cxgb4i_log_info("cdev 0x%p, RECOVERY.\n", cdev); + /* close all connections */ + break; + case CXGB4_STATE_DOWN: + cxgb4i_log_info("cdev 0x%p, DOWN.\n", cdev); + break; + case CXGB4_STATE_DETACH: + cxgb4i_log_info("cdev 0x%p, DETACH.\n", cdev); + break; + default: + cxgb4i_log_info("cdev 0x%p, unknown state %d.\n", cdev, state); + break; + } + return 0; +} + +static int __init cxgb4i_init_module(void) +{ + cxgbi_fill_scsi_host_template(&cxgb4i_host_template); + cxgbi_fill_iscsi_transport(&cxgb4i_iscsi_transport); + cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info); + return 0; +} + +static void __exit cxgb4i_exit_module(void) +{ + cxgb4_unregister_uld(CXGB4_ULD_ISCSI); + cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4); +} + +module_init(cxgb4i_init_module); +module_exit(cxgb4i_exit_module); diff --git a/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h new file mode 100644 index 0000000..ae8fbb4 --- /dev/null +++ b/drivers/scsi/cxgbi/cxgb4i/cxgb4i.h @@ -0,0 +1,42 @@ +/* + * cxgb4i.h: Chelsio T4 iSCSI driver. + * + * Copyright (c) 2010 Chelsio Communications, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + * + * Written by: Karen Xie (kxie@xxxxxxxxxxx) + * Written by: Rakesh Ranjan (rranjan@xxxxxxxxxxx) + */ + +#ifndef __CXGB4I_H__ +#define __CXGB4I_H__ + +#include "../libcxgbi.h" + +#define CXGB4I_SCSI_HOST_QDEPTH 1024 +#define CXGB4I_MAX_CONN 16384 +#define CXGB4I_MAX_TARGET CXGB4I_MAX_CONN +#define CXGB4I_MAX_LUN 0x1000 + +/* for TX: a skb must have a headroom of at least TX_HEADER_LEN bytes */ +#define CXGB4I_TX_HEADER_LEN \ + (sizeof(struct fw_ofld_tx_data_wr) + sizeof(struct sge_opaque_hdr)) + +typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *); + +struct cpl_rx_data_ddp { + union opcode_tid ot; + __be16 urg; + __be16 len; + __be32 seq; + union { + __be32 nxt_seq; + __be32 ddp_report; + }; + __be32 ulp_crc; + __be32 ddpvld; +}; +#endif /* __CXGB4I_H__ */ -- To unsubscribe from this list: send the line "unsubscribe linux-scsi" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
