drivers/scsi/cxgbi/cxgb3i/cxgb3i.c

   1 /*
   2  * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
   3  *
   4  * Copyright (C) 2003-2008 Chelsio Communications.  All rights reserved.
   5  *
   6  * This program is distributed in the hope that it will be useful, but WITHOUT
   7  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
   8  * FITNESS FOR A PARTICULAR PURPOSE.  See the LICENSE file included in this
   9  * release for licensing terms and conditions.
  10  *
  11  * Written by:  Dimitris Michailidis (dm@chelsio.com)
  12  *              Karen Xie (kxie@chelsio.com)
  13  */
  14
  15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
  16
  17 #include <linux/version.h>
  18 #include <linux/module.h>
  19 #include <linux/moduleparam.h>
  20 #include <scsi/scsi_host.h>
  21
  22 #include "common.h"
  23 #include "t3_cpl.h"
  24 #include "t3cdev.h"
  25 #include "cxgb3_defs.h"
  26 #include "cxgb3_ctl_defs.h"
  27 #include "cxgb3_offload.h"
  28 #include "firmware_exports.h"
  29 #include "cxgb3i.h"
  30
  31 static unsigned int dbg_level;
  32 #include "../libcxgbi.h"
  33
  34 #define DRV_MODULE_NAME         "cxgb3i"
  35 #define DRV_MODULE_DESC         "Chelsio T3 iSCSI Driver"
  36 #define DRV_MODULE_VERSION      "2.0.0"
  37 #define DRV_MODULE_RELDATE      "Jun. 2010"
  38
  39 static char version[] =
  40         DRV_MODULE_DESC " " DRV_MODULE_NAME
  41         " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  42
  43 MODULE_AUTHOR("Chelsio Communications, Inc.");
  44 MODULE_DESCRIPTION(DRV_MODULE_DESC);
  45 MODULE_VERSION(DRV_MODULE_VERSION);
  46 MODULE_LICENSE("GPL");
  47
  48 module_param(dbg_level, uint, 0644);
  49 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)");
  50
  51 static int cxgb3i_rcv_win = 256 * 1024;
  52 module_param(cxgb3i_rcv_win, int, 0644);
  53 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)");
  54
  55 static int cxgb3i_snd_win = 128 * 1024;
  56 module_param(cxgb3i_snd_win, int, 0644);
  57 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)");
  58
  59 static int cxgb3i_rx_credit_thres = 10 * 1024;
  60 module_param(cxgb3i_rx_credit_thres, int, 0644);
  61 MODULE_PARM_DESC(rx_credit_thres,
  62                  "RX credits return threshold in bytes (default=10KB)");
  63
  64 static unsigned int cxgb3i_max_connect = 8 * 1024;
  65 module_param(cxgb3i_max_connect, uint, 0644);
  66 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)");
  67
  68 static unsigned int cxgb3i_sport_base = 20000;
  69 module_param(cxgb3i_sport_base, uint, 0644);
  70 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)");
  71
  72 static void cxgb3i_dev_open(struct t3cdev *);
  73 static void cxgb3i_dev_close(struct t3cdev *);
  74 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32);
  75
  76 static struct cxgb3_client t3_client = {
  77         .name = DRV_MODULE_NAME,
  78         .handlers = cxgb3i_cpl_handlers,
  79         .add = cxgb3i_dev_open,
  80         .remove = cxgb3i_dev_close,
  81         .event_handler = cxgb3i_dev_event_handler,
  82 };
  83
  84 static struct scsi_host_template cxgb3i_host_template = {
  85         .module         = THIS_MODULE,
  86         .name           = DRV_MODULE_NAME,
  87         .proc_name      = DRV_MODULE_NAME,
  88         .can_queue      = CXGB3I_SCSI_HOST_QDEPTH,
  89         .queuecommand   = iscsi_queuecommand,
  90         .change_queue_depth = iscsi_change_queue_depth,
  91         .sg_tablesize   = SG_ALL,
  92         .max_sectors    = 0xFFFF,
  93         .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
  94         .eh_abort_handler = iscsi_eh_abort,
  95         .eh_device_reset_handler = iscsi_eh_device_reset,
  96         .eh_target_reset_handler = iscsi_eh_recover_target,
  97         .target_alloc   = iscsi_target_alloc,
  98         .use_clustering = DISABLE_CLUSTERING,
  99         .this_id        = -1,
 100 };
 101
 102 static struct iscsi_transport cxgb3i_iscsi_transport = {
 103         .owner          = THIS_MODULE,
 104         .name           = DRV_MODULE_NAME,
 105         /* owner and name should be set already */
 106         .caps           = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
 107                                 | CAP_DATADGST | CAP_DIGEST_OFFLOAD |
 108                                 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
 109         .param_mask     = ISCSI_MAX_RECV_DLENGTH | ISCSI_MAX_XMIT_DLENGTH |
 110                                 ISCSI_HDRDGST_EN | ISCSI_DATADGST_EN |
 111                                 ISCSI_INITIAL_R2T_EN | ISCSI_MAX_R2T |
 112                                 ISCSI_IMM_DATA_EN | ISCSI_FIRST_BURST |
 113                                 ISCSI_MAX_BURST | ISCSI_PDU_INORDER_EN |
 114                                 ISCSI_DATASEQ_INORDER_EN | ISCSI_ERL |
 115                                 ISCSI_CONN_PORT | ISCSI_CONN_ADDRESS |
 116                                 ISCSI_EXP_STATSN | ISCSI_PERSISTENT_PORT |
 117                                 ISCSI_PERSISTENT_ADDRESS |
 118                                 ISCSI_TARGET_NAME | ISCSI_TPGT |
 119                                 ISCSI_USERNAME | ISCSI_PASSWORD |
 120                                 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
 121                                 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
 122                                 ISCSI_LU_RESET_TMO | ISCSI_TGT_RESET_TMO |
 123                                 ISCSI_PING_TMO | ISCSI_RECV_TMO |
 124                                 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
 125         .host_param_mask        = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
 126                                 ISCSI_HOST_INITIATOR_NAME |
 127                                 ISCSI_HOST_NETDEV_NAME,
 128         .get_host_param = cxgbi_get_host_param,
 129         .set_host_param = cxgbi_set_host_param,
 130         /* session management */
 131         .create_session = cxgbi_create_session,
 132         .destroy_session        = cxgbi_destroy_session,
 133         .get_session_param = iscsi_session_get_param,
 134         /* connection management */
 135         .create_conn    = cxgbi_create_conn,
 136         .bind_conn      = cxgbi_bind_conn,
 137         .destroy_conn   = iscsi_tcp_conn_teardown,
 138         .start_conn     = iscsi_conn_start,
 139         .stop_conn      = iscsi_conn_stop,
 140         .get_conn_param = iscsi_conn_get_param,
 141         .set_param      = cxgbi_set_conn_param,
 142         .get_stats      = cxgbi_get_conn_stats,
 143         /* pdu xmit req from user space */
 144         .send_pdu       = iscsi_conn_send_pdu,
 145         /* task */
 146         .init_task      = iscsi_tcp_task_init,
 147         .xmit_task      = iscsi_tcp_task_xmit,
 148         .cleanup_task   = cxgbi_cleanup_task,
 149         /* pdu */
 150         .alloc_pdu      = cxgbi_conn_alloc_pdu,
 151         .init_pdu       = cxgbi_conn_init_pdu,
 152         .xmit_pdu       = cxgbi_conn_xmit_pdu,
 153         .parse_pdu_itt  = cxgbi_parse_pdu_itt,
 154         /* TCP connect/disconnect */
 155         .get_ep_param   = cxgbi_get_ep_param,
 156         .ep_connect     = cxgbi_ep_connect,
 157         .ep_poll        = cxgbi_ep_poll,
 158         .ep_disconnect  = cxgbi_ep_disconnect,
 159         /* Error recovery timeout call */
 160         .session_recovery_timedout = iscsi_session_recovery_timedout,
 161 };
 162
 163 static struct scsi_transport_template *cxgb3i_stt;
 164
 165 /*
 166  * CPL (Chelsio Protocol Language) defines a message passing interface between
 167  * the host driver and Chelsio asic.
 168  * The section below implments CPLs that related to iscsi tcp connection
 169  * open/close/abort and data send/receive.
 170  */
 171
 172 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion);
 173
 174 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
 175                               const struct l2t_entry *e)
 176 {
 177         unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win);
 178         struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head;
 179
 180         skb->priority = CPL_PRIORITY_SETUP;
 181
 182         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 183         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid));
 184         req->local_port = csk->saddr.sin_port;
 185         req->peer_port = csk->daddr.sin_port;
 186         req->local_ip = csk->saddr.sin_addr.s_addr;
 187         req->peer_ip = csk->daddr.sin_addr.s_addr;
 188
 189         req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS |
 190                         V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) |
 191                         V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx));
 192         req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) |
 193                         V_RCV_BUFSIZ(cxgb3i_rcv_win>>10));
 194
 195         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 196                 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
 197                 csk, csk->state, csk->flags, csk->atid,
 198                 &req->local_ip, ntohs(req->local_port),
 199                 &req->peer_ip, ntohs(req->peer_port),
 200                 csk->mss_idx, e->idx, e->smt_idx);
 201
 202         l2t_send(csk->cdev->lldev, skb, csk->l2t);
 203 }
 204
 205 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
 206 {
 207         cxgbi_sock_act_open_req_arp_failure(NULL, skb);
 208 }
 209
 210 /*
 211  * CPL connection close request: host ->
 212  *
 213  * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
 214  * the write queue (i.e., after any unsent txt data).
 215  */
 216 static void send_close_req(struct cxgbi_sock *csk)
 217 {
 218         struct sk_buff *skb = csk->cpl_close;
 219         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 220         unsigned int tid = csk->tid;
 221
 222         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 223                 "csk 0x%p,%u,0x%lx,%u.\n",
 224                 csk, csk->state, csk->flags, csk->tid);
 225
 226         csk->cpl_close = NULL;
 227         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
 228         req->wr.wr_lo = htonl(V_WR_TID(tid));
 229         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 230         req->rsvd = htonl(csk->write_seq);
 231
 232         cxgbi_sock_skb_entail(csk, skb);
 233         if (csk->state >= CTP_ESTABLISHED)
 234                 push_tx_frames(csk, 1);
 235 }
 236
 237 /*
 238  * CPL connection abort request: host ->
 239  *
 240  * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
 241  * for the same connection and also that we do not try to send a message
 242  * after the connection has closed.
 243  */
 244 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb)
 245 {
 246         struct cpl_abort_req *req = cplhdr(skb);
 247
 248         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 249                 "t3dev 0x%p, tid %u, skb 0x%p.\n",
 250                 tdev, GET_TID(req), skb);
 251         req->cmd = CPL_ABORT_NO_RST;
 252         cxgb3_ofld_send(tdev, skb);
 253 }
 254
 255 static void send_abort_req(struct cxgbi_sock *csk)
 256 {
 257         struct sk_buff *skb = csk->cpl_abort_req;
 258         struct cpl_abort_req *req;
 259
 260         if (unlikely(csk->state == CTP_ABORTING || !skb))
 261                 return;
 262         cxgbi_sock_set_state(csk, CTP_ABORTING);
 263         cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
 264         /* Purge the send queue so we don't send anything after an abort. */
 265         cxgbi_sock_purge_write_queue(csk);
 266
 267         csk->cpl_abort_req = NULL;
 268         req = (struct cpl_abort_req *)skb->head;
 269         skb->priority = CPL_PRIORITY_DATA;
 270         set_arp_failure_handler(skb, abort_arp_failure);
 271         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
 272         req->wr.wr_lo = htonl(V_WR_TID(csk->tid));
 273         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
 274         req->rsvd0 = htonl(csk->snd_nxt);
 275         req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
 276         req->cmd = CPL_ABORT_SEND_RST;
 277
 278         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 279                 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
 280                 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
 281                 req->rsvd1);
 282
 283         l2t_send(csk->cdev->lldev, skb, csk->l2t);
 284 }
 285
 286 /*
 287  * CPL connection abort reply: host ->
 288  *
 289  * Send an ABORT_RPL message in response of the ABORT_REQ received.
 290  */
 291 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
 292 {
 293         struct sk_buff *skb = csk->cpl_abort_rpl;
 294         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 295
 296         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 297                 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
 298                 csk, csk->state, csk->flags, csk->tid, rst_status);
 299
 300         csk->cpl_abort_rpl = NULL;
 301         skb->priority = CPL_PRIORITY_DATA;
 302         rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
 303         rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid));
 304         OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
 305         rpl->cmd = rst_status;
 306         cxgb3_ofld_send(csk->cdev->lldev, skb);
 307 }
 308
 309 /*
 310  * CPL connection rx data ack: host ->
 311  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 312  * credits sent.
 313  */
 314 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
 315 {
 316         struct sk_buff *skb;
 317         struct cpl_rx_data_ack *req;
 318         u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
 319
 320         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 321                 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
 322                 csk, csk->state, csk->flags, csk->tid, credits, dack);
 323
 324         skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
 325         if (!skb) {
 326                 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
 327                 return 0;
 328         }
 329         req = (struct cpl_rx_data_ack *)skb->head;
 330         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 331         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid));
 332         req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) |
 333                                 V_RX_CREDITS(credits));
 334         skb->priority = CPL_PRIORITY_ACK;
 335         cxgb3_ofld_send(csk->cdev->lldev, skb);
 336         return credits;
 337 }
 338
 339 /*
 340  * CPL connection tx data: host ->
 341  *
 342  * Send iscsi PDU via TX_DATA CPL message. Returns the number of
 343  * credits sent.
 344  * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
 345  * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
 346  */
 347
 348 static unsigned int wrlen __read_mostly;
 349 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
 350
 351 static void init_wr_tab(unsigned int wr_len)
 352 {
 353         int i;
 354
 355         if (skb_wrs[1])         /* already initialized */
 356                 return;
 357         for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
 358                 int sgl_len = (3 * i) / 2 + (i & 1);
 359
 360                 sgl_len += 3;
 361                 skb_wrs[i] = (sgl_len <= wr_len
 362                               ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
 363         }
 364         wrlen = wr_len * 8;
 365 }
 366
 367 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
 368                                    int len, int req_completion)
 369 {
 370         struct tx_data_wr *req;
 371         struct l2t_entry *l2t = csk->l2t;
 372
 373         skb_reset_transport_header(skb);
 374         req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
 375         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) |
 376                         (req_completion ? F_WR_COMPL : 0));
 377         req->wr_lo = htonl(V_WR_TID(csk->tid));
 378         /* len includes the length of any HW ULP additions */
 379         req->len = htonl(len);
 380         /* V_TX_ULP_SUBMODE sets both the mode and submode */
 381         req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) |
 382                            V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1)));
 383         req->sndseq = htonl(csk->snd_nxt);
 384         req->param = htonl(V_TX_PORT(l2t->smt_idx));
 385
 386         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 387                 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
 388                                     V_TX_CPU_IDX(csk->rss_qid));
 389                 /* sendbuffer is in units of 32KB. */
 390                 req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15));
 391                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 392         }
 393 }
 394
 395 /**
 396  * push_tx_frames -- start transmit
 397  * @c3cn: the offloaded connection
 398  * @req_completion: request wr_ack or not
 399  *
 400  * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
 401  * connection's send queue and sends them on to T3.  Must be called with the
 402  * connection's lock held.  Returns the amount of send buffer space that was
 403  * freed as a result of sending queued data to T3.
 404  */
 405
 406 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb)
 407 {
 408         kfree_skb(skb);
 409 }
 410
 411 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
 412 {
 413         int total_size = 0;
 414         struct sk_buff *skb;
 415
 416         if (unlikely(csk->state < CTP_ESTABLISHED ||
 417                 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
 418                         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 419                                 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
 420                                 csk, csk->state, csk->flags, csk->tid);
 421                 return 0;
 422         }
 423
 424         while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
 425                 int len = skb->len;     /* length before skb_push */
 426                 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
 427                 int wrs_needed = skb_wrs[frags];
 428
 429                 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
 430                         wrs_needed = 1;
 431
 432                 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
 433
 434                 if (csk->wr_cred < wrs_needed) {
 435                         log_debug(1 << CXGBI_DBG_PDU_TX,
 436                                 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
 437                                 csk, skb->len, skb->data_len, frags,
 438                                 wrs_needed, csk->wr_cred);
 439                         break;
 440                 }
 441
 442                 __skb_unlink(skb, &csk->write_queue);
 443                 skb->priority = CPL_PRIORITY_DATA;
 444                 skb->csum = wrs_needed; /* remember this until the WR_ACK */
 445                 csk->wr_cred -= wrs_needed;
 446                 csk->wr_una_cred += wrs_needed;
 447                 cxgbi_sock_enqueue_wr(csk, skb);
 448
 449                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 450                         "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
 451                         "left %u, unack %u.\n",
 452                         csk, skb->len, skb->data_len, frags, skb->csum,
 453                         csk->wr_cred, csk->wr_una_cred);
 454
 455                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
 456                         if ((req_completion &&
 457                                 csk->wr_una_cred == wrs_needed) ||
 458                              csk->wr_una_cred >= csk->wr_max_cred / 2) {
 459                                 req_completion = 1;
 460                                 csk->wr_una_cred = 0;
 461                         }
 462                         len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
 463                         make_tx_data_wr(csk, skb, len, req_completion);
 464                         csk->snd_nxt += len;
 465                         cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
 466                 }
 467                 total_size += skb->truesize;
 468                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 469                         "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
 470                         csk, csk->tid, skb);
 471                 set_arp_failure_handler(skb, arp_failure_skb_discard);
 472                 l2t_send(csk->cdev->lldev, skb, csk->l2t);
 473         }
 474         return total_size;
 475 }
 476
 477 /*
 478  * Process a CPL_ACT_ESTABLISH message: -> host
 479  * Updates connection state from an active establish CPL message.  Runs with
 480  * the connection lock held.
 481  */
 482
 483 static inline void free_atid(struct cxgbi_sock *csk)
 484 {
 485         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
 486                 cxgb3_free_atid(csk->cdev->lldev, csk->atid);
 487                 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
 488                 cxgbi_sock_put(csk);
 489         }
 490 }
 491
 492 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
 493 {
 494         struct cxgbi_sock *csk = ctx;
 495         struct cpl_act_establish *req = cplhdr(skb);
 496         unsigned int tid = GET_TID(req);
 497         unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
 498         u32 rcv_isn = ntohl(req->rcv_isn);      /* real RCV_ISN + 1 */
 499
 500         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 501                 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
 502                 atid, atid, csk, csk->state, csk->flags, rcv_isn);
 503
 504         cxgbi_sock_get(csk);
 505         cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
 506         csk->tid = tid;
 507         cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid);
 508
 509         free_atid(csk);
 510
 511         csk->rss_qid = G_QNUM(ntohs(skb->csum));
 512
 513         spin_lock_bh(&csk->lock);
 514         if (csk->retry_timer.function) {
 515                 del_timer(&csk->retry_timer);
 516                 csk->retry_timer.function = NULL;
 517         }
 518
 519         if (unlikely(csk->state != CTP_ACTIVE_OPEN))
 520                 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
 521                         csk, csk->state, csk->flags, csk->tid);
 522
 523         csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
 524         if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10))
 525                 csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10);
 526
 527         cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 528
 529         if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
 530                 /* upper layer has requested closing */
 531                 send_abort_req(csk);
 532         else {
 533                 if (skb_queue_len(&csk->write_queue))
 534                         push_tx_frames(csk, 1);
 535                 cxgbi_conn_tx_open(csk);
 536         }
 537
 538         spin_unlock_bh(&csk->lock);
 539         __kfree_skb(skb);
 540         return 0;
 541 }
 542
 543 /*
 544  * Process a CPL_ACT_OPEN_RPL message: -> host
 545  * Handle active open failures.
 546  */
 547 static int act_open_rpl_status_to_errno(int status)
 548 {
 549         switch (status) {
 550         case CPL_ERR_CONN_RESET:
 551                 return -ECONNREFUSED;
 552         case CPL_ERR_ARP_MISS:
 553                 return -EHOSTUNREACH;
 554         case CPL_ERR_CONN_TIMEDOUT:
 555                 return -ETIMEDOUT;
 556         case CPL_ERR_TCAM_FULL:
 557                 return -ENOMEM;
 558         case CPL_ERR_CONN_EXIST:
 559                 return -EADDRINUSE;
 560         default:
 561                 return -EIO;
 562         }
 563 }
 564
 565 static void act_open_retry_timer(unsigned long data)
 566 {
 567         struct sk_buff *skb;
 568         struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
 569
 570         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 571                 "csk 0x%p,%u,0x%lx,%u.\n",
 572                 csk, csk->state, csk->flags, csk->tid);
 573
 574         cxgbi_sock_get(csk);
 575         spin_lock_bh(&csk->lock);
 576         skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
 577         if (!skb)
 578                 cxgbi_sock_fail_act_open(csk, -ENOMEM);
 579         else {
 580                 skb->sk = (struct sock *)csk;
 581                 set_arp_failure_handler(skb, act_open_arp_failure);
 582                 send_act_open_req(csk, skb, csk->l2t);
 583         }
 584         spin_unlock_bh(&csk->lock);
 585         cxgbi_sock_put(csk);
 586 }
 587
 588 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
 589 {
 590         struct cxgbi_sock *csk = ctx;
 591         struct cpl_act_open_rpl *rpl = cplhdr(skb);
 592
 593         pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
 594                 csk, csk->state, csk->flags, csk->atid, rpl->status,
 595                 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
 596                 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
 597
 598         if (rpl->status != CPL_ERR_TCAM_FULL &&
 599             rpl->status != CPL_ERR_CONN_EXIST &&
 600             rpl->status != CPL_ERR_ARP_MISS)
 601                 cxgb3_queue_tid_release(tdev, GET_TID(rpl));
 602
 603         cxgbi_sock_get(csk);
 604         spin_lock_bh(&csk->lock);
 605         if (rpl->status == CPL_ERR_CONN_EXIST &&
 606             csk->retry_timer.function != act_open_retry_timer) {
 607                 csk->retry_timer.function = act_open_retry_timer;
 608                 mod_timer(&csk->retry_timer, jiffies + HZ / 2);
 609         } else
 610                 cxgbi_sock_fail_act_open(csk,
 611                                 act_open_rpl_status_to_errno(rpl->status));
 612
 613         spin_unlock_bh(&csk->lock);
 614         cxgbi_sock_put(csk);
 615         __kfree_skb(skb);
 616         return 0;
 617 }
 618
 619 /*
 620  * Process PEER_CLOSE CPL messages: -> host
 621  * Handle peer FIN.
 622  */
 623 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 624 {
 625         struct cxgbi_sock *csk = ctx;
 626
 627         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 628                 "csk 0x%p,%u,0x%lx,%u.\n",
 629                 csk, csk->state, csk->flags, csk->tid);
 630
 631         cxgbi_sock_rcv_peer_close(csk);
 632         __kfree_skb(skb);
 633         return 0;
 634 }
 635
 636 /*
 637  * Process CLOSE_CONN_RPL CPL message: -> host
 638  * Process a peer ACK to our FIN.
 639  */
 640 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
 641                             void *ctx)
 642 {
 643         struct cxgbi_sock *csk = ctx;
 644         struct cpl_close_con_rpl *rpl = cplhdr(skb);
 645
 646         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 647                 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
 648                 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt));
 649
 650         cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
 651         __kfree_skb(skb);
 652         return 0;
 653 }
 654
 655 /*
 656  * Process ABORT_REQ_RSS CPL message: -> host
 657  * Process abort requests.  If we are waiting for an ABORT_RPL we ignore this
 658  * request except that we need to reply to it.
 659  */
 660
 661 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
 662                                  int *need_rst)
 663 {
 664         switch (abort_reason) {
 665         case CPL_ERR_BAD_SYN: /* fall through */
 666         case CPL_ERR_CONN_RESET:
 667                 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET;
 668         case CPL_ERR_XMIT_TIMEDOUT:
 669         case CPL_ERR_PERSIST_TIMEDOUT:
 670         case CPL_ERR_FINWAIT2_TIMEDOUT:
 671         case CPL_ERR_KEEPALIVE_TIMEDOUT:
 672                 return -ETIMEDOUT;
 673         default:
 674                 return -EIO;
 675         }
 676 }
 677
 678 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 679 {
 680         const struct cpl_abort_req_rss *req = cplhdr(skb);
 681         struct cxgbi_sock *csk = ctx;
 682         int rst_status = CPL_ABORT_NO_RST;
 683
 684         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 685                 "csk 0x%p,%u,0x%lx,%u.\n",
 686                 csk, csk->state, csk->flags, csk->tid);
 687
 688         if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
 689             req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
 690                 goto done;
 691         }
 692
 693         cxgbi_sock_get(csk);
 694         spin_lock_bh(&csk->lock);
 695
 696         if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
 697                 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
 698                 cxgbi_sock_set_state(csk, CTP_ABORTING);
 699                 goto out;
 700         }
 701
 702         cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
 703         send_abort_rpl(csk, rst_status);
 704
 705         if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
 706                 csk->err = abort_status_to_errno(csk, req->status, &rst_status);
 707                 cxgbi_sock_closed(csk);
 708         }
 709
 710 out:
 711         spin_unlock_bh(&csk->lock);
 712         cxgbi_sock_put(csk);
 713 done:
 714         __kfree_skb(skb);
 715         return 0;
 716 }
 717
 718 /*
 719  * Process ABORT_RPL_RSS CPL message: -> host
 720  * Process abort replies.  We only process these messages if we anticipate
 721  * them as the coordination between SW and HW in this area is somewhat lacking
 722  * and sometimes we get ABORT_RPLs after we are done with the connection that
 723  * originated the ABORT_REQ.
 724  */
 725 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 726 {
 727         struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
 728         struct cxgbi_sock *csk = ctx;
 729
 730         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 731                 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
 732                 rpl->status, csk, csk ? csk->state : 0,
 733                 csk ? csk->flags : 0UL);
 734         /*
 735          * Ignore replies to post-close aborts indicating that the abort was
 736          * requested too late.  These connections are terminated when we get
 737          * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
 738          * arrives the TID is either no longer used or it has been recycled.
 739          */
 740         if (rpl->status == CPL_ERR_ABORT_FAILED)
 741                 goto rel_skb;
 742         /*
 743          * Sometimes we've already closed the connection, e.g., a post-close
 744          * abort races with ABORT_REQ_RSS, the latter frees the connection
 745          * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
 746          * but FW turns the ABORT_REQ into a regular one and so we get
 747          * ABORT_RPL_RSS with status 0 and no connection.
 748          */
 749         if (csk)
 750                 cxgbi_sock_rcv_abort_rpl(csk);
 751 rel_skb:
 752         __kfree_skb(skb);
 753         return 0;
 754 }
 755
 756 /*
 757  * Process RX_ISCSI_HDR CPL message: -> host
 758  * Handle received PDUs, the payload could be DDP'ed. If not, the payload
 759  * follow after the bhs.
 760  */
 761 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
 762 {
 763         struct cxgbi_sock *csk = ctx;
 764         struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
 765         struct cpl_iscsi_hdr_norss data_cpl;
 766         struct cpl_rx_data_ddp_norss ddp_cpl;
 767         unsigned int hdr_len, data_len, status;
 768         unsigned int len;
 769         int err;
 770
 771         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 772                 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
 773                 csk, csk->state, csk->flags, csk->tid, skb, skb->len);
 774
 775         spin_lock_bh(&csk->lock);
 776
 777         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
 778                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 779                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
 780                         csk, csk->state, csk->flags, csk->tid);
 781                 if (csk->state != CTP_ABORTING)
 782                         goto abort_conn;
 783                 else
 784                         goto discard;
 785         }
 786
 787         cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq);
 788         cxgbi_skcb_flags(skb) = 0;
 789
 790         skb_reset_transport_header(skb);
 791         __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
 792
 793         len = hdr_len = ntohs(hdr_cpl->len);
 794         /* msg coalesce is off or not enough data received */
 795         if (skb->len <= hdr_len) {
 796                 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
 797                         csk->cdev->ports[csk->port_id]->name, csk->tid,
 798                         skb->len, hdr_len);
 799                 goto abort_conn;
 800         }
 801         cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED);
 802
 803         err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
 804                             sizeof(ddp_cpl));
 805         if (err < 0) {
 806                 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
 807                         csk->cdev->ports[csk->port_id]->name, csk->tid,
 808                         skb->len, sizeof(ddp_cpl), err);
 809                 goto abort_conn;
 810         }
 811
 812         cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS);
 813         cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len);
 814         cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
 815         status = ntohl(ddp_cpl.ddp_status);
 816
 817         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 818                 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
 819                 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status);
 820
 821         if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT))
 822                 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR);
 823         if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT))
 824                 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR);
 825         if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT))
 826                 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR);
 827
 828         if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
 829                 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
 830                 if (err < 0) {
 831                         pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
 832                                 csk->cdev->ports[csk->port_id]->name,
 833                                 csk->tid, sizeof(data_cpl), skb->len, err);
 834                         goto abort_conn;
 835                 }
 836                 data_len = ntohs(data_cpl.len);
 837                 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX,
 838                         "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
 839                         skb, data_len, cxgbi_skcb_rx_pdulen(skb), status);
 840                 len += sizeof(data_cpl) + data_len;
 841         } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT))
 842                 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD);
 843
 844         csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb);
 845         __pskb_trim(skb, len);
 846         __skb_queue_tail(&csk->receive_queue, skb);
 847         cxgbi_conn_pdu_ready(csk);
 848
 849         spin_unlock_bh(&csk->lock);
 850         return 0;
 851
 852 abort_conn:
 853         send_abort_req(csk);
 854 discard:
 855         spin_unlock_bh(&csk->lock);
 856         __kfree_skb(skb);
 857         return 0;
 858 }
 859
 860 /*
 861  * Process TX_DATA_ACK CPL messages: -> host
 862  * Process an acknowledgment of WR completion.  Advance snd_una and send the
 863  * next batch of work requests from the write queue.
 864  */
 865 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
 866 {
 867         struct cxgbi_sock *csk = ctx;
 868         struct cpl_wr_ack *hdr = cplhdr(skb);
 869
 870         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 871                 "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
 872                 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits));
 873
 874         cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1);
 875         __kfree_skb(skb);
 876         return 0;
 877 }
 878
 879 /*
 880  * for each connection, pre-allocate skbs needed for close/abort requests. So
 881  * that we can service the request right away.
 882  */
 883 static int alloc_cpls(struct cxgbi_sock *csk)
 884 {
 885         csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0,
 886                                         GFP_KERNEL);
 887         if (!csk->cpl_close)
 888                 return -ENOMEM;
 889         csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0,
 890                                         GFP_KERNEL);
 891         if (!csk->cpl_abort_req)
 892                 goto free_cpl_skbs;
 893
 894         csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0,
 895                                         GFP_KERNEL);
 896         if (!csk->cpl_abort_rpl)
 897                 goto free_cpl_skbs;
 898
 899         return 0;
 900
 901 free_cpl_skbs:
 902         cxgbi_sock_free_cpl_skbs(csk);
 903         return -ENOMEM;
 904 }
 905
 906 /**
 907  * release_offload_resources - release offload resource
 908  * @c3cn: the offloaded iscsi tcp connection.
 909  * Release resources held by an offload connection (TID, L2T entry, etc.)
 910  */
 911 static void l2t_put(struct cxgbi_sock *csk)
 912 {
 913         struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
 914
 915         if (csk->l2t) {
 916                 l2t_release(L2DATA(t3dev), csk->l2t);
 917                 csk->l2t = NULL;
 918                 cxgbi_sock_put(csk);
 919         }
 920 }
 921
 922 static void release_offload_resources(struct cxgbi_sock *csk)
 923 {
 924         struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev;
 925
 926         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 927                 "csk 0x%p,%u,0x%lx,%u.\n",
 928                 csk, csk->state, csk->flags, csk->tid);
 929
 930         csk->rss_qid = 0;
 931         cxgbi_sock_free_cpl_skbs(csk);
 932
 933         if (csk->wr_cred != csk->wr_max_cred) {
 934                 cxgbi_sock_purge_wr_queue(csk);
 935                 cxgbi_sock_reset_wr_list(csk);
 936         }
 937         l2t_put(csk);
 938         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
 939                 free_atid(csk);
 940         else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
 941                 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid);
 942                 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
 943                 cxgbi_sock_put(csk);
 944         }
 945         csk->dst = NULL;
 946         csk->cdev = NULL;
 947 }
 948
 949 static void update_address(struct cxgbi_hba *chba)
 950 {
 951         if (chba->ipv4addr) {
 952                 if (chba->vdev &&
 953                     chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) {
 954                         cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr);
 955                         cxgb3i_set_private_ipv4addr(chba->ndev, 0);
 956                         pr_info("%s set %pI4.\n",
 957                                 chba->vdev->name, &chba->ipv4addr);
 958                 } else if (chba->ipv4addr !=
 959                                 cxgb3i_get_private_ipv4addr(chba->ndev)) {
 960                         cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr);
 961                         pr_info("%s set %pI4.\n",
 962                                 chba->ndev->name, &chba->ipv4addr);
 963                 }
 964         } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) {
 965                 if (chba->vdev)
 966                         cxgb3i_set_private_ipv4addr(chba->vdev, 0);
 967                 cxgb3i_set_private_ipv4addr(chba->ndev, 0);
 968         }
 969 }
 970
 971 static int init_act_open(struct cxgbi_sock *csk)
 972 {
 973         struct dst_entry *dst = csk->dst;
 974         struct cxgbi_device *cdev = csk->cdev;
 975         struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
 976         struct net_device *ndev = cdev->ports[csk->port_id];
 977         struct cxgbi_hba *chba = cdev->hbas[csk->port_id];
 978         struct sk_buff *skb = NULL;
 979
 980         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 981                 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags);
 982
 983         update_address(chba);
 984         if (chba->ipv4addr)
 985                 csk->saddr.sin_addr.s_addr = chba->ipv4addr;
 986
 987         csk->rss_qid = 0;
 988         csk->l2t = t3_l2t_get(t3dev, dst_get_neighbour(dst), ndev);
 989         if (!csk->l2t) {
 990                 pr_err("NO l2t available.\n");
 991                 return -EINVAL;
 992         }
 993         cxgbi_sock_get(csk);
 994
 995         csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk);
 996         if (csk->atid < 0) {
 997                 pr_err("NO atid available.\n");
 998                 goto rel_resource;
 999         }
1000         cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
1001         cxgbi_sock_get(csk);
1002
1003         skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
1004         if (!skb)
1005                 goto rel_resource;
1006         skb->sk = (struct sock *)csk;
1007         set_arp_failure_handler(skb, act_open_arp_failure);
1008
1009         csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1;
1010         csk->wr_una_cred = 0;
1011         csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst));
1012         cxgbi_sock_reset_wr_list(csk);
1013         csk->err = 0;
1014
1015         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1016                 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1017                 csk, csk->state, csk->flags,
1018                 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
1019                 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port));
1020
1021         cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1022         send_act_open_req(csk, skb, csk->l2t);
1023         return 0;
1024
1025 rel_resource:
1026         if (skb)
1027                 __kfree_skb(skb);
1028         return -EINVAL;
1029 }
1030
1031 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = {
1032         [CPL_ACT_ESTABLISH] = do_act_establish,
1033         [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1034         [CPL_PEER_CLOSE] = do_peer_close,
1035         [CPL_ABORT_REQ_RSS] = do_abort_req,
1036         [CPL_ABORT_RPL_RSS] = do_abort_rpl,
1037         [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1038         [CPL_TX_DMA_ACK] = do_wr_ack,
1039         [CPL_ISCSI_HDR] = do_iscsi_hdr,
1040 };
1041
1042 /**
1043  * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1044  * @cdev:       cxgbi adapter
1045  */
1046 int cxgb3i_ofld_init(struct cxgbi_device *cdev)
1047 {
1048         struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev;
1049         struct adap_ports port;
1050         struct ofld_page_info rx_page_info;
1051         unsigned int wr_len;
1052         int rc;
1053
1054         if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 ||
1055             t3dev->ctl(t3dev, GET_PORTS, &port) < 0 ||
1056             t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) {
1057                 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev);
1058                 return -EINVAL;
1059         }
1060
1061         if (cxgb3i_max_connect > CXGBI_MAX_CONN)
1062                 cxgb3i_max_connect = CXGBI_MAX_CONN;
1063
1064         rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base,
1065                                         cxgb3i_max_connect);
1066         if (rc < 0)
1067                 return rc;
1068
1069         init_wr_tab(wr_len);
1070         cdev->csk_release_offload_resources = release_offload_resources;
1071         cdev->csk_push_tx_frames = push_tx_frames;
1072         cdev->csk_send_abort_req = send_abort_req;
1073         cdev->csk_send_close_req = send_close_req;
1074         cdev->csk_send_rx_credits = send_rx_credits;
1075         cdev->csk_alloc_cpls = alloc_cpls;
1076         cdev->csk_init_act_open = init_act_open;
1077
1078         pr_info("cdev 0x%p, offload up, added.\n", cdev);
1079         return 0;
1080 }
1081
1082 /*
1083  * functions to program the pagepod in h/w
1084  */
1085 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr)
1086 {
1087         struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head;
1088
1089         memset(req, 0, sizeof(*req));
1090
1091         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
1092         req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) |
1093                                    V_ULPTX_CMD(ULP_MEM_WRITE));
1094         req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) |
1095                          V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1));
1096 }
1097
1098 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
1099                         unsigned int idx, unsigned int npods,
1100                                 struct cxgbi_gather_list *gl)
1101 {
1102         struct cxgbi_device *cdev = csk->cdev;
1103         struct cxgbi_ddp_info *ddp = cdev->ddp;
1104         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
1105         int i;
1106
1107         log_debug(1 << CXGBI_DBG_DDP,
1108                 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
1109                 csk, idx, npods, gl);
1110
1111         for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
1112                 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1113                                                 PPOD_SIZE, 0, GFP_ATOMIC);
1114
1115                 if (!skb)
1116                         return -ENOMEM;
1117
1118                 ulp_mem_io_set_hdr(skb, pm_addr);
1119                 cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head +
1120                                         sizeof(struct ulp_mem_io)),
1121                                    hdr, gl, i * PPOD_PAGES_MAX);
1122                 skb->priority = CPL_PRIORITY_CONTROL;
1123                 cxgb3_ofld_send(cdev->lldev, skb);
1124         }
1125         return 0;
1126 }
1127
1128 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
1129                           unsigned int idx, unsigned int npods)
1130 {
1131         struct cxgbi_device *cdev = chba->cdev;
1132         struct cxgbi_ddp_info *ddp = cdev->ddp;
1133         unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit;
1134         int i;
1135
1136         log_debug(1 << CXGBI_DBG_DDP,
1137                 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
1138                 cdev, idx, npods, tag);
1139
1140         for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) {
1141                 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) +
1142                                                 PPOD_SIZE, 0, GFP_ATOMIC);
1143
1144                 if (!skb) {
1145                         pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
1146                                 tag, idx, i, npods);
1147                         continue;
1148                 }
1149                 ulp_mem_io_set_hdr(skb, pm_addr);
1150                 skb->priority = CPL_PRIORITY_CONTROL;
1151                 cxgb3_ofld_send(cdev->lldev, skb);
1152         }
1153 }
1154
1155 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk,
1156                                        unsigned int tid, int pg_idx, bool reply)
1157 {
1158         struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1159                                         GFP_KERNEL);
1160         struct cpl_set_tcb_field *req;
1161         u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0;
1162
1163         log_debug(1 << CXGBI_DBG_DDP,
1164                 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx);
1165         if (!skb)
1166                 return -ENOMEM;
1167
1168         /* set up ulp submode and page size */
1169         req = (struct cpl_set_tcb_field *)skb->head;
1170         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1171         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1172         req->reply = V_NO_REPLY(reply ? 0 : 1);
1173         req->cpu_idx = 0;
1174         req->word = htons(31);
1175         req->mask = cpu_to_be64(0xF0000000);
1176         req->val = cpu_to_be64(val << 28);
1177         skb->priority = CPL_PRIORITY_CONTROL;
1178
1179         cxgb3_ofld_send(csk->cdev->lldev, skb);
1180         return 0;
1181 }
1182
1183 /**
1184  * cxgb3i_setup_conn_digest - setup conn. digest setting
1185  * @csk: cxgb tcp socket
1186  * @tid: connection id
1187  * @hcrc: header digest enabled
1188  * @dcrc: data digest enabled
1189  * @reply: request reply from h/w
1190  * set up the iscsi digest settings for a connection identified by tid
1191  */
1192 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1193                              int hcrc, int dcrc, int reply)
1194 {
1195         struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0,
1196                                         GFP_KERNEL);
1197         struct cpl_set_tcb_field *req;
1198         u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0);
1199
1200         log_debug(1 << CXGBI_DBG_DDP,
1201                 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc);
1202         if (!skb)
1203                 return -ENOMEM;
1204
1205         /* set up ulp submode and page size */
1206         req = (struct cpl_set_tcb_field *)skb->head;
1207         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1208         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1209         req->reply = V_NO_REPLY(reply ? 0 : 1);
1210         req->cpu_idx = 0;
1211         req->word = htons(31);
1212         req->mask = cpu_to_be64(0x0F000000);
1213         req->val = cpu_to_be64(val << 24);
1214         skb->priority = CPL_PRIORITY_CONTROL;
1215
1216         cxgb3_ofld_send(csk->cdev->lldev, skb);
1217         return 0;
1218 }
1219
1220 /**
1221  * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource
1222  * @cdev: cxgb3i adapter
1223  * release all the resource held by the ddp pagepod manager for a given
1224  * adapter if needed
1225  */
1226
1227 static void t3_ddp_cleanup(struct cxgbi_device *cdev)
1228 {
1229         struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1230
1231         if (cxgbi_ddp_cleanup(cdev)) {
1232                 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev);
1233                 tdev->ulp_iscsi = NULL;
1234         }
1235 }
1236
1237 /**
1238  * ddp_init - initialize the cxgb3 adapter's ddp resource
1239  * @cdev: cxgb3i adapter
1240  * initialize the ddp pagepod manager for a given adapter
1241  */
1242 static int cxgb3i_ddp_init(struct cxgbi_device *cdev)
1243 {
1244         struct t3cdev *tdev = (struct t3cdev *)cdev->lldev;
1245         struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi;
1246         struct ulp_iscsi_info uinfo;
1247         unsigned int pgsz_factor[4];
1248         int err;
1249
1250         if (ddp) {
1251                 kref_get(&ddp->refcnt);
1252                 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
1253                         tdev, tdev->ulp_iscsi);
1254                 cdev->ddp = ddp;
1255                 return -EALREADY;
1256         }
1257
1258         err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo);
1259         if (err < 0) {
1260                 pr_err("%s, failed to get iscsi param err=%d.\n",
1261                          tdev->name, err);
1262                 return err;
1263         }
1264
1265         err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit,
1266                         uinfo.max_txsz, uinfo.max_rxsz);
1267         if (err < 0)
1268                 return err;
1269
1270         ddp = cdev->ddp;
1271
1272         uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
1273         cxgbi_ddp_page_size_factor(pgsz_factor);
1274         uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT);
1275
1276         err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo);
1277         if (err < 0) {
1278                 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
1279                         tdev->name, err);
1280                 cxgbi_ddp_cleanup(cdev);
1281                 return err;
1282         }
1283         tdev->ulp_iscsi = ddp;
1284
1285         cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1286         cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1287         cdev->csk_ddp_set = ddp_set_map;
1288         cdev->csk_ddp_clear = ddp_clear_map;
1289
1290         pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1291                 "%u/%u.\n",
1292                 tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
1293                 ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz,
1294                 ddp->max_rxsz, uinfo.max_rxsz);
1295         return 0;
1296 }
1297
1298 static void cxgb3i_dev_close(struct t3cdev *t3dev)
1299 {
1300         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1301
1302         if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) {
1303                 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0);
1304                 return;
1305         }
1306
1307         cxgbi_device_unregister(cdev);
1308 }
1309
1310 /**
1311  * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1312  * @t3dev: t3cdev adapter
1313  */
1314 static void cxgb3i_dev_open(struct t3cdev *t3dev)
1315 {
1316         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1317         struct adapter *adapter = tdev2adap(t3dev);
1318         int i, err;
1319
1320         if (cdev) {
1321                 pr_info("0x%p, updating.\n", cdev);
1322                 return;
1323         }
1324
1325         cdev = cxgbi_device_register(0, adapter->params.nports);
1326         if (!cdev) {
1327                 pr_warn("device 0x%p register failed.\n", t3dev);
1328                 return;
1329         }
1330
1331         cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET;
1332         cdev->lldev = t3dev;
1333         cdev->pdev = adapter->pdev;
1334         cdev->ports = adapter->port;
1335         cdev->nports = adapter->params.nports;
1336         cdev->mtus = adapter->params.mtus;
1337         cdev->nmtus = NMTUS;
1338         cdev->snd_win = cxgb3i_snd_win;
1339         cdev->rcv_win = cxgb3i_rcv_win;
1340         cdev->rx_credit_thres = cxgb3i_rx_credit_thres;
1341         cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN;
1342         cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss);
1343         cdev->dev_ddp_cleanup = t3_ddp_cleanup;
1344         cdev->itp = &cxgb3i_iscsi_transport;
1345
1346         err = cxgb3i_ddp_init(cdev);
1347         if (err) {
1348                 pr_info("0x%p ddp init failed\n", cdev);
1349                 goto err_out;
1350         }
1351
1352         err = cxgb3i_ofld_init(cdev);
1353         if (err) {
1354                 pr_info("0x%p offload init failed\n", cdev);
1355                 goto err_out;
1356         }
1357
1358         err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN,
1359                                 &cxgb3i_host_template, cxgb3i_stt);
1360         if (err)
1361                 goto err_out;
1362
1363         for (i = 0; i < cdev->nports; i++)
1364                 cdev->hbas[i]->ipv4addr =
1365                         cxgb3i_get_private_ipv4addr(cdev->ports[i]);
1366
1367         pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1368                 cdev, cdev ? cdev->flags : 0, t3dev, err);
1369         return;
1370
1371 err_out:
1372         cxgbi_device_unregister(cdev);
1373 }
1374
1375 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port)
1376 {
1377         struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev);
1378
1379         log_debug(1 << CXGBI_DBG_TOE,
1380                 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1381                 t3dev, cdev, event, port);
1382         if (!cdev)
1383                 return;
1384
1385         switch (event) {
1386         case OFFLOAD_STATUS_DOWN:
1387                 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET;
1388                 break;
1389         case OFFLOAD_STATUS_UP:
1390                 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET;
1391                 break;
1392         }
1393 }
1394
1395 /**
1396  * cxgb3i_init_module - module init entry point
1397  *
1398  * initialize any driver wide global data structures and register itself
1399  *      with the cxgb3 module
1400  */
1401 static int __init cxgb3i_init_module(void)
1402 {
1403         int rc;
1404
1405         printk(KERN_INFO "%s", version);
1406
1407         rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1408         if (rc < 0)
1409                 return rc;
1410
1411         cxgb3_register_client(&t3_client);
1412         return 0;
1413 }
1414
1415 /**
1416  * cxgb3i_exit_module - module cleanup/exit entry point
1417  *
1418  * go through the driver hba list and for each hba, release any resource held.
1419  *      and unregisters iscsi transport and the cxgb3 module
1420  */
1421 static void __exit cxgb3i_exit_module(void)
1422 {
1423         cxgb3_unregister_client(&t3_client);
1424         cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3);
1425         cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt);
1426 }
1427
1428 module_init(cxgb3i_init_module);
1429 module_exit(cxgb3i_exit_module);