drivers/scsi/libiscsi.c

   1 /*
   2  * iSCSI lib functions
   3  *
   4  * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
   5  * Copyright (C) 2004 - 2006 Mike Christie
   6  * Copyright (C) 2004 - 2005 Dmitry Yusupov
   7  * Copyright (C) 2004 - 2005 Alex Aizman
   8  * maintained by open-iscsi@googlegroups.com
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  23  */
  24 #include <linux/types.h>
  25 #include <linux/kfifo.h>
  26 #include <linux/delay.h>
  27 #include <linux/log2.h>
  28 #include <asm/unaligned.h>
  29 #include <net/tcp.h>
  30 #include <scsi/scsi_cmnd.h>
  31 #include <scsi/scsi_device.h>
  32 #include <scsi/scsi_eh.h>
  33 #include <scsi/scsi_tcq.h>
  34 #include <scsi/scsi_host.h>
  35 #include <scsi/scsi.h>
  36 #include <scsi/iscsi_proto.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_transport_iscsi.h>
  39 #include <scsi/libiscsi.h>
  40
  41 static int iscsi_dbg_lib;
  42 module_param_named(debug_libiscsi, iscsi_dbg_lib, int, S_IRUGO | S_IWUSR);
  43 MODULE_PARM_DESC(debug_libiscsi, "Turn on debugging for libiscsi module. "
  44                  "Set to 1 to turn on, and zero to turn off. Default "
  45                  "is off.");
  46
  47 #define ISCSI_DBG_CONN(_conn, dbg_fmt, arg...)                  \
  48         do {                                                    \
  49                 if (iscsi_dbg_lib)                              \
  50                         iscsi_conn_printk(KERN_INFO, _conn,     \
  51                                              "%s " dbg_fmt,     \
  52                                              __func__, ##arg);  \
  53         } while (0);
  54
  55 #define ISCSI_DBG_SESSION(_session, dbg_fmt, arg...)                    \
  56         do {                                                            \
  57                 if (iscsi_dbg_lib)                                      \
  58                         iscsi_session_printk(KERN_INFO, _session,       \
  59                                              "%s " dbg_fmt,             \
  60                                              __func__, ##arg);          \
  61         } while (0);
  62
  63 /* Serial Number Arithmetic, 32 bits, less than, RFC1982 */
  64 #define SNA32_CHECK 2147483648UL
  65
  66 static int iscsi_sna_lt(u32 n1, u32 n2)
  67 {
  68         return n1 != n2 && ((n1 < n2 && (n2 - n1 < SNA32_CHECK)) ||
  69                             (n1 > n2 && (n2 - n1 < SNA32_CHECK)));
  70 }
  71
  72 /* Serial Number Arithmetic, 32 bits, less than, RFC1982 */
  73 static int iscsi_sna_lte(u32 n1, u32 n2)
  74 {
  75         return n1 == n2 || ((n1 < n2 && (n2 - n1 < SNA32_CHECK)) ||
  76                             (n1 > n2 && (n2 - n1 < SNA32_CHECK)));
  77 }
  78
  79 inline void iscsi_conn_queue_work(struct iscsi_conn *conn)
  80 {
  81         struct Scsi_Host *shost = conn->session->host;
  82         struct iscsi_host *ihost = shost_priv(shost);
  83
  84         queue_work(ihost->workq, &conn->xmitwork);
  85 }
  86 EXPORT_SYMBOL_GPL(iscsi_conn_queue_work);
  87
  88 void
  89 iscsi_update_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
  90 {
  91         uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
  92         uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
  93
  94         /*
  95          * standard specifies this check for when to update expected and
  96          * max sequence numbers
  97          */
  98         if (iscsi_sna_lt(max_cmdsn, exp_cmdsn - 1))
  99                 return;
 100
 101         if (exp_cmdsn != session->exp_cmdsn &&
 102             !iscsi_sna_lt(exp_cmdsn, session->exp_cmdsn))
 103                 session->exp_cmdsn = exp_cmdsn;
 104
 105         if (max_cmdsn != session->max_cmdsn &&
 106             !iscsi_sna_lt(max_cmdsn, session->max_cmdsn)) {
 107                 session->max_cmdsn = max_cmdsn;
 108                 /*
 109                  * if the window closed with IO queued, then kick the
 110                  * xmit thread
 111                  */
 112                 if (!list_empty(&session->leadconn->xmitqueue) ||
 113                     !list_empty(&session->leadconn->mgmtqueue)) {
 114                         if (!(session->tt->caps & CAP_DATA_PATH_OFFLOAD))
 115                                 iscsi_conn_queue_work(session->leadconn);
 116                 }
 117         }
 118 }
 119 EXPORT_SYMBOL_GPL(iscsi_update_cmdsn);
 120
 121 /**
 122  * iscsi_prep_data_out_pdu - initialize Data-Out
 123  * @task: scsi command task
 124  * @r2t: R2T info
 125  * @hdr: iscsi data in pdu
 126  *
 127  * Notes:
 128  *      Initialize Data-Out within this R2T sequence and finds
 129  *      proper data_offset within this SCSI command.
 130  *
 131  *      This function is called with connection lock taken.
 132  **/
 133 void iscsi_prep_data_out_pdu(struct iscsi_task *task, struct iscsi_r2t_info *r2t,
 134                            struct iscsi_data *hdr)
 135 {
 136         struct iscsi_conn *conn = task->conn;
 137         unsigned int left = r2t->data_length - r2t->sent;
 138
 139         task->hdr_len = sizeof(struct iscsi_data);
 140
 141         memset(hdr, 0, sizeof(struct iscsi_data));
 142         hdr->ttt = r2t->ttt;
 143         hdr->datasn = cpu_to_be32(r2t->datasn);
 144         r2t->datasn++;
 145         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
 146         memcpy(hdr->lun, task->lun, sizeof(hdr->lun));
 147         hdr->itt = task->hdr_itt;
 148         hdr->exp_statsn = r2t->exp_statsn;
 149         hdr->offset = cpu_to_be32(r2t->data_offset + r2t->sent);
 150         if (left > conn->max_xmit_dlength) {
 151                 hton24(hdr->dlength, conn->max_xmit_dlength);
 152                 r2t->data_count = conn->max_xmit_dlength;
 153                 hdr->flags = 0;
 154         } else {
 155                 hton24(hdr->dlength, left);
 156                 r2t->data_count = left;
 157                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 158         }
 159         conn->dataout_pdus_cnt++;
 160 }
 161 EXPORT_SYMBOL_GPL(iscsi_prep_data_out_pdu);
 162
 163 static int iscsi_add_hdr(struct iscsi_task *task, unsigned len)
 164 {
 165         unsigned exp_len = task->hdr_len + len;
 166
 167         if (exp_len > task->hdr_max) {
 168                 WARN_ON(1);
 169                 return -EINVAL;
 170         }
 171
 172         WARN_ON(len & (ISCSI_PAD_LEN - 1)); /* caller must pad the AHS */
 173         task->hdr_len = exp_len;
 174         return 0;
 175 }
 176
 177 /*
 178  * make an extended cdb AHS
 179  */
 180 static int iscsi_prep_ecdb_ahs(struct iscsi_task *task)
 181 {
 182         struct scsi_cmnd *cmd = task->sc;
 183         unsigned rlen, pad_len;
 184         unsigned short ahslength;
 185         struct iscsi_ecdb_ahdr *ecdb_ahdr;
 186         int rc;
 187
 188         ecdb_ahdr = iscsi_next_hdr(task);
 189         rlen = cmd->cmd_len - ISCSI_CDB_SIZE;
 190
 191         BUG_ON(rlen > sizeof(ecdb_ahdr->ecdb));
 192         ahslength = rlen + sizeof(ecdb_ahdr->reserved);
 193
 194         pad_len = iscsi_padding(rlen);
 195
 196         rc = iscsi_add_hdr(task, sizeof(ecdb_ahdr->ahslength) +
 197                            sizeof(ecdb_ahdr->ahstype) + ahslength + pad_len);
 198         if (rc)
 199                 return rc;
 200
 201         if (pad_len)
 202                 memset(&ecdb_ahdr->ecdb[rlen], 0, pad_len);
 203
 204         ecdb_ahdr->ahslength = cpu_to_be16(ahslength);
 205         ecdb_ahdr->ahstype = ISCSI_AHSTYPE_CDB;
 206         ecdb_ahdr->reserved = 0;
 207         memcpy(ecdb_ahdr->ecdb, cmd->cmnd + ISCSI_CDB_SIZE, rlen);
 208
 209         ISCSI_DBG_SESSION(task->conn->session,
 210                           "iscsi_prep_ecdb_ahs: varlen_cdb_len %d "
 211                           "rlen %d pad_len %d ahs_length %d iscsi_headers_size "
 212                           "%u\n", cmd->cmd_len, rlen, pad_len, ahslength,
 213                           task->hdr_len);
 214         return 0;
 215 }
 216
 217 static int iscsi_prep_bidi_ahs(struct iscsi_task *task)
 218 {
 219         struct scsi_cmnd *sc = task->sc;
 220         struct iscsi_rlength_ahdr *rlen_ahdr;
 221         int rc;
 222
 223         rlen_ahdr = iscsi_next_hdr(task);
 224         rc = iscsi_add_hdr(task, sizeof(*rlen_ahdr));
 225         if (rc)
 226                 return rc;
 227
 228         rlen_ahdr->ahslength =
 229                 cpu_to_be16(sizeof(rlen_ahdr->read_length) +
 230                                                   sizeof(rlen_ahdr->reserved));
 231         rlen_ahdr->ahstype = ISCSI_AHSTYPE_RLENGTH;
 232         rlen_ahdr->reserved = 0;
 233         rlen_ahdr->read_length = cpu_to_be32(scsi_in(sc)->length);
 234
 235         ISCSI_DBG_SESSION(task->conn->session,
 236                           "bidi-in rlen_ahdr->read_length(%d) "
 237                           "rlen_ahdr->ahslength(%d)\n",
 238                           be32_to_cpu(rlen_ahdr->read_length),
 239                           be16_to_cpu(rlen_ahdr->ahslength));
 240         return 0;
 241 }
 242
 243 /**
 244  * iscsi_prep_scsi_cmd_pdu - prep iscsi scsi cmd pdu
 245  * @task: iscsi task
 246  *
 247  * Prep basic iSCSI PDU fields for a scsi cmd pdu. The LLD should set
 248  * fields like dlength or final based on how much data it sends
 249  */
 250 static int iscsi_prep_scsi_cmd_pdu(struct iscsi_task *task)
 251 {
 252         struct iscsi_conn *conn = task->conn;
 253         struct iscsi_session *session = conn->session;
 254         struct scsi_cmnd *sc = task->sc;
 255         struct iscsi_cmd *hdr;
 256         unsigned hdrlength, cmd_len;
 257         itt_t itt;
 258         int rc;
 259
 260         if (conn->session->tt->alloc_pdu) {
 261                 rc = conn->session->tt->alloc_pdu(task, ISCSI_OP_SCSI_CMD);
 262                 if (rc)
 263                         return rc;
 264         }
 265         hdr = (struct iscsi_cmd *) task->hdr;
 266         itt = hdr->itt;
 267         memset(hdr, 0, sizeof(*hdr));
 268
 269         if (session->tt->parse_pdu_itt)
 270                 hdr->itt = task->hdr_itt = itt;
 271         else
 272                 hdr->itt = task->hdr_itt = build_itt(task->itt,
 273                                                      task->conn->session->age);
 274         task->hdr_len = 0;
 275         rc = iscsi_add_hdr(task, sizeof(*hdr));
 276         if (rc)
 277                 return rc;
 278         hdr->opcode = ISCSI_OP_SCSI_CMD;
 279         hdr->flags = ISCSI_ATTR_SIMPLE;
 280         int_to_scsilun(sc->device->lun, (struct scsi_lun *)hdr->lun);
 281         memcpy(task->lun, hdr->lun, sizeof(task->lun));
 282         hdr->cmdsn = task->cmdsn = cpu_to_be32(session->cmdsn);
 283         session->cmdsn++;
 284         hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
 285         cmd_len = sc->cmd_len;
 286         if (cmd_len < ISCSI_CDB_SIZE)
 287                 memset(&hdr->cdb[cmd_len], 0, ISCSI_CDB_SIZE - cmd_len);
 288         else if (cmd_len > ISCSI_CDB_SIZE) {
 289                 rc = iscsi_prep_ecdb_ahs(task);
 290                 if (rc)
 291                         return rc;
 292                 cmd_len = ISCSI_CDB_SIZE;
 293         }
 294         memcpy(hdr->cdb, sc->cmnd, cmd_len);
 295
 296         task->imm_count = 0;
 297         if (scsi_bidi_cmnd(sc)) {
 298                 hdr->flags |= ISCSI_FLAG_CMD_READ;
 299                 rc = iscsi_prep_bidi_ahs(task);
 300                 if (rc)
 301                         return rc;
 302         }
 303         if (sc->sc_data_direction == DMA_TO_DEVICE) {
 304                 unsigned out_len = scsi_out(sc)->length;
 305                 struct iscsi_r2t_info *r2t = &task->unsol_r2t;
 306
 307                 hdr->data_length = cpu_to_be32(out_len);
 308                 hdr->flags |= ISCSI_FLAG_CMD_WRITE;
 309                 /*
 310                  * Write counters:
 311                  *
 312                  *      imm_count       bytes to be sent right after
 313                  *                      SCSI PDU Header
 314                  *
 315                  *      unsol_count     bytes(as Data-Out) to be sent
 316                  *                      without R2T ack right after
 317                  *                      immediate data
 318                  *
 319                  *      r2t data_length bytes to be sent via R2T ack's
 320                  *
 321                  *      pad_count       bytes to be sent as zero-padding
 322                  */
 323                 memset(r2t, 0, sizeof(*r2t));
 324
 325                 if (session->imm_data_en) {
 326                         if (out_len >= session->first_burst)
 327                                 task->imm_count = min(session->first_burst,
 328                                                         conn->max_xmit_dlength);
 329                         else
 330                                 task->imm_count = min(out_len,
 331                                                         conn->max_xmit_dlength);
 332                         hton24(hdr->dlength, task->imm_count);
 333                 } else
 334                         zero_data(hdr->dlength);
 335
 336                 if (!session->initial_r2t_en) {
 337                         r2t->data_length = min(session->first_burst, out_len) -
 338                                                task->imm_count;
 339                         r2t->data_offset = task->imm_count;
 340                         r2t->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
 341                         r2t->exp_statsn = cpu_to_be32(conn->exp_statsn);
 342                 }
 343
 344                 if (!task->unsol_r2t.data_length)
 345                         /* No unsolicit Data-Out's */
 346                         hdr->flags |= ISCSI_FLAG_CMD_FINAL;
 347         } else {
 348                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
 349                 zero_data(hdr->dlength);
 350                 hdr->data_length = cpu_to_be32(scsi_in(sc)->length);
 351
 352                 if (sc->sc_data_direction == DMA_FROM_DEVICE)
 353                         hdr->flags |= ISCSI_FLAG_CMD_READ;
 354         }
 355
 356         /* calculate size of additional header segments (AHSs) */
 357         hdrlength = task->hdr_len - sizeof(*hdr);
 358
 359         WARN_ON(hdrlength & (ISCSI_PAD_LEN-1));
 360         hdrlength /= ISCSI_PAD_LEN;
 361
 362         WARN_ON(hdrlength >= 256);
 363         hdr->hlength = hdrlength & 0xFF;
 364
 365         if (session->tt->init_task && session->tt->init_task(task))
 366                 return -EIO;
 367
 368         task->state = ISCSI_TASK_RUNNING;
 369         list_move_tail(&task->running, &conn->run_list);
 370
 371         conn->scsicmd_pdus_cnt++;
 372         ISCSI_DBG_SESSION(session, "iscsi prep [%s cid %d sc %p cdb 0x%x "
 373                           "itt 0x%x len %d bidi_len %d cmdsn %d win %d]\n",
 374                           scsi_bidi_cmnd(sc) ? "bidirectional" :
 375                           sc->sc_data_direction == DMA_TO_DEVICE ?
 376                           "write" : "read", conn->id, sc, sc->cmnd[0],
 377                           task->itt, scsi_bufflen(sc),
 378                           scsi_bidi_cmnd(sc) ? scsi_in(sc)->length : 0,
 379                           session->cmdsn,
 380                           session->max_cmdsn - session->exp_cmdsn + 1);
 381         return 0;
 382 }
 383
 384 /**
 385  * iscsi_complete_command - finish a task
 386  * @task: iscsi cmd task
 387  *
 388  * Must be called with session lock.
 389  * This function returns the scsi command to scsi-ml or cleans
 390  * up mgmt tasks then returns the task to the pool.
 391  */
 392 static void iscsi_complete_command(struct iscsi_task *task)
 393 {
 394         struct iscsi_conn *conn = task->conn;
 395         struct iscsi_session *session = conn->session;
 396         struct scsi_cmnd *sc = task->sc;
 397
 398         session->tt->cleanup_task(task);
 399         list_del_init(&task->running);
 400         task->state = ISCSI_TASK_COMPLETED;
 401         task->sc = NULL;
 402
 403         if (conn->task == task)
 404                 conn->task = NULL;
 405         /*
 406          * login task is preallocated so do not free
 407          */
 408         if (conn->login_task == task)
 409                 return;
 410
 411         __kfifo_put(session->cmdpool.queue, (void*)&task, sizeof(void*));
 412
 413         if (conn->ping_task == task)
 414                 conn->ping_task = NULL;
 415
 416         if (sc) {
 417                 task->sc = NULL;
 418                 /* SCSI eh reuses commands to verify us */
 419                 sc->SCp.ptr = NULL;
 420                 /*
 421                  * queue command may call this to free the task, but
 422                  * not have setup the sc callback
 423                  */
 424                 if (sc->scsi_done)
 425                         sc->scsi_done(sc);
 426         }
 427 }
 428
 429 void __iscsi_get_task(struct iscsi_task *task)
 430 {
 431         atomic_inc(&task->refcount);
 432 }
 433 EXPORT_SYMBOL_GPL(__iscsi_get_task);
 434
 435 static void __iscsi_put_task(struct iscsi_task *task)
 436 {
 437         if (atomic_dec_and_test(&task->refcount))
 438                 iscsi_complete_command(task);
 439 }
 440
 441 void iscsi_put_task(struct iscsi_task *task)
 442 {
 443         struct iscsi_session *session = task->conn->session;
 444
 445         spin_lock_bh(&session->lock);
 446         __iscsi_put_task(task);
 447         spin_unlock_bh(&session->lock);
 448 }
 449 EXPORT_SYMBOL_GPL(iscsi_put_task);
 450
 451 /*
 452  * session lock must be held
 453  */
 454 static void fail_command(struct iscsi_conn *conn, struct iscsi_task *task,
 455                          int err)
 456 {
 457         struct scsi_cmnd *sc;
 458
 459         sc = task->sc;
 460         if (!sc)
 461                 return;
 462
 463         if (task->state == ISCSI_TASK_PENDING)
 464                 /*
 465                  * cmd never made it to the xmit thread, so we should not count
 466                  * the cmd in the sequencing
 467                  */
 468                 conn->session->queued_cmdsn--;
 469
 470         sc->result = err;
 471         if (!scsi_bidi_cmnd(sc))
 472                 scsi_set_resid(sc, scsi_bufflen(sc));
 473         else {
 474                 scsi_out(sc)->resid = scsi_out(sc)->length;
 475                 scsi_in(sc)->resid = scsi_in(sc)->length;
 476         }
 477
 478         if (conn->task == task)
 479                 conn->task = NULL;
 480         /* release ref from queuecommand */
 481         __iscsi_put_task(task);
 482 }
 483
 484 static int iscsi_prep_mgmt_task(struct iscsi_conn *conn,
 485                                 struct iscsi_task *task)
 486 {
 487         struct iscsi_session *session = conn->session;
 488         struct iscsi_hdr *hdr = task->hdr;
 489         struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
 490
 491         if (conn->session->state == ISCSI_STATE_LOGGING_OUT)
 492                 return -ENOTCONN;
 493
 494         if (hdr->opcode != (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) &&
 495             hdr->opcode != (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
 496                 nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
 497         /*
 498          * pre-format CmdSN for outgoing PDU.
 499          */
 500         nop->cmdsn = cpu_to_be32(session->cmdsn);
 501         if (hdr->itt != RESERVED_ITT) {
 502                 /*
 503                  * TODO: We always use immediate, so we never hit this.
 504                  * If we start to send tmfs or nops as non-immediate then
 505                  * we should start checking the cmdsn numbers for mgmt tasks.
 506                  */
 507                 if (conn->c_stage == ISCSI_CONN_STARTED &&
 508                     !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
 509                         session->queued_cmdsn++;
 510                         session->cmdsn++;
 511                 }
 512         }
 513
 514         if (session->tt->init_task && session->tt->init_task(task))
 515                 return -EIO;
 516
 517         if ((hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
 518                 session->state = ISCSI_STATE_LOGGING_OUT;
 519
 520         task->state = ISCSI_TASK_RUNNING;
 521         list_move_tail(&task->running, &conn->mgmt_run_list);
 522         ISCSI_DBG_SESSION(session, "mgmtpdu [op 0x%x hdr->itt 0x%x "
 523                           "datalen %d]\n", hdr->opcode & ISCSI_OPCODE_MASK,
 524                           hdr->itt, task->data_count);
 525         return 0;
 526 }
 527
 528 static struct iscsi_task *
 529 __iscsi_conn_send_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 530                       char *data, uint32_t data_size)
 531 {
 532         struct iscsi_session *session = conn->session;
 533         struct iscsi_task *task;
 534         itt_t itt;
 535
 536         if (session->state == ISCSI_STATE_TERMINATE)
 537                 return NULL;
 538
 539         if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
 540             hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
 541                 /*
 542                  * Login and Text are sent serially, in
 543                  * request-followed-by-response sequence.
 544                  * Same task can be used. Same ITT must be used.
 545                  * Note that login_task is preallocated at conn_create().
 546                  */
 547                 task = conn->login_task;
 548         else {
 549                 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
 550                 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
 551
 552                 if (!__kfifo_get(session->cmdpool.queue,
 553                                  (void*)&task, sizeof(void*)))
 554                         return NULL;
 555         }
 556         /*
 557          * released in complete pdu for task we expect a response for, and
 558          * released by the lld when it has transmitted the task for
 559          * pdus we do not expect a response for.
 560          */
 561         atomic_set(&task->refcount, 1);
 562         task->conn = conn;
 563         task->sc = NULL;
 564
 565         if (data_size) {
 566                 memcpy(task->data, data, data_size);
 567                 task->data_count = data_size;
 568         } else
 569                 task->data_count = 0;
 570
 571         if (conn->session->tt->alloc_pdu) {
 572                 if (conn->session->tt->alloc_pdu(task, hdr->opcode)) {
 573                         iscsi_conn_printk(KERN_ERR, conn, "Could not allocate "
 574                                          "pdu for mgmt task.\n");
 575                         goto requeue_task;
 576                 }
 577         }
 578
 579         itt = task->hdr->itt;
 580         task->hdr_len = sizeof(struct iscsi_hdr);
 581         memcpy(task->hdr, hdr, sizeof(struct iscsi_hdr));
 582
 583         if (hdr->itt != RESERVED_ITT) {
 584                 if (session->tt->parse_pdu_itt)
 585                         task->hdr->itt = itt;
 586                 else
 587                         task->hdr->itt = build_itt(task->itt,
 588                                                    task->conn->session->age);
 589         }
 590
 591         INIT_LIST_HEAD(&task->running);
 592         list_add_tail(&task->running, &conn->mgmtqueue);
 593
 594         if (session->tt->caps & CAP_DATA_PATH_OFFLOAD) {
 595                 if (iscsi_prep_mgmt_task(conn, task))
 596                         goto free_task;
 597
 598                 if (session->tt->xmit_task(task))
 599                         goto free_task;
 600
 601         } else
 602                 iscsi_conn_queue_work(conn);
 603
 604         return task;
 605
 606 free_task:
 607         __iscsi_put_task(task);
 608         return NULL;
 609
 610 requeue_task:
 611         if (task != conn->login_task)
 612                 __kfifo_put(session->cmdpool.queue, (void*)&task,
 613                             sizeof(void*));
 614         return NULL;
 615 }
 616
 617 int iscsi_conn_send_pdu(struct iscsi_cls_conn *cls_conn, struct iscsi_hdr *hdr,
 618                         char *data, uint32_t data_size)
 619 {
 620         struct iscsi_conn *conn = cls_conn->dd_data;
 621         struct iscsi_session *session = conn->session;
 622         int err = 0;
 623
 624         spin_lock_bh(&session->lock);
 625         if (!__iscsi_conn_send_pdu(conn, hdr, data, data_size))
 626                 err = -EPERM;
 627         spin_unlock_bh(&session->lock);
 628         return err;
 629 }
 630 EXPORT_SYMBOL_GPL(iscsi_conn_send_pdu);
 631
 632 /**
 633  * iscsi_cmd_rsp - SCSI Command Response processing
 634  * @conn: iscsi connection
 635  * @hdr: iscsi header
 636  * @task: scsi command task
 637  * @data: cmd data buffer
 638  * @datalen: len of buffer
 639  *
 640  * iscsi_cmd_rsp sets up the scsi_cmnd fields based on the PDU and
 641  * then completes the command and task.
 642  **/
 643 static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 644                                struct iscsi_task *task, char *data,
 645                                int datalen)
 646 {
 647         struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)hdr;
 648         struct iscsi_session *session = conn->session;
 649         struct scsi_cmnd *sc = task->sc;
 650
 651         iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
 652         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 653
 654         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 655
 656         if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
 657                 sc->result = DID_ERROR << 16;
 658                 goto out;
 659         }
 660
 661         if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION) {
 662                 uint16_t senselen;
 663
 664                 if (datalen < 2) {
 665 invalid_datalen:
 666                         iscsi_conn_printk(KERN_ERR,  conn,
 667                                          "Got CHECK_CONDITION but invalid data "
 668                                          "buffer size of %d\n", datalen);
 669                         sc->result = DID_BAD_TARGET << 16;
 670                         goto out;
 671                 }
 672
 673                 senselen = get_unaligned_be16(data);
 674                 if (datalen < senselen)
 675                         goto invalid_datalen;
 676
 677                 memcpy(sc->sense_buffer, data + 2,
 678                        min_t(uint16_t, senselen, SCSI_SENSE_BUFFERSIZE));
 679                 ISCSI_DBG_SESSION(session, "copied %d bytes of sense\n",
 680                                   min_t(uint16_t, senselen,
 681                                   SCSI_SENSE_BUFFERSIZE));
 682         }
 683
 684         if (rhdr->flags & (ISCSI_FLAG_CMD_BIDI_UNDERFLOW |
 685                            ISCSI_FLAG_CMD_BIDI_OVERFLOW)) {
 686                 int res_count = be32_to_cpu(rhdr->bi_residual_count);
 687
 688                 if (scsi_bidi_cmnd(sc) && res_count > 0 &&
 689                                 (rhdr->flags & ISCSI_FLAG_CMD_BIDI_OVERFLOW ||
 690                                  res_count <= scsi_in(sc)->length))
 691                         scsi_in(sc)->resid = res_count;
 692                 else
 693                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 694         }
 695
 696         if (rhdr->flags & (ISCSI_FLAG_CMD_UNDERFLOW |
 697                            ISCSI_FLAG_CMD_OVERFLOW)) {
 698                 int res_count = be32_to_cpu(rhdr->residual_count);
 699
 700                 if (res_count > 0 &&
 701                     (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW ||
 702                      res_count <= scsi_bufflen(sc)))
 703                         /* write side for bidi or uni-io set_resid */
 704                         scsi_set_resid(sc, res_count);
 705                 else
 706                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 707         }
 708 out:
 709         ISCSI_DBG_SESSION(session, "done [sc %p res %d itt 0x%x]\n",
 710                           sc, sc->result, task->itt);
 711         conn->scsirsp_pdus_cnt++;
 712
 713         __iscsi_put_task(task);
 714 }
 715
 716 /**
 717  * iscsi_data_in_rsp - SCSI Data-In Response processing
 718  * @conn: iscsi connection
 719  * @hdr:  iscsi pdu
 720  * @task: scsi command task
 721  **/
 722 static void
 723 iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 724                   struct iscsi_task *task)
 725 {
 726         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)hdr;
 727         struct scsi_cmnd *sc = task->sc;
 728
 729         if (!(rhdr->flags & ISCSI_FLAG_DATA_STATUS))
 730                 return;
 731
 732         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 733         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 734         if (rhdr->flags & (ISCSI_FLAG_DATA_UNDERFLOW |
 735                            ISCSI_FLAG_DATA_OVERFLOW)) {
 736                 int res_count = be32_to_cpu(rhdr->residual_count);
 737
 738                 if (res_count > 0 &&
 739                     (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW ||
 740                      res_count <= scsi_in(sc)->length))
 741                         scsi_in(sc)->resid = res_count;
 742                 else
 743                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 744         }
 745
 746         conn->scsirsp_pdus_cnt++;
 747         __iscsi_put_task(task);
 748 }
 749
 750 static void iscsi_tmf_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
 751 {
 752         struct iscsi_tm_rsp *tmf = (struct iscsi_tm_rsp *)hdr;
 753
 754         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 755         conn->tmfrsp_pdus_cnt++;
 756
 757         if (conn->tmf_state != TMF_QUEUED)
 758                 return;
 759
 760         if (tmf->response == ISCSI_TMF_RSP_COMPLETE)
 761                 conn->tmf_state = TMF_SUCCESS;
 762         else if (tmf->response == ISCSI_TMF_RSP_NO_TASK)
 763                 conn->tmf_state = TMF_NOT_FOUND;
 764         else
 765                 conn->tmf_state = TMF_FAILED;
 766         wake_up(&conn->ehwait);
 767 }
 768
 769 static void iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
 770 {
 771         struct iscsi_nopout hdr;
 772         struct iscsi_task *task;
 773
 774         if (!rhdr && conn->ping_task)
 775                 return;
 776
 777         memset(&hdr, 0, sizeof(struct iscsi_nopout));
 778         hdr.opcode = ISCSI_OP_NOOP_OUT | ISCSI_OP_IMMEDIATE;
 779         hdr.flags = ISCSI_FLAG_CMD_FINAL;
 780
 781         if (rhdr) {
 782                 memcpy(hdr.lun, rhdr->lun, 8);
 783                 hdr.ttt = rhdr->ttt;
 784                 hdr.itt = RESERVED_ITT;
 785         } else
 786                 hdr.ttt = RESERVED_ITT;
 787
 788         task = __iscsi_conn_send_pdu(conn, (struct iscsi_hdr *)&hdr, NULL, 0);
 789         if (!task)
 790                 iscsi_conn_printk(KERN_ERR, conn, "Could not send nopout\n");
 791         else if (!rhdr) {
 792                 /* only track our nops */
 793                 conn->ping_task = task;
 794                 conn->last_ping = jiffies;
 795         }
 796 }
 797
 798 static int iscsi_handle_reject(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 799                                char *data, int datalen)
 800 {
 801         struct iscsi_reject *reject = (struct iscsi_reject *)hdr;
 802         struct iscsi_hdr rejected_pdu;
 803
 804         conn->exp_statsn = be32_to_cpu(reject->statsn) + 1;
 805
 806         if (reject->reason == ISCSI_REASON_DATA_DIGEST_ERROR) {
 807                 if (ntoh24(reject->dlength) > datalen)
 808                         return ISCSI_ERR_PROTO;
 809
 810                 if (ntoh24(reject->dlength) >= sizeof(struct iscsi_hdr)) {
 811                         memcpy(&rejected_pdu, data, sizeof(struct iscsi_hdr));
 812                         iscsi_conn_printk(KERN_ERR, conn,
 813                                           "pdu (op 0x%x) rejected "
 814                                           "due to DataDigest error.\n",
 815                                           rejected_pdu.opcode);
 816                 }
 817         }
 818         return 0;
 819 }
 820
 821 /**
 822  * iscsi_itt_to_task - look up task by itt
 823  * @conn: iscsi connection
 824  * @itt: itt
 825  *
 826  * This should be used for mgmt tasks like login and nops, or if
 827  * the LDD's itt space does not include the session age.
 828  *
 829  * The session lock must be held.
 830  */
 831 static struct iscsi_task *iscsi_itt_to_task(struct iscsi_conn *conn, itt_t itt)
 832 {
 833         struct iscsi_session *session = conn->session;
 834         int i;
 835
 836         if (itt == RESERVED_ITT)
 837                 return NULL;
 838
 839         if (session->tt->parse_pdu_itt)
 840                 session->tt->parse_pdu_itt(conn, itt, &i, NULL);
 841         else
 842                 i = get_itt(itt);
 843         if (i >= session->cmds_max)
 844                 return NULL;
 845
 846         return session->cmds[i];
 847 }
 848
 849 /**
 850  * __iscsi_complete_pdu - complete pdu
 851  * @conn: iscsi conn
 852  * @hdr: iscsi header
 853  * @data: data buffer
 854  * @datalen: len of data buffer
 855  *
 856  * Completes pdu processing by freeing any resources allocated at
 857  * queuecommand or send generic. session lock must be held and verify
 858  * itt must have been called.
 859  */
 860 int __iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 861                          char *data, int datalen)
 862 {
 863         struct iscsi_session *session = conn->session;
 864         int opcode = hdr->opcode & ISCSI_OPCODE_MASK, rc = 0;
 865         struct iscsi_task *task;
 866         uint32_t itt;
 867
 868         conn->last_recv = jiffies;
 869         rc = iscsi_verify_itt(conn, hdr->itt);
 870         if (rc)
 871                 return rc;
 872
 873         if (hdr->itt != RESERVED_ITT)
 874                 itt = get_itt(hdr->itt);
 875         else
 876                 itt = ~0U;
 877
 878         ISCSI_DBG_SESSION(session, "[op 0x%x cid %d itt 0x%x len %d]\n",
 879                           opcode, conn->id, itt, datalen);
 880
 881         if (itt == ~0U) {
 882                 iscsi_update_cmdsn(session, (struct iscsi_nopin*)hdr);
 883
 884                 switch(opcode) {
 885                 case ISCSI_OP_NOOP_IN:
 886                         if (datalen) {
 887                                 rc = ISCSI_ERR_PROTO;
 888                                 break;
 889                         }
 890
 891                         if (hdr->ttt == cpu_to_be32(ISCSI_RESERVED_TAG))
 892                                 break;
 893
 894                         iscsi_send_nopout(conn, (struct iscsi_nopin*)hdr);
 895                         break;
 896                 case ISCSI_OP_REJECT:
 897                         rc = iscsi_handle_reject(conn, hdr, data, datalen);
 898                         break;
 899                 case ISCSI_OP_ASYNC_EVENT:
 900                         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 901                         if (iscsi_recv_pdu(conn->cls_conn, hdr, data, datalen))
 902                                 rc = ISCSI_ERR_CONN_FAILED;
 903                         break;
 904                 default:
 905                         rc = ISCSI_ERR_BAD_OPCODE;
 906                         break;
 907                 }
 908                 goto out;
 909         }
 910
 911         switch(opcode) {
 912         case ISCSI_OP_SCSI_CMD_RSP:
 913         case ISCSI_OP_SCSI_DATA_IN:
 914                 task = iscsi_itt_to_ctask(conn, hdr->itt);
 915                 if (!task)
 916                         return ISCSI_ERR_BAD_ITT;
 917                 break;
 918         case ISCSI_OP_R2T:
 919                 /*
 920                  * LLD handles R2Ts if they need to.
 921                  */
 922                 return 0;
 923         case ISCSI_OP_LOGOUT_RSP:
 924         case ISCSI_OP_LOGIN_RSP:
 925         case ISCSI_OP_TEXT_RSP:
 926         case ISCSI_OP_SCSI_TMFUNC_RSP:
 927         case ISCSI_OP_NOOP_IN:
 928                 task = iscsi_itt_to_task(conn, hdr->itt);
 929                 if (!task)
 930                         return ISCSI_ERR_BAD_ITT;
 931                 break;
 932         default:
 933                 return ISCSI_ERR_BAD_OPCODE;
 934         }
 935
 936         switch(opcode) {
 937         case ISCSI_OP_SCSI_CMD_RSP:
 938                 iscsi_scsi_cmd_rsp(conn, hdr, task, data, datalen);
 939                 break;
 940         case ISCSI_OP_SCSI_DATA_IN:
 941                 iscsi_data_in_rsp(conn, hdr, task);
 942                 break;
 943         case ISCSI_OP_LOGOUT_RSP:
 944                 iscsi_update_cmdsn(session, (struct iscsi_nopin*)hdr);
 945                 if (datalen) {
 946                         rc = ISCSI_ERR_PROTO;
 947                         break;
 948                 }
 949                 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 950                 goto recv_pdu;
 951         case ISCSI_OP_LOGIN_RSP:
 952         case ISCSI_OP_TEXT_RSP:
 953                 iscsi_update_cmdsn(session, (struct iscsi_nopin*)hdr);
 954                 /*
 955                  * login related PDU's exp_statsn is handled in
 956                  * userspace
 957                  */
 958                 goto recv_pdu;
 959         case ISCSI_OP_SCSI_TMFUNC_RSP:
 960                 iscsi_update_cmdsn(session, (struct iscsi_nopin*)hdr);
 961                 if (datalen) {
 962                         rc = ISCSI_ERR_PROTO;
 963                         break;
 964                 }
 965
 966                 iscsi_tmf_rsp(conn, hdr);
 967                 __iscsi_put_task(task);
 968                 break;
 969         case ISCSI_OP_NOOP_IN:
 970                 iscsi_update_cmdsn(session, (struct iscsi_nopin*)hdr);
 971                 if (hdr->ttt != cpu_to_be32(ISCSI_RESERVED_TAG) || datalen) {
 972                         rc = ISCSI_ERR_PROTO;
 973                         break;
 974                 }
 975                 conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 976
 977                 if (conn->ping_task != task)
 978                         /*
 979                          * If this is not in response to one of our
 980                          * nops then it must be from userspace.
 981                          */
 982                         goto recv_pdu;
 983
 984                 mod_timer(&conn->transport_timer, jiffies + conn->recv_timeout);
 985                 __iscsi_put_task(task);
 986                 break;
 987         default:
 988                 rc = ISCSI_ERR_BAD_OPCODE;
 989                 break;
 990         }
 991
 992 out:
 993         return rc;
 994 recv_pdu:
 995         if (iscsi_recv_pdu(conn->cls_conn, hdr, data, datalen))
 996                 rc = ISCSI_ERR_CONN_FAILED;
 997         __iscsi_put_task(task);
 998         return rc;
 999 }
1000 EXPORT_SYMBOL_GPL(__iscsi_complete_pdu);
1001
1002 int iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
1003                        char *data, int datalen)
1004 {
1005         int rc;
1006
1007         spin_lock(&conn->session->lock);
1008         rc = __iscsi_complete_pdu(conn, hdr, data, datalen);
1009         spin_unlock(&conn->session->lock);
1010         return rc;
1011 }
1012 EXPORT_SYMBOL_GPL(iscsi_complete_pdu);
1013
1014 int iscsi_verify_itt(struct iscsi_conn *conn, itt_t itt)
1015 {
1016         struct iscsi_session *session = conn->session;
1017         int age = 0, i = 0;
1018
1019         if (itt == RESERVED_ITT)
1020                 return 0;
1021
1022         if (session->tt->parse_pdu_itt)
1023                 session->tt->parse_pdu_itt(conn, itt, &i, &age);
1024         else {
1025                 i = get_itt(itt);
1026                 age = ((__force u32)itt >> ISCSI_AGE_SHIFT) & ISCSI_AGE_MASK;
1027         }
1028
1029         if (age != session->age) {
1030                 iscsi_conn_printk(KERN_ERR, conn,
1031                                   "received itt %x expected session age (%x)\n",
1032                                   (__force u32)itt, session->age);
1033                 return ISCSI_ERR_BAD_ITT;
1034         }
1035
1036         if (i >= session->cmds_max) {
1037                 iscsi_conn_printk(KERN_ERR, conn,
1038                                   "received invalid itt index %u (max cmds "
1039                                    "%u.\n", i, session->cmds_max);
1040                 return ISCSI_ERR_BAD_ITT;
1041         }
1042         return 0;
1043 }
1044 EXPORT_SYMBOL_GPL(iscsi_verify_itt);
1045
1046 /**
1047  * iscsi_itt_to_ctask - look up ctask by itt
1048  * @conn: iscsi connection
1049  * @itt: itt
1050  *
1051  * This should be used for cmd tasks.
1052  *
1053  * The session lock must be held.
1054  */
1055 struct iscsi_task *iscsi_itt_to_ctask(struct iscsi_conn *conn, itt_t itt)
1056 {
1057         struct iscsi_task *task;
1058
1059         if (iscsi_verify_itt(conn, itt))
1060                 return NULL;
1061
1062         task = iscsi_itt_to_task(conn, itt);
1063         if (!task || !task->sc)
1064                 return NULL;
1065
1066         if (task->sc->SCp.phase != conn->session->age) {
1067                 iscsi_session_printk(KERN_ERR, conn->session,
1068                                   "task's session age %d, expected %d\n",
1069                                   task->sc->SCp.phase, conn->session->age);
1070                 return NULL;
1071         }
1072
1073         return task;
1074 }
1075 EXPORT_SYMBOL_GPL(iscsi_itt_to_ctask);
1076
1077 void iscsi_session_failure(struct iscsi_session *session,
1078                            enum iscsi_err err)
1079 {
1080         struct iscsi_conn *conn;
1081         struct device *dev;
1082         unsigned long flags;
1083
1084         spin_lock_irqsave(&session->lock, flags);
1085         conn = session->leadconn;
1086         if (session->state == ISCSI_STATE_TERMINATE || !conn) {
1087                 spin_unlock_irqrestore(&session->lock, flags);
1088                 return;
1089         }
1090
1091         dev = get_device(&conn->cls_conn->dev);
1092         spin_unlock_irqrestore(&session->lock, flags);
1093         if (!dev)
1094                 return;
1095         /*
1096          * if the host is being removed bypass the connection
1097          * recovery initialization because we are going to kill
1098          * the session.
1099          */
1100         if (err == ISCSI_ERR_INVALID_HOST)
1101                 iscsi_conn_error_event(conn->cls_conn, err);
1102         else
1103                 iscsi_conn_failure(conn, err);
1104         put_device(dev);
1105 }
1106 EXPORT_SYMBOL_GPL(iscsi_session_failure);
1107
1108 void iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
1109 {
1110         struct iscsi_session *session = conn->session;
1111         unsigned long flags;
1112
1113         spin_lock_irqsave(&session->lock, flags);
1114         if (session->state == ISCSI_STATE_FAILED) {
1115                 spin_unlock_irqrestore(&session->lock, flags);
1116                 return;
1117         }
1118
1119         if (conn->stop_stage == 0)
1120                 session->state = ISCSI_STATE_FAILED;
1121         spin_unlock_irqrestore(&session->lock, flags);
1122
1123         set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1124         set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
1125         iscsi_conn_error_event(conn->cls_conn, err);
1126 }
1127 EXPORT_SYMBOL_GPL(iscsi_conn_failure);
1128
1129 static int iscsi_check_cmdsn_window_closed(struct iscsi_conn *conn)
1130 {
1131         struct iscsi_session *session = conn->session;
1132
1133         /*
1134          * Check for iSCSI window and take care of CmdSN wrap-around
1135          */
1136         if (!iscsi_sna_lte(session->queued_cmdsn, session->max_cmdsn)) {
1137                 ISCSI_DBG_SESSION(session, "iSCSI CmdSN closed. ExpCmdSn "
1138                                   "%u MaxCmdSN %u CmdSN %u/%u\n",
1139                                   session->exp_cmdsn, session->max_cmdsn,
1140                                   session->cmdsn, session->queued_cmdsn);
1141                 return -ENOSPC;
1142         }
1143         return 0;
1144 }
1145
1146 static int iscsi_xmit_task(struct iscsi_conn *conn)
1147 {
1148         struct iscsi_task *task = conn->task;
1149         int rc;
1150
1151         __iscsi_get_task(task);
1152         spin_unlock_bh(&conn->session->lock);
1153         rc = conn->session->tt->xmit_task(task);
1154         spin_lock_bh(&conn->session->lock);
1155         __iscsi_put_task(task);
1156         if (!rc)
1157                 /* done with this task */
1158                 conn->task = NULL;
1159         return rc;
1160 }
1161
1162 /**
1163  * iscsi_requeue_task - requeue task to run from session workqueue
1164  * @task: task to requeue
1165  *
1166  * LLDs that need to run a task from the session workqueue should call
1167  * this. The session lock must be held. This should only be called
1168  * by software drivers.
1169  */
1170 void iscsi_requeue_task(struct iscsi_task *task)
1171 {
1172         struct iscsi_conn *conn = task->conn;
1173
1174         list_move_tail(&task->running, &conn->requeue);
1175         iscsi_conn_queue_work(conn);
1176 }
1177 EXPORT_SYMBOL_GPL(iscsi_requeue_task);
1178
1179 /**
1180  * iscsi_data_xmit - xmit any command into the scheduled connection
1181  * @conn: iscsi connection
1182  *
1183  * Notes:
1184  *      The function can return -EAGAIN in which case the caller must
1185  *      re-schedule it again later or recover. '0' return code means
1186  *      successful xmit.
1187  **/
1188 static int iscsi_data_xmit(struct iscsi_conn *conn)
1189 {
1190         int rc = 0;
1191
1192         spin_lock_bh(&conn->session->lock);
1193         if (unlikely(conn->suspend_tx)) {
1194                 ISCSI_DBG_SESSION(conn->session, "Tx suspended!\n");
1195                 spin_unlock_bh(&conn->session->lock);
1196                 return -ENODATA;
1197         }
1198
1199         if (conn->task) {
1200                 rc = iscsi_xmit_task(conn);
1201                 if (rc)
1202                         goto again;
1203         }
1204
1205         /*
1206          * process mgmt pdus like nops before commands since we should
1207          * only have one nop-out as a ping from us and targets should not
1208          * overflow us with nop-ins
1209          */
1210 check_mgmt:
1211         while (!list_empty(&conn->mgmtqueue)) {
1212                 conn->task = list_entry(conn->mgmtqueue.next,
1213                                          struct iscsi_task, running);
1214                 if (iscsi_prep_mgmt_task(conn, conn->task)) {
1215                         __iscsi_put_task(conn->task);
1216                         conn->task = NULL;
1217                         continue;
1218                 }
1219                 rc = iscsi_xmit_task(conn);
1220                 if (rc)
1221                         goto again;
1222         }
1223
1224         /* process pending command queue */
1225         while (!list_empty(&conn->xmitqueue)) {
1226                 if (conn->tmf_state == TMF_QUEUED)
1227                         break;
1228
1229                 conn->task = list_entry(conn->xmitqueue.next,
1230                                          struct iscsi_task, running);
1231                 if (conn->session->state == ISCSI_STATE_LOGGING_OUT) {
1232                         fail_command(conn, conn->task, DID_IMM_RETRY << 16);
1233                         continue;
1234                 }
1235                 rc = iscsi_prep_scsi_cmd_pdu(conn->task);
1236                 if (rc) {
1237                         if (rc == -ENOMEM) {
1238                                 conn->task = NULL;
1239                                 goto again;
1240                         } else
1241                                 fail_command(conn, conn->task, DID_ABORT << 16);
1242                         continue;
1243                 }
1244                 rc = iscsi_xmit_task(conn);
1245                 if (rc)
1246                         goto again;
1247                 /*
1248                  * we could continuously get new task requests so
1249                  * we need to check the mgmt queue for nops that need to
1250                  * be sent to aviod starvation
1251                  */
1252                 if (!list_empty(&conn->mgmtqueue))
1253                         goto check_mgmt;
1254         }
1255
1256         while (!list_empty(&conn->requeue)) {
1257                 if (conn->session->fast_abort && conn->tmf_state != TMF_INITIAL)
1258                         break;
1259
1260                 /*
1261                  * we always do fastlogout - conn stop code will clean up.
1262                  */
1263                 if (conn->session->state == ISCSI_STATE_LOGGING_OUT)
1264                         break;
1265
1266                 conn->task = list_entry(conn->requeue.next,
1267                                          struct iscsi_task, running);
1268                 conn->task->state = ISCSI_TASK_RUNNING;
1269                 list_move_tail(conn->requeue.next, &conn->run_list);
1270                 rc = iscsi_xmit_task(conn);
1271                 if (rc)
1272                         goto again;
1273                 if (!list_empty(&conn->mgmtqueue))
1274                         goto check_mgmt;
1275         }
1276         spin_unlock_bh(&conn->session->lock);
1277         return -ENODATA;
1278
1279 again:
1280         if (unlikely(conn->suspend_tx))
1281                 rc = -ENODATA;
1282         spin_unlock_bh(&conn->session->lock);
1283         return rc;
1284 }
1285
1286 static void iscsi_xmitworker(struct work_struct *work)
1287 {
1288         struct iscsi_conn *conn =
1289                 container_of(work, struct iscsi_conn, xmitwork);
1290         int rc;
1291         /*
1292          * serialize Xmit worker on a per-connection basis.
1293          */
1294         do {
1295                 rc = iscsi_data_xmit(conn);
1296         } while (rc >= 0 || rc == -EAGAIN);
1297 }
1298
1299 static inline struct iscsi_task *iscsi_alloc_task(struct iscsi_conn *conn,
1300                                                   struct scsi_cmnd *sc)
1301 {
1302         struct iscsi_task *task;
1303
1304         if (!__kfifo_get(conn->session->cmdpool.queue,
1305                          (void *) &task, sizeof(void *)))
1306                 return NULL;
1307
1308         sc->SCp.phase = conn->session->age;
1309         sc->SCp.ptr = (char *) task;
1310
1311         atomic_set(&task->refcount, 1);
1312         task->state = ISCSI_TASK_PENDING;
1313         task->conn = conn;
1314         task->sc = sc;
1315         INIT_LIST_HEAD(&task->running);
1316         return task;
1317 }
1318
1319 enum {
1320         FAILURE_BAD_HOST = 1,
1321         FAILURE_SESSION_FAILED,
1322         FAILURE_SESSION_FREED,
1323         FAILURE_WINDOW_CLOSED,
1324         FAILURE_OOM,
1325         FAILURE_SESSION_TERMINATE,
1326         FAILURE_SESSION_IN_RECOVERY,
1327         FAILURE_SESSION_RECOVERY_TIMEOUT,
1328         FAILURE_SESSION_LOGGING_OUT,
1329         FAILURE_SESSION_NOT_READY,
1330 };
1331
1332 int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
1333 {
1334         struct iscsi_cls_session *cls_session;
1335         struct Scsi_Host *host;
1336         int reason = 0;
1337         struct iscsi_session *session;
1338         struct iscsi_conn *conn;
1339         struct iscsi_task *task = NULL;
1340
1341         sc->scsi_done = done;
1342         sc->result = 0;
1343         sc->SCp.ptr = NULL;
1344
1345         host = sc->device->host;
1346         spin_unlock(host->host_lock);
1347
1348         cls_session = starget_to_session(scsi_target(sc->device));
1349         session = cls_session->dd_data;
1350         spin_lock(&session->lock);
1351
1352         reason = iscsi_session_chkready(cls_session);
1353         if (reason) {
1354                 sc->result = reason;
1355                 goto fault;
1356         }
1357
1358         /*
1359          * ISCSI_STATE_FAILED is a temp. state. The recovery
1360          * code will decide what is best to do with command queued
1361          * during this time
1362          */
1363         if (session->state != ISCSI_STATE_LOGGED_IN &&
1364             session->state != ISCSI_STATE_FAILED) {
1365                 /*
1366                  * to handle the race between when we set the recovery state
1367                  * and block the session we requeue here (commands could
1368                  * be entering our queuecommand while a block is starting
1369                  * up because the block code is not locked)
1370                  */
1371                 switch (session->state) {
1372                 case ISCSI_STATE_IN_RECOVERY:
1373                         reason = FAILURE_SESSION_IN_RECOVERY;
1374                         goto reject;
1375                 case ISCSI_STATE_LOGGING_OUT:
1376                         reason = FAILURE_SESSION_LOGGING_OUT;
1377                         goto reject;
1378                 case ISCSI_STATE_RECOVERY_FAILED:
1379                         reason = FAILURE_SESSION_RECOVERY_TIMEOUT;
1380                         sc->result = DID_TRANSPORT_FAILFAST << 16;
1381                         break;
1382                 case ISCSI_STATE_TERMINATE:
1383                         reason = FAILURE_SESSION_TERMINATE;
1384                         sc->result = DID_NO_CONNECT << 16;
1385                         break;
1386                 default:
1387                         reason = FAILURE_SESSION_FREED;
1388                         sc->result = DID_NO_CONNECT << 16;
1389                 }
1390                 goto fault;
1391         }
1392
1393         conn = session->leadconn;
1394         if (!conn) {
1395                 reason = FAILURE_SESSION_FREED;
1396                 sc->result = DID_NO_CONNECT << 16;
1397                 goto fault;
1398         }
1399
1400         if (iscsi_check_cmdsn_window_closed(conn)) {
1401                 reason = FAILURE_WINDOW_CLOSED;
1402                 goto reject;
1403         }
1404
1405         task = iscsi_alloc_task(conn, sc);
1406         if (!task) {
1407                 reason = FAILURE_OOM;
1408                 goto reject;
1409         }
1410         list_add_tail(&task->running, &conn->xmitqueue);
1411
1412         if (session->tt->caps & CAP_DATA_PATH_OFFLOAD) {
1413                 reason = iscsi_prep_scsi_cmd_pdu(task);
1414                 if (reason) {
1415                         if (reason == -ENOMEM) {
1416                                 reason = FAILURE_OOM;
1417                                 goto prepd_reject;
1418                         } else {
1419                                 sc->result = DID_ABORT << 16;
1420                                 goto prepd_fault;
1421                         }
1422                 }
1423                 if (session->tt->xmit_task(task)) {
1424                         reason = FAILURE_SESSION_NOT_READY;
1425                         goto prepd_reject;
1426                 }
1427         } else
1428                 iscsi_conn_queue_work(conn);
1429
1430         session->queued_cmdsn++;
1431         spin_unlock(&session->lock);
1432         spin_lock(host->host_lock);
1433         return 0;
1434
1435 prepd_reject:
1436         sc->scsi_done = NULL;
1437         iscsi_complete_command(task);
1438 reject:
1439         spin_unlock(&session->lock);
1440         ISCSI_DBG_SESSION(session, "cmd 0x%x rejected (%d)\n",
1441                           sc->cmnd[0], reason);
1442         spin_lock(host->host_lock);
1443         return SCSI_MLQUEUE_TARGET_BUSY;
1444
1445 prepd_fault:
1446         sc->scsi_done = NULL;
1447         iscsi_complete_command(task);
1448 fault:
1449         spin_unlock(&session->lock);
1450         ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n",
1451                           sc->cmnd[0], reason);
1452         if (!scsi_bidi_cmnd(sc))
1453                 scsi_set_resid(sc, scsi_bufflen(sc));
1454         else {
1455                 scsi_out(sc)->resid = scsi_out(sc)->length;
1456                 scsi_in(sc)->resid = scsi_in(sc)->length;
1457         }
1458         done(sc);
1459         spin_lock(host->host_lock);
1460         return 0;
1461 }
1462 EXPORT_SYMBOL_GPL(iscsi_queuecommand);
1463
1464 int iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
1465 {
1466         scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
1467         return sdev->queue_depth;
1468 }
1469 EXPORT_SYMBOL_GPL(iscsi_change_queue_depth);
1470
1471 int iscsi_target_alloc(struct scsi_target *starget)
1472 {
1473         struct iscsi_cls_session *cls_session = starget_to_session(starget);
1474         struct iscsi_session *session = cls_session->dd_data;
1475
1476         starget->can_queue = session->scsi_cmds_max;
1477         return 0;
1478 }
1479 EXPORT_SYMBOL_GPL(iscsi_target_alloc);
1480
1481 void iscsi_session_recovery_timedout(struct iscsi_cls_session *cls_session)
1482 {
1483         struct iscsi_session *session = cls_session->dd_data;
1484
1485         spin_lock_bh(&session->lock);
1486         if (session->state != ISCSI_STATE_LOGGED_IN) {
1487                 session->state = ISCSI_STATE_RECOVERY_FAILED;
1488                 if (session->leadconn)
1489                         wake_up(&session->leadconn->ehwait);
1490         }
1491         spin_unlock_bh(&session->lock);
1492 }
1493 EXPORT_SYMBOL_GPL(iscsi_session_recovery_timedout);
1494
1495 int iscsi_eh_target_reset(struct scsi_cmnd *sc)
1496 {
1497         struct iscsi_cls_session *cls_session;
1498         struct iscsi_session *session;
1499         struct iscsi_conn *conn;
1500
1501         cls_session = starget_to_session(scsi_target(sc->device));
1502         session = cls_session->dd_data;
1503         conn = session->leadconn;
1504
1505         mutex_lock(&session->eh_mutex);
1506         spin_lock_bh(&session->lock);
1507         if (session->state == ISCSI_STATE_TERMINATE) {
1508 failed:
1509                 iscsi_session_printk(KERN_INFO, session,
1510                                      "failing target reset: Could not log "
1511                                      "back into target [age %d]\n",
1512                                      session->age);
1513                 spin_unlock_bh(&session->lock);
1514                 mutex_unlock(&session->eh_mutex);
1515                 return FAILED;
1516         }
1517
1518         spin_unlock_bh(&session->lock);
1519         mutex_unlock(&session->eh_mutex);
1520         /*
1521          * we drop the lock here but the leadconn cannot be destoyed while
1522          * we are in the scsi eh
1523          */
1524         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1525
1526         ISCSI_DBG_SESSION(session, "wait for relogin\n");
1527         wait_event_interruptible(conn->ehwait,
1528                                  session->state == ISCSI_STATE_TERMINATE ||
1529                                  session->state == ISCSI_STATE_LOGGED_IN ||
1530                                  session->state == ISCSI_STATE_RECOVERY_FAILED);
1531         if (signal_pending(current))
1532                 flush_signals(current);
1533
1534         mutex_lock(&session->eh_mutex);
1535         spin_lock_bh(&session->lock);
1536         if (session->state == ISCSI_STATE_LOGGED_IN)
1537                 iscsi_session_printk(KERN_INFO, session,
1538                                      "target reset succeeded\n");
1539         else
1540                 goto failed;
1541         spin_unlock_bh(&session->lock);
1542         mutex_unlock(&session->eh_mutex);
1543         return SUCCESS;
1544 }
1545 EXPORT_SYMBOL_GPL(iscsi_eh_target_reset);
1546
1547 static void iscsi_tmf_timedout(unsigned long data)
1548 {
1549         struct iscsi_conn *conn = (struct iscsi_conn *)data;
1550         struct iscsi_session *session = conn->session;
1551
1552         spin_lock(&session->lock);
1553         if (conn->tmf_state == TMF_QUEUED) {
1554                 conn->tmf_state = TMF_TIMEDOUT;
1555                 ISCSI_DBG_SESSION(session, "tmf timedout\n");
1556                 /* unblock eh_abort() */
1557                 wake_up(&conn->ehwait);
1558         }
1559         spin_unlock(&session->lock);
1560 }
1561
1562 static int iscsi_exec_task_mgmt_fn(struct iscsi_conn *conn,
1563                                    struct iscsi_tm *hdr, int age,
1564                                    int timeout)
1565 {
1566         struct iscsi_session *session = conn->session;
1567         struct iscsi_task *task;
1568
1569         task = __iscsi_conn_send_pdu(conn, (struct iscsi_hdr *)hdr,
1570                                       NULL, 0);
1571         if (!task) {
1572                 spin_unlock_bh(&session->lock);
1573                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1574                 spin_lock_bh(&session->lock);
1575                 ISCSI_DBG_SESSION(session, "tmf exec failure\n");
1576                 return -EPERM;
1577         }
1578         conn->tmfcmd_pdus_cnt++;
1579         conn->tmf_timer.expires = timeout * HZ + jiffies;
1580         conn->tmf_timer.function = iscsi_tmf_timedout;
1581         conn->tmf_timer.data = (unsigned long)conn;
1582         add_timer(&conn->tmf_timer);
1583         ISCSI_DBG_SESSION(session, "tmf set timeout\n");
1584
1585         spin_unlock_bh(&session->lock);
1586         mutex_unlock(&session->eh_mutex);
1587
1588         /*
1589          * block eh thread until:
1590          *
1591          * 1) tmf response
1592          * 2) tmf timeout
1593          * 3) session is terminated or restarted or userspace has
1594          * given up on recovery
1595          */
1596         wait_event_interruptible(conn->ehwait, age != session->age ||
1597                                  session->state != ISCSI_STATE_LOGGED_IN ||
1598                                  conn->tmf_state != TMF_QUEUED);
1599         if (signal_pending(current))
1600                 flush_signals(current);
1601         del_timer_sync(&conn->tmf_timer);
1602
1603         mutex_lock(&session->eh_mutex);
1604         spin_lock_bh(&session->lock);
1605         /* if the session drops it will clean up the task */
1606         if (age != session->age ||
1607             session->state != ISCSI_STATE_LOGGED_IN)
1608                 return -ENOTCONN;
1609         return 0;
1610 }
1611
1612 /*
1613  * Fail commands. session lock held and recv side suspended and xmit
1614  * thread flushed
1615  */
1616 static void fail_all_commands(struct iscsi_conn *conn, unsigned lun,
1617                               int error)
1618 {
1619         struct iscsi_task *task, *tmp;
1620
1621         if (conn->task) {
1622                 if (lun == -1 ||
1623                     (conn->task->sc && conn->task->sc->device->lun == lun))
1624                         conn->task = NULL;
1625         }
1626
1627         /* flush pending */
1628         list_for_each_entry_safe(task, tmp, &conn->xmitqueue, running) {
1629                 if (lun == task->sc->device->lun || lun == -1) {
1630                         ISCSI_DBG_SESSION(conn->session,
1631                                           "failing pending sc %p itt 0x%x\n",
1632                                           task->sc, task->itt);
1633                         fail_command(conn, task, error << 16);
1634                 }
1635         }
1636
1637         list_for_each_entry_safe(task, tmp, &conn->requeue, running) {
1638                 if (lun == task->sc->device->lun || lun == -1) {
1639                         ISCSI_DBG_SESSION(conn->session,
1640                                           "failing requeued sc %p itt 0x%x\n",
1641                                           task->sc, task->itt);
1642                         fail_command(conn, task, error << 16);
1643                 }
1644         }
1645
1646         /* fail all other running */
1647         list_for_each_entry_safe(task, tmp, &conn->run_list, running) {
1648                 if (lun == task->sc->device->lun || lun == -1) {
1649                         ISCSI_DBG_SESSION(conn->session,
1650                                          "failing in progress sc %p itt 0x%x\n",
1651                                          task->sc, task->itt);
1652                         fail_command(conn, task, error << 16);
1653                 }
1654         }
1655 }
1656
1657 void iscsi_suspend_tx(struct iscsi_conn *conn)
1658 {
1659         struct Scsi_Host *shost = conn->session->host;
1660         struct iscsi_host *ihost = shost_priv(shost);
1661
1662         set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1663         if (!(conn->session->tt->caps & CAP_DATA_PATH_OFFLOAD))
1664                 flush_workqueue(ihost->workq);
1665 }
1666 EXPORT_SYMBOL_GPL(iscsi_suspend_tx);
1667
1668 static void iscsi_start_tx(struct iscsi_conn *conn)
1669 {
1670         clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
1671         if (!(conn->session->tt->caps & CAP_DATA_PATH_OFFLOAD))
1672                 iscsi_conn_queue_work(conn);
1673 }
1674
1675 static enum blk_eh_timer_return iscsi_eh_cmd_timed_out(struct scsi_cmnd *scmd)
1676 {
1677         struct iscsi_cls_session *cls_session;
1678         struct iscsi_session *session;
1679         struct iscsi_conn *conn;
1680         enum blk_eh_timer_return rc = BLK_EH_NOT_HANDLED;
1681
1682         cls_session = starget_to_session(scsi_target(scmd->device));
1683         session = cls_session->dd_data;
1684
1685         ISCSI_DBG_SESSION(session, "scsi cmd %p timedout\n", scmd);
1686
1687         spin_lock(&session->lock);
1688         if (session->state != ISCSI_STATE_LOGGED_IN) {
1689                 /*
1690                  * We are probably in the middle of iscsi recovery so let
1691                  * that complete and handle the error.
1692                  */
1693                 rc = BLK_EH_RESET_TIMER;
1694                 goto done;
1695         }
1696
1697         conn = session->leadconn;
1698         if (!conn) {
1699                 /* In the middle of shuting down */
1700                 rc = BLK_EH_RESET_TIMER;
1701                 goto done;
1702         }
1703
1704         if (!conn->recv_timeout && !conn->ping_timeout)
1705                 goto done;
1706         /*
1707          * if the ping timedout then we are in the middle of cleaning up
1708          * and can let the iscsi eh handle it
1709          */
1710         if (time_before_eq(conn->last_recv + (conn->recv_timeout * HZ) +
1711                             (conn->ping_timeout * HZ), jiffies))
1712                 rc = BLK_EH_RESET_TIMER;
1713         /*
1714          * if we are about to check the transport then give the command
1715          * more time
1716          */
1717         if (time_before_eq(conn->last_recv + (conn->recv_timeout * HZ),
1718                            jiffies))
1719                 rc = BLK_EH_RESET_TIMER;
1720         /* if in the middle of checking the transport then give us more time */
1721         if (conn->ping_task)
1722                 rc = BLK_EH_RESET_TIMER;
1723 done:
1724         spin_unlock(&session->lock);
1725         ISCSI_DBG_SESSION(session, "return %s\n", rc == BLK_EH_RESET_TIMER ?
1726                           "timer reset" : "nh");
1727         return rc;
1728 }
1729
1730 static void iscsi_check_transport_timeouts(unsigned long data)
1731 {
1732         struct iscsi_conn *conn = (struct iscsi_conn *)data;
1733         struct iscsi_session *session = conn->session;
1734         unsigned long recv_timeout, next_timeout = 0, last_recv;
1735
1736         spin_lock(&session->lock);
1737         if (session->state != ISCSI_STATE_LOGGED_IN)
1738                 goto done;
1739
1740         recv_timeout = conn->recv_timeout;
1741         if (!recv_timeout)
1742                 goto done;
1743
1744         recv_timeout *= HZ;
1745         last_recv = conn->last_recv;
1746         if (conn->ping_task &&
1747             time_before_eq(conn->last_ping + (conn->ping_timeout * HZ),
1748                            jiffies)) {
1749                 iscsi_conn_printk(KERN_ERR, conn, "ping timeout of %d secs "
1750                                   "expired, last rx %lu, last ping %lu, "
1751                                   "now %lu\n", conn->ping_timeout, last_recv,
1752                                   conn->last_ping, jiffies);
1753                 spin_unlock(&session->lock);
1754                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1755                 return;
1756         }
1757
1758         if (time_before_eq(last_recv + recv_timeout, jiffies)) {
1759                 /* send a ping to try to provoke some traffic */
1760                 ISCSI_DBG_CONN(conn, "Sending nopout as ping\n");
1761                 iscsi_send_nopout(conn, NULL);
1762                 next_timeout = conn->last_ping + (conn->ping_timeout * HZ);
1763         } else
1764                 next_timeout = last_recv + recv_timeout;
1765
1766         ISCSI_DBG_CONN(conn, "Setting next tmo %lu\n", next_timeout);
1767         mod_timer(&conn->transport_timer, next_timeout);
1768 done:
1769         spin_unlock(&session->lock);
1770 }
1771
1772 static void iscsi_prep_abort_task_pdu(struct iscsi_task *task,
1773                                       struct iscsi_tm *hdr)
1774 {
1775         memset(hdr, 0, sizeof(*hdr));
1776         hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
1777         hdr->flags = ISCSI_TM_FUNC_ABORT_TASK & ISCSI_FLAG_TM_FUNC_MASK;
1778         hdr->flags |= ISCSI_FLAG_CMD_FINAL;
1779         memcpy(hdr->lun, task->lun, sizeof(hdr->lun));
1780         hdr->rtt = task->hdr_itt;
1781         hdr->refcmdsn = task->cmdsn;
1782 }
1783
1784 int iscsi_eh_abort(struct scsi_cmnd *sc)
1785 {
1786         struct iscsi_cls_session *cls_session;
1787         struct iscsi_session *session;
1788         struct iscsi_conn *conn;
1789         struct iscsi_task *task;
1790         struct iscsi_tm *hdr;
1791         int rc, age;
1792
1793         cls_session = starget_to_session(scsi_target(sc->device));
1794         session = cls_session->dd_data;
1795
1796         mutex_lock(&session->eh_mutex);
1797         spin_lock_bh(&session->lock);
1798         /*
1799          * if session was ISCSI_STATE_IN_RECOVERY then we may not have
1800          * got the command.
1801          */
1802         if (!sc->SCp.ptr) {
1803                 ISCSI_DBG_SESSION(session, "sc never reached iscsi layer or "
1804                                   "it completed.\n");
1805                 spin_unlock_bh(&session->lock);
1806                 mutex_unlock(&session->eh_mutex);
1807                 return SUCCESS;
1808         }
1809
1810         /*
1811          * If we are not logged in or we have started a new session
1812          * then let the host reset code handle this
1813          */
1814         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN ||
1815             sc->SCp.phase != session->age) {
1816                 spin_unlock_bh(&session->lock);
1817                 mutex_unlock(&session->eh_mutex);
1818                 return FAILED;
1819         }
1820
1821         conn = session->leadconn;
1822         conn->eh_abort_cnt++;
1823         age = session->age;
1824
1825         task = (struct iscsi_task *)sc->SCp.ptr;
1826         ISCSI_DBG_SESSION(session, "aborting [sc %p itt 0x%x]\n",
1827                           sc, task->itt);
1828
1829         /* task completed before time out */
1830         if (!task->sc) {
1831                 ISCSI_DBG_SESSION(session, "sc completed while abort in "
1832                                   "progress\n");
1833                 goto success;
1834         }
1835
1836         if (task->state == ISCSI_TASK_PENDING) {
1837                 fail_command(conn, task, DID_ABORT << 16);
1838                 goto success;
1839         }
1840
1841         /* only have one tmf outstanding at a time */
1842         if (conn->tmf_state != TMF_INITIAL)
1843                 goto failed;
1844         conn->tmf_state = TMF_QUEUED;
1845
1846         hdr = &conn->tmhdr;
1847         iscsi_prep_abort_task_pdu(task, hdr);
1848
1849         if (iscsi_exec_task_mgmt_fn(conn, hdr, age, session->abort_timeout)) {
1850                 rc = FAILED;
1851                 goto failed;
1852         }
1853
1854         switch (conn->tmf_state) {
1855         case TMF_SUCCESS:
1856                 spin_unlock_bh(&session->lock);
1857                 /*
1858                  * stop tx side incase the target had sent a abort rsp but
1859                  * the initiator was still writing out data.
1860                  */
1861                 iscsi_suspend_tx(conn);
1862                 /*
1863                  * we do not stop the recv side because targets have been
1864                  * good and have never sent us a successful tmf response
1865                  * then sent more data for the cmd.
1866                  */
1867                 spin_lock(&session->lock);
1868                 fail_command(conn, task, DID_ABORT << 16);
1869                 conn->tmf_state = TMF_INITIAL;
1870                 spin_unlock(&session->lock);
1871                 iscsi_start_tx(conn);
1872                 goto success_unlocked;
1873         case TMF_TIMEDOUT:
1874                 spin_unlock_bh(&session->lock);
1875                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1876                 goto failed_unlocked;
1877         case TMF_NOT_FOUND:
1878                 if (!sc->SCp.ptr) {
1879                         conn->tmf_state = TMF_INITIAL;
1880                         /* task completed before tmf abort response */
1881                         ISCSI_DBG_SESSION(session, "sc completed while abort "
1882                                           "in progress\n");
1883                         goto success;
1884                 }
1885                 /* fall through */
1886         default:
1887                 conn->tmf_state = TMF_INITIAL;
1888                 goto failed;
1889         }
1890
1891 success:
1892         spin_unlock_bh(&session->lock);
1893 success_unlocked:
1894         ISCSI_DBG_SESSION(session, "abort success [sc %p itt 0x%x]\n",
1895                           sc, task->itt);
1896         mutex_unlock(&session->eh_mutex);
1897         return SUCCESS;
1898
1899 failed:
1900         spin_unlock_bh(&session->lock);
1901 failed_unlocked:
1902         ISCSI_DBG_SESSION(session, "abort failed [sc %p itt 0x%x]\n", sc,
1903                           task ? task->itt : 0);
1904         mutex_unlock(&session->eh_mutex);
1905         return FAILED;
1906 }
1907 EXPORT_SYMBOL_GPL(iscsi_eh_abort);
1908
1909 static void iscsi_prep_lun_reset_pdu(struct scsi_cmnd *sc, struct iscsi_tm *hdr)
1910 {
1911         memset(hdr, 0, sizeof(*hdr));
1912         hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
1913         hdr->flags = ISCSI_TM_FUNC_LOGICAL_UNIT_RESET & ISCSI_FLAG_TM_FUNC_MASK;
1914         hdr->flags |= ISCSI_FLAG_CMD_FINAL;
1915         int_to_scsilun(sc->device->lun, (struct scsi_lun *)hdr->lun);
1916         hdr->rtt = RESERVED_ITT;
1917 }
1918
1919 int iscsi_eh_device_reset(struct scsi_cmnd *sc)
1920 {
1921         struct iscsi_cls_session *cls_session;
1922         struct iscsi_session *session;
1923         struct iscsi_conn *conn;
1924         struct iscsi_tm *hdr;
1925         int rc = FAILED;
1926
1927         cls_session = starget_to_session(scsi_target(sc->device));
1928         session = cls_session->dd_data;
1929
1930         ISCSI_DBG_SESSION(session, "LU Reset [sc %p lun %u]\n",
1931                           sc, sc->device->lun);
1932
1933         mutex_lock(&session->eh_mutex);
1934         spin_lock_bh(&session->lock);
1935         /*
1936          * Just check if we are not logged in. We cannot check for
1937          * the phase because the reset could come from a ioctl.
1938          */
1939         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN)
1940                 goto unlock;
1941         conn = session->leadconn;
1942
1943         /* only have one tmf outstanding at a time */
1944         if (conn->tmf_state != TMF_INITIAL)
1945                 goto unlock;
1946         conn->tmf_state = TMF_QUEUED;
1947
1948         hdr = &conn->tmhdr;
1949         iscsi_prep_lun_reset_pdu(sc, hdr);
1950
1951         if (iscsi_exec_task_mgmt_fn(conn, hdr, session->age,
1952                                     session->lu_reset_timeout)) {
1953                 rc = FAILED;
1954                 goto unlock;
1955         }
1956
1957         switch (conn->tmf_state) {
1958         case TMF_SUCCESS:
1959                 break;
1960         case TMF_TIMEDOUT:
1961                 spin_unlock_bh(&session->lock);
1962                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1963                 goto done;
1964         default:
1965                 conn->tmf_state = TMF_INITIAL;
1966                 goto unlock;
1967         }
1968
1969         rc = SUCCESS;
1970         spin_unlock_bh(&session->lock);
1971
1972         iscsi_suspend_tx(conn);
1973
1974         spin_lock_bh(&session->lock);
1975         fail_all_commands(conn, sc->device->lun, DID_ERROR);
1976         conn->tmf_state = TMF_INITIAL;
1977         spin_unlock_bh(&session->lock);
1978
1979         iscsi_start_tx(conn);
1980         goto done;
1981
1982 unlock:
1983         spin_unlock_bh(&session->lock);
1984 done:
1985         ISCSI_DBG_SESSION(session, "dev reset result = %s\n",
1986                          rc == SUCCESS ? "SUCCESS" : "FAILED");
1987         mutex_unlock(&session->eh_mutex);
1988         return rc;
1989 }
1990 EXPORT_SYMBOL_GPL(iscsi_eh_device_reset);
1991
1992 /*
1993  * Pre-allocate a pool of @max items of @item_size. By default, the pool
1994  * should be accessed via kfifo_{get,put} on q->queue.
1995  * Optionally, the caller can obtain the array of object pointers
1996  * by passing in a non-NULL @items pointer
1997  */
1998 int
1999 iscsi_pool_init(struct iscsi_pool *q, int max, void ***items, int item_size)
2000 {
2001         int i, num_arrays = 1;
2002
2003         memset(q, 0, sizeof(*q));
2004
2005         q->max = max;
2006
2007         /* If the user passed an items pointer, he wants a copy of
2008          * the array. */
2009         if (items)
2010                 num_arrays++;
2011         q->pool = kzalloc(num_arrays * max * sizeof(void*), GFP_KERNEL);
2012         if (q->pool == NULL)
2013                 return -ENOMEM;
2014
2015         q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2016                               GFP_KERNEL, NULL);
2017         if (IS_ERR(q->queue)) {
2018                 q->queue = NULL;
2019                 goto enomem;
2020         }
2021
2022         for (i = 0; i < max; i++) {
2023                 q->pool[i] = kzalloc(item_size, GFP_KERNEL);
2024                 if (q->pool[i] == NULL) {
2025                         q->max = i;
2026                         goto enomem;
2027                 }
2028                 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2029         }
2030
2031         if (items) {
2032                 *items = q->pool + max;
2033                 memcpy(*items, q->pool, max * sizeof(void *));
2034         }
2035
2036         return 0;
2037
2038 enomem:
2039         iscsi_pool_free(q);
2040         return -ENOMEM;
2041 }
2042 EXPORT_SYMBOL_GPL(iscsi_pool_init);
2043
2044 void iscsi_pool_free(struct iscsi_pool *q)
2045 {
2046         int i;
2047
2048         for (i = 0; i < q->max; i++)
2049                 kfree(q->pool[i]);
2050         kfree(q->pool);
2051         kfree(q->queue);
2052 }
2053 EXPORT_SYMBOL_GPL(iscsi_pool_free);
2054
2055 /**
2056  * iscsi_host_add - add host to system
2057  * @shost: scsi host
2058  * @pdev: parent device
2059  *
2060  * This should be called by partial offload and software iscsi drivers
2061  * to add a host to the system.
2062  */
2063 int iscsi_host_add(struct Scsi_Host *shost, struct device *pdev)
2064 {
2065         if (!shost->can_queue)
2066                 shost->can_queue = ISCSI_DEF_XMIT_CMDS_MAX;
2067
2068         if (!shost->cmd_per_lun)
2069                 shost->cmd_per_lun = ISCSI_DEF_CMD_PER_LUN;
2070
2071         if (!shost->transportt->eh_timed_out)
2072                 shost->transportt->eh_timed_out = iscsi_eh_cmd_timed_out;
2073         return scsi_add_host(shost, pdev);
2074 }
2075 EXPORT_SYMBOL_GPL(iscsi_host_add);
2076
2077 /**
2078  * iscsi_host_alloc - allocate a host and driver data
2079  * @sht: scsi host template
2080  * @dd_data_size: driver host data size
2081  * @xmit_can_sleep: bool indicating if LLD will queue IO from a work queue
2082  *
2083  * This should be called by partial offload and software iscsi drivers.
2084  * To access the driver specific memory use the iscsi_host_priv() macro.
2085  */
2086 struct Scsi_Host *iscsi_host_alloc(struct scsi_host_template *sht,
2087                                    int dd_data_size, bool xmit_can_sleep)
2088 {
2089         struct Scsi_Host *shost;
2090         struct iscsi_host *ihost;
2091
2092         shost = scsi_host_alloc(sht, sizeof(struct iscsi_host) + dd_data_size);
2093         if (!shost)
2094                 return NULL;
2095         ihost = shost_priv(shost);
2096
2097         if (xmit_can_sleep) {
2098                 snprintf(ihost->workq_name, sizeof(ihost->workq_name),
2099                         "iscsi_q_%d", shost->host_no);
2100                 ihost->workq = create_singlethread_workqueue(ihost->workq_name);
2101                 if (!ihost->workq)
2102                         goto free_host;
2103         }
2104
2105         spin_lock_init(&ihost->lock);
2106         ihost->state = ISCSI_HOST_SETUP;
2107         ihost->num_sessions = 0;
2108         init_waitqueue_head(&ihost->session_removal_wq);
2109         return shost;
2110
2111 free_host:
2112         scsi_host_put(shost);
2113         return NULL;
2114 }
2115 EXPORT_SYMBOL_GPL(iscsi_host_alloc);
2116
2117 static void iscsi_notify_host_removed(struct iscsi_cls_session *cls_session)
2118 {
2119         iscsi_session_failure(cls_session->dd_data, ISCSI_ERR_INVALID_HOST);
2120 }
2121
2122 /**
2123  * iscsi_host_remove - remove host and sessions
2124  * @shost: scsi host
2125  *
2126  * If there are any sessions left, this will initiate the removal and wait
2127  * for the completion.
2128  */
2129 void iscsi_host_remove(struct Scsi_Host *shost)
2130 {
2131         struct iscsi_host *ihost = shost_priv(shost);
2132         unsigned long flags;
2133
2134         spin_lock_irqsave(&ihost->lock, flags);
2135         ihost->state = ISCSI_HOST_REMOVED;
2136         spin_unlock_irqrestore(&ihost->lock, flags);
2137
2138         iscsi_host_for_each_session(shost, iscsi_notify_host_removed);
2139         wait_event_interruptible(ihost->session_removal_wq,
2140                                  ihost->num_sessions == 0);
2141         if (signal_pending(current))
2142                 flush_signals(current);
2143
2144         scsi_remove_host(shost);
2145         if (ihost->workq)
2146                 destroy_workqueue(ihost->workq);
2147 }
2148 EXPORT_SYMBOL_GPL(iscsi_host_remove);
2149
2150 void iscsi_host_free(struct Scsi_Host *shost)
2151 {
2152         struct iscsi_host *ihost = shost_priv(shost);
2153
2154         kfree(ihost->netdev);
2155         kfree(ihost->hwaddress);
2156         kfree(ihost->initiatorname);
2157         scsi_host_put(shost);
2158 }
2159 EXPORT_SYMBOL_GPL(iscsi_host_free);
2160
2161 static void iscsi_host_dec_session_cnt(struct Scsi_Host *shost)
2162 {
2163         struct iscsi_host *ihost = shost_priv(shost);
2164         unsigned long flags;
2165
2166         shost = scsi_host_get(shost);
2167         if (!shost) {
2168                 printk(KERN_ERR "Invalid state. Cannot notify host removal "
2169                       "of session teardown event because host already "
2170                       "removed.\n");
2171                 return;
2172         }
2173
2174         spin_lock_irqsave(&ihost->lock, flags);
2175         ihost->num_sessions--;
2176         if (ihost->num_sessions == 0)
2177                 wake_up(&ihost->session_removal_wq);
2178         spin_unlock_irqrestore(&ihost->lock, flags);
2179         scsi_host_put(shost);
2180 }
2181
2182 /**
2183  * iscsi_session_setup - create iscsi cls session and host and session
2184  * @iscsit: iscsi transport template
2185  * @shost: scsi host
2186  * @cmds_max: session can queue
2187  * @cmd_task_size: LLD task private data size
2188  * @initial_cmdsn: initial CmdSN
2189  *
2190  * This can be used by software iscsi_transports that allocate
2191  * a session per scsi host.
2192  *
2193  * Callers should set cmds_max to the largest total numer (mgmt + scsi) of
2194  * tasks they support. The iscsi layer reserves ISCSI_MGMT_CMDS_MAX tasks
2195  * for nop handling and login/logout requests.
2196  */
2197 struct iscsi_cls_session *
2198 iscsi_session_setup(struct iscsi_transport *iscsit, struct Scsi_Host *shost,
2199                     uint16_t cmds_max, int cmd_task_size,
2200                     uint32_t initial_cmdsn, unsigned int id)
2201 {
2202         struct iscsi_host *ihost = shost_priv(shost);
2203         struct iscsi_session *session;
2204         struct iscsi_cls_session *cls_session;
2205         int cmd_i, scsi_cmds, total_cmds = cmds_max;
2206         unsigned long flags;
2207
2208         spin_lock_irqsave(&ihost->lock, flags);
2209         if (ihost->state == ISCSI_HOST_REMOVED) {
2210                 spin_unlock_irqrestore(&ihost->lock, flags);
2211                 return NULL;
2212         }
2213         ihost->num_sessions++;
2214         spin_unlock_irqrestore(&ihost->lock, flags);
2215
2216         if (!total_cmds)
2217                 total_cmds = ISCSI_DEF_XMIT_CMDS_MAX;
2218         /*
2219          * The iscsi layer needs some tasks for nop handling and tmfs,
2220          * so the cmds_max must at least be greater than ISCSI_MGMT_CMDS_MAX
2221          * + 1 command for scsi IO.
2222          */
2223         if (total_cmds < ISCSI_TOTAL_CMDS_MIN) {
2224                 printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue "
2225                        "must be a power of two that is at least %d.\n",
2226                        total_cmds, ISCSI_TOTAL_CMDS_MIN);
2227                 goto dec_session_count;
2228         }
2229
2230         if (total_cmds > ISCSI_TOTAL_CMDS_MAX) {
2231                 printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue "
2232                        "must be a power of 2 less than or equal to %d.\n",
2233                        cmds_max, ISCSI_TOTAL_CMDS_MAX);
2234                 total_cmds = ISCSI_TOTAL_CMDS_MAX;
2235         }
2236
2237         if (!is_power_of_2(total_cmds)) {
2238                 printk(KERN_ERR "iscsi: invalid can_queue of %d. can_queue "
2239                        "must be a power of 2.\n", total_cmds);
2240                 total_cmds = rounddown_pow_of_two(total_cmds);
2241                 if (total_cmds < ISCSI_TOTAL_CMDS_MIN)
2242                         return NULL;
2243                 printk(KERN_INFO "iscsi: Rounding can_queue to %d.\n",
2244                        total_cmds);
2245         }
2246         scsi_cmds = total_cmds - ISCSI_MGMT_CMDS_MAX;
2247
2248         cls_session = iscsi_alloc_session(shost, iscsit,
2249                                           sizeof(struct iscsi_session));
2250         if (!cls_session)
2251                 goto dec_session_count;
2252         session = cls_session->dd_data;
2253         session->cls_session = cls_session;
2254         session->host = shost;
2255         session->state = ISCSI_STATE_FREE;
2256         session->fast_abort = 1;
2257         session->lu_reset_timeout = 15;
2258         session->abort_timeout = 10;
2259         session->scsi_cmds_max = scsi_cmds;
2260         session->cmds_max = total_cmds;
2261         session->queued_cmdsn = session->cmdsn = initial_cmdsn;
2262         session->exp_cmdsn = initial_cmdsn + 1;
2263         session->max_cmdsn = initial_cmdsn + 1;
2264         session->max_r2t = 1;
2265         session->tt = iscsit;
2266         mutex_init(&session->eh_mutex);
2267         spin_lock_init(&session->lock);
2268
2269         /* initialize SCSI PDU commands pool */
2270         if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
2271                             (void***)&session->cmds,
2272                             cmd_task_size + sizeof(struct iscsi_task)))
2273                 goto cmdpool_alloc_fail;
2274
2275         /* pre-format cmds pool with ITT */
2276         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
2277                 struct iscsi_task *task = session->cmds[cmd_i];
2278
2279                 if (cmd_task_size)
2280                         task->dd_data = &task[1];
2281                 task->itt = cmd_i;
2282                 INIT_LIST_HEAD(&task->running);
2283         }
2284
2285         if (!try_module_get(iscsit->owner))
2286                 goto module_get_fail;
2287
2288         if (iscsi_add_session(cls_session, id))
2289                 goto cls_session_fail;
2290
2291         return cls_session;
2292
2293 cls_session_fail:
2294         module_put(iscsit->owner);
2295 module_get_fail:
2296         iscsi_pool_free(&session->cmdpool);
2297 cmdpool_alloc_fail:
2298         iscsi_free_session(cls_session);
2299 dec_session_count:
2300         iscsi_host_dec_session_cnt(shost);
2301         return NULL;
2302 }
2303 EXPORT_SYMBOL_GPL(iscsi_session_setup);
2304
2305 /**
2306  * iscsi_session_teardown - destroy session, host, and cls_session
2307  * @cls_session: iscsi session
2308  *
2309  * The driver must have called iscsi_remove_session before
2310  * calling this.
2311  */
2312 void iscsi_session_teardown(struct iscsi_cls_session *cls_session)
2313 {
2314         struct iscsi_session *session = cls_session->dd_data;
2315         struct module *owner = cls_session->transport->owner;
2316         struct Scsi_Host *shost = session->host;
2317
2318         iscsi_pool_free(&session->cmdpool);
2319
2320         kfree(session->password);
2321         kfree(session->password_in);
2322         kfree(session->username);
2323         kfree(session->username_in);
2324         kfree(session->targetname);
2325         kfree(session->initiatorname);
2326         kfree(session->ifacename);
2327
2328         iscsi_destroy_session(cls_session);
2329         iscsi_host_dec_session_cnt(shost);
2330         module_put(owner);
2331 }
2332 EXPORT_SYMBOL_GPL(iscsi_session_teardown);
2333
2334 /**
2335  * iscsi_conn_setup - create iscsi_cls_conn and iscsi_conn
2336  * @cls_session: iscsi_cls_session
2337  * @dd_size: private driver data size
2338  * @conn_idx: cid
2339  */
2340 struct iscsi_cls_conn *
2341 iscsi_conn_setup(struct iscsi_cls_session *cls_session, int dd_size,
2342                  uint32_t conn_idx)
2343 {
2344         struct iscsi_session *session = cls_session->dd_data;
2345         struct iscsi_conn *conn;
2346         struct iscsi_cls_conn *cls_conn;
2347         char *data;
2348
2349         cls_conn = iscsi_create_conn(cls_session, sizeof(*conn) + dd_size,
2350                                      conn_idx);
2351         if (!cls_conn)
2352                 return NULL;
2353         conn = cls_conn->dd_data;
2354         memset(conn, 0, sizeof(*conn) + dd_size);
2355
2356         conn->dd_data = cls_conn->dd_data + sizeof(*conn);
2357         conn->session = session;
2358         conn->cls_conn = cls_conn;
2359         conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2360         conn->id = conn_idx;
2361         conn->exp_statsn = 0;
2362         conn->tmf_state = TMF_INITIAL;
2363
2364         init_timer(&conn->transport_timer);
2365         conn->transport_timer.data = (unsigned long)conn;
2366         conn->transport_timer.function = iscsi_check_transport_timeouts;
2367
2368         INIT_LIST_HEAD(&conn->run_list);
2369         INIT_LIST_HEAD(&conn->mgmt_run_list);
2370         INIT_LIST_HEAD(&conn->mgmtqueue);
2371         INIT_LIST_HEAD(&conn->xmitqueue);
2372         INIT_LIST_HEAD(&conn->requeue);
2373         INIT_WORK(&conn->xmitwork, iscsi_xmitworker);
2374
2375         /* allocate login_task used for the login/text sequences */
2376         spin_lock_bh(&session->lock);
2377         if (!__kfifo_get(session->cmdpool.queue,
2378                          (void*)&conn->login_task,
2379                          sizeof(void*))) {
2380                 spin_unlock_bh(&session->lock);
2381                 goto login_task_alloc_fail;
2382         }
2383         spin_unlock_bh(&session->lock);
2384
2385         data = (char *) __get_free_pages(GFP_KERNEL,
2386                                          get_order(ISCSI_DEF_MAX_RECV_SEG_LEN));
2387         if (!data)
2388                 goto login_task_data_alloc_fail;
2389         conn->login_task->data = conn->data = data;
2390
2391         init_timer(&conn->tmf_timer);
2392         init_waitqueue_head(&conn->ehwait);
2393
2394         return cls_conn;
2395
2396 login_task_data_alloc_fail:
2397         __kfifo_put(session->cmdpool.queue, (void*)&conn->login_task,
2398                     sizeof(void*));
2399 login_task_alloc_fail:
2400         iscsi_destroy_conn(cls_conn);
2401         return NULL;
2402 }
2403 EXPORT_SYMBOL_GPL(iscsi_conn_setup);
2404
2405 /**
2406  * iscsi_conn_teardown - teardown iscsi connection
2407  * cls_conn: iscsi class connection
2408  *
2409  * TODO: we may need to make this into a two step process
2410  * like scsi-mls remove + put host
2411  */
2412 void iscsi_conn_teardown(struct iscsi_cls_conn *cls_conn)
2413 {
2414         struct iscsi_conn *conn = cls_conn->dd_data;
2415         struct iscsi_session *session = conn->session;
2416         unsigned long flags;
2417
2418         del_timer_sync(&conn->transport_timer);
2419
2420         spin_lock_bh(&session->lock);
2421         conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2422         if (session->leadconn == conn) {
2423                 /*
2424                  * leading connection? then give up on recovery.
2425                  */
2426                 session->state = ISCSI_STATE_TERMINATE;
2427                 wake_up(&conn->ehwait);
2428         }
2429         spin_unlock_bh(&session->lock);
2430
2431         /*
2432          * Block until all in-progress commands for this connection
2433          * time out or fail.
2434          */
2435         for (;;) {
2436                 spin_lock_irqsave(session->host->host_lock, flags);
2437                 if (!session->host->host_busy) { /* OK for ERL == 0 */
2438                         spin_unlock_irqrestore(session->host->host_lock, flags);
2439                         break;
2440                 }
2441                 spin_unlock_irqrestore(session->host->host_lock, flags);
2442                 msleep_interruptible(500);
2443                 iscsi_conn_printk(KERN_INFO, conn, "iscsi conn_destroy(): "
2444                                   "host_busy %d host_failed %d\n",
2445                                   session->host->host_busy,
2446                                   session->host->host_failed);
2447                 /*
2448                  * force eh_abort() to unblock
2449                  */
2450                 wake_up(&conn->ehwait);
2451         }
2452
2453         /* flush queued up work because we free the connection below */
2454         iscsi_suspend_tx(conn);
2455
2456         spin_lock_bh(&session->lock);
2457         free_pages((unsigned long) conn->data,
2458                    get_order(ISCSI_DEF_MAX_RECV_SEG_LEN));
2459         kfree(conn->persistent_address);
2460         __kfifo_put(session->cmdpool.queue, (void*)&conn->login_task,
2461                     sizeof(void*));
2462         if (session->leadconn == conn)
2463                 session->leadconn = NULL;
2464         spin_unlock_bh(&session->lock);
2465
2466         iscsi_destroy_conn(cls_conn);
2467 }
2468 EXPORT_SYMBOL_GPL(iscsi_conn_teardown);
2469
2470 int iscsi_conn_start(struct iscsi_cls_conn *cls_conn)
2471 {
2472         struct iscsi_conn *conn = cls_conn->dd_data;
2473         struct iscsi_session *session = conn->session;
2474
2475         if (!session) {
2476                 iscsi_conn_printk(KERN_ERR, conn,
2477                                   "can't start unbound connection\n");
2478                 return -EPERM;
2479         }
2480
2481         if ((session->imm_data_en || !session->initial_r2t_en) &&
2482              session->first_burst > session->max_burst) {
2483                 iscsi_conn_printk(KERN_INFO, conn, "invalid burst lengths: "
2484                                   "first_burst %d max_burst %d\n",
2485                                   session->first_burst, session->max_burst);
2486                 return -EINVAL;
2487         }
2488
2489         if (conn->ping_timeout && !conn->recv_timeout) {
2490                 iscsi_conn_printk(KERN_ERR, conn, "invalid recv timeout of "
2491                                   "zero. Using 5 seconds\n.");
2492                 conn->recv_timeout = 5;
2493         }
2494
2495         if (conn->recv_timeout && !conn->ping_timeout) {
2496                 iscsi_conn_printk(KERN_ERR, conn, "invalid ping timeout of "
2497                                   "zero. Using 5 seconds.\n");
2498                 conn->ping_timeout = 5;
2499         }
2500
2501         spin_lock_bh(&session->lock);
2502         conn->c_stage = ISCSI_CONN_STARTED;
2503         session->state = ISCSI_STATE_LOGGED_IN;
2504         session->queued_cmdsn = session->cmdsn;
2505
2506         conn->last_recv = jiffies;
2507         conn->last_ping = jiffies;
2508         if (conn->recv_timeout && conn->ping_timeout)
2509                 mod_timer(&conn->transport_timer,
2510                           jiffies + (conn->recv_timeout * HZ));
2511
2512         switch(conn->stop_stage) {
2513         case STOP_CONN_RECOVER:
2514                 /*
2515                  * unblock eh_abort() if it is blocked. re-try all
2516                  * commands after successful recovery
2517                  */
2518                 conn->stop_stage = 0;
2519                 conn->tmf_state = TMF_INITIAL;
2520                 session->age++;
2521                 if (session->age == 16)
2522                         session->age = 0;
2523                 break;
2524         case STOP_CONN_TERM:
2525                 conn->stop_stage = 0;
2526                 break;
2527         default:
2528                 break;
2529         }
2530         spin_unlock_bh(&session->lock);
2531
2532         iscsi_unblock_session(session->cls_session);
2533         wake_up(&conn->ehwait);
2534         return 0;
2535 }
2536 EXPORT_SYMBOL_GPL(iscsi_conn_start);
2537
2538 static void
2539 flush_control_queues(struct iscsi_session *session, struct iscsi_conn *conn)
2540 {
2541         struct iscsi_task *task, *tmp;
2542
2543         /* handle pending */
2544         list_for_each_entry_safe(task, tmp, &conn->mgmtqueue, running) {
2545                 ISCSI_DBG_SESSION(session, "flushing pending mgmt task "
2546                                   "itt 0x%x\n", task->itt);
2547                 /* release ref from prep task */
2548                 __iscsi_put_task(task);
2549         }
2550
2551         /* handle running */
2552         list_for_each_entry_safe(task, tmp, &conn->mgmt_run_list, running) {
2553                 ISCSI_DBG_SESSION(session, "flushing running mgmt task "
2554                                   "itt 0x%x\n", task->itt);
2555                 /* release ref from prep task */
2556                 __iscsi_put_task(task);
2557         }
2558
2559         conn->task = NULL;
2560 }
2561
2562 static void iscsi_start_session_recovery(struct iscsi_session *session,
2563                                          struct iscsi_conn *conn, int flag)
2564 {
2565         int old_stop_stage;
2566
2567         del_timer_sync(&conn->transport_timer);
2568
2569         mutex_lock(&session->eh_mutex);
2570         spin_lock_bh(&session->lock);
2571         if (conn->stop_stage == STOP_CONN_TERM) {
2572                 spin_unlock_bh(&session->lock);
2573                 mutex_unlock(&session->eh_mutex);
2574                 return;
2575         }
2576
2577         /*
2578          * When this is called for the in_login state, we only want to clean
2579          * up the login task and connection. We do not need to block and set
2580          * the recovery state again
2581          */
2582         if (flag == STOP_CONN_TERM)
2583                 session->state = ISCSI_STATE_TERMINATE;
2584         else if (conn->stop_stage != STOP_CONN_RECOVER)
2585                 session->state = ISCSI_STATE_IN_RECOVERY;
2586
2587         old_stop_stage = conn->stop_stage;
2588         conn->stop_stage = flag;
2589         conn->c_stage = ISCSI_CONN_STOPPED;
2590         spin_unlock_bh(&session->lock);
2591
2592         iscsi_suspend_tx(conn);
2593         /*
2594          * for connection level recovery we should not calculate
2595          * header digest. conn->hdr_size used for optimization
2596          * in hdr_extract() and will be re-negotiated at
2597          * set_param() time.
2598          */
2599         if (flag == STOP_CONN_RECOVER) {
2600                 conn->hdrdgst_en = 0;
2601                 conn->datadgst_en = 0;
2602                 if (session->state == ISCSI_STATE_IN_RECOVERY &&
2603                     old_stop_stage != STOP_CONN_RECOVER) {
2604                         ISCSI_DBG_SESSION(session, "blocking session\n");
2605                         iscsi_block_session(session->cls_session);
2606                 }
2607         }
2608
2609         /*
2610          * flush queues.
2611          */
2612         spin_lock_bh(&session->lock);
2613         if (flag == STOP_CONN_RECOVER)
2614                 fail_all_commands(conn, -1, DID_TRANSPORT_DISRUPTED);
2615         else
2616                 fail_all_commands(conn, -1, DID_ERROR);
2617         flush_control_queues(session, conn);
2618         spin_unlock_bh(&session->lock);
2619         mutex_unlock(&session->eh_mutex);
2620 }
2621
2622 void iscsi_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
2623 {
2624         struct iscsi_conn *conn = cls_conn->dd_data;
2625         struct iscsi_session *session = conn->session;
2626
2627         switch (flag) {
2628         case STOP_CONN_RECOVER:
2629         case STOP_CONN_TERM:
2630                 iscsi_start_session_recovery(session, conn, flag);
2631                 break;
2632         default:
2633                 iscsi_conn_printk(KERN_ERR, conn,
2634                                   "invalid stop flag %d\n", flag);
2635         }
2636 }
2637 EXPORT_SYMBOL_GPL(iscsi_conn_stop);
2638
2639 int iscsi_conn_bind(struct iscsi_cls_session *cls_session,
2640                     struct iscsi_cls_conn *cls_conn, int is_leading)
2641 {
2642         struct iscsi_session *session = cls_session->dd_data;
2643         struct iscsi_conn *conn = cls_conn->dd_data;
2644
2645         spin_lock_bh(&session->lock);
2646         if (is_leading)
2647                 session->leadconn = conn;
2648         spin_unlock_bh(&session->lock);
2649
2650         /*
2651          * Unblock xmitworker(), Login Phase will pass through.
2652          */
2653         clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
2654         clear_bit(ISCSI_SUSPEND_BIT, &conn->suspend_tx);
2655         return 0;
2656 }
2657 EXPORT_SYMBOL_GPL(iscsi_conn_bind);
2658
2659 static int iscsi_switch_str_param(char **param, char *new_val_buf)
2660 {
2661         char *new_val;
2662
2663         if (*param) {
2664                 if (!strcmp(*param, new_val_buf))
2665                         return 0;
2666         }
2667
2668         new_val = kstrdup(new_val_buf, GFP_NOIO);
2669         if (!new_val)
2670                 return -ENOMEM;
2671
2672         kfree(*param);
2673         *param = new_val;
2674         return 0;
2675 }
2676
2677 int iscsi_set_param(struct iscsi_cls_conn *cls_conn,
2678                     enum iscsi_param param, char *buf, int buflen)
2679 {
2680         struct iscsi_conn *conn = cls_conn->dd_data;
2681         struct iscsi_session *session = conn->session;
2682         uint32_t value;
2683
2684         switch(param) {
2685         case ISCSI_PARAM_FAST_ABORT:
2686                 sscanf(buf, "%d", &session->fast_abort);
2687                 break;
2688         case ISCSI_PARAM_ABORT_TMO:
2689                 sscanf(buf, "%d", &session->abort_timeout);
2690                 break;
2691         case ISCSI_PARAM_LU_RESET_TMO:
2692                 sscanf(buf, "%d", &session->lu_reset_timeout);
2693                 break;
2694         case ISCSI_PARAM_PING_TMO:
2695                 sscanf(buf, "%d", &conn->ping_timeout);
2696                 break;
2697         case ISCSI_PARAM_RECV_TMO:
2698                 sscanf(buf, "%d", &conn->recv_timeout);
2699                 break;
2700         case ISCSI_PARAM_MAX_RECV_DLENGTH:
2701                 sscanf(buf, "%d", &conn->max_recv_dlength);
2702                 break;
2703         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2704                 sscanf(buf, "%d", &conn->max_xmit_dlength);
2705                 break;
2706         case ISCSI_PARAM_HDRDGST_EN:
2707                 sscanf(buf, "%d", &conn->hdrdgst_en);
2708                 break;
2709         case ISCSI_PARAM_DATADGST_EN:
2710                 sscanf(buf, "%d", &conn->datadgst_en);
2711                 break;
2712         case ISCSI_PARAM_INITIAL_R2T_EN:
2713                 sscanf(buf, "%d", &session->initial_r2t_en);
2714                 break;
2715         case ISCSI_PARAM_MAX_R2T:
2716                 sscanf(buf, "%d", &session->max_r2t);
2717                 break;
2718         case ISCSI_PARAM_IMM_DATA_EN:
2719                 sscanf(buf, "%d", &session->imm_data_en);
2720                 break;
2721         case ISCSI_PARAM_FIRST_BURST:
2722                 sscanf(buf, "%d", &session->first_burst);
2723                 break;
2724         case ISCSI_PARAM_MAX_BURST:
2725                 sscanf(buf, "%d", &session->max_burst);
2726                 break;
2727         case ISCSI_PARAM_PDU_INORDER_EN:
2728                 sscanf(buf, "%d", &session->pdu_inorder_en);
2729                 break;
2730         case ISCSI_PARAM_DATASEQ_INORDER_EN:
2731                 sscanf(buf, "%d", &session->dataseq_inorder_en);
2732                 break;
2733         case ISCSI_PARAM_ERL:
2734                 sscanf(buf, "%d", &session->erl);
2735                 break;
2736         case ISCSI_PARAM_IFMARKER_EN:
2737                 sscanf(buf, "%d", &value);
2738                 BUG_ON(value);
2739                 break;
2740         case ISCSI_PARAM_OFMARKER_EN:
2741                 sscanf(buf, "%d", &value);
2742                 BUG_ON(value);
2743                 break;
2744         case ISCSI_PARAM_EXP_STATSN:
2745                 sscanf(buf, "%u", &conn->exp_statsn);
2746                 break;
2747         case ISCSI_PARAM_USERNAME:
2748                 return iscsi_switch_str_param(&session->username, buf);
2749         case ISCSI_PARAM_USERNAME_IN:
2750                 return iscsi_switch_str_param(&session->username_in, buf);
2751         case ISCSI_PARAM_PASSWORD:
2752                 return iscsi_switch_str_param(&session->password, buf);
2753         case ISCSI_PARAM_PASSWORD_IN:
2754                 return iscsi_switch_str_param(&session->password_in, buf);
2755         case ISCSI_PARAM_TARGET_NAME:
2756                 return iscsi_switch_str_param(&session->targetname, buf);
2757         case ISCSI_PARAM_TPGT:
2758                 sscanf(buf, "%d", &session->tpgt);
2759                 break;
2760         case ISCSI_PARAM_PERSISTENT_PORT:
2761                 sscanf(buf, "%d", &conn->persistent_port);
2762                 break;
2763         case ISCSI_PARAM_PERSISTENT_ADDRESS:
2764                 return iscsi_switch_str_param(&conn->persistent_address, buf);
2765         case ISCSI_PARAM_IFACE_NAME:
2766                 return iscsi_switch_str_param(&session->ifacename, buf);
2767         case ISCSI_PARAM_INITIATOR_NAME:
2768                 return iscsi_switch_str_param(&session->initiatorname, buf);
2769         default:
2770                 return -ENOSYS;
2771         }
2772
2773         return 0;
2774 }
2775 EXPORT_SYMBOL_GPL(iscsi_set_param);
2776
2777 int iscsi_session_get_param(struct iscsi_cls_session *cls_session,
2778                             enum iscsi_param param, char *buf)
2779 {
2780         struct iscsi_session *session = cls_session->dd_data;
2781         int len;
2782
2783         switch(param) {
2784         case ISCSI_PARAM_FAST_ABORT:
2785                 len = sprintf(buf, "%d\n", session->fast_abort);
2786                 break;
2787         case ISCSI_PARAM_ABORT_TMO:
2788                 len = sprintf(buf, "%d\n", session->abort_timeout);
2789                 break;
2790         case ISCSI_PARAM_LU_RESET_TMO:
2791                 len = sprintf(buf, "%d\n", session->lu_reset_timeout);
2792                 break;
2793         case ISCSI_PARAM_INITIAL_R2T_EN:
2794                 len = sprintf(buf, "%d\n", session->initial_r2t_en);
2795                 break;
2796         case ISCSI_PARAM_MAX_R2T:
2797                 len = sprintf(buf, "%hu\n", session->max_r2t);
2798                 break;
2799         case ISCSI_PARAM_IMM_DATA_EN:
2800                 len = sprintf(buf, "%d\n", session->imm_data_en);
2801                 break;
2802         case ISCSI_PARAM_FIRST_BURST:
2803                 len = sprintf(buf, "%u\n", session->first_burst);
2804                 break;
2805         case ISCSI_PARAM_MAX_BURST:
2806                 len = sprintf(buf, "%u\n", session->max_burst);
2807                 break;
2808         case ISCSI_PARAM_PDU_INORDER_EN:
2809                 len = sprintf(buf, "%d\n", session->pdu_inorder_en);
2810                 break;
2811         case ISCSI_PARAM_DATASEQ_INORDER_EN:
2812                 len = sprintf(buf, "%d\n", session->dataseq_inorder_en);
2813                 break;
2814         case ISCSI_PARAM_ERL:
2815                 len = sprintf(buf, "%d\n", session->erl);
2816                 break;
2817         case ISCSI_PARAM_TARGET_NAME:
2818                 len = sprintf(buf, "%s\n", session->targetname);
2819                 break;
2820         case ISCSI_PARAM_TPGT:
2821                 len = sprintf(buf, "%d\n", session->tpgt);
2822                 break;
2823         case ISCSI_PARAM_USERNAME:
2824                 len = sprintf(buf, "%s\n", session->username);
2825                 break;
2826         case ISCSI_PARAM_USERNAME_IN:
2827                 len = sprintf(buf, "%s\n", session->username_in);
2828                 break;
2829         case ISCSI_PARAM_PASSWORD:
2830                 len = sprintf(buf, "%s\n", session->password);
2831                 break;
2832         case ISCSI_PARAM_PASSWORD_IN:
2833                 len = sprintf(buf, "%s\n", session->password_in);
2834                 break;
2835         case ISCSI_PARAM_IFACE_NAME:
2836                 len = sprintf(buf, "%s\n", session->ifacename);
2837                 break;
2838         case ISCSI_PARAM_INITIATOR_NAME:
2839                 len = sprintf(buf, "%s\n", session->initiatorname);
2840                 break;
2841         default:
2842                 return -ENOSYS;
2843         }
2844
2845         return len;
2846 }
2847 EXPORT_SYMBOL_GPL(iscsi_session_get_param);
2848
2849 int iscsi_conn_get_param(struct iscsi_cls_conn *cls_conn,
2850                          enum iscsi_param param, char *buf)
2851 {
2852         struct iscsi_conn *conn = cls_conn->dd_data;
2853         int len;
2854
2855         switch(param) {
2856         case ISCSI_PARAM_PING_TMO:
2857                 len = sprintf(buf, "%u\n", conn->ping_timeout);
2858                 break;
2859         case ISCSI_PARAM_RECV_TMO:
2860                 len = sprintf(buf, "%u\n", conn->recv_timeout);
2861                 break;
2862         case ISCSI_PARAM_MAX_RECV_DLENGTH:
2863                 len = sprintf(buf, "%u\n", conn->max_recv_dlength);
2864                 break;
2865         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
2866                 len = sprintf(buf, "%u\n", conn->max_xmit_dlength);
2867                 break;
2868         case ISCSI_PARAM_HDRDGST_EN:
2869                 len = sprintf(buf, "%d\n", conn->hdrdgst_en);
2870                 break;
2871         case ISCSI_PARAM_DATADGST_EN:
2872                 len = sprintf(buf, "%d\n", conn->datadgst_en);
2873                 break;
2874         case ISCSI_PARAM_IFMARKER_EN:
2875                 len = sprintf(buf, "%d\n", conn->ifmarker_en);
2876                 break;
2877         case ISCSI_PARAM_OFMARKER_EN:
2878                 len = sprintf(buf, "%d\n", conn->ofmarker_en);
2879                 break;
2880         case ISCSI_PARAM_EXP_STATSN:
2881                 len = sprintf(buf, "%u\n", conn->exp_statsn);
2882                 break;
2883         case ISCSI_PARAM_PERSISTENT_PORT:
2884                 len = sprintf(buf, "%d\n", conn->persistent_port);
2885                 break;
2886         case ISCSI_PARAM_PERSISTENT_ADDRESS:
2887                 len = sprintf(buf, "%s\n", conn->persistent_address);
2888                 break;
2889         default:
2890                 return -ENOSYS;
2891         }
2892
2893         return len;
2894 }
2895 EXPORT_SYMBOL_GPL(iscsi_conn_get_param);
2896
2897 int iscsi_host_get_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2898                          char *buf)
2899 {
2900         struct iscsi_host *ihost = shost_priv(shost);
2901         int len;
2902
2903         switch (param) {
2904         case ISCSI_HOST_PARAM_NETDEV_NAME:
2905                 len = sprintf(buf, "%s\n", ihost->netdev);
2906                 break;
2907         case ISCSI_HOST_PARAM_HWADDRESS:
2908                 len = sprintf(buf, "%s\n", ihost->hwaddress);
2909                 break;
2910         case ISCSI_HOST_PARAM_INITIATOR_NAME:
2911                 len = sprintf(buf, "%s\n", ihost->initiatorname);
2912                 break;
2913         case ISCSI_HOST_PARAM_IPADDRESS:
2914                 len = sprintf(buf, "%s\n", ihost->local_address);
2915                 break;
2916         default:
2917                 return -ENOSYS;
2918         }
2919
2920         return len;
2921 }
2922 EXPORT_SYMBOL_GPL(iscsi_host_get_param);
2923
2924 int iscsi_host_set_param(struct Scsi_Host *shost, enum iscsi_host_param param,
2925                          char *buf, int buflen)
2926 {
2927         struct iscsi_host *ihost = shost_priv(shost);
2928
2929         switch (param) {
2930         case ISCSI_HOST_PARAM_NETDEV_NAME:
2931                 return iscsi_switch_str_param(&ihost->netdev, buf);
2932         case ISCSI_HOST_PARAM_HWADDRESS:
2933                 return iscsi_switch_str_param(&ihost->hwaddress, buf);
2934         case ISCSI_HOST_PARAM_INITIATOR_NAME:
2935                 return iscsi_switch_str_param(&ihost->initiatorname, buf);
2936         default:
2937                 return -ENOSYS;
2938         }
2939
2940         return 0;
2941 }
2942 EXPORT_SYMBOL_GPL(iscsi_host_set_param);
2943
2944 MODULE_AUTHOR("Mike Christie");
2945 MODULE_DESCRIPTION("iSCSI library functions");
2946 MODULE_LICENSE("GPL");