drivers/target/iscsi/iscsi_target.c

   1 /*******************************************************************************
   2  * This file contains main functions related to the iSCSI Target Core Driver.
   3  *
   4  * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   5  *
   6  * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   7  *
   8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published by
  12  * the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  ******************************************************************************/
  20
  21 #include <linux/string.h>
  22 #include <linux/kthread.h>
  23 #include <linux/crypto.h>
  24 #include <linux/completion.h>
  25 #include <linux/module.h>
  26 #include <asm/unaligned.h>
  27 #include <scsi/scsi_device.h>
  28 #include <scsi/iscsi_proto.h>
  29 #include <target/target_core_base.h>
  30 #include <target/target_core_tmr.h>
  31 #include <target/target_core_transport.h>
  32
  33 #include "iscsi_target_core.h"
  34 #include "iscsi_target_parameters.h"
  35 #include "iscsi_target_seq_pdu_list.h"
  36 #include "iscsi_target_tq.h"
  37 #include "iscsi_target_configfs.h"
  38 #include "iscsi_target_datain_values.h"
  39 #include "iscsi_target_erl0.h"
  40 #include "iscsi_target_erl1.h"
  41 #include "iscsi_target_erl2.h"
  42 #include "iscsi_target_login.h"
  43 #include "iscsi_target_tmr.h"
  44 #include "iscsi_target_tpg.h"
  45 #include "iscsi_target_util.h"
  46 #include "iscsi_target.h"
  47 #include "iscsi_target_device.h"
  48 #include "iscsi_target_stat.h"
  49
  50 static LIST_HEAD(g_tiqn_list);
  51 static LIST_HEAD(g_np_list);
  52 static DEFINE_SPINLOCK(tiqn_lock);
  53 static DEFINE_SPINLOCK(np_lock);
  54
  55 static struct idr tiqn_idr;
  56 struct idr sess_idr;
  57 struct mutex auth_id_lock;
  58 spinlock_t sess_idr_lock;
  59
  60 struct iscsit_global *iscsit_global;
  61
  62 struct kmem_cache *lio_cmd_cache;
  63 struct kmem_cache *lio_qr_cache;
  64 struct kmem_cache *lio_dr_cache;
  65 struct kmem_cache *lio_ooo_cache;
  66 struct kmem_cache *lio_r2t_cache;
  67
  68 static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  69                         unsigned char *buf, u32);
  70 static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *);
  71
  72 struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  73 {
  74         struct iscsi_tiqn *tiqn = NULL;
  75
  76         spin_lock(&tiqn_lock);
  77         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  78                 if (!strcmp(tiqn->tiqn, buf)) {
  79
  80                         spin_lock(&tiqn->tiqn_state_lock);
  81                         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  82                                 tiqn->tiqn_access_count++;
  83                                 spin_unlock(&tiqn->tiqn_state_lock);
  84                                 spin_unlock(&tiqn_lock);
  85                                 return tiqn;
  86                         }
  87                         spin_unlock(&tiqn->tiqn_state_lock);
  88                 }
  89         }
  90         spin_unlock(&tiqn_lock);
  91
  92         return NULL;
  93 }
  94
  95 static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  96 {
  97         spin_lock(&tiqn->tiqn_state_lock);
  98         if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  99                 tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
 100                 spin_unlock(&tiqn->tiqn_state_lock);
 101                 return 0;
 102         }
 103         spin_unlock(&tiqn->tiqn_state_lock);
 104
 105         return -1;
 106 }
 107
 108 void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
 109 {
 110         spin_lock(&tiqn->tiqn_state_lock);
 111         tiqn->tiqn_access_count--;
 112         spin_unlock(&tiqn->tiqn_state_lock);
 113 }
 114
 115 /*
 116  * Note that IQN formatting is expected to be done in userspace, and
 117  * no explict IQN format checks are done here.
 118  */
 119 struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
 120 {
 121         struct iscsi_tiqn *tiqn = NULL;
 122         int ret;
 123
 124         if (strlen(buf) >= ISCSI_IQN_LEN) {
 125                 pr_err("Target IQN exceeds %d bytes\n",
 126                                 ISCSI_IQN_LEN);
 127                 return ERR_PTR(-EINVAL);
 128         }
 129
 130         tiqn = kzalloc(sizeof(struct iscsi_tiqn), GFP_KERNEL);
 131         if (!tiqn) {
 132                 pr_err("Unable to allocate struct iscsi_tiqn\n");
 133                 return ERR_PTR(-ENOMEM);
 134         }
 135
 136         sprintf(tiqn->tiqn, "%s", buf);
 137         INIT_LIST_HEAD(&tiqn->tiqn_list);
 138         INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
 139         spin_lock_init(&tiqn->tiqn_state_lock);
 140         spin_lock_init(&tiqn->tiqn_tpg_lock);
 141         spin_lock_init(&tiqn->sess_err_stats.lock);
 142         spin_lock_init(&tiqn->login_stats.lock);
 143         spin_lock_init(&tiqn->logout_stats.lock);
 144
 145         if (!idr_pre_get(&tiqn_idr, GFP_KERNEL)) {
 146                 pr_err("idr_pre_get() for tiqn_idr failed\n");
 147                 kfree(tiqn);
 148                 return ERR_PTR(-ENOMEM);
 149         }
 150         tiqn->tiqn_state = TIQN_STATE_ACTIVE;
 151
 152         spin_lock(&tiqn_lock);
 153         ret = idr_get_new(&tiqn_idr, NULL, &tiqn->tiqn_index);
 154         if (ret < 0) {
 155                 pr_err("idr_get_new() failed for tiqn->tiqn_index\n");
 156                 spin_unlock(&tiqn_lock);
 157                 kfree(tiqn);
 158                 return ERR_PTR(ret);
 159         }
 160         list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
 161         spin_unlock(&tiqn_lock);
 162
 163         pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
 164
 165         return tiqn;
 166
 167 }
 168
 169 static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
 170 {
 171         /*
 172          * Wait for accesses to said struct iscsi_tiqn to end.
 173          */
 174         spin_lock(&tiqn->tiqn_state_lock);
 175         while (tiqn->tiqn_access_count != 0) {
 176                 spin_unlock(&tiqn->tiqn_state_lock);
 177                 msleep(10);
 178                 spin_lock(&tiqn->tiqn_state_lock);
 179         }
 180         spin_unlock(&tiqn->tiqn_state_lock);
 181 }
 182
 183 void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
 184 {
 185         /*
 186          * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
 187          * while holding tiqn->tiqn_state_lock.  This means that all subsequent
 188          * attempts to access this struct iscsi_tiqn will fail from both transport
 189          * fabric and control code paths.
 190          */
 191         if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
 192                 pr_err("iscsit_set_tiqn_shutdown() failed\n");
 193                 return;
 194         }
 195
 196         iscsit_wait_for_tiqn(tiqn);
 197
 198         spin_lock(&tiqn_lock);
 199         list_del(&tiqn->tiqn_list);
 200         idr_remove(&tiqn_idr, tiqn->tiqn_index);
 201         spin_unlock(&tiqn_lock);
 202
 203         pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
 204                         tiqn->tiqn);
 205         kfree(tiqn);
 206 }
 207
 208 int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 209 {
 210         int ret;
 211         /*
 212          * Determine if the network portal is accepting storage traffic.
 213          */
 214         spin_lock_bh(&np->np_thread_lock);
 215         if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 216                 spin_unlock_bh(&np->np_thread_lock);
 217                 return -1;
 218         }
 219         if (np->np_login_tpg) {
 220                 pr_err("np->np_login_tpg() is not NULL!\n");
 221                 spin_unlock_bh(&np->np_thread_lock);
 222                 return -1;
 223         }
 224         spin_unlock_bh(&np->np_thread_lock);
 225         /*
 226          * Determine if the portal group is accepting storage traffic.
 227          */
 228         spin_lock_bh(&tpg->tpg_state_lock);
 229         if (tpg->tpg_state != TPG_STATE_ACTIVE) {
 230                 spin_unlock_bh(&tpg->tpg_state_lock);
 231                 return -1;
 232         }
 233         spin_unlock_bh(&tpg->tpg_state_lock);
 234
 235         /*
 236          * Here we serialize access across the TIQN+TPG Tuple.
 237          */
 238         ret = mutex_lock_interruptible(&tpg->np_login_lock);
 239         if ((ret != 0) || signal_pending(current))
 240                 return -1;
 241
 242         spin_lock_bh(&np->np_thread_lock);
 243         np->np_login_tpg = tpg;
 244         spin_unlock_bh(&np->np_thread_lock);
 245
 246         return 0;
 247 }
 248
 249 int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
 250 {
 251         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 252
 253         spin_lock_bh(&np->np_thread_lock);
 254         np->np_login_tpg = NULL;
 255         spin_unlock_bh(&np->np_thread_lock);
 256
 257         mutex_unlock(&tpg->np_login_lock);
 258
 259         if (tiqn)
 260                 iscsit_put_tiqn_for_login(tiqn);
 261
 262         return 0;
 263 }
 264
 265 static struct iscsi_np *iscsit_get_np(
 266         struct __kernel_sockaddr_storage *sockaddr,
 267         int network_transport)
 268 {
 269         struct sockaddr_in *sock_in, *sock_in_e;
 270         struct sockaddr_in6 *sock_in6, *sock_in6_e;
 271         struct iscsi_np *np;
 272         int ip_match = 0;
 273         u16 port;
 274
 275         spin_lock_bh(&np_lock);
 276         list_for_each_entry(np, &g_np_list, np_list) {
 277                 spin_lock(&np->np_thread_lock);
 278                 if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
 279                         spin_unlock(&np->np_thread_lock);
 280                         continue;
 281                 }
 282
 283                 if (sockaddr->ss_family == AF_INET6) {
 284                         sock_in6 = (struct sockaddr_in6 *)sockaddr;
 285                         sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
 286
 287                         if (!memcmp((void *)&sock_in6->sin6_addr.in6_u,
 288                                     (void *)&sock_in6_e->sin6_addr.in6_u,
 289                                     sizeof(struct in6_addr)))
 290                                 ip_match = 1;
 291
 292                         port = ntohs(sock_in6->sin6_port);
 293                 } else {
 294                         sock_in = (struct sockaddr_in *)sockaddr;
 295                         sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
 296
 297                         if (sock_in->sin_addr.s_addr ==
 298                             sock_in_e->sin_addr.s_addr)
 299                                 ip_match = 1;
 300
 301                         port = ntohs(sock_in->sin_port);
 302                 }
 303
 304                 if ((ip_match == 1) && (np->np_port == port) &&
 305                     (np->np_network_transport == network_transport)) {
 306                         /*
 307                          * Increment the np_exports reference count now to
 308                          * prevent iscsit_del_np() below from being called
 309                          * while iscsi_tpg_add_network_portal() is called.
 310                          */
 311                         np->np_exports++;
 312                         spin_unlock(&np->np_thread_lock);
 313                         spin_unlock_bh(&np_lock);
 314                         return np;
 315                 }
 316                 spin_unlock(&np->np_thread_lock);
 317         }
 318         spin_unlock_bh(&np_lock);
 319
 320         return NULL;
 321 }
 322
 323 struct iscsi_np *iscsit_add_np(
 324         struct __kernel_sockaddr_storage *sockaddr,
 325         char *ip_str,
 326         int network_transport)
 327 {
 328         struct sockaddr_in *sock_in;
 329         struct sockaddr_in6 *sock_in6;
 330         struct iscsi_np *np;
 331         int ret;
 332         /*
 333          * Locate the existing struct iscsi_np if already active..
 334          */
 335         np = iscsit_get_np(sockaddr, network_transport);
 336         if (np)
 337                 return np;
 338
 339         np = kzalloc(sizeof(struct iscsi_np), GFP_KERNEL);
 340         if (!np) {
 341                 pr_err("Unable to allocate memory for struct iscsi_np\n");
 342                 return ERR_PTR(-ENOMEM);
 343         }
 344
 345         np->np_flags |= NPF_IP_NETWORK;
 346         if (sockaddr->ss_family == AF_INET6) {
 347                 sock_in6 = (struct sockaddr_in6 *)sockaddr;
 348                 snprintf(np->np_ip, IPV6_ADDRESS_SPACE, "%s", ip_str);
 349                 np->np_port = ntohs(sock_in6->sin6_port);
 350         } else {
 351                 sock_in = (struct sockaddr_in *)sockaddr;
 352                 sprintf(np->np_ip, "%s", ip_str);
 353                 np->np_port = ntohs(sock_in->sin_port);
 354         }
 355
 356         np->np_network_transport = network_transport;
 357         spin_lock_init(&np->np_thread_lock);
 358         init_completion(&np->np_restart_comp);
 359         INIT_LIST_HEAD(&np->np_list);
 360
 361         ret = iscsi_target_setup_login_socket(np, sockaddr);
 362         if (ret != 0) {
 363                 kfree(np);
 364                 return ERR_PTR(ret);
 365         }
 366
 367         np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
 368         if (IS_ERR(np->np_thread)) {
 369                 pr_err("Unable to create kthread: iscsi_np\n");
 370                 ret = PTR_ERR(np->np_thread);
 371                 kfree(np);
 372                 return ERR_PTR(ret);
 373         }
 374         /*
 375          * Increment the np_exports reference count now to prevent
 376          * iscsit_del_np() below from being run while a new call to
 377          * iscsi_tpg_add_network_portal() for a matching iscsi_np is
 378          * active.  We don't need to hold np->np_thread_lock at this
 379          * point because iscsi_np has not been added to g_np_list yet.
 380          */
 381         np->np_exports = 1;
 382
 383         spin_lock_bh(&np_lock);
 384         list_add_tail(&np->np_list, &g_np_list);
 385         spin_unlock_bh(&np_lock);
 386
 387         pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n",
 388                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 389                 "TCP" : "SCTP");
 390
 391         return np;
 392 }
 393
 394 int iscsit_reset_np_thread(
 395         struct iscsi_np *np,
 396         struct iscsi_tpg_np *tpg_np,
 397         struct iscsi_portal_group *tpg)
 398 {
 399         spin_lock_bh(&np->np_thread_lock);
 400         if (tpg && tpg_np) {
 401                 /*
 402                  * The reset operation need only be performed when the
 403                  * passed struct iscsi_portal_group has a login in progress
 404                  * to one of the network portals.
 405                  */
 406                 if (tpg_np->tpg_np->np_login_tpg != tpg) {
 407                         spin_unlock_bh(&np->np_thread_lock);
 408                         return 0;
 409                 }
 410         }
 411         if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
 412                 spin_unlock_bh(&np->np_thread_lock);
 413                 return 0;
 414         }
 415         np->np_thread_state = ISCSI_NP_THREAD_RESET;
 416
 417         if (np->np_thread) {
 418                 spin_unlock_bh(&np->np_thread_lock);
 419                 send_sig(SIGINT, np->np_thread, 1);
 420                 wait_for_completion(&np->np_restart_comp);
 421                 spin_lock_bh(&np->np_thread_lock);
 422         }
 423         spin_unlock_bh(&np->np_thread_lock);
 424
 425         return 0;
 426 }
 427
 428 int iscsit_del_np_comm(struct iscsi_np *np)
 429 {
 430         if (!np->np_socket)
 431                 return 0;
 432
 433         /*
 434          * Some network transports allocate their own struct sock->file,
 435          * see  if we need to free any additional allocated resources.
 436          */
 437         if (np->np_flags & NPF_SCTP_STRUCT_FILE) {
 438                 kfree(np->np_socket->file);
 439                 np->np_socket->file = NULL;
 440         }
 441
 442         sock_release(np->np_socket);
 443         return 0;
 444 }
 445
 446 int iscsit_del_np(struct iscsi_np *np)
 447 {
 448         spin_lock_bh(&np->np_thread_lock);
 449         np->np_exports--;
 450         if (np->np_exports) {
 451                 spin_unlock_bh(&np->np_thread_lock);
 452                 return 0;
 453         }
 454         np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
 455         spin_unlock_bh(&np->np_thread_lock);
 456
 457         if (np->np_thread) {
 458                 /*
 459                  * We need to send the signal to wakeup Linux/Net
 460                  * which may be sleeping in sock_accept()..
 461                  */
 462                 send_sig(SIGINT, np->np_thread, 1);
 463                 kthread_stop(np->np_thread);
 464         }
 465         iscsit_del_np_comm(np);
 466
 467         spin_lock_bh(&np_lock);
 468         list_del(&np->np_list);
 469         spin_unlock_bh(&np_lock);
 470
 471         pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n",
 472                 np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ?
 473                 "TCP" : "SCTP");
 474
 475         kfree(np);
 476         return 0;
 477 }
 478
 479 static int __init iscsi_target_init_module(void)
 480 {
 481         int ret = 0;
 482
 483         pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
 484
 485         iscsit_global = kzalloc(sizeof(struct iscsit_global), GFP_KERNEL);
 486         if (!iscsit_global) {
 487                 pr_err("Unable to allocate memory for iscsit_global\n");
 488                 return -1;
 489         }
 490         mutex_init(&auth_id_lock);
 491         spin_lock_init(&sess_idr_lock);
 492         idr_init(&tiqn_idr);
 493         idr_init(&sess_idr);
 494
 495         ret = iscsi_target_register_configfs();
 496         if (ret < 0)
 497                 goto out;
 498
 499         ret = iscsi_thread_set_init();
 500         if (ret < 0)
 501                 goto configfs_out;
 502
 503         if (iscsi_allocate_thread_sets(TARGET_THREAD_SET_COUNT) !=
 504                         TARGET_THREAD_SET_COUNT) {
 505                 pr_err("iscsi_allocate_thread_sets() returned"
 506                         " unexpected value!\n");
 507                 goto ts_out1;
 508         }
 509
 510         lio_cmd_cache = kmem_cache_create("lio_cmd_cache",
 511                         sizeof(struct iscsi_cmd), __alignof__(struct iscsi_cmd),
 512                         0, NULL);
 513         if (!lio_cmd_cache) {
 514                 pr_err("Unable to kmem_cache_create() for"
 515                                 " lio_cmd_cache\n");
 516                 goto ts_out2;
 517         }
 518
 519         lio_qr_cache = kmem_cache_create("lio_qr_cache",
 520                         sizeof(struct iscsi_queue_req),
 521                         __alignof__(struct iscsi_queue_req), 0, NULL);
 522         if (!lio_qr_cache) {
 523                 pr_err("nable to kmem_cache_create() for"
 524                                 " lio_qr_cache\n");
 525                 goto cmd_out;
 526         }
 527
 528         lio_dr_cache = kmem_cache_create("lio_dr_cache",
 529                         sizeof(struct iscsi_datain_req),
 530                         __alignof__(struct iscsi_datain_req), 0, NULL);
 531         if (!lio_dr_cache) {
 532                 pr_err("Unable to kmem_cache_create() for"
 533                                 " lio_dr_cache\n");
 534                 goto qr_out;
 535         }
 536
 537         lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
 538                         sizeof(struct iscsi_ooo_cmdsn),
 539                         __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
 540         if (!lio_ooo_cache) {
 541                 pr_err("Unable to kmem_cache_create() for"
 542                                 " lio_ooo_cache\n");
 543                 goto dr_out;
 544         }
 545
 546         lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
 547                         sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
 548                         0, NULL);
 549         if (!lio_r2t_cache) {
 550                 pr_err("Unable to kmem_cache_create() for"
 551                                 " lio_r2t_cache\n");
 552                 goto ooo_out;
 553         }
 554
 555         if (iscsit_load_discovery_tpg() < 0)
 556                 goto r2t_out;
 557
 558         return ret;
 559 r2t_out:
 560         kmem_cache_destroy(lio_r2t_cache);
 561 ooo_out:
 562         kmem_cache_destroy(lio_ooo_cache);
 563 dr_out:
 564         kmem_cache_destroy(lio_dr_cache);
 565 qr_out:
 566         kmem_cache_destroy(lio_qr_cache);
 567 cmd_out:
 568         kmem_cache_destroy(lio_cmd_cache);
 569 ts_out2:
 570         iscsi_deallocate_thread_sets();
 571 ts_out1:
 572         iscsi_thread_set_free();
 573 configfs_out:
 574         iscsi_target_deregister_configfs();
 575 out:
 576         kfree(iscsit_global);
 577         return -ENOMEM;
 578 }
 579
 580 static void __exit iscsi_target_cleanup_module(void)
 581 {
 582         iscsi_deallocate_thread_sets();
 583         iscsi_thread_set_free();
 584         iscsit_release_discovery_tpg();
 585         kmem_cache_destroy(lio_cmd_cache);
 586         kmem_cache_destroy(lio_qr_cache);
 587         kmem_cache_destroy(lio_dr_cache);
 588         kmem_cache_destroy(lio_ooo_cache);
 589         kmem_cache_destroy(lio_r2t_cache);
 590
 591         iscsi_target_deregister_configfs();
 592
 593         kfree(iscsit_global);
 594 }
 595
 596 int iscsit_add_reject(
 597         u8 reason,
 598         int fail_conn,
 599         unsigned char *buf,
 600         struct iscsi_conn *conn)
 601 {
 602         struct iscsi_cmd *cmd;
 603         struct iscsi_reject *hdr;
 604         int ret;
 605
 606         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
 607         if (!cmd)
 608                 return -1;
 609
 610         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 611         if (fail_conn)
 612                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 613
 614         hdr     = (struct iscsi_reject *) cmd->pdu;
 615         hdr->reason = reason;
 616
 617         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 618         if (!cmd->buf_ptr) {
 619                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 620                 iscsit_release_cmd(cmd);
 621                 return -1;
 622         }
 623
 624         spin_lock_bh(&conn->cmd_lock);
 625         list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
 626         spin_unlock_bh(&conn->cmd_lock);
 627
 628         cmd->i_state = ISTATE_SEND_REJECT;
 629         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 630
 631         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 632         if (ret != 0)
 633                 return -1;
 634
 635         return (!fail_conn) ? 0 : -1;
 636 }
 637
 638 int iscsit_add_reject_from_cmd(
 639         u8 reason,
 640         int fail_conn,
 641         int add_to_conn,
 642         unsigned char *buf,
 643         struct iscsi_cmd *cmd)
 644 {
 645         struct iscsi_conn *conn;
 646         struct iscsi_reject *hdr;
 647         int ret;
 648
 649         if (!cmd->conn) {
 650                 pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
 651                                 cmd->init_task_tag);
 652                 return -1;
 653         }
 654         conn = cmd->conn;
 655
 656         cmd->iscsi_opcode = ISCSI_OP_REJECT;
 657         if (fail_conn)
 658                 cmd->cmd_flags |= ICF_REJECT_FAIL_CONN;
 659
 660         hdr     = (struct iscsi_reject *) cmd->pdu;
 661         hdr->reason = reason;
 662
 663         cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
 664         if (!cmd->buf_ptr) {
 665                 pr_err("Unable to allocate memory for cmd->buf_ptr\n");
 666                 iscsit_release_cmd(cmd);
 667                 return -1;
 668         }
 669
 670         if (add_to_conn) {
 671                 spin_lock_bh(&conn->cmd_lock);
 672                 list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
 673                 spin_unlock_bh(&conn->cmd_lock);
 674         }
 675
 676         cmd->i_state = ISTATE_SEND_REJECT;
 677         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
 678
 679         ret = wait_for_completion_interruptible(&cmd->reject_comp);
 680         if (ret != 0)
 681                 return -1;
 682
 683         return (!fail_conn) ? 0 : -1;
 684 }
 685
 686 /*
 687  * Map some portion of the allocated scatterlist to an iovec, suitable for
 688  * kernel sockets to copy data in/out. This handles both pages and slab-allocated
 689  * buffers, since we have been tricky and mapped t_mem_sg to the buffer in
 690  * either case (see iscsit_alloc_buffs)
 691  */
 692 static int iscsit_map_iovec(
 693         struct iscsi_cmd *cmd,
 694         struct kvec *iov,
 695         u32 data_offset,
 696         u32 data_length)
 697 {
 698         u32 i = 0;
 699         struct scatterlist *sg;
 700         unsigned int page_off;
 701
 702         /*
 703          * We have a private mapping of the allocated pages in t_mem_sg.
 704          * At this point, we also know each contains a page.
 705          */
 706         sg = &cmd->t_mem_sg[data_offset / PAGE_SIZE];
 707         page_off = (data_offset % PAGE_SIZE);
 708
 709         cmd->first_data_sg = sg;
 710         cmd->first_data_sg_off = page_off;
 711
 712         while (data_length) {
 713                 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
 714
 715                 iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
 716                 iov[i].iov_len = cur_len;
 717
 718                 data_length -= cur_len;
 719                 page_off = 0;
 720                 sg = sg_next(sg);
 721                 i++;
 722         }
 723
 724         cmd->kmapped_nents = i;
 725
 726         return i;
 727 }
 728
 729 static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
 730 {
 731         u32 i;
 732         struct scatterlist *sg;
 733
 734         sg = cmd->first_data_sg;
 735
 736         for (i = 0; i < cmd->kmapped_nents; i++)
 737                 kunmap(sg_page(&sg[i]));
 738 }
 739
 740 static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
 741 {
 742         struct iscsi_cmd *cmd;
 743
 744         conn->exp_statsn = exp_statsn;
 745
 746         spin_lock_bh(&conn->cmd_lock);
 747         list_for_each_entry(cmd, &conn->conn_cmd_list, i_list) {
 748                 spin_lock(&cmd->istate_lock);
 749                 if ((cmd->i_state == ISTATE_SENT_STATUS) &&
 750                     (cmd->stat_sn < exp_statsn)) {
 751                         cmd->i_state = ISTATE_REMOVE;
 752                         spin_unlock(&cmd->istate_lock);
 753                         iscsit_add_cmd_to_immediate_queue(cmd, conn,
 754                                                 cmd->i_state);
 755                         continue;
 756                 }
 757                 spin_unlock(&cmd->istate_lock);
 758         }
 759         spin_unlock_bh(&conn->cmd_lock);
 760 }
 761
 762 static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
 763 {
 764         u32 iov_count = (cmd->se_cmd.t_data_nents == 0) ? 1 :
 765                                 cmd->se_cmd.t_data_nents;
 766
 767         iov_count += ISCSI_IOV_DATA_BUFFER;
 768
 769         cmd->iov_data = kzalloc(iov_count * sizeof(struct kvec), GFP_KERNEL);
 770         if (!cmd->iov_data) {
 771                 pr_err("Unable to allocate cmd->iov_data\n");
 772                 return -ENOMEM;
 773         }
 774
 775         cmd->orig_iov_data_count = iov_count;
 776         return 0;
 777 }
 778
 779 static int iscsit_alloc_buffs(struct iscsi_cmd *cmd)
 780 {
 781         struct scatterlist *sgl;
 782         u32 length = cmd->se_cmd.data_length;
 783         int nents = DIV_ROUND_UP(length, PAGE_SIZE);
 784         int i = 0, ret;
 785         /*
 786          * If no SCSI payload is present, allocate the default iovecs used for
 787          * iSCSI PDU Header
 788          */
 789         if (!length)
 790                 return iscsit_allocate_iovecs(cmd);
 791
 792         sgl = kzalloc(sizeof(*sgl) * nents, GFP_KERNEL);
 793         if (!sgl)
 794                 return -ENOMEM;
 795
 796         sg_init_table(sgl, nents);
 797
 798         while (length) {
 799                 int buf_size = min_t(int, length, PAGE_SIZE);
 800                 struct page *page;
 801
 802                 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 803                 if (!page)
 804                         goto page_alloc_failed;
 805
 806                 sg_set_page(&sgl[i], page, buf_size, 0);
 807
 808                 length -= buf_size;
 809                 i++;
 810         }
 811
 812         cmd->t_mem_sg = sgl;
 813         cmd->t_mem_sg_nents = nents;
 814
 815         /* BIDI ops not supported */
 816
 817         /* Tell the core about our preallocated memory */
 818         transport_generic_map_mem_to_cmd(&cmd->se_cmd, sgl, nents, NULL, 0);
 819         /*
 820          * Allocate iovecs for SCSI payload after transport_generic_map_mem_to_cmd
 821          * so that cmd->se_cmd.t_tasks_se_num has been set.
 822          */
 823         ret = iscsit_allocate_iovecs(cmd);
 824         if (ret < 0)
 825                 goto page_alloc_failed;
 826
 827         return 0;
 828
 829 page_alloc_failed:
 830         while (i >= 0) {
 831                 __free_page(sg_page(&sgl[i]));
 832                 i--;
 833         }
 834         kfree(cmd->t_mem_sg);
 835         cmd->t_mem_sg = NULL;
 836         return -ENOMEM;
 837 }
 838
 839 static int iscsit_handle_scsi_cmd(
 840         struct iscsi_conn *conn,
 841         unsigned char *buf)
 842 {
 843         int     data_direction, cmdsn_ret = 0, immed_ret, ret, transport_ret;
 844         int     dump_immediate_data = 0, send_check_condition = 0, payload_length;
 845         struct iscsi_cmd        *cmd = NULL;
 846         struct iscsi_scsi_req *hdr;
 847
 848         spin_lock_bh(&conn->sess->session_stats_lock);
 849         conn->sess->cmd_pdus++;
 850         if (conn->sess->se_sess->se_node_acl) {
 851                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
 852                 conn->sess->se_sess->se_node_acl->num_cmds++;
 853                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
 854         }
 855         spin_unlock_bh(&conn->sess->session_stats_lock);
 856
 857         hdr                     = (struct iscsi_scsi_req *) buf;
 858         payload_length          = ntoh24(hdr->dlength);
 859         hdr->itt                = be32_to_cpu(hdr->itt);
 860         hdr->data_length        = be32_to_cpu(hdr->data_length);
 861         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
 862         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
 863
 864         /* FIXME; Add checks for AdditionalHeaderSegment */
 865
 866         if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
 867             !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
 868                 pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
 869                                 " not set. Bad iSCSI Initiator.\n");
 870                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 871                                 buf, conn);
 872         }
 873
 874         if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
 875              (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
 876                 /*
 877                  * Vmware ESX v3.0 uses a modified Cisco Initiator (v3.4.2)
 878                  * that adds support for RESERVE/RELEASE.  There is a bug
 879                  * add with this new functionality that sets R/W bits when
 880                  * neither CDB carries any READ or WRITE datapayloads.
 881                  */
 882                 if ((hdr->cdb[0] == 0x16) || (hdr->cdb[0] == 0x17)) {
 883                         hdr->flags &= ~ISCSI_FLAG_CMD_READ;
 884                         hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
 885                         goto done;
 886                 }
 887
 888                 pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
 889                         " set when Expected Data Transfer Length is 0 for"
 890                         " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]);
 891                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 892                                 buf, conn);
 893         }
 894 done:
 895
 896         if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
 897             !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
 898                 pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
 899                         " MUST be set if Expected Data Transfer Length is not 0."
 900                         " Bad iSCSI Initiator\n");
 901                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 902                                 buf, conn);
 903         }
 904
 905         if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
 906             (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
 907                 pr_err("Bidirectional operations not supported!\n");
 908                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 909                                 buf, conn);
 910         }
 911
 912         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
 913                 pr_err("Illegally set Immediate Bit in iSCSI Initiator"
 914                                 " Scsi Command PDU.\n");
 915                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 916                                 buf, conn);
 917         }
 918
 919         if (payload_length && !conn->sess->sess_ops->ImmediateData) {
 920                 pr_err("ImmediateData=No but DataSegmentLength=%u,"
 921                         " protocol error.\n", payload_length);
 922                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 923                                 buf, conn);
 924         }
 925
 926         if ((hdr->data_length == payload_length) &&
 927             (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
 928                 pr_err("Expected Data Transfer Length and Length of"
 929                         " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
 930                         " bit is not set protocol error\n");
 931                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 932                                 buf, conn);
 933         }
 934
 935         if (payload_length > hdr->data_length) {
 936                 pr_err("DataSegmentLength: %u is greater than"
 937                         " EDTL: %u, protocol error.\n", payload_length,
 938                                 hdr->data_length);
 939                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 940                                 buf, conn);
 941         }
 942
 943         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
 944                 pr_err("DataSegmentLength: %u is greater than"
 945                         " MaxRecvDataSegmentLength: %u, protocol error.\n",
 946                         payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
 947                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
 948                                 buf, conn);
 949         }
 950
 951         if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
 952                 pr_err("DataSegmentLength: %u is greater than"
 953                         " FirstBurstLength: %u, protocol error.\n",
 954                         payload_length, conn->sess->sess_ops->FirstBurstLength);
 955                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1,
 956                                         buf, conn);
 957         }
 958
 959         data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
 960                          (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
 961                           DMA_NONE;
 962
 963         cmd = iscsit_allocate_se_cmd(conn, hdr->data_length, data_direction,
 964                                 (hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK));
 965         if (!cmd)
 966                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
 967                                         buf, conn);
 968
 969         pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
 970                 " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
 971                 hdr->cmdsn, hdr->data_length, payload_length, conn->cid);
 972
 973         cmd->iscsi_opcode       = ISCSI_OP_SCSI_CMD;
 974         cmd->i_state            = ISTATE_NEW_CMD;
 975         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
 976         cmd->immediate_data     = (payload_length) ? 1 : 0;
 977         cmd->unsolicited_data   = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
 978                                      (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
 979         if (cmd->unsolicited_data)
 980                 cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
 981
 982         conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
 983         if (hdr->flags & ISCSI_FLAG_CMD_READ) {
 984                 spin_lock_bh(&conn->sess->ttt_lock);
 985                 cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 986                 if (cmd->targ_xfer_tag == 0xFFFFFFFF)
 987                         cmd->targ_xfer_tag = conn->sess->targ_xfer_tag++;
 988                 spin_unlock_bh(&conn->sess->ttt_lock);
 989         } else if (hdr->flags & ISCSI_FLAG_CMD_WRITE)
 990                 cmd->targ_xfer_tag = 0xFFFFFFFF;
 991         cmd->cmd_sn             = hdr->cmdsn;
 992         cmd->exp_stat_sn        = hdr->exp_statsn;
 993         cmd->first_burst_len    = payload_length;
 994
 995         if (cmd->data_direction == DMA_FROM_DEVICE) {
 996                 struct iscsi_datain_req *dr;
 997
 998                 dr = iscsit_allocate_datain_req();
 999                 if (!dr)
1000                         return iscsit_add_reject_from_cmd(
1001                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1002                                         1, 1, buf, cmd);
1003
1004                 iscsit_attach_datain_req(cmd, dr);
1005         }
1006
1007         /*
1008          * The CDB is going to an se_device_t.
1009          */
1010         ret = iscsit_get_lun_for_cmd(cmd, hdr->cdb,
1011                                 get_unaligned_le64(&hdr->lun));
1012         if (ret < 0) {
1013                 if (cmd->se_cmd.scsi_sense_reason == TCM_NON_EXISTENT_LUN) {
1014                         pr_debug("Responding to non-acl'ed,"
1015                                 " non-existent or non-exported iSCSI LUN:"
1016                                 " 0x%016Lx\n", get_unaligned_le64(&hdr->lun));
1017                 }
1018                 send_check_condition = 1;
1019                 goto attach_cmd;
1020         }
1021         /*
1022          * The Initiator Node has access to the LUN (the addressing method
1023          * is handled inside of iscsit_get_lun_for_cmd()).  Now it's time to
1024          * allocate 1->N transport tasks (depending on sector count and
1025          * maximum request size the physical HBA(s) can handle.
1026          */
1027         transport_ret = transport_generic_allocate_tasks(&cmd->se_cmd, hdr->cdb);
1028         if (transport_ret == -ENOMEM) {
1029                 return iscsit_add_reject_from_cmd(
1030                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1031                                 1, 1, buf, cmd);
1032         } else if (transport_ret == -EINVAL) {
1033                 /*
1034                  * Unsupported SAM Opcode.  CHECK_CONDITION will be sent
1035                  * in iscsit_execute_cmd() during the CmdSN OOO Execution
1036                  * Mechinism.
1037                  */
1038                 send_check_condition = 1;
1039         } else {
1040                 cmd->data_length = cmd->se_cmd.data_length;
1041
1042                 if (iscsit_decide_list_to_build(cmd, payload_length) < 0)
1043                         return iscsit_add_reject_from_cmd(
1044                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1045                                 1, 1, buf, cmd);
1046         }
1047
1048 attach_cmd:
1049         spin_lock_bh(&conn->cmd_lock);
1050         list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
1051         spin_unlock_bh(&conn->cmd_lock);
1052         /*
1053          * Check if we need to delay processing because of ALUA
1054          * Active/NonOptimized primary access state..
1055          */
1056         core_alua_check_nonop_delay(&cmd->se_cmd);
1057         /*
1058          * Allocate and setup SGL used with transport_generic_map_mem_to_cmd().
1059          * also call iscsit_allocate_iovecs()
1060          */
1061         ret = iscsit_alloc_buffs(cmd);
1062         if (ret < 0)
1063                 return iscsit_add_reject_from_cmd(
1064                                 ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1065                                 1, 1, buf, cmd);
1066         /*
1067          * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
1068          * the Immediate Bit is not set, and no Immediate
1069          * Data is attached.
1070          *
1071          * A PDU/CmdSN carrying Immediate Data can only
1072          * be processed after the DataCRC has passed.
1073          * If the DataCRC fails, the CmdSN MUST NOT
1074          * be acknowledged. (See below)
1075          */
1076         if (!cmd->immediate_data) {
1077                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1078                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1079                         return 0;
1080                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1081                         return iscsit_add_reject_from_cmd(
1082                                 ISCSI_REASON_PROTOCOL_ERROR,
1083                                 1, 0, buf, cmd);
1084         }
1085
1086         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1087
1088         /*
1089          * If no Immediate Data is attached, it's OK to return now.
1090          */
1091         if (!cmd->immediate_data) {
1092                 if (send_check_condition)
1093                         return 0;
1094
1095                 if (cmd->unsolicited_data) {
1096                         iscsit_set_dataout_sequence_values(cmd);
1097
1098                         spin_lock_bh(&cmd->dataout_timeout_lock);
1099                         iscsit_start_dataout_timer(cmd, cmd->conn);
1100                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1101                 }
1102
1103                 return 0;
1104         }
1105
1106         /*
1107          * Early CHECK_CONDITIONs never make it to the transport processing
1108          * thread.  They are processed in CmdSN order by
1109          * iscsit_check_received_cmdsn() below.
1110          */
1111         if (send_check_condition) {
1112                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1113                 dump_immediate_data = 1;
1114                 goto after_immediate_data;
1115         }
1116         /*
1117          * Call directly into transport_generic_new_cmd() to perform
1118          * the backend memory allocation.
1119          */
1120         ret = transport_generic_new_cmd(&cmd->se_cmd);
1121         if (ret < 0) {
1122                 immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
1123                 dump_immediate_data = 1;
1124                 goto after_immediate_data;
1125         }
1126
1127         immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length);
1128 after_immediate_data:
1129         if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
1130                 /*
1131                  * A PDU/CmdSN carrying Immediate Data passed
1132                  * DataCRC, check against ExpCmdSN/MaxCmdSN if
1133                  * Immediate Bit is not set.
1134                  */
1135                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1136                 /*
1137                  * Special case for Unsupported SAM WRITE Opcodes
1138                  * and ImmediateData=Yes.
1139                  */
1140                 if (dump_immediate_data) {
1141                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
1142                                 return -1;
1143                 } else if (cmd->unsolicited_data) {
1144                         iscsit_set_dataout_sequence_values(cmd);
1145
1146                         spin_lock_bh(&cmd->dataout_timeout_lock);
1147                         iscsit_start_dataout_timer(cmd, cmd->conn);
1148                         spin_unlock_bh(&cmd->dataout_timeout_lock);
1149                 }
1150
1151                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1152                         return iscsit_add_reject_from_cmd(
1153                                 ISCSI_REASON_PROTOCOL_ERROR,
1154                                 1, 0, buf, cmd);
1155
1156         } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
1157                 /*
1158                  * Immediate Data failed DataCRC and ERL>=1,
1159                  * silently drop this PDU and let the initiator
1160                  * plug the CmdSN gap.
1161                  *
1162                  * FIXME: Send Unsolicited NOPIN with reserved
1163                  * TTT here to help the initiator figure out
1164                  * the missing CmdSN, although they should be
1165                  * intelligent enough to determine the missing
1166                  * CmdSN and issue a retry to plug the sequence.
1167                  */
1168                 cmd->i_state = ISTATE_REMOVE;
1169                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1170         } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
1171                 return -1;
1172
1173         return 0;
1174 }
1175
1176 static u32 iscsit_do_crypto_hash_sg(
1177         struct hash_desc *hash,
1178         struct iscsi_cmd *cmd,
1179         u32 data_offset,
1180         u32 data_length,
1181         u32 padding,
1182         u8 *pad_bytes)
1183 {
1184         u32 data_crc;
1185         u32 i;
1186         struct scatterlist *sg;
1187         unsigned int page_off;
1188
1189         crypto_hash_init(hash);
1190
1191         sg = cmd->first_data_sg;
1192         page_off = cmd->first_data_sg_off;
1193
1194         i = 0;
1195         while (data_length) {
1196                 u32 cur_len = min_t(u32, data_length, (sg[i].length - page_off));
1197
1198                 crypto_hash_update(hash, &sg[i], cur_len);
1199
1200                 data_length -= cur_len;
1201                 page_off = 0;
1202                 i++;
1203         }
1204
1205         if (padding) {
1206                 struct scatterlist pad_sg;
1207
1208                 sg_init_one(&pad_sg, pad_bytes, padding);
1209                 crypto_hash_update(hash, &pad_sg, padding);
1210         }
1211         crypto_hash_final(hash, (u8 *) &data_crc);
1212
1213         return data_crc;
1214 }
1215
1216 static void iscsit_do_crypto_hash_buf(
1217         struct hash_desc *hash,
1218         unsigned char *buf,
1219         u32 payload_length,
1220         u32 padding,
1221         u8 *pad_bytes,
1222         u8 *data_crc)
1223 {
1224         struct scatterlist sg;
1225
1226         crypto_hash_init(hash);
1227
1228         sg_init_one(&sg, (u8 *)buf, payload_length);
1229         crypto_hash_update(hash, &sg, payload_length);
1230
1231         if (padding) {
1232                 sg_init_one(&sg, pad_bytes, padding);
1233                 crypto_hash_update(hash, &sg, padding);
1234         }
1235         crypto_hash_final(hash, data_crc);
1236 }
1237
1238 static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
1239 {
1240         int iov_ret, ooo_cmdsn = 0, ret;
1241         u8 data_crc_failed = 0;
1242         u32 checksum, iov_count = 0, padding = 0, rx_got = 0;
1243         u32 rx_size = 0, payload_length;
1244         struct iscsi_cmd *cmd = NULL;
1245         struct se_cmd *se_cmd;
1246         struct iscsi_data *hdr;
1247         struct kvec *iov;
1248         unsigned long flags;
1249
1250         hdr                     = (struct iscsi_data *) buf;
1251         payload_length          = ntoh24(hdr->dlength);
1252         hdr->itt                = be32_to_cpu(hdr->itt);
1253         hdr->ttt                = be32_to_cpu(hdr->ttt);
1254         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1255         hdr->datasn             = be32_to_cpu(hdr->datasn);
1256         hdr->offset             = be32_to_cpu(hdr->offset);
1257
1258         if (!payload_length) {
1259                 pr_err("DataOUT payload is ZERO, protocol error.\n");
1260                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1261                                         buf, conn);
1262         }
1263
1264         /* iSCSI write */
1265         spin_lock_bh(&conn->sess->session_stats_lock);
1266         conn->sess->rx_data_octets += payload_length;
1267         if (conn->sess->se_sess->se_node_acl) {
1268                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
1269                 conn->sess->se_sess->se_node_acl->write_bytes += payload_length;
1270                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
1271         }
1272         spin_unlock_bh(&conn->sess->session_stats_lock);
1273
1274         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1275                 pr_err("DataSegmentLength: %u is greater than"
1276                         " MaxRecvDataSegmentLength: %u\n", payload_length,
1277                         conn->conn_ops->MaxRecvDataSegmentLength);
1278                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1279                                         buf, conn);
1280         }
1281
1282         cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt,
1283                         payload_length);
1284         if (!cmd)
1285                 return 0;
1286
1287         pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
1288                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
1289                 hdr->itt, hdr->ttt, hdr->datasn, hdr->offset,
1290                 payload_length, conn->cid);
1291
1292         if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
1293                 pr_err("Command ITT: 0x%08x received DataOUT after"
1294                         " last DataOUT received, dumping payload\n",
1295                         cmd->init_task_tag);
1296                 return iscsit_dump_data_payload(conn, payload_length, 1);
1297         }
1298
1299         if (cmd->data_direction != DMA_TO_DEVICE) {
1300                 pr_err("Command ITT: 0x%08x received DataOUT for a"
1301                         " NON-WRITE command.\n", cmd->init_task_tag);
1302                 return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR,
1303                                 1, 0, buf, cmd);
1304         }
1305         se_cmd = &cmd->se_cmd;
1306         iscsit_mod_dataout_timer(cmd);
1307
1308         if ((hdr->offset + payload_length) > cmd->data_length) {
1309                 pr_err("DataOut Offset: %u, Length %u greater than"
1310                         " iSCSI Command EDTL %u, protocol error.\n",
1311                         hdr->offset, payload_length, cmd->data_length);
1312                 return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID,
1313                                 1, 0, buf, cmd);
1314         }
1315
1316         if (cmd->unsolicited_data) {
1317                 int dump_unsolicited_data = 0;
1318
1319                 if (conn->sess->sess_ops->InitialR2T) {
1320                         pr_err("Received unexpected unsolicited data"
1321                                 " while InitialR2T=Yes, protocol error.\n");
1322                         transport_send_check_condition_and_sense(&cmd->se_cmd,
1323                                         TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
1324                         return -1;
1325                 }
1326                 /*
1327                  * Special case for dealing with Unsolicited DataOUT
1328                  * and Unsupported SAM WRITE Opcodes and SE resource allocation
1329                  * failures;
1330                  */
1331
1332                 /* Something's amiss if we're not in WRITE_PENDING state... */
1333                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1334                 WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
1335                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1336
1337                 spin_lock_irqsave(&se_cmd->t_state_lock, flags);
1338                 if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) ||
1339                      (se_cmd->se_cmd_flags & SCF_SCSI_CDB_EXCEPTION))
1340                         dump_unsolicited_data = 1;
1341                 spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
1342
1343                 if (dump_unsolicited_data) {
1344                         /*
1345                          * Check if a delayed TASK_ABORTED status needs to
1346                          * be sent now if the ISCSI_FLAG_CMD_FINAL has been
1347                          * received with the unsolicitied data out.
1348                          */
1349                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1350                                 iscsit_stop_dataout_timer(cmd);
1351
1352                         transport_check_aborted_status(se_cmd,
1353                                         (hdr->flags & ISCSI_FLAG_CMD_FINAL));
1354                         return iscsit_dump_data_payload(conn, payload_length, 1);
1355                 }
1356         } else {
1357                 /*
1358                  * For the normal solicited data path:
1359                  *
1360                  * Check for a delayed TASK_ABORTED status and dump any
1361                  * incoming data out payload if one exists.  Also, when the
1362                  * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
1363                  * data out sequence, we decrement outstanding_r2ts.  Once
1364                  * outstanding_r2ts reaches zero, go ahead and send the delayed
1365                  * TASK_ABORTED status.
1366                  */
1367                 if (atomic_read(&se_cmd->t_transport_aborted) != 0) {
1368                         if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
1369                                 if (--cmd->outstanding_r2ts < 1) {
1370                                         iscsit_stop_dataout_timer(cmd);
1371                                         transport_check_aborted_status(
1372                                                         se_cmd, 1);
1373                                 }
1374
1375                         return iscsit_dump_data_payload(conn, payload_length, 1);
1376                 }
1377         }
1378         /*
1379          * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and
1380          * within-command recovery checks before receiving the payload.
1381          */
1382         ret = iscsit_check_pre_dataout(cmd, buf);
1383         if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)
1384                 return 0;
1385         else if (ret == DATAOUT_CANNOT_RECOVER)
1386                 return -1;
1387
1388         rx_size += payload_length;
1389         iov = &cmd->iov_data[0];
1390
1391         iov_ret = iscsit_map_iovec(cmd, iov, hdr->offset, payload_length);
1392         if (iov_ret < 0)
1393                 return -1;
1394
1395         iov_count += iov_ret;
1396
1397         padding = ((-payload_length) & 3);
1398         if (padding != 0) {
1399                 iov[iov_count].iov_base = cmd->pad_bytes;
1400                 iov[iov_count++].iov_len = padding;
1401                 rx_size += padding;
1402                 pr_debug("Receiving %u padding bytes.\n", padding);
1403         }
1404
1405         if (conn->conn_ops->DataDigest) {
1406                 iov[iov_count].iov_base = &checksum;
1407                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
1408                 rx_size += ISCSI_CRC_LEN;
1409         }
1410
1411         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
1412
1413         iscsit_unmap_iovec(cmd);
1414
1415         if (rx_got != rx_size)
1416                 return -1;
1417
1418         if (conn->conn_ops->DataDigest) {
1419                 u32 data_crc;
1420
1421                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
1422                                                     hdr->offset, payload_length, padding,
1423                                                     cmd->pad_bytes);
1424
1425                 if (checksum != data_crc) {
1426                         pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1427                                 " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
1428                                 " does not match computed 0x%08x\n",
1429                                 hdr->itt, hdr->offset, payload_length,
1430                                 hdr->datasn, checksum, data_crc);
1431                         data_crc_failed = 1;
1432                 } else {
1433                         pr_debug("Got CRC32C DataDigest 0x%08x for"
1434                                 " %u bytes of Data Out\n", checksum,
1435                                 payload_length);
1436                 }
1437         }
1438         /*
1439          * Increment post receive data and CRC values or perform
1440          * within-command recovery.
1441          */
1442         ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed);
1443         if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY))
1444                 return 0;
1445         else if (ret == DATAOUT_SEND_R2T) {
1446                 iscsit_set_dataout_sequence_values(cmd);
1447                 iscsit_build_r2ts_for_cmd(cmd, conn, 0);
1448         } else if (ret == DATAOUT_SEND_TO_TRANSPORT) {
1449                 /*
1450                  * Handle extra special case for out of order
1451                  * Unsolicited Data Out.
1452                  */
1453                 spin_lock_bh(&cmd->istate_lock);
1454                 ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
1455                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1456                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1457                 spin_unlock_bh(&cmd->istate_lock);
1458
1459                 iscsit_stop_dataout_timer(cmd);
1460                 return (!ooo_cmdsn) ? transport_generic_handle_data(
1461                                         &cmd->se_cmd) : 0;
1462         } else /* DATAOUT_CANNOT_RECOVER */
1463                 return -1;
1464
1465         return 0;
1466 }
1467
1468 static int iscsit_handle_nop_out(
1469         struct iscsi_conn *conn,
1470         unsigned char *buf)
1471 {
1472         unsigned char *ping_data = NULL;
1473         int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size;
1474         u32 checksum, data_crc, padding = 0, payload_length;
1475         u64 lun;
1476         struct iscsi_cmd *cmd = NULL;
1477         struct kvec *iov = NULL;
1478         struct iscsi_nopout *hdr;
1479
1480         hdr                     = (struct iscsi_nopout *) buf;
1481         payload_length          = ntoh24(hdr->dlength);
1482         lun                     = get_unaligned_le64(&hdr->lun);
1483         hdr->itt                = be32_to_cpu(hdr->itt);
1484         hdr->ttt                = be32_to_cpu(hdr->ttt);
1485         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1486         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1487
1488         if ((hdr->itt == 0xFFFFFFFF) && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1489                 pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
1490                         " not set, protocol error.\n");
1491                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1492                                         buf, conn);
1493         }
1494
1495         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1496                 pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
1497                         " greater than MaxRecvDataSegmentLength: %u, protocol"
1498                         " error.\n", payload_length,
1499                         conn->conn_ops->MaxRecvDataSegmentLength);
1500                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1501                                         buf, conn);
1502         }
1503
1504         pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x,"
1505                 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
1506                 (hdr->itt == 0xFFFFFFFF) ? "Response" : "Request",
1507                 hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
1508                 payload_length);
1509         /*
1510          * This is not a response to a Unsolicited NopIN, which means
1511          * it can either be a NOPOUT ping request (with a valid ITT),
1512          * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
1513          * Either way, make sure we allocate an struct iscsi_cmd, as both
1514          * can contain ping data.
1515          */
1516         if (hdr->ttt == 0xFFFFFFFF) {
1517                 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1518                 if (!cmd)
1519                         return iscsit_add_reject(
1520                                         ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1521                                         1, buf, conn);
1522
1523                 cmd->iscsi_opcode       = ISCSI_OP_NOOP_OUT;
1524                 cmd->i_state            = ISTATE_SEND_NOPIN;
1525                 cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
1526                                                 1 : 0);
1527                 conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
1528                 cmd->targ_xfer_tag      = 0xFFFFFFFF;
1529                 cmd->cmd_sn             = hdr->cmdsn;
1530                 cmd->exp_stat_sn        = hdr->exp_statsn;
1531                 cmd->data_direction     = DMA_NONE;
1532         }
1533
1534         if (payload_length && (hdr->ttt == 0xFFFFFFFF)) {
1535                 rx_size = payload_length;
1536                 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1537                 if (!ping_data) {
1538                         pr_err("Unable to allocate memory for"
1539                                 " NOPOUT ping data.\n");
1540                         ret = -1;
1541                         goto out;
1542                 }
1543
1544                 iov = &cmd->iov_misc[0];
1545                 iov[niov].iov_base      = ping_data;
1546                 iov[niov++].iov_len     = payload_length;
1547
1548                 padding = ((-payload_length) & 3);
1549                 if (padding != 0) {
1550                         pr_debug("Receiving %u additional bytes"
1551                                 " for padding.\n", padding);
1552                         iov[niov].iov_base      = &cmd->pad_bytes;
1553                         iov[niov++].iov_len     = padding;
1554                         rx_size += padding;
1555                 }
1556                 if (conn->conn_ops->DataDigest) {
1557                         iov[niov].iov_base      = &checksum;
1558                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1559                         rx_size += ISCSI_CRC_LEN;
1560                 }
1561
1562                 rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
1563                 if (rx_got != rx_size) {
1564                         ret = -1;
1565                         goto out;
1566                 }
1567
1568                 if (conn->conn_ops->DataDigest) {
1569                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1570                                         ping_data, payload_length,
1571                                         padding, cmd->pad_bytes,
1572                                         (u8 *)&data_crc);
1573
1574                         if (checksum != data_crc) {
1575                                 pr_err("Ping data CRC32C DataDigest"
1576                                 " 0x%08x does not match computed 0x%08x\n",
1577                                         checksum, data_crc);
1578                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1579                                         pr_err("Unable to recover from"
1580                                         " NOPOUT Ping DataCRC failure while in"
1581                                                 " ERL=0.\n");
1582                                         ret = -1;
1583                                         goto out;
1584                                 } else {
1585                                         /*
1586                                          * Silently drop this PDU and let the
1587                                          * initiator plug the CmdSN gap.
1588                                          */
1589                                         pr_debug("Dropping NOPOUT"
1590                                         " Command CmdSN: 0x%08x due to"
1591                                         " DataCRC error.\n", hdr->cmdsn);
1592                                         ret = 0;
1593                                         goto out;
1594                                 }
1595                         } else {
1596                                 pr_debug("Got CRC32C DataDigest"
1597                                 " 0x%08x for %u bytes of ping data.\n",
1598                                         checksum, payload_length);
1599                         }
1600                 }
1601
1602                 ping_data[payload_length] = '\0';
1603                 /*
1604                  * Attach ping data to struct iscsi_cmd->buf_ptr.
1605                  */
1606                 cmd->buf_ptr = (void *)ping_data;
1607                 cmd->buf_ptr_size = payload_length;
1608
1609                 pr_debug("Got %u bytes of NOPOUT ping"
1610                         " data.\n", payload_length);
1611                 pr_debug("Ping Data: \"%s\"\n", ping_data);
1612         }
1613
1614         if (hdr->itt != 0xFFFFFFFF) {
1615                 if (!cmd) {
1616                         pr_err("Checking CmdSN for NOPOUT,"
1617                                 " but cmd is NULL!\n");
1618                         return -1;
1619                 }
1620                 /*
1621                  * Initiator is expecting a NopIN ping reply,
1622                  */
1623                 spin_lock_bh(&conn->cmd_lock);
1624                 list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
1625                 spin_unlock_bh(&conn->cmd_lock);
1626
1627                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1628
1629                 if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
1630                         iscsit_add_cmd_to_response_queue(cmd, conn,
1631                                         cmd->i_state);
1632                         return 0;
1633                 }
1634
1635                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1636                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
1637                         ret = 0;
1638                         goto ping_out;
1639                 }
1640                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1641                         return iscsit_add_reject_from_cmd(
1642                                         ISCSI_REASON_PROTOCOL_ERROR,
1643                                         1, 0, buf, cmd);
1644
1645                 return 0;
1646         }
1647
1648         if (hdr->ttt != 0xFFFFFFFF) {
1649                 /*
1650                  * This was a response to a unsolicited NOPIN ping.
1651                  */
1652                 cmd = iscsit_find_cmd_from_ttt(conn, hdr->ttt);
1653                 if (!cmd)
1654                         return -1;
1655
1656                 iscsit_stop_nopin_response_timer(conn);
1657
1658                 cmd->i_state = ISTATE_REMOVE;
1659                 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
1660                 iscsit_start_nopin_timer(conn);
1661         } else {
1662                 /*
1663                  * Initiator is not expecting a NOPIN is response.
1664                  * Just ignore for now.
1665                  *
1666                  * iSCSI v19-91 10.18
1667                  * "A NOP-OUT may also be used to confirm a changed
1668                  *  ExpStatSN if another PDU will not be available
1669                  *  for a long time."
1670                  */
1671                 ret = 0;
1672                 goto out;
1673         }
1674
1675         return 0;
1676 out:
1677         if (cmd)
1678                 iscsit_release_cmd(cmd);
1679 ping_out:
1680         kfree(ping_data);
1681         return ret;
1682 }
1683
1684 static int iscsit_handle_task_mgt_cmd(
1685         struct iscsi_conn *conn,
1686         unsigned char *buf)
1687 {
1688         struct iscsi_cmd *cmd;
1689         struct se_tmr_req *se_tmr;
1690         struct iscsi_tmr_req *tmr_req;
1691         struct iscsi_tm *hdr;
1692         u32 payload_length;
1693         int out_of_order_cmdsn = 0;
1694         int ret;
1695         u8 function;
1696
1697         hdr                     = (struct iscsi_tm *) buf;
1698         payload_length          = ntoh24(hdr->dlength);
1699         hdr->itt                = be32_to_cpu(hdr->itt);
1700         hdr->rtt                = be32_to_cpu(hdr->rtt);
1701         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1702         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1703         hdr->refcmdsn           = be32_to_cpu(hdr->refcmdsn);
1704         hdr->exp_datasn         = be32_to_cpu(hdr->exp_datasn);
1705         hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
1706         function = hdr->flags;
1707
1708         pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
1709                 " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
1710                 " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
1711                 hdr->rtt, hdr->refcmdsn, conn->cid);
1712
1713         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1714             ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1715              (hdr->rtt != ISCSI_RESERVED_TAG))) {
1716                 pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
1717                 hdr->rtt = ISCSI_RESERVED_TAG;
1718         }
1719
1720         if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
1721                         !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1722                 pr_err("Task Management Request TASK_REASSIGN not"
1723                         " issued as immediate command, bad iSCSI Initiator"
1724                                 "implementation\n");
1725                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1726                                         buf, conn);
1727         }
1728         if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
1729             (hdr->refcmdsn != ISCSI_RESERVED_TAG))
1730                 hdr->refcmdsn = ISCSI_RESERVED_TAG;
1731
1732         cmd = iscsit_allocate_se_cmd_for_tmr(conn, function);
1733         if (!cmd)
1734                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1735                                         1, buf, conn);
1736
1737         cmd->iscsi_opcode       = ISCSI_OP_SCSI_TMFUNC;
1738         cmd->i_state            = ISTATE_SEND_TASKMGTRSP;
1739         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1740         cmd->init_task_tag      = hdr->itt;
1741         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1742         cmd->cmd_sn             = hdr->cmdsn;
1743         cmd->exp_stat_sn        = hdr->exp_statsn;
1744         se_tmr                  = cmd->se_cmd.se_tmr_req;
1745         tmr_req                 = cmd->tmr_req;
1746         /*
1747          * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
1748          */
1749         if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
1750                 ret = iscsit_get_lun_for_tmr(cmd,
1751                                 get_unaligned_le64(&hdr->lun));
1752                 if (ret < 0) {
1753                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1754                         se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
1755                         goto attach;
1756                 }
1757         }
1758
1759         switch (function) {
1760         case ISCSI_TM_FUNC_ABORT_TASK:
1761                 se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
1762                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE) {
1763                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1764                         goto attach;
1765                 }
1766                 break;
1767         case ISCSI_TM_FUNC_ABORT_TASK_SET:
1768         case ISCSI_TM_FUNC_CLEAR_ACA:
1769         case ISCSI_TM_FUNC_CLEAR_TASK_SET:
1770         case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
1771                 break;
1772         case ISCSI_TM_FUNC_TARGET_WARM_RESET:
1773                 if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
1774                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1775                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1776                         goto attach;
1777                 }
1778                 break;
1779         case ISCSI_TM_FUNC_TARGET_COLD_RESET:
1780                 if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
1781                         cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1782                         se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
1783                         goto attach;
1784                 }
1785                 break;
1786         case ISCSI_TM_FUNC_TASK_REASSIGN:
1787                 se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
1788                 /*
1789                  * Perform sanity checks on the ExpDataSN only if the
1790                  * TASK_REASSIGN was successful.
1791                  */
1792                 if (se_tmr->response != ISCSI_TMF_RSP_COMPLETE)
1793                         break;
1794
1795                 if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
1796                         return iscsit_add_reject_from_cmd(
1797                                         ISCSI_REASON_BOOKMARK_INVALID, 1, 1,
1798                                         buf, cmd);
1799                 break;
1800         default:
1801                 pr_err("Unknown TMR function: 0x%02x, protocol"
1802                         " error.\n", function);
1803                 cmd->se_cmd.se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1804                 se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
1805                 goto attach;
1806         }
1807
1808         if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
1809             (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
1810                 se_tmr->call_transport = 1;
1811 attach:
1812         spin_lock_bh(&conn->cmd_lock);
1813         list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
1814         spin_unlock_bh(&conn->cmd_lock);
1815
1816         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1817                 int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1818                 if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP)
1819                         out_of_order_cmdsn = 1;
1820                 else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
1821                         return 0;
1822                 else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1823                         return iscsit_add_reject_from_cmd(
1824                                         ISCSI_REASON_PROTOCOL_ERROR,
1825                                         1, 0, buf, cmd);
1826         }
1827         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1828
1829         if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
1830                 return 0;
1831         /*
1832          * Found the referenced task, send to transport for processing.
1833          */
1834         if (se_tmr->call_transport)
1835                 return transport_generic_handle_tmr(&cmd->se_cmd);
1836
1837         /*
1838          * Could not find the referenced LUN, task, or Task Management
1839          * command not authorized or supported.  Change state and
1840          * let the tx_thread send the response.
1841          *
1842          * For connection recovery, this is also the default action for
1843          * TMR TASK_REASSIGN.
1844          */
1845         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
1846         return 0;
1847 }
1848
1849 /* #warning FIXME: Support Text Command parameters besides SendTargets */
1850 static int iscsit_handle_text_cmd(
1851         struct iscsi_conn *conn,
1852         unsigned char *buf)
1853 {
1854         char *text_ptr, *text_in;
1855         int cmdsn_ret, niov = 0, rx_got, rx_size;
1856         u32 checksum = 0, data_crc = 0, payload_length;
1857         u32 padding = 0, pad_bytes = 0, text_length = 0;
1858         struct iscsi_cmd *cmd;
1859         struct kvec iov[3];
1860         struct iscsi_text *hdr;
1861
1862         hdr                     = (struct iscsi_text *) buf;
1863         payload_length          = ntoh24(hdr->dlength);
1864         hdr->itt                = be32_to_cpu(hdr->itt);
1865         hdr->ttt                = be32_to_cpu(hdr->ttt);
1866         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
1867         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
1868
1869         if (payload_length > conn->conn_ops->MaxRecvDataSegmentLength) {
1870                 pr_err("Unable to accept text parameter length: %u"
1871                         "greater than MaxRecvDataSegmentLength %u.\n",
1872                        payload_length, conn->conn_ops->MaxRecvDataSegmentLength);
1873                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
1874                                         buf, conn);
1875         }
1876
1877         pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
1878                 " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
1879                 hdr->exp_statsn, payload_length);
1880
1881         rx_size = text_length = payload_length;
1882         if (text_length) {
1883                 text_in = kzalloc(text_length, GFP_KERNEL);
1884                 if (!text_in) {
1885                         pr_err("Unable to allocate memory for"
1886                                 " incoming text parameters\n");
1887                         return -1;
1888                 }
1889
1890                 memset(iov, 0, 3 * sizeof(struct kvec));
1891                 iov[niov].iov_base      = text_in;
1892                 iov[niov++].iov_len     = text_length;
1893
1894                 padding = ((-payload_length) & 3);
1895                 if (padding != 0) {
1896                         iov[niov].iov_base = &pad_bytes;
1897                         iov[niov++].iov_len  = padding;
1898                         rx_size += padding;
1899                         pr_debug("Receiving %u additional bytes"
1900                                         " for padding.\n", padding);
1901                 }
1902                 if (conn->conn_ops->DataDigest) {
1903                         iov[niov].iov_base      = &checksum;
1904                         iov[niov++].iov_len     = ISCSI_CRC_LEN;
1905                         rx_size += ISCSI_CRC_LEN;
1906                 }
1907
1908                 rx_got = rx_data(conn, &iov[0], niov, rx_size);
1909                 if (rx_got != rx_size) {
1910                         kfree(text_in);
1911                         return -1;
1912                 }
1913
1914                 if (conn->conn_ops->DataDigest) {
1915                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
1916                                         text_in, text_length,
1917                                         padding, (u8 *)&pad_bytes,
1918                                         (u8 *)&data_crc);
1919
1920                         if (checksum != data_crc) {
1921                                 pr_err("Text data CRC32C DataDigest"
1922                                         " 0x%08x does not match computed"
1923                                         " 0x%08x\n", checksum, data_crc);
1924                                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1925                                         pr_err("Unable to recover from"
1926                                         " Text Data digest failure while in"
1927                                                 " ERL=0.\n");
1928                                         kfree(text_in);
1929                                         return -1;
1930                                 } else {
1931                                         /*
1932                                          * Silently drop this PDU and let the
1933                                          * initiator plug the CmdSN gap.
1934                                          */
1935                                         pr_debug("Dropping Text"
1936                                         " Command CmdSN: 0x%08x due to"
1937                                         " DataCRC error.\n", hdr->cmdsn);
1938                                         kfree(text_in);
1939                                         return 0;
1940                                 }
1941                         } else {
1942                                 pr_debug("Got CRC32C DataDigest"
1943                                         " 0x%08x for %u bytes of text data.\n",
1944                                                 checksum, text_length);
1945                         }
1946                 }
1947                 text_in[text_length - 1] = '\0';
1948                 pr_debug("Successfully read %d bytes of text"
1949                                 " data.\n", text_length);
1950
1951                 if (strncmp("SendTargets", text_in, 11) != 0) {
1952                         pr_err("Received Text Data that is not"
1953                                 " SendTargets, cannot continue.\n");
1954                         kfree(text_in);
1955                         return -1;
1956                 }
1957                 text_ptr = strchr(text_in, '=');
1958                 if (!text_ptr) {
1959                         pr_err("No \"=\" separator found in Text Data,"
1960                                 "  cannot continue.\n");
1961                         kfree(text_in);
1962                         return -1;
1963                 }
1964                 if (strncmp("=All", text_ptr, 4) != 0) {
1965                         pr_err("Unable to locate All value for"
1966                                 " SendTargets key,  cannot continue.\n");
1967                         kfree(text_in);
1968                         return -1;
1969                 }
1970 /*#warning Support SendTargets=(iSCSI Target Name/Nothing) values. */
1971                 kfree(text_in);
1972         }
1973
1974         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
1975         if (!cmd)
1976                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1977                                         1, buf, conn);
1978
1979         cmd->iscsi_opcode       = ISCSI_OP_TEXT;
1980         cmd->i_state            = ISTATE_SEND_TEXTRSP;
1981         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
1982         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
1983         cmd->targ_xfer_tag      = 0xFFFFFFFF;
1984         cmd->cmd_sn             = hdr->cmdsn;
1985         cmd->exp_stat_sn        = hdr->exp_statsn;
1986         cmd->data_direction     = DMA_NONE;
1987
1988         spin_lock_bh(&conn->cmd_lock);
1989         list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
1990         spin_unlock_bh(&conn->cmd_lock);
1991
1992         iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
1993
1994         if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
1995                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
1996                 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
1997                         return iscsit_add_reject_from_cmd(
1998                                         ISCSI_REASON_PROTOCOL_ERROR,
1999                                         1, 0, buf, cmd);
2000
2001                 return 0;
2002         }
2003
2004         return iscsit_execute_cmd(cmd, 0);
2005 }
2006
2007 int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2008 {
2009         struct iscsi_conn *conn_p;
2010         struct iscsi_session *sess = conn->sess;
2011
2012         pr_debug("Received logout request CLOSESESSION on CID: %hu"
2013                 " for SID: %u.\n", conn->cid, conn->sess->sid);
2014
2015         atomic_set(&sess->session_logout, 1);
2016         atomic_set(&conn->conn_logout_remove, 1);
2017         conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
2018
2019         iscsit_inc_conn_usage_count(conn);
2020         iscsit_inc_session_usage_count(sess);
2021
2022         spin_lock_bh(&sess->conn_lock);
2023         list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
2024                 if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
2025                         continue;
2026
2027                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2028                 conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2029         }
2030         spin_unlock_bh(&sess->conn_lock);
2031
2032         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2033
2034         return 0;
2035 }
2036
2037 int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2038 {
2039         struct iscsi_conn *l_conn;
2040         struct iscsi_session *sess = conn->sess;
2041
2042         pr_debug("Received logout request CLOSECONNECTION for CID:"
2043                 " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2044
2045         /*
2046          * A Logout Request with a CLOSECONNECTION reason code for a CID
2047          * can arrive on a connection with a differing CID.
2048          */
2049         if (conn->cid == cmd->logout_cid) {
2050                 spin_lock_bh(&conn->state_lock);
2051                 pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
2052                 conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
2053
2054                 atomic_set(&conn->conn_logout_remove, 1);
2055                 conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
2056                 iscsit_inc_conn_usage_count(conn);
2057
2058                 spin_unlock_bh(&conn->state_lock);
2059         } else {
2060                 /*
2061                  * Handle all different cid CLOSECONNECTION requests in
2062                  * iscsit_logout_post_handler_diffcid() as to give enough
2063                  * time for any non immediate command's CmdSN to be
2064                  * acknowledged on the connection in question.
2065                  *
2066                  * Here we simply make sure the CID is still around.
2067                  */
2068                 l_conn = iscsit_get_conn_from_cid(sess,
2069                                 cmd->logout_cid);
2070                 if (!l_conn) {
2071                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2072                         iscsit_add_cmd_to_response_queue(cmd, conn,
2073                                         cmd->i_state);
2074                         return 0;
2075                 }
2076
2077                 iscsit_dec_conn_usage_count(l_conn);
2078         }
2079
2080         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2081
2082         return 0;
2083 }
2084
2085 int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2086 {
2087         struct iscsi_session *sess = conn->sess;
2088
2089         pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
2090                 " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
2091
2092         if (sess->sess_ops->ErrorRecoveryLevel != 2) {
2093                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2094                         " while ERL!=2.\n");
2095                 cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
2096                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2097                 return 0;
2098         }
2099
2100         if (conn->cid == cmd->logout_cid) {
2101                 pr_err("Received Logout Request REMOVECONNFORRECOVERY"
2102                         " with CID: %hu on CID: %hu, implementation error.\n",
2103                                 cmd->logout_cid, conn->cid);
2104                 cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
2105                 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2106                 return 0;
2107         }
2108
2109         iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
2110
2111         return 0;
2112 }
2113
2114 static int iscsit_handle_logout_cmd(
2115         struct iscsi_conn *conn,
2116         unsigned char *buf)
2117 {
2118         int cmdsn_ret, logout_remove = 0;
2119         u8 reason_code = 0;
2120         struct iscsi_cmd *cmd;
2121         struct iscsi_logout *hdr;
2122         struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
2123
2124         hdr                     = (struct iscsi_logout *) buf;
2125         reason_code             = (hdr->flags & 0x7f);
2126         hdr->itt                = be32_to_cpu(hdr->itt);
2127         hdr->cid                = be16_to_cpu(hdr->cid);
2128         hdr->cmdsn              = be32_to_cpu(hdr->cmdsn);
2129         hdr->exp_statsn = be32_to_cpu(hdr->exp_statsn);
2130
2131         if (tiqn) {
2132                 spin_lock(&tiqn->logout_stats.lock);
2133                 if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
2134                         tiqn->logout_stats.normal_logouts++;
2135                 else
2136                         tiqn->logout_stats.abnormal_logouts++;
2137                 spin_unlock(&tiqn->logout_stats.lock);
2138         }
2139
2140         pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
2141                 " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
2142                 hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
2143                 hdr->cid, conn->cid);
2144
2145         if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
2146                 pr_err("Received logout request on connection that"
2147                         " is not in logged in state, ignoring request.\n");
2148                 return 0;
2149         }
2150
2151         cmd = iscsit_allocate_cmd(conn, GFP_KERNEL);
2152         if (!cmd)
2153                 return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1,
2154                                         buf, conn);
2155
2156         cmd->iscsi_opcode       = ISCSI_OP_LOGOUT;
2157         cmd->i_state            = ISTATE_SEND_LOGOUTRSP;
2158         cmd->immediate_cmd      = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
2159         conn->sess->init_task_tag = cmd->init_task_tag  = hdr->itt;
2160         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2161         cmd->cmd_sn             = hdr->cmdsn;
2162         cmd->exp_stat_sn        = hdr->exp_statsn;
2163         cmd->logout_cid         = hdr->cid;
2164         cmd->logout_reason      = reason_code;
2165         cmd->data_direction     = DMA_NONE;
2166
2167         /*
2168          * We need to sleep in these cases (by returning 1) until the Logout
2169          * Response gets sent in the tx thread.
2170          */
2171         if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
2172            ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
2173             (hdr->cid == conn->cid)))
2174                 logout_remove = 1;
2175
2176         spin_lock_bh(&conn->cmd_lock);
2177         list_add_tail(&cmd->i_list, &conn->conn_cmd_list);
2178         spin_unlock_bh(&conn->cmd_lock);
2179
2180         if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
2181                 iscsit_ack_from_expstatsn(conn, hdr->exp_statsn);
2182
2183         /*
2184          * Immediate commands are executed, well, immediately.
2185          * Non-Immediate Logout Commands are executed in CmdSN order.
2186          */
2187         if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
2188                 int ret = iscsit_execute_cmd(cmd, 0);
2189
2190                 if (ret < 0)
2191                         return ret;
2192         } else {
2193                 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn);
2194                 if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
2195                         logout_remove = 0;
2196                 } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
2197                         return iscsit_add_reject_from_cmd(
2198                                 ISCSI_REASON_PROTOCOL_ERROR,
2199                                 1, 0, buf, cmd);
2200                 }
2201         }
2202
2203         return logout_remove;
2204 }
2205
2206 static int iscsit_handle_snack(
2207         struct iscsi_conn *conn,
2208         unsigned char *buf)
2209 {
2210         u32 unpacked_lun;
2211         u64 lun;
2212         struct iscsi_snack *hdr;
2213
2214         hdr                     = (struct iscsi_snack *) buf;
2215         hdr->flags              &= ~ISCSI_FLAG_CMD_FINAL;
2216         lun                     = get_unaligned_le64(&hdr->lun);
2217         unpacked_lun            = scsilun_to_int((struct scsi_lun *)&lun);
2218         hdr->itt                = be32_to_cpu(hdr->itt);
2219         hdr->ttt                = be32_to_cpu(hdr->ttt);
2220         hdr->exp_statsn         = be32_to_cpu(hdr->exp_statsn);
2221         hdr->begrun             = be32_to_cpu(hdr->begrun);
2222         hdr->runlength          = be32_to_cpu(hdr->runlength);
2223
2224         pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
2225                 " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
2226                 " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
2227                         hdr->begrun, hdr->runlength, conn->cid);
2228
2229         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2230                 pr_err("Initiator sent SNACK request while in"
2231                         " ErrorRecoveryLevel=0.\n");
2232                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2233                                         buf, conn);
2234         }
2235         /*
2236          * SNACK_DATA and SNACK_R2T are both 0,  so check which function to
2237          * call from inside iscsi_send_recovery_datain_or_r2t().
2238          */
2239         switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
2240         case 0:
2241                 return iscsit_handle_recovery_datain_or_r2t(conn, buf,
2242                         hdr->itt, hdr->ttt, hdr->begrun, hdr->runlength);
2243         case ISCSI_FLAG_SNACK_TYPE_STATUS:
2244                 return iscsit_handle_status_snack(conn, hdr->itt, hdr->ttt,
2245                         hdr->begrun, hdr->runlength);
2246         case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
2247                 return iscsit_handle_data_ack(conn, hdr->ttt, hdr->begrun,
2248                         hdr->runlength);
2249         case ISCSI_FLAG_SNACK_TYPE_RDATA:
2250                 /* FIXME: Support R-Data SNACK */
2251                 pr_err("R-Data SNACK Not Supported.\n");
2252                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2253                                         buf, conn);
2254         default:
2255                 pr_err("Unknown SNACK type 0x%02x, protocol"
2256                         " error.\n", hdr->flags & 0x0f);
2257                 return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
2258                                         buf, conn);
2259         }
2260
2261         return 0;
2262 }
2263
2264 static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
2265 {
2266         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
2267             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
2268                 wait_for_completion_interruptible_timeout(
2269                                         &conn->rx_half_close_comp,
2270                                         ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
2271         }
2272 }
2273
2274 static int iscsit_handle_immediate_data(
2275         struct iscsi_cmd *cmd,
2276         unsigned char *buf,
2277         u32 length)
2278 {
2279         int iov_ret, rx_got = 0, rx_size = 0;
2280         u32 checksum, iov_count = 0, padding = 0;
2281         struct iscsi_conn *conn = cmd->conn;
2282         struct kvec *iov;
2283
2284         iov_ret = iscsit_map_iovec(cmd, cmd->iov_data, cmd->write_data_done, length);
2285         if (iov_ret < 0)
2286                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2287
2288         rx_size = length;
2289         iov_count = iov_ret;
2290         iov = &cmd->iov_data[0];
2291
2292         padding = ((-length) & 3);
2293         if (padding != 0) {
2294                 iov[iov_count].iov_base = cmd->pad_bytes;
2295                 iov[iov_count++].iov_len = padding;
2296                 rx_size += padding;
2297         }
2298
2299         if (conn->conn_ops->DataDigest) {
2300                 iov[iov_count].iov_base         = &checksum;
2301                 iov[iov_count++].iov_len        = ISCSI_CRC_LEN;
2302                 rx_size += ISCSI_CRC_LEN;
2303         }
2304
2305         rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
2306
2307         iscsit_unmap_iovec(cmd);
2308
2309         if (rx_got != rx_size) {
2310                 iscsit_rx_thread_wait_for_tcp(conn);
2311                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2312         }
2313
2314         if (conn->conn_ops->DataDigest) {
2315                 u32 data_crc;
2316
2317                 data_crc = iscsit_do_crypto_hash_sg(&conn->conn_rx_hash, cmd,
2318                                                     cmd->write_data_done, length, padding,
2319                                                     cmd->pad_bytes);
2320
2321                 if (checksum != data_crc) {
2322                         pr_err("ImmediateData CRC32C DataDigest 0x%08x"
2323                                 " does not match computed 0x%08x\n", checksum,
2324                                 data_crc);
2325
2326                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
2327                                 pr_err("Unable to recover from"
2328                                         " Immediate Data digest failure while"
2329                                         " in ERL=0.\n");
2330                                 iscsit_add_reject_from_cmd(
2331                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2332                                                 1, 0, buf, cmd);
2333                                 return IMMEDIATE_DATA_CANNOT_RECOVER;
2334                         } else {
2335                                 iscsit_add_reject_from_cmd(
2336                                                 ISCSI_REASON_DATA_DIGEST_ERROR,
2337                                                 0, 0, buf, cmd);
2338                                 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
2339                         }
2340                 } else {
2341                         pr_debug("Got CRC32C DataDigest 0x%08x for"
2342                                 " %u bytes of Immediate Data\n", checksum,
2343                                 length);
2344                 }
2345         }
2346
2347         cmd->write_data_done += length;
2348
2349         if (cmd->write_data_done == cmd->data_length) {
2350                 spin_lock_bh(&cmd->istate_lock);
2351                 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
2352                 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
2353                 spin_unlock_bh(&cmd->istate_lock);
2354         }
2355
2356         return IMMEDIATE_DATA_NORMAL_OPERATION;
2357 }
2358
2359 /*
2360  *      Called with sess->conn_lock held.
2361  */
2362 /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
2363         with active network interface */
2364 static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
2365 {
2366         struct iscsi_cmd *cmd;
2367         struct iscsi_conn *conn_p;
2368
2369         /*
2370          * Only send a Asynchronous Message on connections whos network
2371          * interface is still functional.
2372          */
2373         list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
2374                 if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
2375                         iscsit_inc_conn_usage_count(conn_p);
2376                         break;
2377                 }
2378         }
2379
2380         if (!conn_p)
2381                 return;
2382
2383         cmd = iscsit_allocate_cmd(conn_p, GFP_KERNEL);
2384         if (!cmd) {
2385                 iscsit_dec_conn_usage_count(conn_p);
2386                 return;
2387         }
2388
2389         cmd->logout_cid = conn->cid;
2390         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2391         cmd->i_state = ISTATE_SEND_ASYNCMSG;
2392
2393         spin_lock_bh(&conn_p->cmd_lock);
2394         list_add_tail(&cmd->i_list, &conn_p->conn_cmd_list);
2395         spin_unlock_bh(&conn_p->cmd_lock);
2396
2397         iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
2398         iscsit_dec_conn_usage_count(conn_p);
2399 }
2400
2401 static int iscsit_send_conn_drop_async_message(
2402         struct iscsi_cmd *cmd,
2403         struct iscsi_conn *conn)
2404 {
2405         struct iscsi_async *hdr;
2406
2407         cmd->tx_size = ISCSI_HDR_LEN;
2408         cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
2409
2410         hdr                     = (struct iscsi_async *) cmd->pdu;
2411         hdr->opcode             = ISCSI_OP_ASYNC_EVENT;
2412         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
2413         cmd->init_task_tag      = 0xFFFFFFFF;
2414         cmd->targ_xfer_tag      = 0xFFFFFFFF;
2415         put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
2416         cmd->stat_sn            = conn->stat_sn++;
2417         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2418         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2419         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2420         hdr->async_event        = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
2421         hdr->param1             = cpu_to_be16(cmd->logout_cid);
2422         hdr->param2             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
2423         hdr->param3             = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
2424
2425         if (conn->conn_ops->HeaderDigest) {
2426                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2427
2428                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2429                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2430                                 0, NULL, (u8 *)header_digest);
2431
2432                 cmd->tx_size += ISCSI_CRC_LEN;
2433                 pr_debug("Attaching CRC32C HeaderDigest to"
2434                         " Async Message 0x%08x\n", *header_digest);
2435         }
2436
2437         cmd->iov_misc[0].iov_base       = cmd->pdu;
2438         cmd->iov_misc[0].iov_len        = cmd->tx_size;
2439         cmd->iov_misc_count             = 1;
2440
2441         pr_debug("Sending Connection Dropped Async Message StatSN:"
2442                 " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
2443                         cmd->logout_cid, conn->cid);
2444         return 0;
2445 }
2446
2447 static int iscsit_send_data_in(
2448         struct iscsi_cmd *cmd,
2449         struct iscsi_conn *conn,
2450         int *eodr)
2451 {
2452         int iov_ret = 0, set_statsn = 0;
2453         u32 iov_count = 0, tx_size = 0;
2454         struct iscsi_datain datain;
2455         struct iscsi_datain_req *dr;
2456         struct iscsi_data_rsp *hdr;
2457         struct kvec *iov;
2458
2459         memset(&datain, 0, sizeof(struct iscsi_datain));
2460         dr = iscsit_get_datain_values(cmd, &datain);
2461         if (!dr) {
2462                 pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
2463                                 cmd->init_task_tag);
2464                 return -1;
2465         }
2466
2467         /*
2468          * Be paranoid and double check the logic for now.
2469          */
2470         if ((datain.offset + datain.length) > cmd->data_length) {
2471                 pr_err("Command ITT: 0x%08x, datain.offset: %u and"
2472                         " datain.length: %u exceeds cmd->data_length: %u\n",
2473                         cmd->init_task_tag, datain.offset, datain.length,
2474                                 cmd->data_length);
2475                 return -1;
2476         }
2477
2478         spin_lock_bh(&conn->sess->session_stats_lock);
2479         conn->sess->tx_data_octets += datain.length;
2480         if (conn->sess->se_sess->se_node_acl) {
2481                 spin_lock(&conn->sess->se_sess->se_node_acl->stats_lock);
2482                 conn->sess->se_sess->se_node_acl->read_bytes += datain.length;
2483                 spin_unlock(&conn->sess->se_sess->se_node_acl->stats_lock);
2484         }
2485         spin_unlock_bh(&conn->sess->session_stats_lock);
2486         /*
2487          * Special case for successfully execution w/ both DATAIN
2488          * and Sense Data.
2489          */
2490         if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
2491             (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
2492                 datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
2493         else {
2494                 if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
2495                     (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
2496                         iscsit_increment_maxcmdsn(cmd, conn->sess);
2497                         cmd->stat_sn = conn->stat_sn++;
2498                         set_statsn = 1;
2499                 } else if (dr->dr_complete ==
2500                                 DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
2501                         set_statsn = 1;
2502         }
2503
2504         hdr     = (struct iscsi_data_rsp *) cmd->pdu;
2505         memset(hdr, 0, ISCSI_HDR_LEN);
2506         hdr->opcode             = ISCSI_OP_SCSI_DATA_IN;
2507         hdr->flags              = datain.flags;
2508         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
2509                 if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
2510                         hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
2511                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2512                 } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
2513                         hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
2514                         hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
2515                 }
2516         }
2517         hton24(hdr->dlength, datain.length);
2518         if (hdr->flags & ISCSI_FLAG_DATA_ACK)
2519                 int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2520                                 (struct scsi_lun *)&hdr->lun);
2521         else
2522                 put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2523
2524         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2525         hdr->ttt                = (hdr->flags & ISCSI_FLAG_DATA_ACK) ?
2526                                    cpu_to_be32(cmd->targ_xfer_tag) :
2527                                    0xFFFFFFFF;
2528         hdr->statsn             = (set_statsn) ? cpu_to_be32(cmd->stat_sn) :
2529                                                 0xFFFFFFFF;
2530         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2531         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2532         hdr->datasn             = cpu_to_be32(datain.data_sn);
2533         hdr->offset             = cpu_to_be32(datain.offset);
2534
2535         iov = &cmd->iov_data[0];
2536         iov[iov_count].iov_base = cmd->pdu;
2537         iov[iov_count++].iov_len        = ISCSI_HDR_LEN;
2538         tx_size += ISCSI_HDR_LEN;
2539
2540         if (conn->conn_ops->HeaderDigest) {
2541                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2542
2543                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2544                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2545                                 0, NULL, (u8 *)header_digest);
2546
2547                 iov[0].iov_len += ISCSI_CRC_LEN;
2548                 tx_size += ISCSI_CRC_LEN;
2549
2550                 pr_debug("Attaching CRC32 HeaderDigest"
2551                         " for DataIN PDU 0x%08x\n", *header_digest);
2552         }
2553
2554         iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length);
2555         if (iov_ret < 0)
2556                 return -1;
2557
2558         iov_count += iov_ret;
2559         tx_size += datain.length;
2560
2561         cmd->padding = ((-datain.length) & 3);
2562         if (cmd->padding) {
2563                 iov[iov_count].iov_base         = cmd->pad_bytes;
2564                 iov[iov_count++].iov_len        = cmd->padding;
2565                 tx_size += cmd->padding;
2566
2567                 pr_debug("Attaching %u padding bytes\n",
2568                                 cmd->padding);
2569         }
2570         if (conn->conn_ops->DataDigest) {
2571                 cmd->data_crc = iscsit_do_crypto_hash_sg(&conn->conn_tx_hash, cmd,
2572                          datain.offset, datain.length, cmd->padding, cmd->pad_bytes);
2573
2574                 iov[iov_count].iov_base = &cmd->data_crc;
2575                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
2576                 tx_size += ISCSI_CRC_LEN;
2577
2578                 pr_debug("Attached CRC32C DataDigest %d bytes, crc"
2579                         " 0x%08x\n", datain.length+cmd->padding, cmd->data_crc);
2580         }
2581
2582         cmd->iov_data_count = iov_count;
2583         cmd->tx_size = tx_size;
2584
2585         pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
2586                 " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
2587                 cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
2588                 ntohl(hdr->offset), datain.length, conn->cid);
2589
2590         if (dr->dr_complete) {
2591                 *eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
2592                                 2 : 1;
2593                 iscsit_free_datain_req(cmd, dr);
2594         }
2595
2596         return 0;
2597 }
2598
2599 static int iscsit_send_logout_response(
2600         struct iscsi_cmd *cmd,
2601         struct iscsi_conn *conn)
2602 {
2603         int niov = 0, tx_size;
2604         struct iscsi_conn *logout_conn = NULL;
2605         struct iscsi_conn_recovery *cr = NULL;
2606         struct iscsi_session *sess = conn->sess;
2607         struct kvec *iov;
2608         struct iscsi_logout_rsp *hdr;
2609         /*
2610          * The actual shutting down of Sessions and/or Connections
2611          * for CLOSESESSION and CLOSECONNECTION Logout Requests
2612          * is done in scsi_logout_post_handler().
2613          */
2614         switch (cmd->logout_reason) {
2615         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
2616                 pr_debug("iSCSI session logout successful, setting"
2617                         " logout response to ISCSI_LOGOUT_SUCCESS.\n");
2618                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2619                 break;
2620         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
2621                 if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
2622                         break;
2623                 /*
2624                  * For CLOSECONNECTION logout requests carrying
2625                  * a matching logout CID -> local CID, the reference
2626                  * for the local CID will have been incremented in
2627                  * iscsi_logout_closeconnection().
2628                  *
2629                  * For CLOSECONNECTION logout requests carrying
2630                  * a different CID than the connection it arrived
2631                  * on, the connection responding to cmd->logout_cid
2632                  * is stopped in iscsit_logout_post_handler_diffcid().
2633                  */
2634
2635                 pr_debug("iSCSI CID: %hu logout on CID: %hu"
2636                         " successful.\n", cmd->logout_cid, conn->cid);
2637                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2638                 break;
2639         case ISCSI_LOGOUT_REASON_RECOVERY:
2640                 if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
2641                     (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
2642                         break;
2643                 /*
2644                  * If the connection is still active from our point of view
2645                  * force connection recovery to occur.
2646                  */
2647                 logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
2648                                 cmd->logout_cid);
2649                 if ((logout_conn)) {
2650                         iscsit_connection_reinstatement_rcfr(logout_conn);
2651                         iscsit_dec_conn_usage_count(logout_conn);
2652                 }
2653
2654                 cr = iscsit_get_inactive_connection_recovery_entry(
2655                                 conn->sess, cmd->logout_cid);
2656                 if (!cr) {
2657                         pr_err("Unable to locate CID: %hu for"
2658                         " REMOVECONNFORRECOVERY Logout Request.\n",
2659                                 cmd->logout_cid);
2660                         cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
2661                         break;
2662                 }
2663
2664                 iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
2665
2666                 pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
2667                         " for recovery for CID: %hu on CID: %hu successful.\n",
2668                                 cmd->logout_cid, conn->cid);
2669                 cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
2670                 break;
2671         default:
2672                 pr_err("Unknown cmd->logout_reason: 0x%02x\n",
2673                                 cmd->logout_reason);
2674                 return -1;
2675         }
2676
2677         tx_size = ISCSI_HDR_LEN;
2678         hdr                     = (struct iscsi_logout_rsp *)cmd->pdu;
2679         memset(hdr, 0, ISCSI_HDR_LEN);
2680         hdr->opcode             = ISCSI_OP_LOGOUT_RSP;
2681         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2682         hdr->response           = cmd->logout_response;
2683         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2684         cmd->stat_sn            = conn->stat_sn++;
2685         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2686
2687         iscsit_increment_maxcmdsn(cmd, conn->sess);
2688         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2689         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2690
2691         iov = &cmd->iov_misc[0];
2692         iov[niov].iov_base      = cmd->pdu;
2693         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2694
2695         if (conn->conn_ops->HeaderDigest) {
2696                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2697
2698                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2699                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2700                                 0, NULL, (u8 *)header_digest);
2701
2702                 iov[0].iov_len += ISCSI_CRC_LEN;
2703                 tx_size += ISCSI_CRC_LEN;
2704                 pr_debug("Attaching CRC32C HeaderDigest to"
2705                         " Logout Response 0x%08x\n", *header_digest);
2706         }
2707         cmd->iov_misc_count = niov;
2708         cmd->tx_size = tx_size;
2709
2710         pr_debug("Sending Logout Response ITT: 0x%08x StatSN:"
2711                 " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
2712                 cmd->init_task_tag, cmd->stat_sn, hdr->response,
2713                 cmd->logout_cid, conn->cid);
2714
2715         return 0;
2716 }
2717
2718 /*
2719  *      Unsolicited NOPIN, either requesting a response or not.
2720  */
2721 static int iscsit_send_unsolicited_nopin(
2722         struct iscsi_cmd *cmd,
2723         struct iscsi_conn *conn,
2724         int want_response)
2725 {
2726         int tx_size = ISCSI_HDR_LEN;
2727         struct iscsi_nopin *hdr;
2728
2729         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2730         memset(hdr, 0, ISCSI_HDR_LEN);
2731         hdr->opcode             = ISCSI_OP_NOOP_IN;
2732         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2733         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2734         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2735         cmd->stat_sn            = conn->stat_sn;
2736         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2737         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2738         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2739
2740         if (conn->conn_ops->HeaderDigest) {
2741                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2742
2743                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2744                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2745                                 0, NULL, (u8 *)header_digest);
2746
2747                 tx_size += ISCSI_CRC_LEN;
2748                 pr_debug("Attaching CRC32C HeaderDigest to"
2749                         " NopIN 0x%08x\n", *header_digest);
2750         }
2751
2752         cmd->iov_misc[0].iov_base       = cmd->pdu;
2753         cmd->iov_misc[0].iov_len        = tx_size;
2754         cmd->iov_misc_count     = 1;
2755         cmd->tx_size            = tx_size;
2756
2757         pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
2758                 " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
2759
2760         return 0;
2761 }
2762
2763 static int iscsit_send_nopin_response(
2764         struct iscsi_cmd *cmd,
2765         struct iscsi_conn *conn)
2766 {
2767         int niov = 0, tx_size;
2768         u32 padding = 0;
2769         struct kvec *iov;
2770         struct iscsi_nopin *hdr;
2771
2772         tx_size = ISCSI_HDR_LEN;
2773         hdr                     = (struct iscsi_nopin *) cmd->pdu;
2774         memset(hdr, 0, ISCSI_HDR_LEN);
2775         hdr->opcode             = ISCSI_OP_NOOP_IN;
2776         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2777         hton24(hdr->dlength, cmd->buf_ptr_size);
2778         put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
2779         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2780         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
2781         cmd->stat_sn            = conn->stat_sn++;
2782         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
2783
2784         iscsit_increment_maxcmdsn(cmd, conn->sess);
2785         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2786         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2787
2788         iov = &cmd->iov_misc[0];
2789         iov[niov].iov_base      = cmd->pdu;
2790         iov[niov++].iov_len     = ISCSI_HDR_LEN;
2791
2792         if (conn->conn_ops->HeaderDigest) {
2793                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2794
2795                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2796                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2797                                 0, NULL, (u8 *)header_digest);
2798
2799                 iov[0].iov_len += ISCSI_CRC_LEN;
2800                 tx_size += ISCSI_CRC_LEN;
2801                 pr_debug("Attaching CRC32C HeaderDigest"
2802                         " to NopIn 0x%08x\n", *header_digest);
2803         }
2804
2805         /*
2806          * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
2807          * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
2808          */
2809         if (cmd->buf_ptr_size) {
2810                 iov[niov].iov_base      = cmd->buf_ptr;
2811                 iov[niov++].iov_len     = cmd->buf_ptr_size;
2812                 tx_size += cmd->buf_ptr_size;
2813
2814                 pr_debug("Echoing back %u bytes of ping"
2815                         " data.\n", cmd->buf_ptr_size);
2816
2817                 padding = ((-cmd->buf_ptr_size) & 3);
2818                 if (padding != 0) {
2819                         iov[niov].iov_base = &cmd->pad_bytes;
2820                         iov[niov++].iov_len = padding;
2821                         tx_size += padding;
2822                         pr_debug("Attaching %u additional"
2823                                 " padding bytes.\n", padding);
2824                 }
2825                 if (conn->conn_ops->DataDigest) {
2826                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2827                                 cmd->buf_ptr, cmd->buf_ptr_size,
2828                                 padding, (u8 *)&cmd->pad_bytes,
2829                                 (u8 *)&cmd->data_crc);
2830
2831                         iov[niov].iov_base = &cmd->data_crc;
2832                         iov[niov++].iov_len = ISCSI_CRC_LEN;
2833                         tx_size += ISCSI_CRC_LEN;
2834                         pr_debug("Attached DataDigest for %u"
2835                                 " bytes of ping data, CRC 0x%08x\n",
2836                                 cmd->buf_ptr_size, cmd->data_crc);
2837                 }
2838         }
2839
2840         cmd->iov_misc_count = niov;
2841         cmd->tx_size = tx_size;
2842
2843         pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:"
2844                 " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag,
2845                 cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
2846
2847         return 0;
2848 }
2849
2850 int iscsit_send_r2t(
2851         struct iscsi_cmd *cmd,
2852         struct iscsi_conn *conn)
2853 {
2854         int tx_size = 0;
2855         struct iscsi_r2t *r2t;
2856         struct iscsi_r2t_rsp *hdr;
2857
2858         r2t = iscsit_get_r2t_from_list(cmd);
2859         if (!r2t)
2860                 return -1;
2861
2862         hdr                     = (struct iscsi_r2t_rsp *) cmd->pdu;
2863         memset(hdr, 0, ISCSI_HDR_LEN);
2864         hdr->opcode             = ISCSI_OP_R2T;
2865         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
2866         int_to_scsilun(cmd->se_cmd.orig_fe_lun,
2867                         (struct scsi_lun *)&hdr->lun);
2868         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
2869         spin_lock_bh(&conn->sess->ttt_lock);
2870         r2t->targ_xfer_tag      = conn->sess->targ_xfer_tag++;
2871         if (r2t->targ_xfer_tag == 0xFFFFFFFF)
2872                 r2t->targ_xfer_tag = conn->sess->targ_xfer_tag++;
2873         spin_unlock_bh(&conn->sess->ttt_lock);
2874         hdr->ttt                = cpu_to_be32(r2t->targ_xfer_tag);
2875         hdr->statsn             = cpu_to_be32(conn->stat_sn);
2876         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
2877         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
2878         hdr->r2tsn              = cpu_to_be32(r2t->r2t_sn);
2879         hdr->data_offset        = cpu_to_be32(r2t->offset);
2880         hdr->data_length        = cpu_to_be32(r2t->xfer_len);
2881
2882         cmd->iov_misc[0].iov_base       = cmd->pdu;
2883         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
2884         tx_size += ISCSI_HDR_LEN;
2885
2886         if (conn->conn_ops->HeaderDigest) {
2887                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
2888
2889                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
2890                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
2891                                 0, NULL, (u8 *)header_digest);
2892
2893                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
2894                 tx_size += ISCSI_CRC_LEN;
2895                 pr_debug("Attaching CRC32 HeaderDigest for R2T"
2896                         " PDU 0x%08x\n", *header_digest);
2897         }
2898
2899         pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
2900                 " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
2901                 (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
2902                 r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
2903                         r2t->offset, r2t->xfer_len, conn->cid);
2904
2905         cmd->iov_misc_count = 1;
2906         cmd->tx_size = tx_size;
2907
2908         spin_lock_bh(&cmd->r2t_lock);
2909         r2t->sent_r2t = 1;
2910         spin_unlock_bh(&cmd->r2t_lock);
2911
2912         return 0;
2913 }
2914
2915 /*
2916  *      type 0: Normal Operation.
2917  *      type 1: Called from Storage Transport.
2918  *      type 2: Called from iscsi_task_reassign_complete_write() for
2919  *              connection recovery.
2920  */
2921 int iscsit_build_r2ts_for_cmd(
2922         struct iscsi_cmd *cmd,
2923         struct iscsi_conn *conn,
2924         int type)
2925 {
2926         int first_r2t = 1;
2927         u32 offset = 0, xfer_len = 0;
2928
2929         spin_lock_bh(&cmd->r2t_lock);
2930         if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
2931                 spin_unlock_bh(&cmd->r2t_lock);
2932                 return 0;
2933         }
2934
2935         if (conn->sess->sess_ops->DataSequenceInOrder && (type != 2))
2936                 if (cmd->r2t_offset < cmd->write_data_done)
2937                         cmd->r2t_offset = cmd->write_data_done;
2938
2939         while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
2940                 if (conn->sess->sess_ops->DataSequenceInOrder) {
2941                         offset = cmd->r2t_offset;
2942
2943                         if (first_r2t && (type == 2)) {
2944                                 xfer_len = ((offset +
2945                                              (conn->sess->sess_ops->MaxBurstLength -
2946                                              cmd->next_burst_len) >
2947                                              cmd->data_length) ?
2948                                             (cmd->data_length - offset) :
2949                                             (conn->sess->sess_ops->MaxBurstLength -
2950                                              cmd->next_burst_len));
2951                         } else {
2952                                 xfer_len = ((offset +
2953                                              conn->sess->sess_ops->MaxBurstLength) >
2954                                              cmd->data_length) ?
2955                                              (cmd->data_length - offset) :
2956                                              conn->sess->sess_ops->MaxBurstLength;
2957                         }
2958                         cmd->r2t_offset += xfer_len;
2959
2960                         if (cmd->r2t_offset == cmd->data_length)
2961                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
2962                 } else {
2963                         struct iscsi_seq *seq;
2964
2965                         seq = iscsit_get_seq_holder_for_r2t(cmd);
2966                         if (!seq) {
2967                                 spin_unlock_bh(&cmd->r2t_lock);
2968                                 return -1;
2969                         }
2970
2971                         offset = seq->offset;
2972                         xfer_len = seq->xfer_len;
2973
2974                         if (cmd->seq_send_order == cmd->seq_count)
2975                                 cmd->cmd_flags |= ICF_SENT_LAST_R2T;
2976                 }
2977                 cmd->outstanding_r2ts++;
2978                 first_r2t = 0;
2979
2980                 if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
2981                         spin_unlock_bh(&cmd->r2t_lock);
2982                         return -1;
2983                 }
2984
2985                 if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
2986                         break;
2987         }
2988         spin_unlock_bh(&cmd->r2t_lock);
2989
2990         return 0;
2991 }
2992
2993 static int iscsit_send_status(
2994         struct iscsi_cmd *cmd,
2995         struct iscsi_conn *conn)
2996 {
2997         u8 iov_count = 0, recovery;
2998         u32 padding = 0, tx_size = 0;
2999         struct iscsi_scsi_rsp *hdr;
3000         struct kvec *iov;
3001
3002         recovery = (cmd->i_state != ISTATE_SEND_STATUS);
3003         if (!recovery)
3004                 cmd->stat_sn = conn->stat_sn++;
3005
3006         spin_lock_bh(&conn->sess->session_stats_lock);
3007         conn->sess->rsp_pdus++;
3008         spin_unlock_bh(&conn->sess->session_stats_lock);
3009
3010         hdr                     = (struct iscsi_scsi_rsp *) cmd->pdu;
3011         memset(hdr, 0, ISCSI_HDR_LEN);
3012         hdr->opcode             = ISCSI_OP_SCSI_CMD_RSP;
3013         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3014         if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
3015                 hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
3016                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3017         } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
3018                 hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
3019                 hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
3020         }
3021         hdr->response           = cmd->iscsi_response;
3022         hdr->cmd_status         = cmd->se_cmd.scsi_status;
3023         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3024         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3025
3026         iscsit_increment_maxcmdsn(cmd, conn->sess);
3027         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3028         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3029
3030         iov = &cmd->iov_misc[0];
3031         iov[iov_count].iov_base = cmd->pdu;
3032         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3033         tx_size += ISCSI_HDR_LEN;
3034
3035         /*
3036          * Attach SENSE DATA payload to iSCSI Response PDU
3037          */
3038         if (cmd->se_cmd.sense_buffer &&
3039            ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
3040             (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
3041                 padding         = -(cmd->se_cmd.scsi_sense_length) & 3;
3042                 hton24(hdr->dlength, cmd->se_cmd.scsi_sense_length);
3043                 iov[iov_count].iov_base = cmd->se_cmd.sense_buffer;
3044                 iov[iov_count++].iov_len =
3045                                 (cmd->se_cmd.scsi_sense_length + padding);
3046                 tx_size += cmd->se_cmd.scsi_sense_length;
3047
3048                 if (padding) {
3049                         memset(cmd->se_cmd.sense_buffer +
3050                                 cmd->se_cmd.scsi_sense_length, 0, padding);
3051                         tx_size += padding;
3052                         pr_debug("Adding %u bytes of padding to"
3053                                 " SENSE.\n", padding);
3054                 }
3055
3056                 if (conn->conn_ops->DataDigest) {
3057                         iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3058                                 cmd->se_cmd.sense_buffer,
3059                                 (cmd->se_cmd.scsi_sense_length + padding),
3060                                 0, NULL, (u8 *)&cmd->data_crc);
3061
3062                         iov[iov_count].iov_base    = &cmd->data_crc;
3063                         iov[iov_count++].iov_len     = ISCSI_CRC_LEN;
3064                         tx_size += ISCSI_CRC_LEN;
3065
3066                         pr_debug("Attaching CRC32 DataDigest for"
3067                                 " SENSE, %u bytes CRC 0x%08x\n",
3068                                 (cmd->se_cmd.scsi_sense_length + padding),
3069                                 cmd->data_crc);
3070                 }
3071
3072                 pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
3073                                 " Response PDU\n",
3074                                 cmd->se_cmd.scsi_sense_length);
3075         }
3076
3077         if (conn->conn_ops->HeaderDigest) {
3078                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3079
3080                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3081                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3082                                 0, NULL, (u8 *)header_digest);
3083
3084                 iov[0].iov_len += ISCSI_CRC_LEN;
3085                 tx_size += ISCSI_CRC_LEN;
3086                 pr_debug("Attaching CRC32 HeaderDigest for Response"
3087                                 " PDU 0x%08x\n", *header_digest);
3088         }
3089
3090         cmd->iov_misc_count = iov_count;
3091         cmd->tx_size = tx_size;
3092
3093         pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
3094                 " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
3095                 (!recovery) ? "" : "Recovery ", cmd->init_task_tag,
3096                 cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid);
3097
3098         return 0;
3099 }
3100
3101 static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
3102 {
3103         switch (se_tmr->response) {
3104         case TMR_FUNCTION_COMPLETE:
3105                 return ISCSI_TMF_RSP_COMPLETE;
3106         case TMR_TASK_DOES_NOT_EXIST:
3107                 return ISCSI_TMF_RSP_NO_TASK;
3108         case TMR_LUN_DOES_NOT_EXIST:
3109                 return ISCSI_TMF_RSP_NO_LUN;
3110         case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3111                 return ISCSI_TMF_RSP_NOT_SUPPORTED;
3112         case TMR_FUNCTION_AUTHORIZATION_FAILED:
3113                 return ISCSI_TMF_RSP_AUTH_FAILED;
3114         case TMR_FUNCTION_REJECTED:
3115         default:
3116                 return ISCSI_TMF_RSP_REJECTED;
3117         }
3118 }
3119
3120 static int iscsit_send_task_mgt_rsp(
3121         struct iscsi_cmd *cmd,
3122         struct iscsi_conn *conn)
3123 {
3124         struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
3125         struct iscsi_tm_rsp *hdr;
3126         u32 tx_size = 0;
3127
3128         hdr                     = (struct iscsi_tm_rsp *) cmd->pdu;
3129         memset(hdr, 0, ISCSI_HDR_LEN);
3130         hdr->opcode             = ISCSI_OP_SCSI_TMFUNC_RSP;
3131         hdr->flags              = ISCSI_FLAG_CMD_FINAL;
3132         hdr->response           = iscsit_convert_tcm_tmr_rsp(se_tmr);
3133         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3134         cmd->stat_sn            = conn->stat_sn++;
3135         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3136
3137         iscsit_increment_maxcmdsn(cmd, conn->sess);
3138         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3139         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3140
3141         cmd->iov_misc[0].iov_base       = cmd->pdu;
3142         cmd->iov_misc[0].iov_len        = ISCSI_HDR_LEN;
3143         tx_size += ISCSI_HDR_LEN;
3144
3145         if (conn->conn_ops->HeaderDigest) {
3146                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3147
3148                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3149                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3150                                 0, NULL, (u8 *)header_digest);
3151
3152                 cmd->iov_misc[0].iov_len += ISCSI_CRC_LEN;
3153                 tx_size += ISCSI_CRC_LEN;
3154                 pr_debug("Attaching CRC32 HeaderDigest for Task"
3155                         " Mgmt Response PDU 0x%08x\n", *header_digest);
3156         }
3157
3158         cmd->iov_misc_count = 1;
3159         cmd->tx_size = tx_size;
3160
3161         pr_debug("Built Task Management Response ITT: 0x%08x,"
3162                 " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
3163                 cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
3164
3165         return 0;
3166 }
3167
3168 static int iscsit_build_sendtargets_response(struct iscsi_cmd *cmd)
3169 {
3170         char *payload = NULL;
3171         struct iscsi_conn *conn = cmd->conn;
3172         struct iscsi_portal_group *tpg;
3173         struct iscsi_tiqn *tiqn;
3174         struct iscsi_tpg_np *tpg_np;
3175         int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
3176         unsigned char buf[256];
3177
3178         buffer_len = (conn->conn_ops->MaxRecvDataSegmentLength > 32768) ?
3179                         32768 : conn->conn_ops->MaxRecvDataSegmentLength;
3180
3181         memset(buf, 0, 256);
3182
3183         payload = kzalloc(buffer_len, GFP_KERNEL);
3184         if (!payload) {
3185                 pr_err("Unable to allocate memory for sendtargets"
3186                                 " response.\n");
3187                 return -ENOMEM;
3188         }
3189
3190         spin_lock(&tiqn_lock);
3191         list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
3192                 len = sprintf(buf, "TargetName=%s", tiqn->tiqn);
3193                 len += 1;
3194
3195                 if ((len + payload_len) > buffer_len) {
3196                         spin_unlock(&tiqn->tiqn_tpg_lock);
3197                         end_of_buf = 1;
3198                         goto eob;
3199                 }
3200                 memcpy((void *)payload + payload_len, buf, len);
3201                 payload_len += len;
3202
3203                 spin_lock(&tiqn->tiqn_tpg_lock);
3204                 list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
3205
3206                         spin_lock(&tpg->tpg_state_lock);
3207                         if ((tpg->tpg_state == TPG_STATE_FREE) ||
3208                             (tpg->tpg_state == TPG_STATE_INACTIVE)) {
3209                                 spin_unlock(&tpg->tpg_state_lock);
3210                                 continue;
3211                         }
3212                         spin_unlock(&tpg->tpg_state_lock);
3213
3214                         spin_lock(&tpg->tpg_np_lock);
3215                         list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
3216                                                 tpg_np_list) {
3217                                 len = sprintf(buf, "TargetAddress="
3218                                         "%s%s%s:%hu,%hu",
3219                                         (tpg_np->tpg_np->np_sockaddr.ss_family == AF_INET6) ?
3220                                         "[" : "", tpg_np->tpg_np->np_ip,
3221                                         (tpg_np->tpg_np->np_sockaddr.ss_family == AF_INET6) ?
3222                                         "]" : "", tpg_np->tpg_np->np_port,
3223                                         tpg->tpgt);
3224                                 len += 1;
3225
3226                                 if ((len + payload_len) > buffer_len) {
3227                                         spin_unlock(&tpg->tpg_np_lock);
3228                                         spin_unlock(&tiqn->tiqn_tpg_lock);
3229                                         end_of_buf = 1;
3230                                         goto eob;
3231                                 }
3232                                 memcpy((void *)payload + payload_len, buf, len);
3233                                 payload_len += len;
3234                         }
3235                         spin_unlock(&tpg->tpg_np_lock);
3236                 }
3237                 spin_unlock(&tiqn->tiqn_tpg_lock);
3238 eob:
3239                 if (end_of_buf)
3240                         break;
3241         }
3242         spin_unlock(&tiqn_lock);
3243
3244         cmd->buf_ptr = payload;
3245
3246         return payload_len;
3247 }
3248
3249 /*
3250  *      FIXME: Add support for F_BIT and C_BIT when the length is longer than
3251  *      MaxRecvDataSegmentLength.
3252  */
3253 static int iscsit_send_text_rsp(
3254         struct iscsi_cmd *cmd,
3255         struct iscsi_conn *conn)
3256 {
3257         struct iscsi_text_rsp *hdr;
3258         struct kvec *iov;
3259         u32 padding = 0, tx_size = 0;
3260         int text_length, iov_count = 0;
3261
3262         text_length = iscsit_build_sendtargets_response(cmd);
3263         if (text_length < 0)
3264                 return text_length;
3265
3266         padding = ((-text_length) & 3);
3267         if (padding != 0) {
3268                 memset(cmd->buf_ptr + text_length, 0, padding);
3269                 pr_debug("Attaching %u additional bytes for"
3270                         " padding.\n", padding);
3271         }
3272
3273         hdr                     = (struct iscsi_text_rsp *) cmd->pdu;
3274         memset(hdr, 0, ISCSI_HDR_LEN);
3275         hdr->opcode             = ISCSI_OP_TEXT_RSP;
3276         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3277         hton24(hdr->dlength, text_length);
3278         hdr->itt                = cpu_to_be32(cmd->init_task_tag);
3279         hdr->ttt                = cpu_to_be32(cmd->targ_xfer_tag);
3280         cmd->stat_sn            = conn->stat_sn++;
3281         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3282
3283         iscsit_increment_maxcmdsn(cmd, conn->sess);
3284         hdr->exp_cmdsn          = cpu_to_be32(conn->sess->exp_cmd_sn);
3285         hdr->max_cmdsn          = cpu_to_be32(conn->sess->max_cmd_sn);
3286
3287         iov = &cmd->iov_misc[0];
3288
3289         iov[iov_count].iov_base = cmd->pdu;
3290         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3291         iov[iov_count].iov_base = cmd->buf_ptr;
3292         iov[iov_count++].iov_len = text_length + padding;
3293
3294         tx_size += (ISCSI_HDR_LEN + text_length + padding);
3295
3296         if (conn->conn_ops->HeaderDigest) {
3297                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3298
3299                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3300                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3301                                 0, NULL, (u8 *)header_digest);
3302
3303                 iov[0].iov_len += ISCSI_CRC_LEN;
3304                 tx_size += ISCSI_CRC_LEN;
3305                 pr_debug("Attaching CRC32 HeaderDigest for"
3306                         " Text Response PDU 0x%08x\n", *header_digest);
3307         }
3308
3309         if (conn->conn_ops->DataDigest) {
3310                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3311                                 cmd->buf_ptr, (text_length + padding),
3312                                 0, NULL, (u8 *)&cmd->data_crc);
3313
3314                 iov[iov_count].iov_base = &cmd->data_crc;
3315                 iov[iov_count++].iov_len = ISCSI_CRC_LEN;
3316                 tx_size += ISCSI_CRC_LEN;
3317
3318                 pr_debug("Attaching DataDigest for %u bytes of text"
3319                         " data, CRC 0x%08x\n", (text_length + padding),
3320                         cmd->data_crc);
3321         }
3322
3323         cmd->iov_misc_count = iov_count;
3324         cmd->tx_size = tx_size;
3325
3326         pr_debug("Built Text Response: ITT: 0x%08x, StatSN: 0x%08x,"
3327                 " Length: %u, CID: %hu\n", cmd->init_task_tag, cmd->stat_sn,
3328                         text_length, conn->cid);
3329         return 0;
3330 }
3331
3332 static int iscsit_send_reject(
3333         struct iscsi_cmd *cmd,
3334         struct iscsi_conn *conn)
3335 {
3336         u32 iov_count = 0, tx_size = 0;
3337         struct iscsi_reject *hdr;
3338         struct kvec *iov;
3339
3340         hdr                     = (struct iscsi_reject *) cmd->pdu;
3341         hdr->opcode             = ISCSI_OP_REJECT;
3342         hdr->flags              |= ISCSI_FLAG_CMD_FINAL;
3343         hton24(hdr->dlength, ISCSI_HDR_LEN);
3344         cmd->stat_sn            = conn->stat_sn++;
3345         hdr->statsn             = cpu_to_be32(cmd->stat_sn);
3346         hdr->exp_cmdsn  = cpu_to_be32(conn->sess->exp_cmd_sn);
3347         hdr->max_cmdsn  = cpu_to_be32(conn->sess->max_cmd_sn);
3348
3349         iov = &cmd->iov_misc[0];
3350
3351         iov[iov_count].iov_base = cmd->pdu;
3352         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3353         iov[iov_count].iov_base = cmd->buf_ptr;
3354         iov[iov_count++].iov_len = ISCSI_HDR_LEN;
3355
3356         tx_size = (ISCSI_HDR_LEN + ISCSI_HDR_LEN);
3357
3358         if (conn->conn_ops->HeaderDigest) {
3359                 u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
3360
3361                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3362                                 (unsigned char *)hdr, ISCSI_HDR_LEN,
3363                                 0, NULL, (u8 *)header_digest);
3364
3365                 iov[0].iov_len += ISCSI_CRC_LEN;
3366                 tx_size += ISCSI_CRC_LEN;
3367                 pr_debug("Attaching CRC32 HeaderDigest for"
3368                         " REJECT PDU 0x%08x\n", *header_digest);
3369         }
3370
3371         if (conn->conn_ops->DataDigest) {
3372                 iscsit_do_crypto_hash_buf(&conn->conn_tx_hash,
3373                                 (unsigned char *)cmd->buf_ptr, ISCSI_HDR_LEN,
3374                                 0, NULL, (u8 *)&cmd->data_crc);
3375
3376                 iov[iov_count].iov_base = &cmd->data_crc;
3377                 iov[iov_count++].iov_len  = ISCSI_CRC_LEN;
3378                 tx_size += ISCSI_CRC_LEN;
3379                 pr_debug("Attaching CRC32 DataDigest for REJECT"
3380                                 " PDU 0x%08x\n", cmd->data_crc);
3381         }
3382
3383         cmd->iov_misc_count = iov_count;
3384         cmd->tx_size = tx_size;
3385
3386         pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
3387                 " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
3388
3389         return 0;
3390 }
3391
3392 static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
3393 {
3394         if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
3395             (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
3396                 wait_for_completion_interruptible_timeout(
3397                                         &conn->tx_half_close_comp,
3398                                         ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
3399         }
3400 }
3401
3402 #ifdef CONFIG_SMP
3403
3404 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3405 {
3406         struct iscsi_thread_set *ts = conn->thread_set;
3407         int ord, cpu;
3408         /*
3409          * thread_id is assigned from iscsit_global->ts_bitmap from
3410          * within iscsi_thread_set.c:iscsi_allocate_thread_sets()
3411          *
3412          * Here we use thread_id to determine which CPU that this
3413          * iSCSI connection's iscsi_thread_set will be scheduled to
3414          * execute upon.
3415          */
3416         ord = ts->thread_id % cpumask_weight(cpu_online_mask);
3417 #if 0
3418         pr_debug(">>>>>>>>>>>>>>>>>>>> Generated ord: %d from"
3419                         " thread_id: %d\n", ord, ts->thread_id);
3420 #endif
3421         for_each_online_cpu(cpu) {
3422                 if (ord-- == 0) {
3423                         cpumask_set_cpu(cpu, conn->conn_cpumask);
3424                         return;
3425                 }
3426         }
3427         /*
3428          * This should never be reached..
3429          */
3430         dump_stack();
3431         cpumask_setall(conn->conn_cpumask);
3432 }
3433
3434 static inline void iscsit_thread_check_cpumask(
3435         struct iscsi_conn *conn,
3436         struct task_struct *p,
3437         int mode)
3438 {
3439         char buf[128];
3440         /*
3441          * mode == 1 signals iscsi_target_tx_thread() usage.
3442          * mode == 0 signals iscsi_target_rx_thread() usage.
3443          */
3444         if (mode == 1) {
3445                 if (!conn->conn_tx_reset_cpumask)
3446                         return;
3447                 conn->conn_tx_reset_cpumask = 0;
3448         } else {
3449                 if (!conn->conn_rx_reset_cpumask)
3450                         return;
3451                 conn->conn_rx_reset_cpumask = 0;
3452         }
3453         /*
3454          * Update the CPU mask for this single kthread so that
3455          * both TX and RX kthreads are scheduled to run on the
3456          * same CPU.
3457          */
3458         memset(buf, 0, 128);
3459         cpumask_scnprintf(buf, 128, conn->conn_cpumask);
3460 #if 0
3461         pr_debug(">>>>>>>>>>>>>> Calling set_cpus_allowed_ptr():"
3462                         " %s for %s\n", buf, p->comm);
3463 #endif
3464         set_cpus_allowed_ptr(p, conn->conn_cpumask);
3465 }
3466
3467 #else
3468
3469 void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
3470 {
3471         return;
3472 }
3473
3474 #define iscsit_thread_check_cpumask(X, Y, Z) ({})
3475 #endif /* CONFIG_SMP */
3476
3477 int iscsi_target_tx_thread(void *arg)
3478 {
3479         u8 state;
3480         int eodr = 0;
3481         int ret = 0;
3482         int sent_status = 0;
3483         int use_misc = 0;
3484         int map_sg = 0;
3485         struct iscsi_cmd *cmd = NULL;
3486         struct iscsi_conn *conn;
3487         struct iscsi_queue_req *qr = NULL;
3488         struct se_cmd *se_cmd;
3489         struct iscsi_thread_set *ts = (struct iscsi_thread_set *)arg;
3490         /*
3491          * Allow ourselves to be interrupted by SIGINT so that a
3492          * connection recovery / failure event can be triggered externally.
3493          */
3494         allow_signal(SIGINT);
3495
3496 restart:
3497         conn = iscsi_tx_thread_pre_handler(ts);
3498         if (!conn)
3499                 goto out;
3500
3501         eodr = map_sg = ret = sent_status = use_misc = 0;
3502
3503         while (!kthread_should_stop()) {
3504                 /*
3505                  * Ensure that both TX and RX per connection kthreads
3506                  * are scheduled to run on the same CPU.
3507                  */
3508                 iscsit_thread_check_cpumask(conn, current, 1);
3509
3510                 schedule_timeout_interruptible(MAX_SCHEDULE_TIMEOUT);
3511
3512                 if ((ts->status == ISCSI_THREAD_SET_RESET) ||
3513                      signal_pending(current))
3514                         goto transport_err;
3515
3516 get_immediate:
3517                 qr = iscsit_get_cmd_from_immediate_queue(conn);
3518                 if (qr) {
3519                         atomic_set(&conn->check_immediate_queue, 0);
3520                         cmd = qr->cmd;
3521                         state = qr->state;
3522                         kmem_cache_free(lio_qr_cache, qr);
3523
3524                         spin_lock_bh(&cmd->istate_lock);
3525                         switch (state) {
3526                         case ISTATE_SEND_R2T:
3527                                 spin_unlock_bh(&cmd->istate_lock);
3528                                 ret = iscsit_send_r2t(cmd, conn);
3529                                 break;
3530                         case ISTATE_REMOVE:
3531                                 spin_unlock_bh(&cmd->istate_lock);
3532
3533                                 if (cmd->data_direction == DMA_TO_DEVICE)
3534                                         iscsit_stop_dataout_timer(cmd);
3535
3536                                 spin_lock_bh(&conn->cmd_lock);
3537                                 list_del(&cmd->i_list);
3538                                 spin_unlock_bh(&conn->cmd_lock);
3539
3540                                 iscsit_free_cmd(cmd);
3541                                 goto get_immediate;
3542                         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3543                                 spin_unlock_bh(&cmd->istate_lock);
3544                                 iscsit_mod_nopin_response_timer(conn);
3545                                 ret = iscsit_send_unsolicited_nopin(cmd,
3546                                                 conn, 1);
3547                                 break;
3548                         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3549                                 spin_unlock_bh(&cmd->istate_lock);
3550                                 ret = iscsit_send_unsolicited_nopin(cmd,
3551                                                 conn, 0);
3552                                 break;
3553                         default:
3554                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3555                                 " 0x%08x, i_state: %d on CID: %hu\n",
3556                                 cmd->iscsi_opcode, cmd->init_task_tag, state,
3557                                 conn->cid);
3558                                 spin_unlock_bh(&cmd->istate_lock);
3559                                 goto transport_err;
3560                         }
3561                         if (ret < 0) {
3562                                 conn->tx_immediate_queue = 0;
3563                                 goto transport_err;
3564                         }
3565
3566                         if (iscsit_send_tx_data(cmd, conn, 1) < 0) {
3567                                 conn->tx_immediate_queue = 0;
3568                                 iscsit_tx_thread_wait_for_tcp(conn);
3569                                 goto transport_err;
3570                         }
3571
3572                         spin_lock_bh(&cmd->istate_lock);
3573                         switch (state) {
3574                         case ISTATE_SEND_R2T:
3575                                 spin_unlock_bh(&cmd->istate_lock);
3576                                 spin_lock_bh(&cmd->dataout_timeout_lock);
3577                                 iscsit_start_dataout_timer(cmd, conn);
3578                                 spin_unlock_bh(&cmd->dataout_timeout_lock);
3579                                 break;
3580                         case ISTATE_SEND_NOPIN_WANT_RESPONSE:
3581                                 cmd->i_state = ISTATE_SENT_NOPIN_WANT_RESPONSE;
3582                                 spin_unlock_bh(&cmd->istate_lock);
3583                                 break;
3584                         case ISTATE_SEND_NOPIN_NO_RESPONSE:
3585                                 cmd->i_state = ISTATE_SENT_STATUS;
3586                                 spin_unlock_bh(&cmd->istate_lock);
3587                                 break;
3588                         default:
3589                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3590                                         " 0x%08x, i_state: %d on CID: %hu\n",
3591                                         cmd->iscsi_opcode, cmd->init_task_tag,
3592                                         state, conn->cid);
3593                                 spin_unlock_bh(&cmd->istate_lock);
3594                                 goto transport_err;
3595                         }
3596                         goto get_immediate;
3597                 } else
3598                         conn->tx_immediate_queue = 0;
3599
3600 get_response:
3601                 qr = iscsit_get_cmd_from_response_queue(conn);
3602                 if (qr) {
3603                         cmd = qr->cmd;
3604                         state = qr->state;
3605                         kmem_cache_free(lio_qr_cache, qr);
3606
3607                         spin_lock_bh(&cmd->istate_lock);
3608 check_rsp_state:
3609                         switch (state) {
3610                         case ISTATE_SEND_DATAIN:
3611                                 spin_unlock_bh(&cmd->istate_lock);
3612                                 ret = iscsit_send_data_in(cmd, conn,
3613                                                           &eodr);
3614                                 map_sg = 1;
3615                                 break;
3616                         case ISTATE_SEND_STATUS:
3617                         case ISTATE_SEND_STATUS_RECOVERY:
3618                                 spin_unlock_bh(&cmd->istate_lock);
3619                                 use_misc = 1;
3620                                 ret = iscsit_send_status(cmd, conn);
3621                                 break;
3622                         case ISTATE_SEND_LOGOUTRSP:
3623                                 spin_unlock_bh(&cmd->istate_lock);
3624                                 use_misc = 1;
3625                                 ret = iscsit_send_logout_response(cmd, conn);
3626                                 break;
3627                         case ISTATE_SEND_ASYNCMSG:
3628                                 spin_unlock_bh(&cmd->istate_lock);
3629                                 use_misc = 1;
3630                                 ret = iscsit_send_conn_drop_async_message(
3631                                                 cmd, conn);
3632                                 break;
3633                         case ISTATE_SEND_NOPIN:
3634                                 spin_unlock_bh(&cmd->istate_lock);
3635                                 use_misc = 1;
3636                                 ret = iscsit_send_nopin_response(cmd, conn);
3637                                 break;
3638                         case ISTATE_SEND_REJECT:
3639                                 spin_unlock_bh(&cmd->istate_lock);
3640                                 use_misc = 1;
3641                                 ret = iscsit_send_reject(cmd, conn);
3642                                 break;
3643                         case ISTATE_SEND_TASKMGTRSP:
3644                                 spin_unlock_bh(&cmd->istate_lock);
3645                                 use_misc = 1;
3646                                 ret = iscsit_send_task_mgt_rsp(cmd, conn);
3647                                 if (ret != 0)
3648                                         break;
3649                                 ret = iscsit_tmr_post_handler(cmd, conn);
3650                                 if (ret != 0)
3651                                         iscsit_fall_back_to_erl0(conn->sess);
3652                                 break;
3653                         case ISTATE_SEND_TEXTRSP:
3654                                 spin_unlock_bh(&cmd->istate_lock);
3655                                 use_misc = 1;
3656                                 ret = iscsit_send_text_rsp(cmd, conn);
3657                                 break;
3658                         default:
3659                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3660                                         " 0x%08x, i_state: %d on CID: %hu\n",
3661                                         cmd->iscsi_opcode, cmd->init_task_tag,
3662                                         state, conn->cid);
3663                                 spin_unlock_bh(&cmd->istate_lock);
3664                                 goto transport_err;
3665                         }
3666                         if (ret < 0) {
3667                                 conn->tx_response_queue = 0;
3668                                 goto transport_err;
3669                         }
3670
3671                         se_cmd = &cmd->se_cmd;
3672
3673                         if (map_sg && !conn->conn_ops->IFMarker) {
3674                                 if (iscsit_fe_sendpage_sg(cmd, conn) < 0) {
3675                                         conn->tx_response_queue = 0;
3676                                         iscsit_tx_thread_wait_for_tcp(conn);
3677                                         iscsit_unmap_iovec(cmd);
3678                                         goto transport_err;
3679                                 }
3680                         } else {
3681                                 if (iscsit_send_tx_data(cmd, conn, use_misc) < 0) {
3682                                         conn->tx_response_queue = 0;
3683                                         iscsit_tx_thread_wait_for_tcp(conn);
3684                                         iscsit_unmap_iovec(cmd);
3685                                         goto transport_err;
3686                                 }
3687                         }
3688                         map_sg = 0;
3689                         iscsit_unmap_iovec(cmd);
3690
3691                         spin_lock_bh(&cmd->istate_lock);
3692                         switch (state) {
3693                         case ISTATE_SEND_DATAIN:
3694                                 if (!eodr)
3695                                         goto check_rsp_state;
3696
3697                                 if (eodr == 1) {
3698                                         cmd->i_state = ISTATE_SENT_LAST_DATAIN;
3699                                         sent_status = 1;
3700                                         eodr = use_misc = 0;
3701                                 } else if (eodr == 2) {
3702                                         cmd->i_state = state =
3703                                                         ISTATE_SEND_STATUS;
3704                                         sent_status = 0;
3705                                         eodr = use_misc = 0;
3706                                         goto check_rsp_state;
3707                                 }
3708                                 break;
3709                         case ISTATE_SEND_STATUS:
3710                                 use_misc = 0;
3711                                 sent_status = 1;
3712                                 break;
3713                         case ISTATE_SEND_ASYNCMSG:
3714                         case ISTATE_SEND_NOPIN:
3715                         case ISTATE_SEND_STATUS_RECOVERY:
3716                         case ISTATE_SEND_TEXTRSP:
3717                                 use_misc = 0;
3718                                 sent_status = 1;
3719                                 break;
3720                         case ISTATE_SEND_REJECT:
3721                                 use_misc = 0;
3722                                 if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) {
3723                                         cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN;
3724                                         spin_unlock_bh(&cmd->istate_lock);
3725                                         complete(&cmd->reject_comp);
3726                                         goto transport_err;
3727                                 }
3728                                 complete(&cmd->reject_comp);
3729                                 break;
3730                         case ISTATE_SEND_TASKMGTRSP:
3731                                 use_misc = 0;
3732                                 sent_status = 1;
3733                                 break;
3734                         case ISTATE_SEND_LOGOUTRSP:
3735                                 spin_unlock_bh(&cmd->istate_lock);
3736                                 if (!iscsit_logout_post_handler(cmd, conn))
3737                                         goto restart;
3738                                 spin_lock_bh(&cmd->istate_lock);
3739                                 use_misc = 0;
3740                                 sent_status = 1;
3741                                 break;
3742                         default:
3743                                 pr_err("Unknown Opcode: 0x%02x ITT:"
3744                                         " 0x%08x, i_state: %d on CID: %hu\n",
3745                                         cmd->iscsi_opcode, cmd->init_task_tag,
3746                                         cmd->i_state, conn->cid);
3747                                 spin_unlock_bh(&cmd->istate_lock);
3748                                 goto transport_err;
3749                         }
3750
3751                         if (sent_status) {
3752                                 cmd->i_state = ISTATE_SENT_STATUS;
3753                                 sent_status = 0;
3754                         }
3755                         spin_unlock_bh(&cmd->istate_lock);
3756
3757                         if (atomic_read(&conn->check_immediate_queue))
3758                                 goto get_immediate;
3759
3760                         goto get_response;
3761                 } else
3762                         conn->tx_response_queue = 0;
3763         }
3764
3765 transport_err:
3766         iscsit_take_action_for_connection_exit(conn);
3767         goto restart;
3768 out:
3769         return 0;
3770 }
3771
3772 int iscsi_target_rx_thread(void *arg)
3773 {
3774         int ret;
3775         u8 buffer[ISCSI_HDR_LEN], opcode;
3776         u32 checksum = 0, digest = 0;
3777         struct iscsi_conn *conn = NULL;
3778         struct iscsi_thread_set *ts = (struct iscsi_thread_set *)arg;
3779         struct kvec iov;
3780         /*
3781          * Allow ourselves to be interrupted by SIGINT so that a
3782          * connection recovery / failure event can be triggered externally.
3783          */
3784         allow_signal(SIGINT);
3785
3786 restart:
3787         conn = iscsi_rx_thread_pre_handler(ts);
3788         if (!conn)
3789                 goto out;
3790
3791         while (!kthread_should_stop()) {
3792                 /*
3793                  * Ensure that both TX and RX per connection kthreads
3794                  * are scheduled to run on the same CPU.
3795                  */
3796                 iscsit_thread_check_cpumask(conn, current, 0);
3797
3798                 memset(buffer, 0, ISCSI_HDR_LEN);
3799                 memset(&iov, 0, sizeof(struct kvec));
3800
3801                 iov.iov_base    = buffer;
3802                 iov.iov_len     = ISCSI_HDR_LEN;
3803
3804                 ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
3805                 if (ret != ISCSI_HDR_LEN) {
3806                         iscsit_rx_thread_wait_for_tcp(conn);
3807                         goto transport_err;
3808                 }
3809
3810                 /*
3811                  * Set conn->bad_hdr for use with REJECT PDUs.
3812                  */
3813                 memcpy(&conn->bad_hdr, &buffer, ISCSI_HDR_LEN);
3814
3815                 if (conn->conn_ops->HeaderDigest) {
3816                         iov.iov_base    = &digest;
3817                         iov.iov_len     = ISCSI_CRC_LEN;
3818
3819                         ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
3820                         if (ret != ISCSI_CRC_LEN) {
3821                                 iscsit_rx_thread_wait_for_tcp(conn);
3822                                 goto transport_err;
3823                         }
3824
3825                         iscsit_do_crypto_hash_buf(&conn->conn_rx_hash,
3826                                         buffer, ISCSI_HDR_LEN,
3827                                         0, NULL, (u8 *)&checksum);
3828
3829                         if (digest != checksum) {
3830                                 pr_err("HeaderDigest CRC32C failed,"
3831                                         " received 0x%08x, computed 0x%08x\n",
3832                                         digest, checksum);
3833                                 /*
3834                                  * Set the PDU to 0xff so it will intentionally
3835                                  * hit default in the switch below.
3836                                  */
3837                                 memset(buffer, 0xff, ISCSI_HDR_LEN);
3838                                 spin_lock_bh(&conn->sess->session_stats_lock);
3839                                 conn->sess->conn_digest_errors++;
3840                                 spin_unlock_bh(&conn->sess->session_stats_lock);
3841                         } else {
3842                                 pr_debug("Got HeaderDigest CRC32C"
3843                                                 " 0x%08x\n", checksum);
3844                         }
3845                 }
3846
3847                 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
3848                         goto transport_err;
3849
3850                 opcode = buffer[0] & ISCSI_OPCODE_MASK;
3851
3852                 if (conn->sess->sess_ops->SessionType &&
3853                    ((!(opcode & ISCSI_OP_TEXT)) ||
3854                     (!(opcode & ISCSI_OP_LOGOUT)))) {
3855                         pr_err("Received illegal iSCSI Opcode: 0x%02x"
3856                         " while in Discovery Session, rejecting.\n", opcode);
3857                         iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1,
3858                                         buffer, conn);
3859                         goto transport_err;
3860                 }
3861
3862                 switch (opcode) {
3863                 case ISCSI_OP_SCSI_CMD:
3864                         if (iscsit_handle_scsi_cmd(conn, buffer) < 0)
3865                                 goto transport_err;
3866                         break;
3867                 case ISCSI_OP_SCSI_DATA_OUT:
3868                         if (iscsit_handle_data_out(conn, buffer) < 0)
3869                                 goto transport_err;
3870                         break;
3871                 case ISCSI_OP_NOOP_OUT:
3872                         if (iscsit_handle_nop_out(conn, buffer) < 0)
3873                                 goto transport_err;
3874                         break;
3875                 case ISCSI_OP_SCSI_TMFUNC:
3876                         if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0)
3877                                 goto transport_err;
3878                         break;
3879                 case ISCSI_OP_TEXT:
3880                         if (iscsit_handle_text_cmd(conn, buffer) < 0)
3881                                 goto transport_err;
3882                         break;
3883                 case ISCSI_OP_LOGOUT:
3884                         ret = iscsit_handle_logout_cmd(conn, buffer);
3885                         if (ret > 0) {
3886                                 wait_for_completion_timeout(&conn->conn_logout_comp,
3887                                                 SECONDS_FOR_LOGOUT_COMP * HZ);
3888                                 goto transport_err;
3889                         } else if (ret < 0)
3890                                 goto transport_err;
3891                         break;
3892                 case ISCSI_OP_SNACK:
3893                         if (iscsit_handle_snack(conn, buffer) < 0)
3894                                 goto transport_err;
3895                         break;
3896                 default:
3897                         pr_err("Got unknown iSCSI OpCode: 0x%02x\n",
3898                                         opcode);
3899                         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
3900                                 pr_err("Cannot recover from unknown"
3901                                 " opcode while ERL=0, closing iSCSI connection"
3902                                 ".\n");
3903                                 goto transport_err;
3904                         }
3905                         if (!conn->conn_ops->OFMarker) {
3906                                 pr_err("Unable to recover from unknown"
3907                                 " opcode while OFMarker=No, closing iSCSI"
3908                                         " connection.\n");
3909                                 goto transport_err;
3910                         }
3911                         if (iscsit_recover_from_unknown_opcode(conn) < 0) {
3912                                 pr_err("Unable to recover from unknown"
3913                                         " opcode, closing iSCSI connection.\n");
3914                                 goto transport_err;
3915                         }
3916                         break;
3917                 }
3918         }
3919
3920 transport_err:
3921         if (!signal_pending(current))
3922                 atomic_set(&conn->transport_failed, 1);
3923         iscsit_take_action_for_connection_exit(conn);
3924         goto restart;
3925 out:
3926         return 0;
3927 }
3928
3929 static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
3930 {
3931         struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
3932         struct iscsi_session *sess = conn->sess;
3933         /*
3934          * We expect this function to only ever be called from either RX or TX
3935          * thread context via iscsit_close_connection() once the other context
3936          * has been reset -> returned sleeping pre-handler state.
3937          */
3938         spin_lock_bh(&conn->cmd_lock);
3939         list_for_each_entry_safe(cmd, cmd_tmp, &conn->conn_cmd_list, i_list) {
3940
3941                 list_del(&cmd->i_list);
3942                 spin_unlock_bh(&conn->cmd_lock);
3943
3944                 iscsit_increment_maxcmdsn(cmd, sess);
3945
3946                 iscsit_free_cmd(cmd);
3947
3948                 spin_lock_bh(&conn->cmd_lock);
3949         }
3950         spin_unlock_bh(&conn->cmd_lock);
3951 }
3952
3953 static void iscsit_stop_timers_for_cmds(
3954         struct iscsi_conn *conn)
3955 {
3956         struct iscsi_cmd *cmd;
3957
3958         spin_lock_bh(&conn->cmd_lock);
3959         list_for_each_entry(cmd, &conn->conn_cmd_list, i_list) {
3960                 if (cmd->data_direction == DMA_TO_DEVICE)
3961                         iscsit_stop_dataout_timer(cmd);
3962         }
3963         spin_unlock_bh(&conn->cmd_lock);
3964 }
3965
3966 int iscsit_close_connection(
3967         struct iscsi_conn *conn)
3968 {
3969         int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
3970         struct iscsi_session    *sess = conn->sess;
3971
3972         pr_debug("Closing iSCSI connection CID %hu on SID:"
3973                 " %u\n", conn->cid, sess->sid);
3974         /*
3975          * Always up conn_logout_comp just in case the RX Thread is sleeping
3976          * and the logout response never got sent because the connection
3977          * failed.
3978          */
3979         complete(&conn->conn_logout_comp);
3980
3981         iscsi_release_thread_set(conn);
3982
3983         iscsit_stop_timers_for_cmds(conn);
3984         iscsit_stop_nopin_response_timer(conn);
3985         iscsit_stop_nopin_timer(conn);
3986         iscsit_free_queue_reqs_for_conn(conn);
3987
3988         /*
3989          * During Connection recovery drop unacknowledged out of order
3990          * commands for this connection, and prepare the other commands
3991          * for realligence.
3992          *
3993          * During normal operation clear the out of order commands (but
3994          * do not free the struct iscsi_ooo_cmdsn's) and release all
3995          * struct iscsi_cmds.
3996          */
3997         if (atomic_read(&conn->connection_recovery)) {
3998                 iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
3999                 iscsit_prepare_cmds_for_realligance(conn);
4000         } else {
4001                 iscsit_clear_ooo_cmdsns_for_conn(conn);
4002                 iscsit_release_commands_from_conn(conn);
4003         }
4004
4005         /*
4006          * Handle decrementing session or connection usage count if
4007          * a logout response was not able to be sent because the
4008          * connection failed.  Fall back to Session Recovery here.
4009          */
4010         if (atomic_read(&conn->conn_logout_remove)) {
4011                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
4012                         iscsit_dec_conn_usage_count(conn);
4013                         iscsit_dec_session_usage_count(sess);
4014                 }
4015                 if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
4016                         iscsit_dec_conn_usage_count(conn);
4017
4018                 atomic_set(&conn->conn_logout_remove, 0);
4019                 atomic_set(&sess->session_reinstatement, 0);
4020                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4021         }
4022
4023         spin_lock_bh(&sess->conn_lock);
4024         list_del(&conn->conn_list);
4025
4026         /*
4027          * Attempt to let the Initiator know this connection failed by
4028          * sending an Connection Dropped Async Message on another
4029          * active connection.
4030          */
4031         if (atomic_read(&conn->connection_recovery))
4032                 iscsit_build_conn_drop_async_message(conn);
4033
4034         spin_unlock_bh(&sess->conn_lock);
4035
4036         /*
4037          * If connection reinstatement is being performed on this connection,
4038          * up the connection reinstatement semaphore that is being blocked on
4039          * in iscsit_cause_connection_reinstatement().
4040          */
4041         spin_lock_bh(&conn->state_lock);
4042         if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
4043                 spin_unlock_bh(&conn->state_lock);
4044                 complete(&conn->conn_wait_comp);
4045                 wait_for_completion(&conn->conn_post_wait_comp);
4046                 spin_lock_bh(&conn->state_lock);
4047         }
4048
4049         /*
4050          * If connection reinstatement is being performed on this connection
4051          * by receiving a REMOVECONNFORRECOVERY logout request, up the
4052          * connection wait rcfr semaphore that is being blocked on
4053          * an iscsit_connection_reinstatement_rcfr().
4054          */
4055         if (atomic_read(&conn->connection_wait_rcfr)) {
4056                 spin_unlock_bh(&conn->state_lock);
4057                 complete(&conn->conn_wait_rcfr_comp);
4058                 wait_for_completion(&conn->conn_post_wait_comp);
4059                 spin_lock_bh(&conn->state_lock);
4060         }
4061         atomic_set(&conn->connection_reinstatement, 1);
4062         spin_unlock_bh(&conn->state_lock);
4063
4064         /*
4065          * If any other processes are accessing this connection pointer we
4066          * must wait until they have completed.
4067          */
4068         iscsit_check_conn_usage_count(conn);
4069
4070         if (conn->conn_rx_hash.tfm)
4071                 crypto_free_hash(conn->conn_rx_hash.tfm);
4072         if (conn->conn_tx_hash.tfm)
4073                 crypto_free_hash(conn->conn_tx_hash.tfm);
4074
4075         if (conn->conn_cpumask)
4076                 free_cpumask_var(conn->conn_cpumask);
4077
4078         kfree(conn->conn_ops);
4079         conn->conn_ops = NULL;
4080
4081         if (conn->sock) {
4082                 if (conn->conn_flags & CONNFLAG_SCTP_STRUCT_FILE) {
4083                         kfree(conn->sock->file);
4084                         conn->sock->file = NULL;
4085                 }
4086                 sock_release(conn->sock);
4087         }
4088         conn->thread_set = NULL;
4089
4090         pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
4091         conn->conn_state = TARG_CONN_STATE_FREE;
4092         kfree(conn);
4093
4094         spin_lock_bh(&sess->conn_lock);
4095         atomic_dec(&sess->nconn);
4096         pr_debug("Decremented iSCSI connection count to %hu from node:"
4097                 " %s\n", atomic_read(&sess->nconn),
4098                 sess->sess_ops->InitiatorName);
4099         /*
4100          * Make sure that if one connection fails in an non ERL=2 iSCSI
4101          * Session that they all fail.
4102          */
4103         if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
4104              !atomic_read(&sess->session_logout))
4105                 atomic_set(&sess->session_fall_back_to_erl0, 1);
4106
4107         /*
4108          * If this was not the last connection in the session, and we are
4109          * performing session reinstatement or falling back to ERL=0, call
4110          * iscsit_stop_session() without sleeping to shutdown the other
4111          * active connections.
4112          */
4113         if (atomic_read(&sess->nconn)) {
4114                 if (!atomic_read(&sess->session_reinstatement) &&
4115                     !atomic_read(&sess->session_fall_back_to_erl0)) {
4116                         spin_unlock_bh(&sess->conn_lock);
4117                         return 0;
4118                 }
4119                 if (!atomic_read(&sess->session_stop_active)) {
4120                         atomic_set(&sess->session_stop_active, 1);
4121                         spin_unlock_bh(&sess->conn_lock);
4122                         iscsit_stop_session(sess, 0, 0);
4123                         return 0;
4124                 }
4125                 spin_unlock_bh(&sess->conn_lock);
4126                 return 0;
4127         }
4128
4129         /*
4130          * If this was the last connection in the session and one of the
4131          * following is occurring:
4132          *
4133          * Session Reinstatement is not being performed, and are falling back
4134          * to ERL=0 call iscsit_close_session().
4135          *
4136          * Session Logout was requested.  iscsit_close_session() will be called
4137          * elsewhere.
4138          *
4139          * Session Continuation is not being performed, start the Time2Retain
4140          * handler and check if sleep_on_sess_wait_sem is active.
4141          */
4142         if (!atomic_read(&sess->session_reinstatement) &&
4143              atomic_read(&sess->session_fall_back_to_erl0)) {
4144                 spin_unlock_bh(&sess->conn_lock);
4145                 iscsit_close_session(sess);
4146
4147                 return 0;
4148         } else if (atomic_read(&sess->session_logout)) {
4149                 pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4150                 sess->session_state = TARG_SESS_STATE_FREE;
4151                 spin_unlock_bh(&sess->conn_lock);
4152
4153                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4154                         complete(&sess->session_wait_comp);
4155
4156                 return 0;
4157         } else {
4158                 pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4159                 sess->session_state = TARG_SESS_STATE_FAILED;
4160
4161                 if (!atomic_read(&sess->session_continuation)) {
4162                         spin_unlock_bh(&sess->conn_lock);
4163                         iscsit_start_time2retain_handler(sess);
4164                 } else
4165                         spin_unlock_bh(&sess->conn_lock);
4166
4167                 if (atomic_read(&sess->sleep_on_sess_wait_comp))
4168                         complete(&sess->session_wait_comp);
4169
4170                 return 0;
4171         }
4172         spin_unlock_bh(&sess->conn_lock);
4173
4174         return 0;
4175 }
4176
4177 int iscsit_close_session(struct iscsi_session *sess)
4178 {
4179         struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
4180         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4181
4182         if (atomic_read(&sess->nconn)) {
4183                 pr_err("%d connection(s) still exist for iSCSI session"
4184                         " to %s\n", atomic_read(&sess->nconn),
4185                         sess->sess_ops->InitiatorName);
4186                 BUG();
4187         }
4188
4189         spin_lock_bh(&se_tpg->session_lock);
4190         atomic_set(&sess->session_logout, 1);
4191         atomic_set(&sess->session_reinstatement, 1);
4192         iscsit_stop_time2retain_timer(sess);
4193         spin_unlock_bh(&se_tpg->session_lock);
4194
4195         /*
4196          * transport_deregister_session_configfs() will clear the
4197          * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
4198          * can be setting it again with __transport_register_session() in
4199          * iscsi_post_login_handler() again after the iscsit_stop_session()
4200          * completes in iscsi_np context.
4201          */
4202         transport_deregister_session_configfs(sess->se_sess);
4203
4204         /*
4205          * If any other processes are accessing this session pointer we must
4206          * wait until they have completed.  If we are in an interrupt (the
4207          * time2retain handler) and contain and active session usage count we
4208          * restart the timer and exit.
4209          */
4210         if (!in_interrupt()) {
4211                 if (iscsit_check_session_usage_count(sess) == 1)
4212                         iscsit_stop_session(sess, 1, 1);
4213         } else {
4214                 if (iscsit_check_session_usage_count(sess) == 2) {
4215                         atomic_set(&sess->session_logout, 0);
4216                         iscsit_start_time2retain_handler(sess);
4217                         return 0;
4218                 }
4219         }
4220
4221         transport_deregister_session(sess->se_sess);
4222
4223         if (sess->sess_ops->ErrorRecoveryLevel == 2)
4224                 iscsit_free_connection_recovery_entires(sess);
4225
4226         iscsit_free_all_ooo_cmdsns(sess);
4227
4228         spin_lock_bh(&se_tpg->session_lock);
4229         pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
4230         sess->session_state = TARG_SESS_STATE_FREE;
4231         pr_debug("Released iSCSI session from node: %s\n",
4232                         sess->sess_ops->InitiatorName);
4233         tpg->nsessions--;
4234         if (tpg->tpg_tiqn)
4235                 tpg->tpg_tiqn->tiqn_nsessions--;
4236
4237         pr_debug("Decremented number of active iSCSI Sessions on"
4238                 " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
4239
4240         spin_lock(&sess_idr_lock);
4241         idr_remove(&sess_idr, sess->session_index);
4242         spin_unlock(&sess_idr_lock);
4243
4244         kfree(sess->sess_ops);
4245         sess->sess_ops = NULL;
4246         spin_unlock_bh(&se_tpg->session_lock);
4247
4248         kfree(sess);
4249         return 0;
4250 }
4251
4252 static void iscsit_logout_post_handler_closesession(
4253         struct iscsi_conn *conn)
4254 {
4255         struct iscsi_session *sess = conn->sess;
4256
4257         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4258         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4259
4260         atomic_set(&conn->conn_logout_remove, 0);
4261         complete(&conn->conn_logout_comp);
4262
4263         iscsit_dec_conn_usage_count(conn);
4264         iscsit_stop_session(sess, 1, 1);
4265         iscsit_dec_session_usage_count(sess);
4266         iscsit_close_session(sess);
4267 }
4268
4269 static void iscsit_logout_post_handler_samecid(
4270         struct iscsi_conn *conn)
4271 {
4272         iscsi_set_thread_clear(conn, ISCSI_CLEAR_TX_THREAD);
4273         iscsi_set_thread_set_signal(conn, ISCSI_SIGNAL_TX_THREAD);
4274
4275         atomic_set(&conn->conn_logout_remove, 0);
4276         complete(&conn->conn_logout_comp);
4277
4278         iscsit_cause_connection_reinstatement(conn, 1);
4279         iscsit_dec_conn_usage_count(conn);
4280 }
4281
4282 static void iscsit_logout_post_handler_diffcid(
4283         struct iscsi_conn *conn,
4284         u16 cid)
4285 {
4286         struct iscsi_conn *l_conn;
4287         struct iscsi_session *sess = conn->sess;
4288
4289         if (!sess)
4290                 return;
4291
4292         spin_lock_bh(&sess->conn_lock);
4293         list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
4294                 if (l_conn->cid == cid) {
4295                         iscsit_inc_conn_usage_count(l_conn);
4296                         break;
4297                 }
4298         }
4299         spin_unlock_bh(&sess->conn_lock);
4300
4301         if (!l_conn)
4302                 return;
4303
4304         if (l_conn->sock)
4305                 l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
4306
4307         spin_lock_bh(&l_conn->state_lock);
4308         pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
4309         l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
4310         spin_unlock_bh(&l_conn->state_lock);
4311
4312         iscsit_cause_connection_reinstatement(l_conn, 1);
4313         iscsit_dec_conn_usage_count(l_conn);
4314 }
4315
4316 /*
4317  *      Return of 0 causes the TX thread to restart.
4318  */
4319 static int iscsit_logout_post_handler(
4320         struct iscsi_cmd *cmd,
4321         struct iscsi_conn *conn)
4322 {
4323         int ret = 0;
4324
4325         switch (cmd->logout_reason) {
4326         case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
4327                 switch (cmd->logout_response) {
4328                 case ISCSI_LOGOUT_SUCCESS:
4329                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4330                 default:
4331                         iscsit_logout_post_handler_closesession(conn);
4332                         break;
4333                 }
4334                 ret = 0;
4335                 break;
4336         case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
4337                 if (conn->cid == cmd->logout_cid) {
4338                         switch (cmd->logout_response) {
4339                         case ISCSI_LOGOUT_SUCCESS:
4340                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4341                         default:
4342                                 iscsit_logout_post_handler_samecid(conn);
4343                                 break;
4344                         }
4345                         ret = 0;
4346                 } else {
4347                         switch (cmd->logout_response) {
4348                         case ISCSI_LOGOUT_SUCCESS:
4349                                 iscsit_logout_post_handler_diffcid(conn,
4350                                         cmd->logout_cid);
4351                                 break;
4352                         case ISCSI_LOGOUT_CID_NOT_FOUND:
4353                         case ISCSI_LOGOUT_CLEANUP_FAILED:
4354                         default:
4355                                 break;
4356                         }
4357                         ret = 1;
4358                 }
4359                 break;
4360         case ISCSI_LOGOUT_REASON_RECOVERY:
4361                 switch (cmd->logout_response) {
4362                 case ISCSI_LOGOUT_SUCCESS:
4363                 case ISCSI_LOGOUT_CID_NOT_FOUND:
4364                 case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
4365                 case ISCSI_LOGOUT_CLEANUP_FAILED:
4366                 default:
4367                         break;
4368                 }
4369                 ret = 1;
4370                 break;
4371         default:
4372                 break;
4373
4374         }
4375         return ret;
4376 }
4377
4378 void iscsit_fail_session(struct iscsi_session *sess)
4379 {
4380         struct iscsi_conn *conn;
4381
4382         spin_lock_bh(&sess->conn_lock);
4383         list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
4384                 pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
4385                 conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
4386         }
4387         spin_unlock_bh(&sess->conn_lock);
4388
4389         pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
4390         sess->session_state = TARG_SESS_STATE_FAILED;
4391 }
4392
4393 int iscsit_free_session(struct iscsi_session *sess)
4394 {
4395         u16 conn_count = atomic_read(&sess->nconn);
4396         struct iscsi_conn *conn, *conn_tmp = NULL;
4397         int is_last;
4398
4399         spin_lock_bh(&sess->conn_lock);
4400         atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4401
4402         list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4403                         conn_list) {
4404                 if (conn_count == 0)
4405                         break;
4406
4407                 if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4408                         is_last = 1;
4409                 } else {
4410                         iscsit_inc_conn_usage_count(conn_tmp);
4411                         is_last = 0;
4412                 }
4413                 iscsit_inc_conn_usage_count(conn);
4414
4415                 spin_unlock_bh(&sess->conn_lock);
4416                 iscsit_cause_connection_reinstatement(conn, 1);
4417                 spin_lock_bh(&sess->conn_lock);
4418
4419                 iscsit_dec_conn_usage_count(conn);
4420                 if (is_last == 0)
4421                         iscsit_dec_conn_usage_count(conn_tmp);
4422
4423                 conn_count--;
4424         }
4425
4426         if (atomic_read(&sess->nconn)) {
4427                 spin_unlock_bh(&sess->conn_lock);
4428                 wait_for_completion(&sess->session_wait_comp);
4429         } else
4430                 spin_unlock_bh(&sess->conn_lock);
4431
4432         iscsit_close_session(sess);
4433         return 0;
4434 }
4435
4436 void iscsit_stop_session(
4437         struct iscsi_session *sess,
4438         int session_sleep,
4439         int connection_sleep)
4440 {
4441         u16 conn_count = atomic_read(&sess->nconn);
4442         struct iscsi_conn *conn, *conn_tmp = NULL;
4443         int is_last;
4444
4445         spin_lock_bh(&sess->conn_lock);
4446         if (session_sleep)
4447                 atomic_set(&sess->sleep_on_sess_wait_comp, 1);
4448
4449         if (connection_sleep) {
4450                 list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
4451                                 conn_list) {
4452                         if (conn_count == 0)
4453                                 break;
4454
4455                         if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
4456                                 is_last = 1;
4457                         } else {
4458                                 iscsit_inc_conn_usage_count(conn_tmp);
4459                                 is_last = 0;
4460                         }
4461                         iscsit_inc_conn_usage_count(conn);
4462
4463                         spin_unlock_bh(&sess->conn_lock);
4464                         iscsit_cause_connection_reinstatement(conn, 1);
4465                         spin_lock_bh(&sess->conn_lock);
4466
4467                         iscsit_dec_conn_usage_count(conn);
4468                         if (is_last == 0)
4469                                 iscsit_dec_conn_usage_count(conn_tmp);
4470                         conn_count--;
4471                 }
4472         } else {
4473                 list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
4474                         iscsit_cause_connection_reinstatement(conn, 0);
4475         }
4476
4477         if (session_sleep && atomic_read(&sess->nconn)) {
4478                 spin_unlock_bh(&sess->conn_lock);
4479                 wait_for_completion(&sess->session_wait_comp);
4480         } else
4481                 spin_unlock_bh(&sess->conn_lock);
4482 }
4483
4484 int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
4485 {
4486         struct iscsi_session *sess;
4487         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
4488         struct se_session *se_sess, *se_sess_tmp;
4489         int session_count = 0;
4490
4491         spin_lock_bh(&se_tpg->session_lock);
4492         if (tpg->nsessions && !force) {
4493                 spin_unlock_bh(&se_tpg->session_lock);
4494                 return -1;
4495         }
4496
4497         list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
4498                         sess_list) {
4499                 sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
4500
4501                 spin_lock(&sess->conn_lock);
4502                 if (atomic_read(&sess->session_fall_back_to_erl0) ||
4503                     atomic_read(&sess->session_logout) ||
4504                     (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
4505                         spin_unlock(&sess->conn_lock);
4506                         continue;
4507                 }
4508                 atomic_set(&sess->session_reinstatement, 1);
4509                 spin_unlock(&sess->conn_lock);
4510                 spin_unlock_bh(&se_tpg->session_lock);
4511
4512                 iscsit_free_session(sess);
4513                 spin_lock_bh(&se_tpg->session_lock);
4514
4515                 session_count++;
4516         }
4517         spin_unlock_bh(&se_tpg->session_lock);
4518
4519         pr_debug("Released %d iSCSI Session(s) from Target Portal"
4520                         " Group: %hu\n", session_count, tpg->tpgt);
4521         return 0;
4522 }
4523
4524 MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
4525 MODULE_VERSION("4.1.x");
4526 MODULE_AUTHOR("nab@Linux-iSCSI.org");
4527 MODULE_LICENSE("GPL");
4528
4529 module_init(iscsi_target_init_module);
4530 module_exit(iscsi_target_cleanup_module);