drivers/infiniband/hw/amso1100/c2_qp.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #include "c2.h"
  39 #include "c2_vq.h"
  40 #include "c2_status.h"
  41
  42 #define C2_MAX_ORD_PER_QP 128
  43 #define C2_MAX_IRD_PER_QP 128
  44
  45 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  46 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  47 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  48
  49 #define NO_SUPPORT -1
  50 static const u8 c2_opcode[] = {
  51         [IB_WR_SEND] = C2_WR_TYPE_SEND,
  52         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  53         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  54         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  55         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  56         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  57         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  58 };
  59
  60 static int to_c2_state(enum ib_qp_state ib_state)
  61 {
  62         switch (ib_state) {
  63         case IB_QPS_RESET:
  64                 return C2_QP_STATE_IDLE;
  65         case IB_QPS_RTS:
  66                 return C2_QP_STATE_RTS;
  67         case IB_QPS_SQD:
  68                 return C2_QP_STATE_CLOSING;
  69         case IB_QPS_SQE:
  70                 return C2_QP_STATE_CLOSING;
  71         case IB_QPS_ERR:
  72                 return C2_QP_STATE_ERROR;
  73         default:
  74                 return -1;
  75         }
  76 }
  77
  78 static int to_ib_state(enum c2_qp_state c2_state)
  79 {
  80         switch (c2_state) {
  81         case C2_QP_STATE_IDLE:
  82                 return IB_QPS_RESET;
  83         case C2_QP_STATE_CONNECTING:
  84                 return IB_QPS_RTR;
  85         case C2_QP_STATE_RTS:
  86                 return IB_QPS_RTS;
  87         case C2_QP_STATE_CLOSING:
  88                 return IB_QPS_SQD;
  89         case C2_QP_STATE_ERROR:
  90                 return IB_QPS_ERR;
  91         case C2_QP_STATE_TERMINATE:
  92                 return IB_QPS_SQE;
  93         default:
  94                 return -1;
  95         }
  96 }
  97
  98 static const char *to_ib_state_str(int ib_state)
  99 {
 100         static const char *state_str[] = {
 101                 "IB_QPS_RESET",
 102                 "IB_QPS_INIT",
 103                 "IB_QPS_RTR",
 104                 "IB_QPS_RTS",
 105                 "IB_QPS_SQD",
 106                 "IB_QPS_SQE",
 107                 "IB_QPS_ERR"
 108         };
 109         if (ib_state < IB_QPS_RESET ||
 110             ib_state > IB_QPS_ERR)
 111                 return "<invalid IB QP state>";
 112
 113         ib_state -= IB_QPS_RESET;
 114         return state_str[ib_state];
 115 }
 116
 117 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 118 {
 119         int new_state = to_ib_state(c2_state);
 120
 121         pr_debug("%s: qp[%p] state modify %s --> %s\n",
 122                __FUNCTION__,
 123                 qp,
 124                 to_ib_state_str(qp->state),
 125                 to_ib_state_str(new_state));
 126         qp->state = new_state;
 127 }
 128
 129 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 130
 131 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 132                  struct ib_qp_attr *attr, int attr_mask)
 133 {
 134         struct c2wr_qp_modify_req wr;
 135         struct c2wr_qp_modify_rep *reply;
 136         struct c2_vq_req *vq_req;
 137         unsigned long flags;
 138         u8 next_state;
 139         int err;
 140
 141         pr_debug("%s:%d qp=%p, %s --> %s\n",
 142                 __FUNCTION__, __LINE__,
 143                 qp,
 144                 to_ib_state_str(qp->state),
 145                 to_ib_state_str(attr->qp_state));
 146
 147         vq_req = vq_req_alloc(c2dev);
 148         if (!vq_req)
 149                 return -ENOMEM;
 150
 151         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 152         wr.hdr.context = (unsigned long) vq_req;
 153         wr.rnic_handle = c2dev->adapter_handle;
 154         wr.qp_handle = qp->adapter_handle;
 155         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 156         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 157         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 158         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159
 160         if (attr_mask & IB_QP_STATE) {
 161                 /* Ensure the state is valid */
 162                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
 163                         return -EINVAL;
 164
 165                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 166
 167                 if (attr->qp_state == IB_QPS_ERR) {
 168                         spin_lock_irqsave(&qp->lock, flags);
 169                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 170                                 pr_debug("Generating CLOSE event for QP-->ERR, "
 171                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 172                                 /* Generate an CLOSE event */
 173                                 vq_req->cm_id = qp->cm_id;
 174                                 vq_req->event = IW_CM_EVENT_CLOSE;
 175                         }
 176                         spin_unlock_irqrestore(&qp->lock, flags);
 177                 }
 178                 next_state =  attr->qp_state;
 179
 180         } else if (attr_mask & IB_QP_CUR_STATE) {
 181
 182                 if (attr->cur_qp_state != IB_QPS_RTR &&
 183                     attr->cur_qp_state != IB_QPS_RTS &&
 184                     attr->cur_qp_state != IB_QPS_SQD &&
 185                     attr->cur_qp_state != IB_QPS_SQE)
 186                         return -EINVAL;
 187                 else
 188                         wr.next_qp_state =
 189                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
 190
 191                 next_state = attr->cur_qp_state;
 192
 193         } else {
 194                 err = 0;
 195                 goto bail0;
 196         }
 197
 198         /* reference the request struct */
 199         vq_req_get(c2dev, vq_req);
 200
 201         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 202         if (err) {
 203                 vq_req_put(c2dev, vq_req);
 204                 goto bail0;
 205         }
 206
 207         err = vq_wait_for_reply(c2dev, vq_req);
 208         if (err)
 209                 goto bail0;
 210
 211         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 212         if (!reply) {
 213                 err = -ENOMEM;
 214                 goto bail0;
 215         }
 216
 217         err = c2_errno(reply);
 218         if (!err)
 219                 qp->state = next_state;
 220 #ifdef DEBUG
 221         else
 222                 pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
 223 #endif
 224         /*
 225          * If we're going to error and generating the event here, then
 226          * we need to remove the reference because there will be no
 227          * close event generated by the adapter
 228         */
 229         spin_lock_irqsave(&qp->lock, flags);
 230         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 231                 qp->cm_id->rem_ref(qp->cm_id);
 232                 qp->cm_id = NULL;
 233         }
 234         spin_unlock_irqrestore(&qp->lock, flags);
 235
 236         vq_repbuf_free(c2dev, reply);
 237       bail0:
 238         vq_req_free(c2dev, vq_req);
 239
 240         pr_debug("%s:%d qp=%p, cur_state=%s\n",
 241                 __FUNCTION__, __LINE__,
 242                 qp,
 243                 to_ib_state_str(qp->state));
 244         return err;
 245 }
 246
 247 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 248                           int ord, int ird)
 249 {
 250         struct c2wr_qp_modify_req wr;
 251         struct c2wr_qp_modify_rep *reply;
 252         struct c2_vq_req *vq_req;
 253         int err;
 254
 255         vq_req = vq_req_alloc(c2dev);
 256         if (!vq_req)
 257                 return -ENOMEM;
 258
 259         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 260         wr.hdr.context = (unsigned long) vq_req;
 261         wr.rnic_handle = c2dev->adapter_handle;
 262         wr.qp_handle = qp->adapter_handle;
 263         wr.ord = cpu_to_be32(ord);
 264         wr.ird = cpu_to_be32(ird);
 265         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 266         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 267         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 268
 269         /* reference the request struct */
 270         vq_req_get(c2dev, vq_req);
 271
 272         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 273         if (err) {
 274                 vq_req_put(c2dev, vq_req);
 275                 goto bail0;
 276         }
 277
 278         err = vq_wait_for_reply(c2dev, vq_req);
 279         if (err)
 280                 goto bail0;
 281
 282         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 283                 vq_req->reply_msg;
 284         if (!reply) {
 285                 err = -ENOMEM;
 286                 goto bail0;
 287         }
 288
 289         err = c2_errno(reply);
 290         vq_repbuf_free(c2dev, reply);
 291       bail0:
 292         vq_req_free(c2dev, vq_req);
 293         return err;
 294 }
 295
 296 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 297 {
 298         struct c2_vq_req *vq_req;
 299         struct c2wr_qp_destroy_req wr;
 300         struct c2wr_qp_destroy_rep *reply;
 301         unsigned long flags;
 302         int err;
 303
 304         /*
 305          * Allocate a verb request message
 306          */
 307         vq_req = vq_req_alloc(c2dev);
 308         if (!vq_req) {
 309                 return -ENOMEM;
 310         }
 311
 312         /*
 313          * Initialize the WR
 314          */
 315         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 316         wr.hdr.context = (unsigned long) vq_req;
 317         wr.rnic_handle = c2dev->adapter_handle;
 318         wr.qp_handle = qp->adapter_handle;
 319
 320         /*
 321          * reference the request struct.  dereferenced in the int handler.
 322          */
 323         vq_req_get(c2dev, vq_req);
 324
 325         spin_lock_irqsave(&qp->lock, flags);
 326         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 327                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 328                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 329                 /* Generate an CLOSE event */
 330                 vq_req->qp = qp;
 331                 vq_req->cm_id = qp->cm_id;
 332                 vq_req->event = IW_CM_EVENT_CLOSE;
 333         }
 334         spin_unlock_irqrestore(&qp->lock, flags);
 335
 336         /*
 337          * Send WR to adapter
 338          */
 339         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 340         if (err) {
 341                 vq_req_put(c2dev, vq_req);
 342                 goto bail0;
 343         }
 344
 345         /*
 346          * Wait for reply from adapter
 347          */
 348         err = vq_wait_for_reply(c2dev, vq_req);
 349         if (err) {
 350                 goto bail0;
 351         }
 352
 353         /*
 354          * Process reply
 355          */
 356         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 357         if (!reply) {
 358                 err = -ENOMEM;
 359                 goto bail0;
 360         }
 361
 362         spin_lock_irqsave(&qp->lock, flags);
 363         if (qp->cm_id) {
 364                 qp->cm_id->rem_ref(qp->cm_id);
 365                 qp->cm_id = NULL;
 366         }
 367         spin_unlock_irqrestore(&qp->lock, flags);
 368
 369         vq_repbuf_free(c2dev, reply);
 370       bail0:
 371         vq_req_free(c2dev, vq_req);
 372         return err;
 373 }
 374
 375 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 376 {
 377         int ret;
 378
 379         do {
 380                 spin_lock_irq(&c2dev->qp_table.lock);
 381                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
 382                                         c2dev->qp_table.last++, &qp->qpn);
 383                 spin_unlock_irq(&c2dev->qp_table.lock);
 384         } while ((ret == -EAGAIN) &&
 385                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
 386         return ret;
 387 }
 388
 389 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 390 {
 391         spin_lock_irq(&c2dev->qp_table.lock);
 392         idr_remove(&c2dev->qp_table.idr, qpn);
 393         spin_unlock_irq(&c2dev->qp_table.lock);
 394 }
 395
 396 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 397 {
 398         unsigned long flags;
 399         struct c2_qp *qp;
 400
 401         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 402         qp = idr_find(&c2dev->qp_table.idr, qpn);
 403         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 404         return qp;
 405 }
 406
 407 int c2_alloc_qp(struct c2_dev *c2dev,
 408                 struct c2_pd *pd,
 409                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 410 {
 411         struct c2wr_qp_create_req wr;
 412         struct c2wr_qp_create_rep *reply;
 413         struct c2_vq_req *vq_req;
 414         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 415         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 416         unsigned long peer_pa;
 417         u32 q_size, msg_size, mmap_size;
 418         void __iomem *mmap;
 419         int err;
 420
 421         err = c2_alloc_qpn(c2dev, qp);
 422         if (err)
 423                 return err;
 424         qp->ibqp.qp_num = qp->qpn;
 425         qp->ibqp.qp_type = IB_QPT_RC;
 426
 427         /* Allocate the SQ and RQ shared pointers */
 428         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 429                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
 430         if (!qp->sq_mq.shared) {
 431                 err = -ENOMEM;
 432                 goto bail0;
 433         }
 434
 435         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 436                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
 437         if (!qp->rq_mq.shared) {
 438                 err = -ENOMEM;
 439                 goto bail1;
 440         }
 441
 442         /* Allocate the verbs request */
 443         vq_req = vq_req_alloc(c2dev);
 444         if (vq_req == NULL) {
 445                 err = -ENOMEM;
 446                 goto bail2;
 447         }
 448
 449         /* Initialize the work request */
 450         memset(&wr, 0, sizeof(wr));
 451         c2_wr_set_id(&wr, CCWR_QP_CREATE);
 452         wr.hdr.context = (unsigned long) vq_req;
 453         wr.rnic_handle = c2dev->adapter_handle;
 454         wr.sq_cq_handle = send_cq->adapter_handle;
 455         wr.rq_cq_handle = recv_cq->adapter_handle;
 456         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 457         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 458         wr.srq_handle = 0;
 459         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 460                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 461         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 462         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 463         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 464         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 465         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 466         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 467         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 468         wr.pd_id = pd->pd_id;
 469         wr.user_context = (unsigned long) qp;
 470
 471         vq_req_get(c2dev, vq_req);
 472
 473         /* Send the WR to the adapter */
 474         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 475         if (err) {
 476                 vq_req_put(c2dev, vq_req);
 477                 goto bail3;
 478         }
 479
 480         /* Wait for the verb reply  */
 481         err = vq_wait_for_reply(c2dev, vq_req);
 482         if (err) {
 483                 goto bail3;
 484         }
 485
 486         /* Process the reply */
 487         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 488         if (!reply) {
 489                 err = -ENOMEM;
 490                 goto bail3;
 491         }
 492
 493         if ((err = c2_wr_get_result(reply)) != 0) {
 494                 goto bail4;
 495         }
 496
 497         /* Fill in the kernel QP struct */
 498         atomic_set(&qp->refcount, 1);
 499         qp->adapter_handle = reply->qp_handle;
 500         qp->state = IB_QPS_RESET;
 501         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 502         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 503         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 504
 505         /* Initialize the SQ MQ */
 506         q_size = be32_to_cpu(reply->sq_depth);
 507         msg_size = be32_to_cpu(reply->sq_msg_size);
 508         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 509         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 510         mmap = ioremap_nocache(peer_pa, mmap_size);
 511         if (!mmap) {
 512                 err = -ENOMEM;
 513                 goto bail5;
 514         }
 515
 516         c2_mq_req_init(&qp->sq_mq,
 517                        be32_to_cpu(reply->sq_mq_index),
 518                        q_size,
 519                        msg_size,
 520                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 521                        mmap,                            /* peer */
 522                        C2_MQ_ADAPTER_TARGET);
 523
 524         /* Initialize the RQ mq */
 525         q_size = be32_to_cpu(reply->rq_depth);
 526         msg_size = be32_to_cpu(reply->rq_msg_size);
 527         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 528         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 529         mmap = ioremap_nocache(peer_pa, mmap_size);
 530         if (!mmap) {
 531                 err = -ENOMEM;
 532                 goto bail6;
 533         }
 534
 535         c2_mq_req_init(&qp->rq_mq,
 536                        be32_to_cpu(reply->rq_mq_index),
 537                        q_size,
 538                        msg_size,
 539                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 540                        mmap,                            /* peer */
 541                        C2_MQ_ADAPTER_TARGET);
 542
 543         vq_repbuf_free(c2dev, reply);
 544         vq_req_free(c2dev, vq_req);
 545
 546         return 0;
 547
 548       bail6:
 549         iounmap(qp->sq_mq.peer);
 550       bail5:
 551         destroy_qp(c2dev, qp);
 552       bail4:
 553         vq_repbuf_free(c2dev, reply);
 554       bail3:
 555         vq_req_free(c2dev, vq_req);
 556       bail2:
 557         c2_free_mqsp(qp->rq_mq.shared);
 558       bail1:
 559         c2_free_mqsp(qp->sq_mq.shared);
 560       bail0:
 561         c2_free_qpn(c2dev, qp->qpn);
 562         return err;
 563 }
 564
 565 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 566 {
 567         struct c2_cq *send_cq;
 568         struct c2_cq *recv_cq;
 569
 570         send_cq = to_c2cq(qp->ibqp.send_cq);
 571         recv_cq = to_c2cq(qp->ibqp.recv_cq);
 572
 573         /*
 574          * Lock CQs here, so that CQ polling code can do QP lookup
 575          * without taking a lock.
 576          */
 577         spin_lock_irq(&send_cq->lock);
 578         if (send_cq != recv_cq)
 579                 spin_lock(&recv_cq->lock);
 580
 581         c2_free_qpn(c2dev, qp->qpn);
 582
 583         if (send_cq != recv_cq)
 584                 spin_unlock(&recv_cq->lock);
 585         spin_unlock_irq(&send_cq->lock);
 586
 587         /*
 588          * Destory qp in the rnic...
 589          */
 590         destroy_qp(c2dev, qp);
 591
 592         /*
 593          * Mark any unreaped CQEs as null and void.
 594          */
 595         c2_cq_clean(c2dev, qp, send_cq->cqn);
 596         if (send_cq != recv_cq)
 597                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
 598         /*
 599          * Unmap the MQs and return the shared pointers
 600          * to the message pool.
 601          */
 602         iounmap(qp->sq_mq.peer);
 603         iounmap(qp->rq_mq.peer);
 604         c2_free_mqsp(qp->sq_mq.shared);
 605         c2_free_mqsp(qp->rq_mq.shared);
 606
 607         atomic_dec(&qp->refcount);
 608         wait_event(qp->wait, !atomic_read(&qp->refcount));
 609 }
 610
 611 /*
 612  * Function: move_sgl
 613  *
 614  * Description:
 615  * Move an SGL from the user's work request struct into a CCIL Work Request
 616  * message, swapping to WR byte order and ensure the total length doesn't
 617  * overflow.
 618  *
 619  * IN:
 620  * dst          - ptr to CCIL Work Request message SGL memory.
 621  * src          - ptr to the consumers SGL memory.
 622  *
 623  * OUT: none
 624  *
 625  * Return:
 626  * CCIL status codes.
 627  */
 628 static int
 629 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 630          u8 * actual_count)
 631 {
 632         u32 tot = 0;            /* running total */
 633         u8 acount = 0;          /* running total non-0 len sge's */
 634
 635         while (count > 0) {
 636                 /*
 637                  * If the addition of this SGE causes the
 638                  * total SGL length to exceed 2^32-1, then
 639                  * fail-n-bail.
 640                  *
 641                  * If the current total plus the next element length
 642                  * wraps, then it will go negative and be less than the
 643                  * current total...
 644                  */
 645                 if ((tot + src->length) < tot) {
 646                         return -EINVAL;
 647                 }
 648                 /*
 649                  * Bug: 1456 (as well as 1498 & 1643)
 650                  * Skip over any sge's supplied with len=0
 651                  */
 652                 if (src->length) {
 653                         tot += src->length;
 654                         dst->stag = cpu_to_be32(src->lkey);
 655                         dst->to = cpu_to_be64(src->addr);
 656                         dst->length = cpu_to_be32(src->length);
 657                         dst++;
 658                         acount++;
 659                 }
 660                 src++;
 661                 count--;
 662         }
 663
 664         if (acount == 0) {
 665                 /*
 666                  * Bug: 1476 (as well as 1498, 1456 and 1643)
 667                  * Setup the SGL in the WR to make it easier for the RNIC.
 668                  * This way, the FW doesn't have to deal with special cases.
 669                  * Setting length=0 should be sufficient.
 670                  */
 671                 dst->stag = 0;
 672                 dst->to = 0;
 673                 dst->length = 0;
 674         }
 675
 676         *p_len = tot;
 677         *actual_count = acount;
 678         return 0;
 679 }
 680
 681 /*
 682  * Function: c2_activity (private function)
 683  *
 684  * Description:
 685  * Post an mq index to the host->adapter activity fifo.
 686  *
 687  * IN:
 688  * c2dev        - ptr to c2dev structure
 689  * mq_index     - mq index to post
 690  * shared       - value most recently written to shared
 691  *
 692  * OUT:
 693  *
 694  * Return:
 695  * none
 696  */
 697 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 698 {
 699         /*
 700          * First read the register to see if the FIFO is full, and if so,
 701          * spin until it's not.  This isn't perfect -- there is no
 702          * synchronization among the clients of the register, but in
 703          * practice it prevents multiple CPU from hammering the bus
 704          * with PCI RETRY. Note that when this does happen, the card
 705          * cannot get on the bus and the card and system hang in a
 706          * deadlock -- thus the need for this code. [TOT]
 707          */
 708         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) {
 709                 set_current_state(TASK_UNINTERRUPTIBLE);
 710                 schedule_timeout(0);
 711         }
 712
 713         __raw_writel(C2_HINT_MAKE(mq_index, shared),
 714                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 715 }
 716
 717 /*
 718  * Function: qp_wr_post
 719  *
 720  * Description:
 721  * This in-line function allocates a MQ msg, then moves the host-copy of
 722  * the completed WR into msg.  Then it posts the message.
 723  *
 724  * IN:
 725  * q            - ptr to user MQ.
 726  * wr           - ptr to host-copy of the WR.
 727  * qp           - ptr to user qp
 728  * size         - Number of bytes to post.  Assumed to be divisible by 4.
 729  *
 730  * OUT: none
 731  *
 732  * Return:
 733  * CCIL status codes.
 734  */
 735 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 736 {
 737         union c2wr *msg;
 738
 739         msg = c2_mq_alloc(q);
 740         if (msg == NULL) {
 741                 return -EINVAL;
 742         }
 743 #ifdef CCMSGMAGIC
 744         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 745 #endif
 746
 747         /*
 748          * Since all header fields in the WR are the same as the
 749          * CQE, set the following so the adapter need not.
 750          */
 751         c2_wr_set_result(wr, CCERR_PENDING);
 752
 753         /*
 754          * Copy the wr down to the adapter
 755          */
 756         memcpy((void *) msg, (void *) wr, size);
 757
 758         c2_mq_produce(q);
 759         return 0;
 760 }
 761
 762
 763 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 764                  struct ib_send_wr **bad_wr)
 765 {
 766         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 767         struct c2_qp *qp = to_c2qp(ibqp);
 768         union c2wr wr;
 769         int err = 0;
 770
 771         u32 flags;
 772         u32 tot_len;
 773         u8 actual_sge_count;
 774         u32 msg_size;
 775
 776         if (qp->state > IB_QPS_RTS)
 777                 return -EINVAL;
 778
 779         while (ib_wr) {
 780
 781                 flags = 0;
 782                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 783                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 784                         flags |= SQ_SIGNALED;
 785                 }
 786
 787                 switch (ib_wr->opcode) {
 788                 case IB_WR_SEND:
 789                         if (ib_wr->send_flags & IB_SEND_SOLICITED) {
 790                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 791                                 msg_size = sizeof(struct c2wr_send_req);
 792                         } else {
 793                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 794                                 msg_size = sizeof(struct c2wr_send_req);
 795                         }
 796
 797                         wr.sqwr.send.remote_stag = 0;
 798                         msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
 799                         if (ib_wr->num_sge > qp->send_sgl_depth) {
 800                                 err = -EINVAL;
 801                                 break;
 802                         }
 803                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 804                                 flags |= SQ_READ_FENCE;
 805                         }
 806                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 807                                        ib_wr->sg_list,
 808                                        ib_wr->num_sge,
 809                                        &tot_len, &actual_sge_count);
 810                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 811                         c2_wr_set_sge_count(&wr, actual_sge_count);
 812                         break;
 813                 case IB_WR_RDMA_WRITE:
 814                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 815                         msg_size = sizeof(struct c2wr_rdma_write_req) +
 816                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 817                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 818                                 err = -EINVAL;
 819                                 break;
 820                         }
 821                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 822                                 flags |= SQ_READ_FENCE;
 823                         }
 824                         wr.sqwr.rdma_write.remote_stag =
 825                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 826                         wr.sqwr.rdma_write.remote_to =
 827                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 828                         err = move_sgl((struct c2_data_addr *)
 829                                        & (wr.sqwr.rdma_write.data),
 830                                        ib_wr->sg_list,
 831                                        ib_wr->num_sge,
 832                                        &tot_len, &actual_sge_count);
 833                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 834                         c2_wr_set_sge_count(&wr, actual_sge_count);
 835                         break;
 836                 case IB_WR_RDMA_READ:
 837                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 838                         msg_size = sizeof(struct c2wr_rdma_read_req);
 839
 840                         /* IWarp only suppots 1 sge for RDMA reads */
 841                         if (ib_wr->num_sge > 1) {
 842                                 err = -EINVAL;
 843                                 break;
 844                         }
 845
 846                         /*
 847                          * Move the local and remote stag/to/len into the WR.
 848                          */
 849                         wr.sqwr.rdma_read.local_stag =
 850                             cpu_to_be32(ib_wr->sg_list->lkey);
 851                         wr.sqwr.rdma_read.local_to =
 852                             cpu_to_be64(ib_wr->sg_list->addr);
 853                         wr.sqwr.rdma_read.remote_stag =
 854                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 855                         wr.sqwr.rdma_read.remote_to =
 856                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 857                         wr.sqwr.rdma_read.length =
 858                             cpu_to_be32(ib_wr->sg_list->length);
 859                         break;
 860                 default:
 861                         /* error */
 862                         msg_size = 0;
 863                         err = -EINVAL;
 864                         break;
 865                 }
 866
 867                 /*
 868                  * If we had an error on the last wr build, then
 869                  * break out.  Possible errors include bogus WR
 870                  * type, and a bogus SGL length...
 871                  */
 872                 if (err) {
 873                         break;
 874                 }
 875
 876                 /*
 877                  * Store flags
 878                  */
 879                 c2_wr_set_flags(&wr, flags);
 880
 881                 /*
 882                  * Post the puppy!
 883                  */
 884                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 885                 if (err) {
 886                         break;
 887                 }
 888
 889                 /*
 890                  * Enqueue mq index to activity FIFO.
 891                  */
 892                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 893
 894                 ib_wr = ib_wr->next;
 895         }
 896
 897         if (err)
 898                 *bad_wr = ib_wr;
 899         return err;
 900 }
 901
 902 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 903                     struct ib_recv_wr **bad_wr)
 904 {
 905         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 906         struct c2_qp *qp = to_c2qp(ibqp);
 907         union c2wr wr;
 908         int err = 0;
 909
 910         if (qp->state > IB_QPS_RTS)
 911                 return -EINVAL;
 912
 913         /*
 914          * Try and post each work request
 915          */
 916         while (ib_wr) {
 917                 u32 tot_len;
 918                 u8 actual_sge_count;
 919
 920                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
 921                         err = -EINVAL;
 922                         break;
 923                 }
 924
 925                 /*
 926                  * Create local host-copy of the WR
 927                  */
 928                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 929                 c2_wr_set_id(&wr, CCWR_RECV);
 930                 c2_wr_set_flags(&wr, 0);
 931
 932                 /* sge_count is limited to eight bits. */
 933                 BUG_ON(ib_wr->num_sge >= 256);
 934                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 935                                ib_wr->sg_list,
 936                                ib_wr->num_sge, &tot_len, &actual_sge_count);
 937                 c2_wr_set_sge_count(&wr, actual_sge_count);
 938
 939                 /*
 940                  * If we had an error on the last wr build, then
 941                  * break out.  Possible errors include bogus WR
 942                  * type, and a bogus SGL length...
 943                  */
 944                 if (err) {
 945                         break;
 946                 }
 947
 948                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 949                 if (err) {
 950                         break;
 951                 }
 952
 953                 /*
 954                  * Enqueue mq index to activity FIFO
 955                  */
 956                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
 957
 958                 ib_wr = ib_wr->next;
 959         }
 960
 961         if (err)
 962                 *bad_wr = ib_wr;
 963         return err;
 964 }
 965
 966 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
 967 {
 968         spin_lock_init(&c2dev->qp_table.lock);
 969         idr_init(&c2dev->qp_table.idr);
 970 }
 971
 972 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
 973 {
 974         idr_destroy(&c2dev->qp_table.idr);
 975 }