drivers/infiniband/hw/amso1100/c2_qp.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #include <linux/delay.h>
  39 #include <linux/gfp.h>
  40
  41 #include "c2.h"
  42 #include "c2_vq.h"
  43 #include "c2_status.h"
  44
  45 #define C2_MAX_ORD_PER_QP 128
  46 #define C2_MAX_IRD_PER_QP 128
  47
  48 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  49 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  50 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  51
  52 #define NO_SUPPORT -1
  53 static const u8 c2_opcode[] = {
  54         [IB_WR_SEND] = C2_WR_TYPE_SEND,
  55         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  56         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  57         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  58         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  59         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  60         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  61 };
  62
  63 static int to_c2_state(enum ib_qp_state ib_state)
  64 {
  65         switch (ib_state) {
  66         case IB_QPS_RESET:
  67                 return C2_QP_STATE_IDLE;
  68         case IB_QPS_RTS:
  69                 return C2_QP_STATE_RTS;
  70         case IB_QPS_SQD:
  71                 return C2_QP_STATE_CLOSING;
  72         case IB_QPS_SQE:
  73                 return C2_QP_STATE_CLOSING;
  74         case IB_QPS_ERR:
  75                 return C2_QP_STATE_ERROR;
  76         default:
  77                 return -1;
  78         }
  79 }
  80
  81 static int to_ib_state(enum c2_qp_state c2_state)
  82 {
  83         switch (c2_state) {
  84         case C2_QP_STATE_IDLE:
  85                 return IB_QPS_RESET;
  86         case C2_QP_STATE_CONNECTING:
  87                 return IB_QPS_RTR;
  88         case C2_QP_STATE_RTS:
  89                 return IB_QPS_RTS;
  90         case C2_QP_STATE_CLOSING:
  91                 return IB_QPS_SQD;
  92         case C2_QP_STATE_ERROR:
  93                 return IB_QPS_ERR;
  94         case C2_QP_STATE_TERMINATE:
  95                 return IB_QPS_SQE;
  96         default:
  97                 return -1;
  98         }
  99 }
 100
 101 static const char *to_ib_state_str(int ib_state)
 102 {
 103         static const char *state_str[] = {
 104                 "IB_QPS_RESET",
 105                 "IB_QPS_INIT",
 106                 "IB_QPS_RTR",
 107                 "IB_QPS_RTS",
 108                 "IB_QPS_SQD",
 109                 "IB_QPS_SQE",
 110                 "IB_QPS_ERR"
 111         };
 112         if (ib_state < IB_QPS_RESET ||
 113             ib_state > IB_QPS_ERR)
 114                 return "<invalid IB QP state>";
 115
 116         ib_state -= IB_QPS_RESET;
 117         return state_str[ib_state];
 118 }
 119
 120 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 121 {
 122         int new_state = to_ib_state(c2_state);
 123
 124         pr_debug("%s: qp[%p] state modify %s --> %s\n",
 125                __func__,
 126                 qp,
 127                 to_ib_state_str(qp->state),
 128                 to_ib_state_str(new_state));
 129         qp->state = new_state;
 130 }
 131
 132 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 133
 134 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 135                  struct ib_qp_attr *attr, int attr_mask)
 136 {
 137         struct c2wr_qp_modify_req wr;
 138         struct c2wr_qp_modify_rep *reply;
 139         struct c2_vq_req *vq_req;
 140         unsigned long flags;
 141         u8 next_state;
 142         int err;
 143
 144         pr_debug("%s:%d qp=%p, %s --> %s\n",
 145                 __func__, __LINE__,
 146                 qp,
 147                 to_ib_state_str(qp->state),
 148                 to_ib_state_str(attr->qp_state));
 149
 150         vq_req = vq_req_alloc(c2dev);
 151         if (!vq_req)
 152                 return -ENOMEM;
 153
 154         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 155         wr.hdr.context = (unsigned long) vq_req;
 156         wr.rnic_handle = c2dev->adapter_handle;
 157         wr.qp_handle = qp->adapter_handle;
 158         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 160         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 161         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 162
 163         if (attr_mask & IB_QP_STATE) {
 164                 /* Ensure the state is valid */
 165                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
 166                         err = -EINVAL;
 167                         goto bail0;
 168                 }
 169
 170                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 171
 172                 if (attr->qp_state == IB_QPS_ERR) {
 173                         spin_lock_irqsave(&qp->lock, flags);
 174                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 175                                 pr_debug("Generating CLOSE event for QP-->ERR, "
 176                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 177                                 /* Generate an CLOSE event */
 178                                 vq_req->cm_id = qp->cm_id;
 179                                 vq_req->event = IW_CM_EVENT_CLOSE;
 180                         }
 181                         spin_unlock_irqrestore(&qp->lock, flags);
 182                 }
 183                 next_state =  attr->qp_state;
 184
 185         } else if (attr_mask & IB_QP_CUR_STATE) {
 186
 187                 if (attr->cur_qp_state != IB_QPS_RTR &&
 188                     attr->cur_qp_state != IB_QPS_RTS &&
 189                     attr->cur_qp_state != IB_QPS_SQD &&
 190                     attr->cur_qp_state != IB_QPS_SQE) {
 191                         err = -EINVAL;
 192                         goto bail0;
 193                 } else
 194                         wr.next_qp_state =
 195                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
 196
 197                 next_state = attr->cur_qp_state;
 198
 199         } else {
 200                 err = 0;
 201                 goto bail0;
 202         }
 203
 204         /* reference the request struct */
 205         vq_req_get(c2dev, vq_req);
 206
 207         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 208         if (err) {
 209                 vq_req_put(c2dev, vq_req);
 210                 goto bail0;
 211         }
 212
 213         err = vq_wait_for_reply(c2dev, vq_req);
 214         if (err)
 215                 goto bail0;
 216
 217         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 218         if (!reply) {
 219                 err = -ENOMEM;
 220                 goto bail0;
 221         }
 222
 223         err = c2_errno(reply);
 224         if (!err)
 225                 qp->state = next_state;
 226 #ifdef DEBUG
 227         else
 228                 pr_debug("%s: c2_errno=%d\n", __func__, err);
 229 #endif
 230         /*
 231          * If we're going to error and generating the event here, then
 232          * we need to remove the reference because there will be no
 233          * close event generated by the adapter
 234         */
 235         spin_lock_irqsave(&qp->lock, flags);
 236         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 237                 qp->cm_id->rem_ref(qp->cm_id);
 238                 qp->cm_id = NULL;
 239         }
 240         spin_unlock_irqrestore(&qp->lock, flags);
 241
 242         vq_repbuf_free(c2dev, reply);
 243       bail0:
 244         vq_req_free(c2dev, vq_req);
 245
 246         pr_debug("%s:%d qp=%p, cur_state=%s\n",
 247                 __func__, __LINE__,
 248                 qp,
 249                 to_ib_state_str(qp->state));
 250         return err;
 251 }
 252
 253 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 254                           int ord, int ird)
 255 {
 256         struct c2wr_qp_modify_req wr;
 257         struct c2wr_qp_modify_rep *reply;
 258         struct c2_vq_req *vq_req;
 259         int err;
 260
 261         vq_req = vq_req_alloc(c2dev);
 262         if (!vq_req)
 263                 return -ENOMEM;
 264
 265         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 266         wr.hdr.context = (unsigned long) vq_req;
 267         wr.rnic_handle = c2dev->adapter_handle;
 268         wr.qp_handle = qp->adapter_handle;
 269         wr.ord = cpu_to_be32(ord);
 270         wr.ird = cpu_to_be32(ird);
 271         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 272         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 273         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 274
 275         /* reference the request struct */
 276         vq_req_get(c2dev, vq_req);
 277
 278         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 279         if (err) {
 280                 vq_req_put(c2dev, vq_req);
 281                 goto bail0;
 282         }
 283
 284         err = vq_wait_for_reply(c2dev, vq_req);
 285         if (err)
 286                 goto bail0;
 287
 288         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 289                 vq_req->reply_msg;
 290         if (!reply) {
 291                 err = -ENOMEM;
 292                 goto bail0;
 293         }
 294
 295         err = c2_errno(reply);
 296         vq_repbuf_free(c2dev, reply);
 297       bail0:
 298         vq_req_free(c2dev, vq_req);
 299         return err;
 300 }
 301
 302 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 303 {
 304         struct c2_vq_req *vq_req;
 305         struct c2wr_qp_destroy_req wr;
 306         struct c2wr_qp_destroy_rep *reply;
 307         unsigned long flags;
 308         int err;
 309
 310         /*
 311          * Allocate a verb request message
 312          */
 313         vq_req = vq_req_alloc(c2dev);
 314         if (!vq_req) {
 315                 return -ENOMEM;
 316         }
 317
 318         /*
 319          * Initialize the WR
 320          */
 321         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 322         wr.hdr.context = (unsigned long) vq_req;
 323         wr.rnic_handle = c2dev->adapter_handle;
 324         wr.qp_handle = qp->adapter_handle;
 325
 326         /*
 327          * reference the request struct.  dereferenced in the int handler.
 328          */
 329         vq_req_get(c2dev, vq_req);
 330
 331         spin_lock_irqsave(&qp->lock, flags);
 332         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 333                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 334                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 335                 /* Generate an CLOSE event */
 336                 vq_req->qp = qp;
 337                 vq_req->cm_id = qp->cm_id;
 338                 vq_req->event = IW_CM_EVENT_CLOSE;
 339         }
 340         spin_unlock_irqrestore(&qp->lock, flags);
 341
 342         /*
 343          * Send WR to adapter
 344          */
 345         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 346         if (err) {
 347                 vq_req_put(c2dev, vq_req);
 348                 goto bail0;
 349         }
 350
 351         /*
 352          * Wait for reply from adapter
 353          */
 354         err = vq_wait_for_reply(c2dev, vq_req);
 355         if (err) {
 356                 goto bail0;
 357         }
 358
 359         /*
 360          * Process reply
 361          */
 362         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 363         if (!reply) {
 364                 err = -ENOMEM;
 365                 goto bail0;
 366         }
 367
 368         spin_lock_irqsave(&qp->lock, flags);
 369         if (qp->cm_id) {
 370                 qp->cm_id->rem_ref(qp->cm_id);
 371                 qp->cm_id = NULL;
 372         }
 373         spin_unlock_irqrestore(&qp->lock, flags);
 374
 375         vq_repbuf_free(c2dev, reply);
 376       bail0:
 377         vq_req_free(c2dev, vq_req);
 378         return err;
 379 }
 380
 381 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 382 {
 383         int ret;
 384
 385         do {
 386                 spin_lock_irq(&c2dev->qp_table.lock);
 387                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
 388                                         c2dev->qp_table.last++, &qp->qpn);
 389                 spin_unlock_irq(&c2dev->qp_table.lock);
 390         } while ((ret == -EAGAIN) &&
 391                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
 392         return ret;
 393 }
 394
 395 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 396 {
 397         spin_lock_irq(&c2dev->qp_table.lock);
 398         idr_remove(&c2dev->qp_table.idr, qpn);
 399         spin_unlock_irq(&c2dev->qp_table.lock);
 400 }
 401
 402 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 403 {
 404         unsigned long flags;
 405         struct c2_qp *qp;
 406
 407         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 408         qp = idr_find(&c2dev->qp_table.idr, qpn);
 409         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 410         return qp;
 411 }
 412
 413 int c2_alloc_qp(struct c2_dev *c2dev,
 414                 struct c2_pd *pd,
 415                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 416 {
 417         struct c2wr_qp_create_req wr;
 418         struct c2wr_qp_create_rep *reply;
 419         struct c2_vq_req *vq_req;
 420         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 421         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 422         unsigned long peer_pa;
 423         u32 q_size, msg_size, mmap_size;
 424         void __iomem *mmap;
 425         int err;
 426
 427         err = c2_alloc_qpn(c2dev, qp);
 428         if (err)
 429                 return err;
 430         qp->ibqp.qp_num = qp->qpn;
 431         qp->ibqp.qp_type = IB_QPT_RC;
 432
 433         /* Allocate the SQ and RQ shared pointers */
 434         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 435                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
 436         if (!qp->sq_mq.shared) {
 437                 err = -ENOMEM;
 438                 goto bail0;
 439         }
 440
 441         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 442                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
 443         if (!qp->rq_mq.shared) {
 444                 err = -ENOMEM;
 445                 goto bail1;
 446         }
 447
 448         /* Allocate the verbs request */
 449         vq_req = vq_req_alloc(c2dev);
 450         if (vq_req == NULL) {
 451                 err = -ENOMEM;
 452                 goto bail2;
 453         }
 454
 455         /* Initialize the work request */
 456         memset(&wr, 0, sizeof(wr));
 457         c2_wr_set_id(&wr, CCWR_QP_CREATE);
 458         wr.hdr.context = (unsigned long) vq_req;
 459         wr.rnic_handle = c2dev->adapter_handle;
 460         wr.sq_cq_handle = send_cq->adapter_handle;
 461         wr.rq_cq_handle = recv_cq->adapter_handle;
 462         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 463         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 464         wr.srq_handle = 0;
 465         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 466                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 467         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 468         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 469         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 470         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 471         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 472         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 473         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 474         wr.pd_id = pd->pd_id;
 475         wr.user_context = (unsigned long) qp;
 476
 477         vq_req_get(c2dev, vq_req);
 478
 479         /* Send the WR to the adapter */
 480         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 481         if (err) {
 482                 vq_req_put(c2dev, vq_req);
 483                 goto bail3;
 484         }
 485
 486         /* Wait for the verb reply  */
 487         err = vq_wait_for_reply(c2dev, vq_req);
 488         if (err) {
 489                 goto bail3;
 490         }
 491
 492         /* Process the reply */
 493         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 494         if (!reply) {
 495                 err = -ENOMEM;
 496                 goto bail3;
 497         }
 498
 499         if ((err = c2_wr_get_result(reply)) != 0) {
 500                 goto bail4;
 501         }
 502
 503         /* Fill in the kernel QP struct */
 504         atomic_set(&qp->refcount, 1);
 505         qp->adapter_handle = reply->qp_handle;
 506         qp->state = IB_QPS_RESET;
 507         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 508         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 509         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 510         init_waitqueue_head(&qp->wait);
 511
 512         /* Initialize the SQ MQ */
 513         q_size = be32_to_cpu(reply->sq_depth);
 514         msg_size = be32_to_cpu(reply->sq_msg_size);
 515         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 516         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 517         mmap = ioremap_nocache(peer_pa, mmap_size);
 518         if (!mmap) {
 519                 err = -ENOMEM;
 520                 goto bail5;
 521         }
 522
 523         c2_mq_req_init(&qp->sq_mq,
 524                        be32_to_cpu(reply->sq_mq_index),
 525                        q_size,
 526                        msg_size,
 527                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 528                        mmap,                            /* peer */
 529                        C2_MQ_ADAPTER_TARGET);
 530
 531         /* Initialize the RQ mq */
 532         q_size = be32_to_cpu(reply->rq_depth);
 533         msg_size = be32_to_cpu(reply->rq_msg_size);
 534         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 535         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 536         mmap = ioremap_nocache(peer_pa, mmap_size);
 537         if (!mmap) {
 538                 err = -ENOMEM;
 539                 goto bail6;
 540         }
 541
 542         c2_mq_req_init(&qp->rq_mq,
 543                        be32_to_cpu(reply->rq_mq_index),
 544                        q_size,
 545                        msg_size,
 546                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 547                        mmap,                            /* peer */
 548                        C2_MQ_ADAPTER_TARGET);
 549
 550         vq_repbuf_free(c2dev, reply);
 551         vq_req_free(c2dev, vq_req);
 552
 553         return 0;
 554
 555       bail6:
 556         iounmap(qp->sq_mq.peer);
 557       bail5:
 558         destroy_qp(c2dev, qp);
 559       bail4:
 560         vq_repbuf_free(c2dev, reply);
 561       bail3:
 562         vq_req_free(c2dev, vq_req);
 563       bail2:
 564         c2_free_mqsp(qp->rq_mq.shared);
 565       bail1:
 566         c2_free_mqsp(qp->sq_mq.shared);
 567       bail0:
 568         c2_free_qpn(c2dev, qp->qpn);
 569         return err;
 570 }
 571
 572 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 573 {
 574         if (send_cq == recv_cq)
 575                 spin_lock_irq(&send_cq->lock);
 576         else if (send_cq > recv_cq) {
 577                 spin_lock_irq(&send_cq->lock);
 578                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
 579         } else {
 580                 spin_lock_irq(&recv_cq->lock);
 581                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
 582         }
 583 }
 584
 585 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 586 {
 587         if (send_cq == recv_cq)
 588                 spin_unlock_irq(&send_cq->lock);
 589         else if (send_cq > recv_cq) {
 590                 spin_unlock(&recv_cq->lock);
 591                 spin_unlock_irq(&send_cq->lock);
 592         } else {
 593                 spin_unlock(&send_cq->lock);
 594                 spin_unlock_irq(&recv_cq->lock);
 595         }
 596 }
 597
 598 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 599 {
 600         struct c2_cq *send_cq;
 601         struct c2_cq *recv_cq;
 602
 603         send_cq = to_c2cq(qp->ibqp.send_cq);
 604         recv_cq = to_c2cq(qp->ibqp.recv_cq);
 605
 606         /*
 607          * Lock CQs here, so that CQ polling code can do QP lookup
 608          * without taking a lock.
 609          */
 610         c2_lock_cqs(send_cq, recv_cq);
 611         c2_free_qpn(c2dev, qp->qpn);
 612         c2_unlock_cqs(send_cq, recv_cq);
 613
 614         /*
 615          * Destroy qp in the rnic...
 616          */
 617         destroy_qp(c2dev, qp);
 618
 619         /*
 620          * Mark any unreaped CQEs as null and void.
 621          */
 622         c2_cq_clean(c2dev, qp, send_cq->cqn);
 623         if (send_cq != recv_cq)
 624                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
 625         /*
 626          * Unmap the MQs and return the shared pointers
 627          * to the message pool.
 628          */
 629         iounmap(qp->sq_mq.peer);
 630         iounmap(qp->rq_mq.peer);
 631         c2_free_mqsp(qp->sq_mq.shared);
 632         c2_free_mqsp(qp->rq_mq.shared);
 633
 634         atomic_dec(&qp->refcount);
 635         wait_event(qp->wait, !atomic_read(&qp->refcount));
 636 }
 637
 638 /*
 639  * Function: move_sgl
 640  *
 641  * Description:
 642  * Move an SGL from the user's work request struct into a CCIL Work Request
 643  * message, swapping to WR byte order and ensure the total length doesn't
 644  * overflow.
 645  *
 646  * IN:
 647  * dst          - ptr to CCIL Work Request message SGL memory.
 648  * src          - ptr to the consumers SGL memory.
 649  *
 650  * OUT: none
 651  *
 652  * Return:
 653  * CCIL status codes.
 654  */
 655 static int
 656 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 657          u8 * actual_count)
 658 {
 659         u32 tot = 0;            /* running total */
 660         u8 acount = 0;          /* running total non-0 len sge's */
 661
 662         while (count > 0) {
 663                 /*
 664                  * If the addition of this SGE causes the
 665                  * total SGL length to exceed 2^32-1, then
 666                  * fail-n-bail.
 667                  *
 668                  * If the current total plus the next element length
 669                  * wraps, then it will go negative and be less than the
 670                  * current total...
 671                  */
 672                 if ((tot + src->length) < tot) {
 673                         return -EINVAL;
 674                 }
 675                 /*
 676                  * Bug: 1456 (as well as 1498 & 1643)
 677                  * Skip over any sge's supplied with len=0
 678                  */
 679                 if (src->length) {
 680                         tot += src->length;
 681                         dst->stag = cpu_to_be32(src->lkey);
 682                         dst->to = cpu_to_be64(src->addr);
 683                         dst->length = cpu_to_be32(src->length);
 684                         dst++;
 685                         acount++;
 686                 }
 687                 src++;
 688                 count--;
 689         }
 690
 691         if (acount == 0) {
 692                 /*
 693                  * Bug: 1476 (as well as 1498, 1456 and 1643)
 694                  * Setup the SGL in the WR to make it easier for the RNIC.
 695                  * This way, the FW doesn't have to deal with special cases.
 696                  * Setting length=0 should be sufficient.
 697                  */
 698                 dst->stag = 0;
 699                 dst->to = 0;
 700                 dst->length = 0;
 701         }
 702
 703         *p_len = tot;
 704         *actual_count = acount;
 705         return 0;
 706 }
 707
 708 /*
 709  * Function: c2_activity (private function)
 710  *
 711  * Description:
 712  * Post an mq index to the host->adapter activity fifo.
 713  *
 714  * IN:
 715  * c2dev        - ptr to c2dev structure
 716  * mq_index     - mq index to post
 717  * shared       - value most recently written to shared
 718  *
 719  * OUT:
 720  *
 721  * Return:
 722  * none
 723  */
 724 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 725 {
 726         /*
 727          * First read the register to see if the FIFO is full, and if so,
 728          * spin until it's not.  This isn't perfect -- there is no
 729          * synchronization among the clients of the register, but in
 730          * practice it prevents multiple CPU from hammering the bus
 731          * with PCI RETRY. Note that when this does happen, the card
 732          * cannot get on the bus and the card and system hang in a
 733          * deadlock -- thus the need for this code. [TOT]
 734          */
 735         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
 736                 udelay(10);
 737
 738         __raw_writel(C2_HINT_MAKE(mq_index, shared),
 739                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 740 }
 741
 742 /*
 743  * Function: qp_wr_post
 744  *
 745  * Description:
 746  * This in-line function allocates a MQ msg, then moves the host-copy of
 747  * the completed WR into msg.  Then it posts the message.
 748  *
 749  * IN:
 750  * q            - ptr to user MQ.
 751  * wr           - ptr to host-copy of the WR.
 752  * qp           - ptr to user qp
 753  * size         - Number of bytes to post.  Assumed to be divisible by 4.
 754  *
 755  * OUT: none
 756  *
 757  * Return:
 758  * CCIL status codes.
 759  */
 760 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 761 {
 762         union c2wr *msg;
 763
 764         msg = c2_mq_alloc(q);
 765         if (msg == NULL) {
 766                 return -EINVAL;
 767         }
 768 #ifdef CCMSGMAGIC
 769         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 770 #endif
 771
 772         /*
 773          * Since all header fields in the WR are the same as the
 774          * CQE, set the following so the adapter need not.
 775          */
 776         c2_wr_set_result(wr, CCERR_PENDING);
 777
 778         /*
 779          * Copy the wr down to the adapter
 780          */
 781         memcpy((void *) msg, (void *) wr, size);
 782
 783         c2_mq_produce(q);
 784         return 0;
 785 }
 786
 787
 788 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 789                  struct ib_send_wr **bad_wr)
 790 {
 791         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 792         struct c2_qp *qp = to_c2qp(ibqp);
 793         union c2wr wr;
 794         unsigned long lock_flags;
 795         int err = 0;
 796
 797         u32 flags;
 798         u32 tot_len;
 799         u8 actual_sge_count;
 800         u32 msg_size;
 801
 802         if (qp->state > IB_QPS_RTS) {
 803                 err = -EINVAL;
 804                 goto out;
 805         }
 806
 807         while (ib_wr) {
 808
 809                 flags = 0;
 810                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 811                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 812                         flags |= SQ_SIGNALED;
 813                 }
 814
 815                 switch (ib_wr->opcode) {
 816                 case IB_WR_SEND:
 817                 case IB_WR_SEND_WITH_INV:
 818                         if (ib_wr->opcode == IB_WR_SEND) {
 819                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
 820                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 821                                 else
 822                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 823                                 wr.sqwr.send.remote_stag = 0;
 824                         } else {
 825                                 if (ib_wr->send_flags & IB_SEND_SOLICITED)
 826                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
 827                                 else
 828                                         c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
 829                                 wr.sqwr.send.remote_stag =
 830                                         cpu_to_be32(ib_wr->ex.invalidate_rkey);
 831                         }
 832
 833                         msg_size = sizeof(struct c2wr_send_req) +
 834                                 sizeof(struct c2_data_addr) * ib_wr->num_sge;
 835                         if (ib_wr->num_sge > qp->send_sgl_depth) {
 836                                 err = -EINVAL;
 837                                 break;
 838                         }
 839                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 840                                 flags |= SQ_READ_FENCE;
 841                         }
 842                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 843                                        ib_wr->sg_list,
 844                                        ib_wr->num_sge,
 845                                        &tot_len, &actual_sge_count);
 846                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 847                         c2_wr_set_sge_count(&wr, actual_sge_count);
 848                         break;
 849                 case IB_WR_RDMA_WRITE:
 850                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 851                         msg_size = sizeof(struct c2wr_rdma_write_req) +
 852                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 853                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 854                                 err = -EINVAL;
 855                                 break;
 856                         }
 857                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 858                                 flags |= SQ_READ_FENCE;
 859                         }
 860                         wr.sqwr.rdma_write.remote_stag =
 861                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 862                         wr.sqwr.rdma_write.remote_to =
 863                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 864                         err = move_sgl((struct c2_data_addr *)
 865                                        & (wr.sqwr.rdma_write.data),
 866                                        ib_wr->sg_list,
 867                                        ib_wr->num_sge,
 868                                        &tot_len, &actual_sge_count);
 869                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 870                         c2_wr_set_sge_count(&wr, actual_sge_count);
 871                         break;
 872                 case IB_WR_RDMA_READ:
 873                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 874                         msg_size = sizeof(struct c2wr_rdma_read_req);
 875
 876                         /* IWarp only suppots 1 sge for RDMA reads */
 877                         if (ib_wr->num_sge > 1) {
 878                                 err = -EINVAL;
 879                                 break;
 880                         }
 881
 882                         /*
 883                          * Move the local and remote stag/to/len into the WR.
 884                          */
 885                         wr.sqwr.rdma_read.local_stag =
 886                             cpu_to_be32(ib_wr->sg_list->lkey);
 887                         wr.sqwr.rdma_read.local_to =
 888                             cpu_to_be64(ib_wr->sg_list->addr);
 889                         wr.sqwr.rdma_read.remote_stag =
 890                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 891                         wr.sqwr.rdma_read.remote_to =
 892                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 893                         wr.sqwr.rdma_read.length =
 894                             cpu_to_be32(ib_wr->sg_list->length);
 895                         break;
 896                 default:
 897                         /* error */
 898                         msg_size = 0;
 899                         err = -EINVAL;
 900                         break;
 901                 }
 902
 903                 /*
 904                  * If we had an error on the last wr build, then
 905                  * break out.  Possible errors include bogus WR
 906                  * type, and a bogus SGL length...
 907                  */
 908                 if (err) {
 909                         break;
 910                 }
 911
 912                 /*
 913                  * Store flags
 914                  */
 915                 c2_wr_set_flags(&wr, flags);
 916
 917                 /*
 918                  * Post the puppy!
 919                  */
 920                 spin_lock_irqsave(&qp->lock, lock_flags);
 921                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 922                 if (err) {
 923                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 924                         break;
 925                 }
 926
 927                 /*
 928                  * Enqueue mq index to activity FIFO.
 929                  */
 930                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 931                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 932
 933                 ib_wr = ib_wr->next;
 934         }
 935
 936 out:
 937         if (err)
 938                 *bad_wr = ib_wr;
 939         return err;
 940 }
 941
 942 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 943                     struct ib_recv_wr **bad_wr)
 944 {
 945         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 946         struct c2_qp *qp = to_c2qp(ibqp);
 947         union c2wr wr;
 948         unsigned long lock_flags;
 949         int err = 0;
 950
 951         if (qp->state > IB_QPS_RTS) {
 952                 err = -EINVAL;
 953                 goto out;
 954         }
 955
 956         /*
 957          * Try and post each work request
 958          */
 959         while (ib_wr) {
 960                 u32 tot_len;
 961                 u8 actual_sge_count;
 962
 963                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
 964                         err = -EINVAL;
 965                         break;
 966                 }
 967
 968                 /*
 969                  * Create local host-copy of the WR
 970                  */
 971                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 972                 c2_wr_set_id(&wr, CCWR_RECV);
 973                 c2_wr_set_flags(&wr, 0);
 974
 975                 /* sge_count is limited to eight bits. */
 976                 BUG_ON(ib_wr->num_sge >= 256);
 977                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 978                                ib_wr->sg_list,
 979                                ib_wr->num_sge, &tot_len, &actual_sge_count);
 980                 c2_wr_set_sge_count(&wr, actual_sge_count);
 981
 982                 /*
 983                  * If we had an error on the last wr build, then
 984                  * break out.  Possible errors include bogus WR
 985                  * type, and a bogus SGL length...
 986                  */
 987                 if (err) {
 988                         break;
 989                 }
 990
 991                 spin_lock_irqsave(&qp->lock, lock_flags);
 992                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 993                 if (err) {
 994                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 995                         break;
 996                 }
 997
 998                 /*
 999                  * Enqueue mq index to activity FIFO
1000                  */
1001                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
1002                 spin_unlock_irqrestore(&qp->lock, lock_flags);
1003
1004                 ib_wr = ib_wr->next;
1005         }
1006
1007 out:
1008         if (err)
1009                 *bad_wr = ib_wr;
1010         return err;
1011 }
1012
1013 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
1014 {
1015         spin_lock_init(&c2dev->qp_table.lock);
1016         idr_init(&c2dev->qp_table.idr);
1017 }
1018
1019 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1020 {
1021         idr_destroy(&c2dev->qp_table.idr);
1022 }