drivers/infiniband/hw/amso1100/c2_qp.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #include <linux/delay.h>
  39
  40 #include "c2.h"
  41 #include "c2_vq.h"
  42 #include "c2_status.h"
  43
  44 #define C2_MAX_ORD_PER_QP 128
  45 #define C2_MAX_IRD_PER_QP 128
  46
  47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  50
  51 #define NO_SUPPORT -1
  52 static const u8 c2_opcode[] = {
  53         [IB_WR_SEND] = C2_WR_TYPE_SEND,
  54         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  55         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  56         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  57         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  58         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  59         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  60 };
  61
  62 static int to_c2_state(enum ib_qp_state ib_state)
  63 {
  64         switch (ib_state) {
  65         case IB_QPS_RESET:
  66                 return C2_QP_STATE_IDLE;
  67         case IB_QPS_RTS:
  68                 return C2_QP_STATE_RTS;
  69         case IB_QPS_SQD:
  70                 return C2_QP_STATE_CLOSING;
  71         case IB_QPS_SQE:
  72                 return C2_QP_STATE_CLOSING;
  73         case IB_QPS_ERR:
  74                 return C2_QP_STATE_ERROR;
  75         default:
  76                 return -1;
  77         }
  78 }
  79
  80 static int to_ib_state(enum c2_qp_state c2_state)
  81 {
  82         switch (c2_state) {
  83         case C2_QP_STATE_IDLE:
  84                 return IB_QPS_RESET;
  85         case C2_QP_STATE_CONNECTING:
  86                 return IB_QPS_RTR;
  87         case C2_QP_STATE_RTS:
  88                 return IB_QPS_RTS;
  89         case C2_QP_STATE_CLOSING:
  90                 return IB_QPS_SQD;
  91         case C2_QP_STATE_ERROR:
  92                 return IB_QPS_ERR;
  93         case C2_QP_STATE_TERMINATE:
  94                 return IB_QPS_SQE;
  95         default:
  96                 return -1;
  97         }
  98 }
  99
 100 static const char *to_ib_state_str(int ib_state)
 101 {
 102         static const char *state_str[] = {
 103                 "IB_QPS_RESET",
 104                 "IB_QPS_INIT",
 105                 "IB_QPS_RTR",
 106                 "IB_QPS_RTS",
 107                 "IB_QPS_SQD",
 108                 "IB_QPS_SQE",
 109                 "IB_QPS_ERR"
 110         };
 111         if (ib_state < IB_QPS_RESET ||
 112             ib_state > IB_QPS_ERR)
 113                 return "<invalid IB QP state>";
 114
 115         ib_state -= IB_QPS_RESET;
 116         return state_str[ib_state];
 117 }
 118
 119 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 120 {
 121         int new_state = to_ib_state(c2_state);
 122
 123         pr_debug("%s: qp[%p] state modify %s --> %s\n",
 124                __FUNCTION__,
 125                 qp,
 126                 to_ib_state_str(qp->state),
 127                 to_ib_state_str(new_state));
 128         qp->state = new_state;
 129 }
 130
 131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 132
 133 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 134                  struct ib_qp_attr *attr, int attr_mask)
 135 {
 136         struct c2wr_qp_modify_req wr;
 137         struct c2wr_qp_modify_rep *reply;
 138         struct c2_vq_req *vq_req;
 139         unsigned long flags;
 140         u8 next_state;
 141         int err;
 142
 143         pr_debug("%s:%d qp=%p, %s --> %s\n",
 144                 __FUNCTION__, __LINE__,
 145                 qp,
 146                 to_ib_state_str(qp->state),
 147                 to_ib_state_str(attr->qp_state));
 148
 149         vq_req = vq_req_alloc(c2dev);
 150         if (!vq_req)
 151                 return -ENOMEM;
 152
 153         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 154         wr.hdr.context = (unsigned long) vq_req;
 155         wr.rnic_handle = c2dev->adapter_handle;
 156         wr.qp_handle = qp->adapter_handle;
 157         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 158         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 160         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 161
 162         if (attr_mask & IB_QP_STATE) {
 163                 /* Ensure the state is valid */
 164                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
 165                         return -EINVAL;
 166
 167                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 168
 169                 if (attr->qp_state == IB_QPS_ERR) {
 170                         spin_lock_irqsave(&qp->lock, flags);
 171                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 172                                 pr_debug("Generating CLOSE event for QP-->ERR, "
 173                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 174                                 /* Generate an CLOSE event */
 175                                 vq_req->cm_id = qp->cm_id;
 176                                 vq_req->event = IW_CM_EVENT_CLOSE;
 177                         }
 178                         spin_unlock_irqrestore(&qp->lock, flags);
 179                 }
 180                 next_state =  attr->qp_state;
 181
 182         } else if (attr_mask & IB_QP_CUR_STATE) {
 183
 184                 if (attr->cur_qp_state != IB_QPS_RTR &&
 185                     attr->cur_qp_state != IB_QPS_RTS &&
 186                     attr->cur_qp_state != IB_QPS_SQD &&
 187                     attr->cur_qp_state != IB_QPS_SQE)
 188                         return -EINVAL;
 189                 else
 190                         wr.next_qp_state =
 191                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
 192
 193                 next_state = attr->cur_qp_state;
 194
 195         } else {
 196                 err = 0;
 197                 goto bail0;
 198         }
 199
 200         /* reference the request struct */
 201         vq_req_get(c2dev, vq_req);
 202
 203         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 204         if (err) {
 205                 vq_req_put(c2dev, vq_req);
 206                 goto bail0;
 207         }
 208
 209         err = vq_wait_for_reply(c2dev, vq_req);
 210         if (err)
 211                 goto bail0;
 212
 213         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 214         if (!reply) {
 215                 err = -ENOMEM;
 216                 goto bail0;
 217         }
 218
 219         err = c2_errno(reply);
 220         if (!err)
 221                 qp->state = next_state;
 222 #ifdef DEBUG
 223         else
 224                 pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
 225 #endif
 226         /*
 227          * If we're going to error and generating the event here, then
 228          * we need to remove the reference because there will be no
 229          * close event generated by the adapter
 230         */
 231         spin_lock_irqsave(&qp->lock, flags);
 232         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 233                 qp->cm_id->rem_ref(qp->cm_id);
 234                 qp->cm_id = NULL;
 235         }
 236         spin_unlock_irqrestore(&qp->lock, flags);
 237
 238         vq_repbuf_free(c2dev, reply);
 239       bail0:
 240         vq_req_free(c2dev, vq_req);
 241
 242         pr_debug("%s:%d qp=%p, cur_state=%s\n",
 243                 __FUNCTION__, __LINE__,
 244                 qp,
 245                 to_ib_state_str(qp->state));
 246         return err;
 247 }
 248
 249 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 250                           int ord, int ird)
 251 {
 252         struct c2wr_qp_modify_req wr;
 253         struct c2wr_qp_modify_rep *reply;
 254         struct c2_vq_req *vq_req;
 255         int err;
 256
 257         vq_req = vq_req_alloc(c2dev);
 258         if (!vq_req)
 259                 return -ENOMEM;
 260
 261         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 262         wr.hdr.context = (unsigned long) vq_req;
 263         wr.rnic_handle = c2dev->adapter_handle;
 264         wr.qp_handle = qp->adapter_handle;
 265         wr.ord = cpu_to_be32(ord);
 266         wr.ird = cpu_to_be32(ird);
 267         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 268         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 269         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 270
 271         /* reference the request struct */
 272         vq_req_get(c2dev, vq_req);
 273
 274         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 275         if (err) {
 276                 vq_req_put(c2dev, vq_req);
 277                 goto bail0;
 278         }
 279
 280         err = vq_wait_for_reply(c2dev, vq_req);
 281         if (err)
 282                 goto bail0;
 283
 284         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 285                 vq_req->reply_msg;
 286         if (!reply) {
 287                 err = -ENOMEM;
 288                 goto bail0;
 289         }
 290
 291         err = c2_errno(reply);
 292         vq_repbuf_free(c2dev, reply);
 293       bail0:
 294         vq_req_free(c2dev, vq_req);
 295         return err;
 296 }
 297
 298 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 299 {
 300         struct c2_vq_req *vq_req;
 301         struct c2wr_qp_destroy_req wr;
 302         struct c2wr_qp_destroy_rep *reply;
 303         unsigned long flags;
 304         int err;
 305
 306         /*
 307          * Allocate a verb request message
 308          */
 309         vq_req = vq_req_alloc(c2dev);
 310         if (!vq_req) {
 311                 return -ENOMEM;
 312         }
 313
 314         /*
 315          * Initialize the WR
 316          */
 317         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 318         wr.hdr.context = (unsigned long) vq_req;
 319         wr.rnic_handle = c2dev->adapter_handle;
 320         wr.qp_handle = qp->adapter_handle;
 321
 322         /*
 323          * reference the request struct.  dereferenced in the int handler.
 324          */
 325         vq_req_get(c2dev, vq_req);
 326
 327         spin_lock_irqsave(&qp->lock, flags);
 328         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 329                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 330                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 331                 /* Generate an CLOSE event */
 332                 vq_req->qp = qp;
 333                 vq_req->cm_id = qp->cm_id;
 334                 vq_req->event = IW_CM_EVENT_CLOSE;
 335         }
 336         spin_unlock_irqrestore(&qp->lock, flags);
 337
 338         /*
 339          * Send WR to adapter
 340          */
 341         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 342         if (err) {
 343                 vq_req_put(c2dev, vq_req);
 344                 goto bail0;
 345         }
 346
 347         /*
 348          * Wait for reply from adapter
 349          */
 350         err = vq_wait_for_reply(c2dev, vq_req);
 351         if (err) {
 352                 goto bail0;
 353         }
 354
 355         /*
 356          * Process reply
 357          */
 358         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 359         if (!reply) {
 360                 err = -ENOMEM;
 361                 goto bail0;
 362         }
 363
 364         spin_lock_irqsave(&qp->lock, flags);
 365         if (qp->cm_id) {
 366                 qp->cm_id->rem_ref(qp->cm_id);
 367                 qp->cm_id = NULL;
 368         }
 369         spin_unlock_irqrestore(&qp->lock, flags);
 370
 371         vq_repbuf_free(c2dev, reply);
 372       bail0:
 373         vq_req_free(c2dev, vq_req);
 374         return err;
 375 }
 376
 377 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 378 {
 379         int ret;
 380
 381         do {
 382                 spin_lock_irq(&c2dev->qp_table.lock);
 383                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
 384                                         c2dev->qp_table.last++, &qp->qpn);
 385                 spin_unlock_irq(&c2dev->qp_table.lock);
 386         } while ((ret == -EAGAIN) &&
 387                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
 388         return ret;
 389 }
 390
 391 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 392 {
 393         spin_lock_irq(&c2dev->qp_table.lock);
 394         idr_remove(&c2dev->qp_table.idr, qpn);
 395         spin_unlock_irq(&c2dev->qp_table.lock);
 396 }
 397
 398 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 399 {
 400         unsigned long flags;
 401         struct c2_qp *qp;
 402
 403         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 404         qp = idr_find(&c2dev->qp_table.idr, qpn);
 405         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 406         return qp;
 407 }
 408
 409 int c2_alloc_qp(struct c2_dev *c2dev,
 410                 struct c2_pd *pd,
 411                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 412 {
 413         struct c2wr_qp_create_req wr;
 414         struct c2wr_qp_create_rep *reply;
 415         struct c2_vq_req *vq_req;
 416         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 417         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 418         unsigned long peer_pa;
 419         u32 q_size, msg_size, mmap_size;
 420         void __iomem *mmap;
 421         int err;
 422
 423         err = c2_alloc_qpn(c2dev, qp);
 424         if (err)
 425                 return err;
 426         qp->ibqp.qp_num = qp->qpn;
 427         qp->ibqp.qp_type = IB_QPT_RC;
 428
 429         /* Allocate the SQ and RQ shared pointers */
 430         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 431                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
 432         if (!qp->sq_mq.shared) {
 433                 err = -ENOMEM;
 434                 goto bail0;
 435         }
 436
 437         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 438                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
 439         if (!qp->rq_mq.shared) {
 440                 err = -ENOMEM;
 441                 goto bail1;
 442         }
 443
 444         /* Allocate the verbs request */
 445         vq_req = vq_req_alloc(c2dev);
 446         if (vq_req == NULL) {
 447                 err = -ENOMEM;
 448                 goto bail2;
 449         }
 450
 451         /* Initialize the work request */
 452         memset(&wr, 0, sizeof(wr));
 453         c2_wr_set_id(&wr, CCWR_QP_CREATE);
 454         wr.hdr.context = (unsigned long) vq_req;
 455         wr.rnic_handle = c2dev->adapter_handle;
 456         wr.sq_cq_handle = send_cq->adapter_handle;
 457         wr.rq_cq_handle = recv_cq->adapter_handle;
 458         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 459         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 460         wr.srq_handle = 0;
 461         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 462                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 463         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 464         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 465         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 466         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 467         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 468         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 469         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 470         wr.pd_id = pd->pd_id;
 471         wr.user_context = (unsigned long) qp;
 472
 473         vq_req_get(c2dev, vq_req);
 474
 475         /* Send the WR to the adapter */
 476         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 477         if (err) {
 478                 vq_req_put(c2dev, vq_req);
 479                 goto bail3;
 480         }
 481
 482         /* Wait for the verb reply  */
 483         err = vq_wait_for_reply(c2dev, vq_req);
 484         if (err) {
 485                 goto bail3;
 486         }
 487
 488         /* Process the reply */
 489         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 490         if (!reply) {
 491                 err = -ENOMEM;
 492                 goto bail3;
 493         }
 494
 495         if ((err = c2_wr_get_result(reply)) != 0) {
 496                 goto bail4;
 497         }
 498
 499         /* Fill in the kernel QP struct */
 500         atomic_set(&qp->refcount, 1);
 501         qp->adapter_handle = reply->qp_handle;
 502         qp->state = IB_QPS_RESET;
 503         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 504         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 505         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 506
 507         /* Initialize the SQ MQ */
 508         q_size = be32_to_cpu(reply->sq_depth);
 509         msg_size = be32_to_cpu(reply->sq_msg_size);
 510         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 511         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 512         mmap = ioremap_nocache(peer_pa, mmap_size);
 513         if (!mmap) {
 514                 err = -ENOMEM;
 515                 goto bail5;
 516         }
 517
 518         c2_mq_req_init(&qp->sq_mq,
 519                        be32_to_cpu(reply->sq_mq_index),
 520                        q_size,
 521                        msg_size,
 522                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 523                        mmap,                            /* peer */
 524                        C2_MQ_ADAPTER_TARGET);
 525
 526         /* Initialize the RQ mq */
 527         q_size = be32_to_cpu(reply->rq_depth);
 528         msg_size = be32_to_cpu(reply->rq_msg_size);
 529         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 530         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 531         mmap = ioremap_nocache(peer_pa, mmap_size);
 532         if (!mmap) {
 533                 err = -ENOMEM;
 534                 goto bail6;
 535         }
 536
 537         c2_mq_req_init(&qp->rq_mq,
 538                        be32_to_cpu(reply->rq_mq_index),
 539                        q_size,
 540                        msg_size,
 541                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 542                        mmap,                            /* peer */
 543                        C2_MQ_ADAPTER_TARGET);
 544
 545         vq_repbuf_free(c2dev, reply);
 546         vq_req_free(c2dev, vq_req);
 547
 548         return 0;
 549
 550       bail6:
 551         iounmap(qp->sq_mq.peer);
 552       bail5:
 553         destroy_qp(c2dev, qp);
 554       bail4:
 555         vq_repbuf_free(c2dev, reply);
 556       bail3:
 557         vq_req_free(c2dev, vq_req);
 558       bail2:
 559         c2_free_mqsp(qp->rq_mq.shared);
 560       bail1:
 561         c2_free_mqsp(qp->sq_mq.shared);
 562       bail0:
 563         c2_free_qpn(c2dev, qp->qpn);
 564         return err;
 565 }
 566
 567 static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 568 {
 569         if (send_cq == recv_cq)
 570                 spin_lock_irq(&send_cq->lock);
 571         else if (send_cq > recv_cq) {
 572                 spin_lock_irq(&send_cq->lock);
 573                 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
 574         } else {
 575                 spin_lock_irq(&recv_cq->lock);
 576                 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
 577         }
 578 }
 579
 580 static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
 581 {
 582         if (send_cq == recv_cq)
 583                 spin_unlock_irq(&send_cq->lock);
 584         else if (send_cq > recv_cq) {
 585                 spin_unlock(&recv_cq->lock);
 586                 spin_unlock_irq(&send_cq->lock);
 587         } else {
 588                 spin_unlock(&send_cq->lock);
 589                 spin_unlock_irq(&recv_cq->lock);
 590         }
 591 }
 592
 593 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 594 {
 595         struct c2_cq *send_cq;
 596         struct c2_cq *recv_cq;
 597
 598         send_cq = to_c2cq(qp->ibqp.send_cq);
 599         recv_cq = to_c2cq(qp->ibqp.recv_cq);
 600
 601         /*
 602          * Lock CQs here, so that CQ polling code can do QP lookup
 603          * without taking a lock.
 604          */
 605         c2_lock_cqs(send_cq, recv_cq);
 606         c2_free_qpn(c2dev, qp->qpn);
 607         c2_unlock_cqs(send_cq, recv_cq);
 608
 609         /*
 610          * Destory qp in the rnic...
 611          */
 612         destroy_qp(c2dev, qp);
 613
 614         /*
 615          * Mark any unreaped CQEs as null and void.
 616          */
 617         c2_cq_clean(c2dev, qp, send_cq->cqn);
 618         if (send_cq != recv_cq)
 619                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
 620         /*
 621          * Unmap the MQs and return the shared pointers
 622          * to the message pool.
 623          */
 624         iounmap(qp->sq_mq.peer);
 625         iounmap(qp->rq_mq.peer);
 626         c2_free_mqsp(qp->sq_mq.shared);
 627         c2_free_mqsp(qp->rq_mq.shared);
 628
 629         atomic_dec(&qp->refcount);
 630         wait_event(qp->wait, !atomic_read(&qp->refcount));
 631 }
 632
 633 /*
 634  * Function: move_sgl
 635  *
 636  * Description:
 637  * Move an SGL from the user's work request struct into a CCIL Work Request
 638  * message, swapping to WR byte order and ensure the total length doesn't
 639  * overflow.
 640  *
 641  * IN:
 642  * dst          - ptr to CCIL Work Request message SGL memory.
 643  * src          - ptr to the consumers SGL memory.
 644  *
 645  * OUT: none
 646  *
 647  * Return:
 648  * CCIL status codes.
 649  */
 650 static int
 651 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 652          u8 * actual_count)
 653 {
 654         u32 tot = 0;            /* running total */
 655         u8 acount = 0;          /* running total non-0 len sge's */
 656
 657         while (count > 0) {
 658                 /*
 659                  * If the addition of this SGE causes the
 660                  * total SGL length to exceed 2^32-1, then
 661                  * fail-n-bail.
 662                  *
 663                  * If the current total plus the next element length
 664                  * wraps, then it will go negative and be less than the
 665                  * current total...
 666                  */
 667                 if ((tot + src->length) < tot) {
 668                         return -EINVAL;
 669                 }
 670                 /*
 671                  * Bug: 1456 (as well as 1498 & 1643)
 672                  * Skip over any sge's supplied with len=0
 673                  */
 674                 if (src->length) {
 675                         tot += src->length;
 676                         dst->stag = cpu_to_be32(src->lkey);
 677                         dst->to = cpu_to_be64(src->addr);
 678                         dst->length = cpu_to_be32(src->length);
 679                         dst++;
 680                         acount++;
 681                 }
 682                 src++;
 683                 count--;
 684         }
 685
 686         if (acount == 0) {
 687                 /*
 688                  * Bug: 1476 (as well as 1498, 1456 and 1643)
 689                  * Setup the SGL in the WR to make it easier for the RNIC.
 690                  * This way, the FW doesn't have to deal with special cases.
 691                  * Setting length=0 should be sufficient.
 692                  */
 693                 dst->stag = 0;
 694                 dst->to = 0;
 695                 dst->length = 0;
 696         }
 697
 698         *p_len = tot;
 699         *actual_count = acount;
 700         return 0;
 701 }
 702
 703 /*
 704  * Function: c2_activity (private function)
 705  *
 706  * Description:
 707  * Post an mq index to the host->adapter activity fifo.
 708  *
 709  * IN:
 710  * c2dev        - ptr to c2dev structure
 711  * mq_index     - mq index to post
 712  * shared       - value most recently written to shared
 713  *
 714  * OUT:
 715  *
 716  * Return:
 717  * none
 718  */
 719 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 720 {
 721         /*
 722          * First read the register to see if the FIFO is full, and if so,
 723          * spin until it's not.  This isn't perfect -- there is no
 724          * synchronization among the clients of the register, but in
 725          * practice it prevents multiple CPU from hammering the bus
 726          * with PCI RETRY. Note that when this does happen, the card
 727          * cannot get on the bus and the card and system hang in a
 728          * deadlock -- thus the need for this code. [TOT]
 729          */
 730         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
 731                 udelay(10);
 732
 733         __raw_writel(C2_HINT_MAKE(mq_index, shared),
 734                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 735 }
 736
 737 /*
 738  * Function: qp_wr_post
 739  *
 740  * Description:
 741  * This in-line function allocates a MQ msg, then moves the host-copy of
 742  * the completed WR into msg.  Then it posts the message.
 743  *
 744  * IN:
 745  * q            - ptr to user MQ.
 746  * wr           - ptr to host-copy of the WR.
 747  * qp           - ptr to user qp
 748  * size         - Number of bytes to post.  Assumed to be divisible by 4.
 749  *
 750  * OUT: none
 751  *
 752  * Return:
 753  * CCIL status codes.
 754  */
 755 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 756 {
 757         union c2wr *msg;
 758
 759         msg = c2_mq_alloc(q);
 760         if (msg == NULL) {
 761                 return -EINVAL;
 762         }
 763 #ifdef CCMSGMAGIC
 764         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 765 #endif
 766
 767         /*
 768          * Since all header fields in the WR are the same as the
 769          * CQE, set the following so the adapter need not.
 770          */
 771         c2_wr_set_result(wr, CCERR_PENDING);
 772
 773         /*
 774          * Copy the wr down to the adapter
 775          */
 776         memcpy((void *) msg, (void *) wr, size);
 777
 778         c2_mq_produce(q);
 779         return 0;
 780 }
 781
 782
 783 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 784                  struct ib_send_wr **bad_wr)
 785 {
 786         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 787         struct c2_qp *qp = to_c2qp(ibqp);
 788         union c2wr wr;
 789         unsigned long lock_flags;
 790         int err = 0;
 791
 792         u32 flags;
 793         u32 tot_len;
 794         u8 actual_sge_count;
 795         u32 msg_size;
 796
 797         if (qp->state > IB_QPS_RTS)
 798                 return -EINVAL;
 799
 800         while (ib_wr) {
 801
 802                 flags = 0;
 803                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 804                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 805                         flags |= SQ_SIGNALED;
 806                 }
 807
 808                 switch (ib_wr->opcode) {
 809                 case IB_WR_SEND:
 810                         if (ib_wr->send_flags & IB_SEND_SOLICITED) {
 811                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 812                                 msg_size = sizeof(struct c2wr_send_req);
 813                         } else {
 814                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 815                                 msg_size = sizeof(struct c2wr_send_req);
 816                         }
 817
 818                         wr.sqwr.send.remote_stag = 0;
 819                         msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
 820                         if (ib_wr->num_sge > qp->send_sgl_depth) {
 821                                 err = -EINVAL;
 822                                 break;
 823                         }
 824                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 825                                 flags |= SQ_READ_FENCE;
 826                         }
 827                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 828                                        ib_wr->sg_list,
 829                                        ib_wr->num_sge,
 830                                        &tot_len, &actual_sge_count);
 831                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 832                         c2_wr_set_sge_count(&wr, actual_sge_count);
 833                         break;
 834                 case IB_WR_RDMA_WRITE:
 835                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 836                         msg_size = sizeof(struct c2wr_rdma_write_req) +
 837                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 838                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 839                                 err = -EINVAL;
 840                                 break;
 841                         }
 842                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 843                                 flags |= SQ_READ_FENCE;
 844                         }
 845                         wr.sqwr.rdma_write.remote_stag =
 846                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 847                         wr.sqwr.rdma_write.remote_to =
 848                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 849                         err = move_sgl((struct c2_data_addr *)
 850                                        & (wr.sqwr.rdma_write.data),
 851                                        ib_wr->sg_list,
 852                                        ib_wr->num_sge,
 853                                        &tot_len, &actual_sge_count);
 854                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 855                         c2_wr_set_sge_count(&wr, actual_sge_count);
 856                         break;
 857                 case IB_WR_RDMA_READ:
 858                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 859                         msg_size = sizeof(struct c2wr_rdma_read_req);
 860
 861                         /* IWarp only suppots 1 sge for RDMA reads */
 862                         if (ib_wr->num_sge > 1) {
 863                                 err = -EINVAL;
 864                                 break;
 865                         }
 866
 867                         /*
 868                          * Move the local and remote stag/to/len into the WR.
 869                          */
 870                         wr.sqwr.rdma_read.local_stag =
 871                             cpu_to_be32(ib_wr->sg_list->lkey);
 872                         wr.sqwr.rdma_read.local_to =
 873                             cpu_to_be64(ib_wr->sg_list->addr);
 874                         wr.sqwr.rdma_read.remote_stag =
 875                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 876                         wr.sqwr.rdma_read.remote_to =
 877                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 878                         wr.sqwr.rdma_read.length =
 879                             cpu_to_be32(ib_wr->sg_list->length);
 880                         break;
 881                 default:
 882                         /* error */
 883                         msg_size = 0;
 884                         err = -EINVAL;
 885                         break;
 886                 }
 887
 888                 /*
 889                  * If we had an error on the last wr build, then
 890                  * break out.  Possible errors include bogus WR
 891                  * type, and a bogus SGL length...
 892                  */
 893                 if (err) {
 894                         break;
 895                 }
 896
 897                 /*
 898                  * Store flags
 899                  */
 900                 c2_wr_set_flags(&wr, flags);
 901
 902                 /*
 903                  * Post the puppy!
 904                  */
 905                 spin_lock_irqsave(&qp->lock, lock_flags);
 906                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 907                 if (err) {
 908                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 909                         break;
 910                 }
 911
 912                 /*
 913                  * Enqueue mq index to activity FIFO.
 914                  */
 915                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 916                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 917
 918                 ib_wr = ib_wr->next;
 919         }
 920
 921         if (err)
 922                 *bad_wr = ib_wr;
 923         return err;
 924 }
 925
 926 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 927                     struct ib_recv_wr **bad_wr)
 928 {
 929         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 930         struct c2_qp *qp = to_c2qp(ibqp);
 931         union c2wr wr;
 932         unsigned long lock_flags;
 933         int err = 0;
 934
 935         if (qp->state > IB_QPS_RTS)
 936                 return -EINVAL;
 937
 938         /*
 939          * Try and post each work request
 940          */
 941         while (ib_wr) {
 942                 u32 tot_len;
 943                 u8 actual_sge_count;
 944
 945                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
 946                         err = -EINVAL;
 947                         break;
 948                 }
 949
 950                 /*
 951                  * Create local host-copy of the WR
 952                  */
 953                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 954                 c2_wr_set_id(&wr, CCWR_RECV);
 955                 c2_wr_set_flags(&wr, 0);
 956
 957                 /* sge_count is limited to eight bits. */
 958                 BUG_ON(ib_wr->num_sge >= 256);
 959                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 960                                ib_wr->sg_list,
 961                                ib_wr->num_sge, &tot_len, &actual_sge_count);
 962                 c2_wr_set_sge_count(&wr, actual_sge_count);
 963
 964                 /*
 965                  * If we had an error on the last wr build, then
 966                  * break out.  Possible errors include bogus WR
 967                  * type, and a bogus SGL length...
 968                  */
 969                 if (err) {
 970                         break;
 971                 }
 972
 973                 spin_lock_irqsave(&qp->lock, lock_flags);
 974                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 975                 if (err) {
 976                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 977                         break;
 978                 }
 979
 980                 /*
 981                  * Enqueue mq index to activity FIFO
 982                  */
 983                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
 984                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 985
 986                 ib_wr = ib_wr->next;
 987         }
 988
 989         if (err)
 990                 *bad_wr = ib_wr;
 991         return err;
 992 }
 993
 994 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
 995 {
 996         spin_lock_init(&c2dev->qp_table.lock);
 997         idr_init(&c2dev->qp_table.idr);
 998 }
 999
1000 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1001 {
1002         idr_destroy(&c2dev->qp_table.idr);
1003 }