drivers/infiniband/core/verbs.c

   1 /*
   2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   9  *
  10  * This software is available to you under a choice of one of two
  11  * licenses.  You may choose to be licensed under the terms of the GNU
  12  * General Public License (GPL) Version 2, available from the file
  13  * COPYING in the main directory of this source tree, or the
  14  * OpenIB.org BSD license below:
  15  *
  16  *     Redistribution and use in source and binary forms, with or
  17  *     without modification, are permitted provided that the following
  18  *     conditions are met:
  19  *
  20  *      - Redistributions of source code must retain the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer.
  23  *
  24  *      - Redistributions in binary form must reproduce the above
  25  *        copyright notice, this list of conditions and the following
  26  *        disclaimer in the documentation and/or other materials
  27  *        provided with the distribution.
  28  *
  29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  36  * SOFTWARE.
  37  */
  38
  39 #include <linux/errno.h>
  40 #include <linux/err.h>
  41 #include <linux/string.h>
  42 #include <linux/slab.h>
  43
  44 #include <rdma/ib_verbs.h>
  45 #include <rdma/ib_cache.h>
  46
  47 int ib_rate_to_mult(enum ib_rate rate)
  48 {
  49         switch (rate) {
  50         case IB_RATE_2_5_GBPS: return  1;
  51         case IB_RATE_5_GBPS:   return  2;
  52         case IB_RATE_10_GBPS:  return  4;
  53         case IB_RATE_20_GBPS:  return  8;
  54         case IB_RATE_30_GBPS:  return 12;
  55         case IB_RATE_40_GBPS:  return 16;
  56         case IB_RATE_60_GBPS:  return 24;
  57         case IB_RATE_80_GBPS:  return 32;
  58         case IB_RATE_120_GBPS: return 48;
  59         default:               return -1;
  60         }
  61 }
  62 EXPORT_SYMBOL(ib_rate_to_mult);
  63
  64 enum ib_rate mult_to_ib_rate(int mult)
  65 {
  66         switch (mult) {
  67         case 1:  return IB_RATE_2_5_GBPS;
  68         case 2:  return IB_RATE_5_GBPS;
  69         case 4:  return IB_RATE_10_GBPS;
  70         case 8:  return IB_RATE_20_GBPS;
  71         case 12: return IB_RATE_30_GBPS;
  72         case 16: return IB_RATE_40_GBPS;
  73         case 24: return IB_RATE_60_GBPS;
  74         case 32: return IB_RATE_80_GBPS;
  75         case 48: return IB_RATE_120_GBPS;
  76         default: return IB_RATE_PORT_CURRENT;
  77         }
  78 }
  79 EXPORT_SYMBOL(mult_to_ib_rate);
  80
  81 int ib_rate_to_mbps(enum ib_rate rate)
  82 {
  83         switch (rate) {
  84         case IB_RATE_2_5_GBPS: return 2500;
  85         case IB_RATE_5_GBPS:   return 5000;
  86         case IB_RATE_10_GBPS:  return 10000;
  87         case IB_RATE_20_GBPS:  return 20000;
  88         case IB_RATE_30_GBPS:  return 30000;
  89         case IB_RATE_40_GBPS:  return 40000;
  90         case IB_RATE_60_GBPS:  return 60000;
  91         case IB_RATE_80_GBPS:  return 80000;
  92         case IB_RATE_120_GBPS: return 120000;
  93         case IB_RATE_14_GBPS:  return 14062;
  94         case IB_RATE_56_GBPS:  return 56250;
  95         case IB_RATE_112_GBPS: return 112500;
  96         case IB_RATE_168_GBPS: return 168750;
  97         case IB_RATE_25_GBPS:  return 25781;
  98         case IB_RATE_100_GBPS: return 103125;
  99         case IB_RATE_200_GBPS: return 206250;
 100         case IB_RATE_300_GBPS: return 309375;
 101         default:               return -1;
 102         }
 103 }
 104 EXPORT_SYMBOL(ib_rate_to_mbps);
 105
 106 enum rdma_transport_type
 107 rdma_node_get_transport(enum rdma_node_type node_type)
 108 {
 109         switch (node_type) {
 110         case RDMA_NODE_IB_CA:
 111         case RDMA_NODE_IB_SWITCH:
 112         case RDMA_NODE_IB_ROUTER:
 113                 return RDMA_TRANSPORT_IB;
 114         case RDMA_NODE_RNIC:
 115                 return RDMA_TRANSPORT_IWARP;
 116         default:
 117                 BUG();
 118                 return 0;
 119         }
 120 }
 121 EXPORT_SYMBOL(rdma_node_get_transport);
 122
 123 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
 124 {
 125         if (device->get_link_layer)
 126                 return device->get_link_layer(device, port_num);
 127
 128         switch (rdma_node_get_transport(device->node_type)) {
 129         case RDMA_TRANSPORT_IB:
 130                 return IB_LINK_LAYER_INFINIBAND;
 131         case RDMA_TRANSPORT_IWARP:
 132                 return IB_LINK_LAYER_ETHERNET;
 133         default:
 134                 return IB_LINK_LAYER_UNSPECIFIED;
 135         }
 136 }
 137 EXPORT_SYMBOL(rdma_port_get_link_layer);
 138
 139 /* Protection domains */
 140
 141 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 142 {
 143         struct ib_pd *pd;
 144
 145         pd = device->alloc_pd(device, NULL, NULL);
 146
 147         if (!IS_ERR(pd)) {
 148                 pd->device  = device;
 149                 pd->uobject = NULL;
 150                 atomic_set(&pd->usecnt, 0);
 151         }
 152
 153         return pd;
 154 }
 155 EXPORT_SYMBOL(ib_alloc_pd);
 156
 157 int ib_dealloc_pd(struct ib_pd *pd)
 158 {
 159         if (atomic_read(&pd->usecnt))
 160                 return -EBUSY;
 161
 162         return pd->device->dealloc_pd(pd);
 163 }
 164 EXPORT_SYMBOL(ib_dealloc_pd);
 165
 166 /* Address handles */
 167
 168 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 169 {
 170         struct ib_ah *ah;
 171
 172         ah = pd->device->create_ah(pd, ah_attr);
 173
 174         if (!IS_ERR(ah)) {
 175                 ah->device  = pd->device;
 176                 ah->pd      = pd;
 177                 ah->uobject = NULL;
 178                 atomic_inc(&pd->usecnt);
 179         }
 180
 181         return ah;
 182 }
 183 EXPORT_SYMBOL(ib_create_ah);
 184
 185 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
 186                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
 187 {
 188         u32 flow_class;
 189         u16 gid_index;
 190         int ret;
 191
 192         memset(ah_attr, 0, sizeof *ah_attr);
 193         ah_attr->dlid = wc->slid;
 194         ah_attr->sl = wc->sl;
 195         ah_attr->src_path_bits = wc->dlid_path_bits;
 196         ah_attr->port_num = port_num;
 197
 198         if (wc->wc_flags & IB_WC_GRH) {
 199                 ah_attr->ah_flags = IB_AH_GRH;
 200                 ah_attr->grh.dgid = grh->sgid;
 201
 202                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
 203                                          &gid_index);
 204                 if (ret)
 205                         return ret;
 206
 207                 ah_attr->grh.sgid_index = (u8) gid_index;
 208                 flow_class = be32_to_cpu(grh->version_tclass_flow);
 209                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
 210                 ah_attr->grh.hop_limit = 0xFF;
 211                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 212         }
 213         return 0;
 214 }
 215 EXPORT_SYMBOL(ib_init_ah_from_wc);
 216
 217 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
 218                                    struct ib_grh *grh, u8 port_num)
 219 {
 220         struct ib_ah_attr ah_attr;
 221         int ret;
 222
 223         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 224         if (ret)
 225                 return ERR_PTR(ret);
 226
 227         return ib_create_ah(pd, &ah_attr);
 228 }
 229 EXPORT_SYMBOL(ib_create_ah_from_wc);
 230
 231 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 232 {
 233         return ah->device->modify_ah ?
 234                 ah->device->modify_ah(ah, ah_attr) :
 235                 -ENOSYS;
 236 }
 237 EXPORT_SYMBOL(ib_modify_ah);
 238
 239 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 240 {
 241         return ah->device->query_ah ?
 242                 ah->device->query_ah(ah, ah_attr) :
 243                 -ENOSYS;
 244 }
 245 EXPORT_SYMBOL(ib_query_ah);
 246
 247 int ib_destroy_ah(struct ib_ah *ah)
 248 {
 249         struct ib_pd *pd;
 250         int ret;
 251
 252         pd = ah->pd;
 253         ret = ah->device->destroy_ah(ah);
 254         if (!ret)
 255                 atomic_dec(&pd->usecnt);
 256
 257         return ret;
 258 }
 259 EXPORT_SYMBOL(ib_destroy_ah);
 260
 261 /* Shared receive queues */
 262
 263 struct ib_srq *ib_create_srq(struct ib_pd *pd,
 264                              struct ib_srq_init_attr *srq_init_attr)
 265 {
 266         struct ib_srq *srq;
 267
 268         if (!pd->device->create_srq)
 269                 return ERR_PTR(-ENOSYS);
 270
 271         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
 272
 273         if (!IS_ERR(srq)) {
 274                 srq->device        = pd->device;
 275                 srq->pd            = pd;
 276                 srq->uobject       = NULL;
 277                 srq->event_handler = srq_init_attr->event_handler;
 278                 srq->srq_context   = srq_init_attr->srq_context;
 279                 srq->srq_type      = srq_init_attr->srq_type;
 280                 if (srq->srq_type == IB_SRQT_XRC) {
 281                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
 282                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 283                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
 284                         atomic_inc(&srq->ext.xrc.cq->usecnt);
 285                 }
 286                 atomic_inc(&pd->usecnt);
 287                 atomic_set(&srq->usecnt, 0);
 288         }
 289
 290         return srq;
 291 }
 292 EXPORT_SYMBOL(ib_create_srq);
 293
 294 int ib_modify_srq(struct ib_srq *srq,
 295                   struct ib_srq_attr *srq_attr,
 296                   enum ib_srq_attr_mask srq_attr_mask)
 297 {
 298         return srq->device->modify_srq ?
 299                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
 300                 -ENOSYS;
 301 }
 302 EXPORT_SYMBOL(ib_modify_srq);
 303
 304 int ib_query_srq(struct ib_srq *srq,
 305                  struct ib_srq_attr *srq_attr)
 306 {
 307         return srq->device->query_srq ?
 308                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
 309 }
 310 EXPORT_SYMBOL(ib_query_srq);
 311
 312 int ib_destroy_srq(struct ib_srq *srq)
 313 {
 314         struct ib_pd *pd;
 315         enum ib_srq_type srq_type;
 316         struct ib_xrcd *uninitialized_var(xrcd);
 317         struct ib_cq *uninitialized_var(cq);
 318         int ret;
 319
 320         if (atomic_read(&srq->usecnt))
 321                 return -EBUSY;
 322
 323         pd = srq->pd;
 324         srq_type = srq->srq_type;
 325         if (srq_type == IB_SRQT_XRC) {
 326                 xrcd = srq->ext.xrc.xrcd;
 327                 cq = srq->ext.xrc.cq;
 328         }
 329
 330         ret = srq->device->destroy_srq(srq);
 331         if (!ret) {
 332                 atomic_dec(&pd->usecnt);
 333                 if (srq_type == IB_SRQT_XRC) {
 334                         atomic_dec(&xrcd->usecnt);
 335                         atomic_dec(&cq->usecnt);
 336                 }
 337         }
 338
 339         return ret;
 340 }
 341 EXPORT_SYMBOL(ib_destroy_srq);
 342
 343 /* Queue pairs */
 344
 345 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
 346 {
 347         struct ib_qp *qp = context;
 348
 349         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
 350                 event->element.qp->event_handler(event, event->element.qp->qp_context);
 351 }
 352
 353 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
 354 {
 355         mutex_lock(&xrcd->tgt_qp_mutex);
 356         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
 357         mutex_unlock(&xrcd->tgt_qp_mutex);
 358 }
 359
 360 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
 361                                   void (*event_handler)(struct ib_event *, void *),
 362                                   void *qp_context)
 363 {
 364         struct ib_qp *qp;
 365         unsigned long flags;
 366
 367         qp = kzalloc(sizeof *qp, GFP_KERNEL);
 368         if (!qp)
 369                 return ERR_PTR(-ENOMEM);
 370
 371         qp->real_qp = real_qp;
 372         atomic_inc(&real_qp->usecnt);
 373         qp->device = real_qp->device;
 374         qp->event_handler = event_handler;
 375         qp->qp_context = qp_context;
 376         qp->qp_num = real_qp->qp_num;
 377         qp->qp_type = real_qp->qp_type;
 378
 379         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 380         list_add(&qp->open_list, &real_qp->open_list);
 381         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 382
 383         return qp;
 384 }
 385
 386 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 387                          struct ib_qp_open_attr *qp_open_attr)
 388 {
 389         struct ib_qp *qp, *real_qp;
 390
 391         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
 392                 return ERR_PTR(-EINVAL);
 393
 394         qp = ERR_PTR(-EINVAL);
 395         mutex_lock(&xrcd->tgt_qp_mutex);
 396         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
 397                 if (real_qp->qp_num == qp_open_attr->qp_num) {
 398                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
 399                                           qp_open_attr->qp_context);
 400                         break;
 401                 }
 402         }
 403         mutex_unlock(&xrcd->tgt_qp_mutex);
 404         return qp;
 405 }
 406 EXPORT_SYMBOL(ib_open_qp);
 407
 408 struct ib_qp *ib_create_qp(struct ib_pd *pd,
 409                            struct ib_qp_init_attr *qp_init_attr)
 410 {
 411         struct ib_qp *qp, *real_qp;
 412         struct ib_device *device;
 413
 414         device = pd ? pd->device : qp_init_attr->xrcd->device;
 415         qp = device->create_qp(pd, qp_init_attr, NULL);
 416
 417         if (!IS_ERR(qp)) {
 418                 qp->device     = device;
 419                 qp->real_qp    = qp;
 420                 qp->uobject    = NULL;
 421                 qp->qp_type    = qp_init_attr->qp_type;
 422
 423                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
 424                         qp->event_handler = __ib_shared_qp_event_handler;
 425                         qp->qp_context = qp;
 426                         qp->pd = NULL;
 427                         qp->send_cq = qp->recv_cq = NULL;
 428                         qp->srq = NULL;
 429                         qp->xrcd = qp_init_attr->xrcd;
 430                         atomic_inc(&qp_init_attr->xrcd->usecnt);
 431                         INIT_LIST_HEAD(&qp->open_list);
 432                         atomic_set(&qp->usecnt, 0);
 433
 434                         real_qp = qp;
 435                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
 436                                           qp_init_attr->qp_context);
 437                         if (!IS_ERR(qp))
 438                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
 439                         else
 440                                 real_qp->device->destroy_qp(real_qp);
 441                 } else {
 442                         qp->event_handler = qp_init_attr->event_handler;
 443                         qp->qp_context = qp_init_attr->qp_context;
 444                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
 445                                 qp->recv_cq = NULL;
 446                                 qp->srq = NULL;
 447                         } else {
 448                                 qp->recv_cq = qp_init_attr->recv_cq;
 449                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
 450                                 qp->srq = qp_init_attr->srq;
 451                                 if (qp->srq)
 452                                         atomic_inc(&qp_init_attr->srq->usecnt);
 453                         }
 454
 455                         qp->pd      = pd;
 456                         qp->send_cq = qp_init_attr->send_cq;
 457                         qp->xrcd    = NULL;
 458
 459                         atomic_inc(&pd->usecnt);
 460                         atomic_inc(&qp_init_attr->send_cq->usecnt);
 461                 }
 462         }
 463
 464         return qp;
 465 }
 466 EXPORT_SYMBOL(ib_create_qp);
 467
 468 static const struct {
 469         int                     valid;
 470         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
 471         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
 472 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
 473         [IB_QPS_RESET] = {
 474                 [IB_QPS_RESET] = { .valid = 1 },
 475                 [IB_QPS_INIT]  = {
 476                         .valid = 1,
 477                         .req_param = {
 478                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 479                                                 IB_QP_PORT                      |
 480                                                 IB_QP_QKEY),
 481                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 482                                                 IB_QP_PORT                      |
 483                                                 IB_QP_ACCESS_FLAGS),
 484                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 485                                                 IB_QP_PORT                      |
 486                                                 IB_QP_ACCESS_FLAGS),
 487                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 488                                                 IB_QP_PORT                      |
 489                                                 IB_QP_ACCESS_FLAGS),
 490                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 491                                                 IB_QP_PORT                      |
 492                                                 IB_QP_ACCESS_FLAGS),
 493                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 494                                                 IB_QP_QKEY),
 495                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 496                                                 IB_QP_QKEY),
 497                         }
 498                 },
 499         },
 500         [IB_QPS_INIT]  = {
 501                 [IB_QPS_RESET] = { .valid = 1 },
 502                 [IB_QPS_ERR] =   { .valid = 1 },
 503                 [IB_QPS_INIT]  = {
 504                         .valid = 1,
 505                         .opt_param = {
 506                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 507                                                 IB_QP_PORT                      |
 508                                                 IB_QP_QKEY),
 509                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 510                                                 IB_QP_PORT                      |
 511                                                 IB_QP_ACCESS_FLAGS),
 512                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 513                                                 IB_QP_PORT                      |
 514                                                 IB_QP_ACCESS_FLAGS),
 515                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 516                                                 IB_QP_PORT                      |
 517                                                 IB_QP_ACCESS_FLAGS),
 518                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 519                                                 IB_QP_PORT                      |
 520                                                 IB_QP_ACCESS_FLAGS),
 521                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 522                                                 IB_QP_QKEY),
 523                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 524                                                 IB_QP_QKEY),
 525                         }
 526                 },
 527                 [IB_QPS_RTR]   = {
 528                         .valid = 1,
 529                         .req_param = {
 530                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 531                                                 IB_QP_PATH_MTU                  |
 532                                                 IB_QP_DEST_QPN                  |
 533                                                 IB_QP_RQ_PSN),
 534                                 [IB_QPT_RC]  = (IB_QP_AV                        |
 535                                                 IB_QP_PATH_MTU                  |
 536                                                 IB_QP_DEST_QPN                  |
 537                                                 IB_QP_RQ_PSN                    |
 538                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 539                                                 IB_QP_MIN_RNR_TIMER),
 540                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
 541                                                 IB_QP_PATH_MTU                  |
 542                                                 IB_QP_DEST_QPN                  |
 543                                                 IB_QP_RQ_PSN),
 544                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
 545                                                 IB_QP_PATH_MTU                  |
 546                                                 IB_QP_DEST_QPN                  |
 547                                                 IB_QP_RQ_PSN                    |
 548                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 549                                                 IB_QP_MIN_RNR_TIMER),
 550                         },
 551                         .opt_param = {
 552                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
 553                                                  IB_QP_QKEY),
 554                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
 555                                                  IB_QP_ACCESS_FLAGS             |
 556                                                  IB_QP_PKEY_INDEX),
 557                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
 558                                                  IB_QP_ACCESS_FLAGS             |
 559                                                  IB_QP_PKEY_INDEX),
 560                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
 561                                                  IB_QP_ACCESS_FLAGS             |
 562                                                  IB_QP_PKEY_INDEX),
 563                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
 564                                                  IB_QP_ACCESS_FLAGS             |
 565                                                  IB_QP_PKEY_INDEX),
 566                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
 567                                                  IB_QP_QKEY),
 568                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
 569                                                  IB_QP_QKEY),
 570                          }
 571                 }
 572         },
 573         [IB_QPS_RTR]   = {
 574                 [IB_QPS_RESET] = { .valid = 1 },
 575                 [IB_QPS_ERR] =   { .valid = 1 },
 576                 [IB_QPS_RTS]   = {
 577                         .valid = 1,
 578                         .req_param = {
 579                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
 580                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
 581                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
 582                                                 IB_QP_RETRY_CNT                 |
 583                                                 IB_QP_RNR_RETRY                 |
 584                                                 IB_QP_SQ_PSN                    |
 585                                                 IB_QP_MAX_QP_RD_ATOMIC),
 586                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
 587                                                 IB_QP_RETRY_CNT                 |
 588                                                 IB_QP_RNR_RETRY                 |
 589                                                 IB_QP_SQ_PSN                    |
 590                                                 IB_QP_MAX_QP_RD_ATOMIC),
 591                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
 592                                                 IB_QP_SQ_PSN),
 593                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
 594                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
 595                         },
 596                         .opt_param = {
 597                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
 598                                                  IB_QP_QKEY),
 599                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
 600                                                  IB_QP_ALT_PATH                 |
 601                                                  IB_QP_ACCESS_FLAGS             |
 602                                                  IB_QP_PATH_MIG_STATE),
 603                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
 604                                                  IB_QP_ALT_PATH                 |
 605                                                  IB_QP_ACCESS_FLAGS             |
 606                                                  IB_QP_MIN_RNR_TIMER            |
 607                                                  IB_QP_PATH_MIG_STATE),
 608                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
 609                                                  IB_QP_ALT_PATH                 |
 610                                                  IB_QP_ACCESS_FLAGS             |
 611                                                  IB_QP_PATH_MIG_STATE),
 612                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
 613                                                  IB_QP_ALT_PATH                 |
 614                                                  IB_QP_ACCESS_FLAGS             |
 615                                                  IB_QP_MIN_RNR_TIMER            |
 616                                                  IB_QP_PATH_MIG_STATE),
 617                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
 618                                                  IB_QP_QKEY),
 619                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
 620                                                  IB_QP_QKEY),
 621                          }
 622                 }
 623         },
 624         [IB_QPS_RTS]   = {
 625                 [IB_QPS_RESET] = { .valid = 1 },
 626                 [IB_QPS_ERR] =   { .valid = 1 },
 627                 [IB_QPS_RTS]   = {
 628                         .valid = 1,
 629                         .opt_param = {
 630                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 631                                                 IB_QP_QKEY),
 632                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 633                                                 IB_QP_ACCESS_FLAGS              |
 634                                                 IB_QP_ALT_PATH                  |
 635                                                 IB_QP_PATH_MIG_STATE),
 636                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 637                                                 IB_QP_ACCESS_FLAGS              |
 638                                                 IB_QP_ALT_PATH                  |
 639                                                 IB_QP_PATH_MIG_STATE            |
 640                                                 IB_QP_MIN_RNR_TIMER),
 641                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 642                                                 IB_QP_ACCESS_FLAGS              |
 643                                                 IB_QP_ALT_PATH                  |
 644                                                 IB_QP_PATH_MIG_STATE),
 645                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 646                                                 IB_QP_ACCESS_FLAGS              |
 647                                                 IB_QP_ALT_PATH                  |
 648                                                 IB_QP_PATH_MIG_STATE            |
 649                                                 IB_QP_MIN_RNR_TIMER),
 650                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 651                                                 IB_QP_QKEY),
 652                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 653                                                 IB_QP_QKEY),
 654                         }
 655                 },
 656                 [IB_QPS_SQD]   = {
 657                         .valid = 1,
 658                         .opt_param = {
 659                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 660                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 661                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 662                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 663                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
 664                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 665                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
 666                         }
 667                 },
 668         },
 669         [IB_QPS_SQD]   = {
 670                 [IB_QPS_RESET] = { .valid = 1 },
 671                 [IB_QPS_ERR] =   { .valid = 1 },
 672                 [IB_QPS_RTS]   = {
 673                         .valid = 1,
 674                         .opt_param = {
 675                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 676                                                 IB_QP_QKEY),
 677                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 678                                                 IB_QP_ALT_PATH                  |
 679                                                 IB_QP_ACCESS_FLAGS              |
 680                                                 IB_QP_PATH_MIG_STATE),
 681                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 682                                                 IB_QP_ALT_PATH                  |
 683                                                 IB_QP_ACCESS_FLAGS              |
 684                                                 IB_QP_MIN_RNR_TIMER             |
 685                                                 IB_QP_PATH_MIG_STATE),
 686                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 687                                                 IB_QP_ALT_PATH                  |
 688                                                 IB_QP_ACCESS_FLAGS              |
 689                                                 IB_QP_PATH_MIG_STATE),
 690                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 691                                                 IB_QP_ALT_PATH                  |
 692                                                 IB_QP_ACCESS_FLAGS              |
 693                                                 IB_QP_MIN_RNR_TIMER             |
 694                                                 IB_QP_PATH_MIG_STATE),
 695                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 696                                                 IB_QP_QKEY),
 697                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 698                                                 IB_QP_QKEY),
 699                         }
 700                 },
 701                 [IB_QPS_SQD]   = {
 702                         .valid = 1,
 703                         .opt_param = {
 704                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 705                                                 IB_QP_QKEY),
 706                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 707                                                 IB_QP_ALT_PATH                  |
 708                                                 IB_QP_ACCESS_FLAGS              |
 709                                                 IB_QP_PKEY_INDEX                |
 710                                                 IB_QP_PATH_MIG_STATE),
 711                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
 712                                                 IB_QP_AV                        |
 713                                                 IB_QP_TIMEOUT                   |
 714                                                 IB_QP_RETRY_CNT                 |
 715                                                 IB_QP_RNR_RETRY                 |
 716                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 717                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 718                                                 IB_QP_ALT_PATH                  |
 719                                                 IB_QP_ACCESS_FLAGS              |
 720                                                 IB_QP_PKEY_INDEX                |
 721                                                 IB_QP_MIN_RNR_TIMER             |
 722                                                 IB_QP_PATH_MIG_STATE),
 723                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
 724                                                 IB_QP_AV                        |
 725                                                 IB_QP_TIMEOUT                   |
 726                                                 IB_QP_RETRY_CNT                 |
 727                                                 IB_QP_RNR_RETRY                 |
 728                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 729                                                 IB_QP_ALT_PATH                  |
 730                                                 IB_QP_ACCESS_FLAGS              |
 731                                                 IB_QP_PKEY_INDEX                |
 732                                                 IB_QP_PATH_MIG_STATE),
 733                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
 734                                                 IB_QP_AV                        |
 735                                                 IB_QP_TIMEOUT                   |
 736                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 737                                                 IB_QP_ALT_PATH                  |
 738                                                 IB_QP_ACCESS_FLAGS              |
 739                                                 IB_QP_PKEY_INDEX                |
 740                                                 IB_QP_MIN_RNR_TIMER             |
 741                                                 IB_QP_PATH_MIG_STATE),
 742                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 743                                                 IB_QP_QKEY),
 744                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 745                                                 IB_QP_QKEY),
 746                         }
 747                 }
 748         },
 749         [IB_QPS_SQE]   = {
 750                 [IB_QPS_RESET] = { .valid = 1 },
 751                 [IB_QPS_ERR] =   { .valid = 1 },
 752                 [IB_QPS_RTS]   = {
 753                         .valid = 1,
 754                         .opt_param = {
 755                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 756                                                 IB_QP_QKEY),
 757                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 758                                                 IB_QP_ACCESS_FLAGS),
 759                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 760                                                 IB_QP_QKEY),
 761                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 762                                                 IB_QP_QKEY),
 763                         }
 764                 }
 765         },
 766         [IB_QPS_ERR] = {
 767                 [IB_QPS_RESET] = { .valid = 1 },
 768                 [IB_QPS_ERR] =   { .valid = 1 }
 769         }
 770 };
 771
 772 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 773                        enum ib_qp_type type, enum ib_qp_attr_mask mask)
 774 {
 775         enum ib_qp_attr_mask req_param, opt_param;
 776
 777         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
 778             next_state < 0 || next_state > IB_QPS_ERR)
 779                 return 0;
 780
 781         if (mask & IB_QP_CUR_STATE  &&
 782             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
 783             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
 784                 return 0;
 785
 786         if (!qp_state_table[cur_state][next_state].valid)
 787                 return 0;
 788
 789         req_param = qp_state_table[cur_state][next_state].req_param[type];
 790         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
 791
 792         if ((mask & req_param) != req_param)
 793                 return 0;
 794
 795         if (mask & ~(req_param | opt_param | IB_QP_STATE))
 796                 return 0;
 797
 798         return 1;
 799 }
 800 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 801
 802 int ib_modify_qp(struct ib_qp *qp,
 803                  struct ib_qp_attr *qp_attr,
 804                  int qp_attr_mask)
 805 {
 806         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 807 }
 808 EXPORT_SYMBOL(ib_modify_qp);
 809
 810 int ib_query_qp(struct ib_qp *qp,
 811                 struct ib_qp_attr *qp_attr,
 812                 int qp_attr_mask,
 813                 struct ib_qp_init_attr *qp_init_attr)
 814 {
 815         return qp->device->query_qp ?
 816                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
 817                 -ENOSYS;
 818 }
 819 EXPORT_SYMBOL(ib_query_qp);
 820
 821 int ib_close_qp(struct ib_qp *qp)
 822 {
 823         struct ib_qp *real_qp;
 824         unsigned long flags;
 825
 826         real_qp = qp->real_qp;
 827         if (real_qp == qp)
 828                 return -EINVAL;
 829
 830         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 831         list_del(&qp->open_list);
 832         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 833
 834         atomic_dec(&real_qp->usecnt);
 835         kfree(qp);
 836
 837         return 0;
 838 }
 839 EXPORT_SYMBOL(ib_close_qp);
 840
 841 static int __ib_destroy_shared_qp(struct ib_qp *qp)
 842 {
 843         struct ib_xrcd *xrcd;
 844         struct ib_qp *real_qp;
 845         int ret;
 846
 847         real_qp = qp->real_qp;
 848         xrcd = real_qp->xrcd;
 849
 850         mutex_lock(&xrcd->tgt_qp_mutex);
 851         ib_close_qp(qp);
 852         if (atomic_read(&real_qp->usecnt) == 0)
 853                 list_del(&real_qp->xrcd_list);
 854         else
 855                 real_qp = NULL;
 856         mutex_unlock(&xrcd->tgt_qp_mutex);
 857
 858         if (real_qp) {
 859                 ret = ib_destroy_qp(real_qp);
 860                 if (!ret)
 861                         atomic_dec(&xrcd->usecnt);
 862                 else
 863                         __ib_insert_xrcd_qp(xrcd, real_qp);
 864         }
 865
 866         return 0;
 867 }
 868
 869 int ib_destroy_qp(struct ib_qp *qp)
 870 {
 871         struct ib_pd *pd;
 872         struct ib_cq *scq, *rcq;
 873         struct ib_srq *srq;
 874         int ret;
 875
 876         if (atomic_read(&qp->usecnt))
 877                 return -EBUSY;
 878
 879         if (qp->real_qp != qp)
 880                 return __ib_destroy_shared_qp(qp);
 881
 882         pd   = qp->pd;
 883         scq  = qp->send_cq;
 884         rcq  = qp->recv_cq;
 885         srq  = qp->srq;
 886
 887         ret = qp->device->destroy_qp(qp);
 888         if (!ret) {
 889                 if (pd)
 890                         atomic_dec(&pd->usecnt);
 891                 if (scq)
 892                         atomic_dec(&scq->usecnt);
 893                 if (rcq)
 894                         atomic_dec(&rcq->usecnt);
 895                 if (srq)
 896                         atomic_dec(&srq->usecnt);
 897         }
 898
 899         return ret;
 900 }
 901 EXPORT_SYMBOL(ib_destroy_qp);
 902
 903 /* Completion queues */
 904
 905 struct ib_cq *ib_create_cq(struct ib_device *device,
 906                            ib_comp_handler comp_handler,
 907                            void (*event_handler)(struct ib_event *, void *),
 908                            void *cq_context, int cqe, int comp_vector)
 909 {
 910         struct ib_cq *cq;
 911
 912         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
 913
 914         if (!IS_ERR(cq)) {
 915                 cq->device        = device;
 916                 cq->uobject       = NULL;
 917                 cq->comp_handler  = comp_handler;
 918                 cq->event_handler = event_handler;
 919                 cq->cq_context    = cq_context;
 920                 atomic_set(&cq->usecnt, 0);
 921         }
 922
 923         return cq;
 924 }
 925 EXPORT_SYMBOL(ib_create_cq);
 926
 927 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 928 {
 929         return cq->device->modify_cq ?
 930                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
 931 }
 932 EXPORT_SYMBOL(ib_modify_cq);
 933
 934 int ib_destroy_cq(struct ib_cq *cq)
 935 {
 936         if (atomic_read(&cq->usecnt))
 937                 return -EBUSY;
 938
 939         return cq->device->destroy_cq(cq);
 940 }
 941 EXPORT_SYMBOL(ib_destroy_cq);
 942
 943 int ib_resize_cq(struct ib_cq *cq, int cqe)
 944 {
 945         return cq->device->resize_cq ?
 946                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
 947 }
 948 EXPORT_SYMBOL(ib_resize_cq);
 949
 950 /* Memory regions */
 951
 952 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
 953 {
 954         struct ib_mr *mr;
 955
 956         mr = pd->device->get_dma_mr(pd, mr_access_flags);
 957
 958         if (!IS_ERR(mr)) {
 959                 mr->device  = pd->device;
 960                 mr->pd      = pd;
 961                 mr->uobject = NULL;
 962                 atomic_inc(&pd->usecnt);
 963                 atomic_set(&mr->usecnt, 0);
 964         }
 965
 966         return mr;
 967 }
 968 EXPORT_SYMBOL(ib_get_dma_mr);
 969
 970 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
 971                              struct ib_phys_buf *phys_buf_array,
 972                              int num_phys_buf,
 973                              int mr_access_flags,
 974                              u64 *iova_start)
 975 {
 976         struct ib_mr *mr;
 977
 978         if (!pd->device->reg_phys_mr)
 979                 return ERR_PTR(-ENOSYS);
 980
 981         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
 982                                      mr_access_flags, iova_start);
 983
 984         if (!IS_ERR(mr)) {
 985                 mr->device  = pd->device;
 986                 mr->pd      = pd;
 987                 mr->uobject = NULL;
 988                 atomic_inc(&pd->usecnt);
 989                 atomic_set(&mr->usecnt, 0);
 990         }
 991
 992         return mr;
 993 }
 994 EXPORT_SYMBOL(ib_reg_phys_mr);
 995
 996 int ib_rereg_phys_mr(struct ib_mr *mr,
 997                      int mr_rereg_mask,
 998                      struct ib_pd *pd,
 999                      struct ib_phys_buf *phys_buf_array,
1000                      int num_phys_buf,
1001                      int mr_access_flags,
1002                      u64 *iova_start)
1003 {
1004         struct ib_pd *old_pd;
1005         int ret;
1006
1007         if (!mr->device->rereg_phys_mr)
1008                 return -ENOSYS;
1009
1010         if (atomic_read(&mr->usecnt))
1011                 return -EBUSY;
1012
1013         old_pd = mr->pd;
1014
1015         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1016                                         phys_buf_array, num_phys_buf,
1017                                         mr_access_flags, iova_start);
1018
1019         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1020                 atomic_dec(&old_pd->usecnt);
1021                 atomic_inc(&pd->usecnt);
1022         }
1023
1024         return ret;
1025 }
1026 EXPORT_SYMBOL(ib_rereg_phys_mr);
1027
1028 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1029 {
1030         return mr->device->query_mr ?
1031                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1032 }
1033 EXPORT_SYMBOL(ib_query_mr);
1034
1035 int ib_dereg_mr(struct ib_mr *mr)
1036 {
1037         struct ib_pd *pd;
1038         int ret;
1039
1040         if (atomic_read(&mr->usecnt))
1041                 return -EBUSY;
1042
1043         pd = mr->pd;
1044         ret = mr->device->dereg_mr(mr);
1045         if (!ret)
1046                 atomic_dec(&pd->usecnt);
1047
1048         return ret;
1049 }
1050 EXPORT_SYMBOL(ib_dereg_mr);
1051
1052 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1053 {
1054         struct ib_mr *mr;
1055
1056         if (!pd->device->alloc_fast_reg_mr)
1057                 return ERR_PTR(-ENOSYS);
1058
1059         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1060
1061         if (!IS_ERR(mr)) {
1062                 mr->device  = pd->device;
1063                 mr->pd      = pd;
1064                 mr->uobject = NULL;
1065                 atomic_inc(&pd->usecnt);
1066                 atomic_set(&mr->usecnt, 0);
1067         }
1068
1069         return mr;
1070 }
1071 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1072
1073 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1074                                                           int max_page_list_len)
1075 {
1076         struct ib_fast_reg_page_list *page_list;
1077
1078         if (!device->alloc_fast_reg_page_list)
1079                 return ERR_PTR(-ENOSYS);
1080
1081         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1082
1083         if (!IS_ERR(page_list)) {
1084                 page_list->device = device;
1085                 page_list->max_page_list_len = max_page_list_len;
1086         }
1087
1088         return page_list;
1089 }
1090 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1091
1092 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1093 {
1094         page_list->device->free_fast_reg_page_list(page_list);
1095 }
1096 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1097
1098 /* Memory windows */
1099
1100 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
1101 {
1102         struct ib_mw *mw;
1103
1104         if (!pd->device->alloc_mw)
1105                 return ERR_PTR(-ENOSYS);
1106
1107         mw = pd->device->alloc_mw(pd);
1108         if (!IS_ERR(mw)) {
1109                 mw->device  = pd->device;
1110                 mw->pd      = pd;
1111                 mw->uobject = NULL;
1112                 atomic_inc(&pd->usecnt);
1113         }
1114
1115         return mw;
1116 }
1117 EXPORT_SYMBOL(ib_alloc_mw);
1118
1119 int ib_dealloc_mw(struct ib_mw *mw)
1120 {
1121         struct ib_pd *pd;
1122         int ret;
1123
1124         pd = mw->pd;
1125         ret = mw->device->dealloc_mw(mw);
1126         if (!ret)
1127                 atomic_dec(&pd->usecnt);
1128
1129         return ret;
1130 }
1131 EXPORT_SYMBOL(ib_dealloc_mw);
1132
1133 /* "Fast" memory regions */
1134
1135 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1136                             int mr_access_flags,
1137                             struct ib_fmr_attr *fmr_attr)
1138 {
1139         struct ib_fmr *fmr;
1140
1141         if (!pd->device->alloc_fmr)
1142                 return ERR_PTR(-ENOSYS);
1143
1144         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1145         if (!IS_ERR(fmr)) {
1146                 fmr->device = pd->device;
1147                 fmr->pd     = pd;
1148                 atomic_inc(&pd->usecnt);
1149         }
1150
1151         return fmr;
1152 }
1153 EXPORT_SYMBOL(ib_alloc_fmr);
1154
1155 int ib_unmap_fmr(struct list_head *fmr_list)
1156 {
1157         struct ib_fmr *fmr;
1158
1159         if (list_empty(fmr_list))
1160                 return 0;
1161
1162         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1163         return fmr->device->unmap_fmr(fmr_list);
1164 }
1165 EXPORT_SYMBOL(ib_unmap_fmr);
1166
1167 int ib_dealloc_fmr(struct ib_fmr *fmr)
1168 {
1169         struct ib_pd *pd;
1170         int ret;
1171
1172         pd = fmr->pd;
1173         ret = fmr->device->dealloc_fmr(fmr);
1174         if (!ret)
1175                 atomic_dec(&pd->usecnt);
1176
1177         return ret;
1178 }
1179 EXPORT_SYMBOL(ib_dealloc_fmr);
1180
1181 /* Multicast groups */
1182
1183 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1184 {
1185         if (!qp->device->attach_mcast)
1186                 return -ENOSYS;
1187         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1188                 return -EINVAL;
1189
1190         return qp->device->attach_mcast(qp, gid, lid);
1191 }
1192 EXPORT_SYMBOL(ib_attach_mcast);
1193
1194 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1195 {
1196         if (!qp->device->detach_mcast)
1197                 return -ENOSYS;
1198         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1199                 return -EINVAL;
1200
1201         return qp->device->detach_mcast(qp, gid, lid);
1202 }
1203 EXPORT_SYMBOL(ib_detach_mcast);
1204
1205 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1206 {
1207         struct ib_xrcd *xrcd;
1208
1209         if (!device->alloc_xrcd)
1210                 return ERR_PTR(-ENOSYS);
1211
1212         xrcd = device->alloc_xrcd(device, NULL, NULL);
1213         if (!IS_ERR(xrcd)) {
1214                 xrcd->device = device;
1215                 xrcd->inode = NULL;
1216                 atomic_set(&xrcd->usecnt, 0);
1217                 mutex_init(&xrcd->tgt_qp_mutex);
1218                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1219         }
1220
1221         return xrcd;
1222 }
1223 EXPORT_SYMBOL(ib_alloc_xrcd);
1224
1225 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1226 {
1227         struct ib_qp *qp;
1228         int ret;
1229
1230         if (atomic_read(&xrcd->usecnt))
1231                 return -EBUSY;
1232
1233         while (!list_empty(&xrcd->tgt_qp_list)) {
1234                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1235                 ret = ib_destroy_qp(qp);
1236                 if (ret)
1237                         return ret;
1238         }
1239
1240         return xrcd->device->dealloc_xrcd(xrcd);
1241 }
1242 EXPORT_SYMBOL(ib_dealloc_xrcd);