drivers/infiniband/core/verbs.c

   1 /*
   2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   9  *
  10  * This software is available to you under a choice of one of two
  11  * licenses.  You may choose to be licensed under the terms of the GNU
  12  * General Public License (GPL) Version 2, available from the file
  13  * COPYING in the main directory of this source tree, or the
  14  * OpenIB.org BSD license below:
  15  *
  16  *     Redistribution and use in source and binary forms, with or
  17  *     without modification, are permitted provided that the following
  18  *     conditions are met:
  19  *
  20  *      - Redistributions of source code must retain the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer.
  23  *
  24  *      - Redistributions in binary form must reproduce the above
  25  *        copyright notice, this list of conditions and the following
  26  *        disclaimer in the documentation and/or other materials
  27  *        provided with the distribution.
  28  *
  29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  36  * SOFTWARE.
  37  */
  38
  39 #include <linux/errno.h>
  40 #include <linux/err.h>
  41 #include <linux/string.h>
  42 #include <linux/slab.h>
  43
  44 #include <rdma/ib_verbs.h>
  45 #include <rdma/ib_cache.h>
  46
  47 int ib_rate_to_mult(enum ib_rate rate)
  48 {
  49         switch (rate) {
  50         case IB_RATE_2_5_GBPS: return  1;
  51         case IB_RATE_5_GBPS:   return  2;
  52         case IB_RATE_10_GBPS:  return  4;
  53         case IB_RATE_20_GBPS:  return  8;
  54         case IB_RATE_30_GBPS:  return 12;
  55         case IB_RATE_40_GBPS:  return 16;
  56         case IB_RATE_60_GBPS:  return 24;
  57         case IB_RATE_80_GBPS:  return 32;
  58         case IB_RATE_120_GBPS: return 48;
  59         default:               return -1;
  60         }
  61 }
  62 EXPORT_SYMBOL(ib_rate_to_mult);
  63
  64 enum ib_rate mult_to_ib_rate(int mult)
  65 {
  66         switch (mult) {
  67         case 1:  return IB_RATE_2_5_GBPS;
  68         case 2:  return IB_RATE_5_GBPS;
  69         case 4:  return IB_RATE_10_GBPS;
  70         case 8:  return IB_RATE_20_GBPS;
  71         case 12: return IB_RATE_30_GBPS;
  72         case 16: return IB_RATE_40_GBPS;
  73         case 24: return IB_RATE_60_GBPS;
  74         case 32: return IB_RATE_80_GBPS;
  75         case 48: return IB_RATE_120_GBPS;
  76         default: return IB_RATE_PORT_CURRENT;
  77         }
  78 }
  79 EXPORT_SYMBOL(mult_to_ib_rate);
  80
  81 enum rdma_transport_type
  82 rdma_node_get_transport(enum rdma_node_type node_type)
  83 {
  84         switch (node_type) {
  85         case RDMA_NODE_IB_CA:
  86         case RDMA_NODE_IB_SWITCH:
  87         case RDMA_NODE_IB_ROUTER:
  88                 return RDMA_TRANSPORT_IB;
  89         case RDMA_NODE_RNIC:
  90                 return RDMA_TRANSPORT_IWARP;
  91         default:
  92                 BUG();
  93                 return 0;
  94         }
  95 }
  96 EXPORT_SYMBOL(rdma_node_get_transport);
  97
  98 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
  99 {
 100         if (device->get_link_layer)
 101                 return device->get_link_layer(device, port_num);
 102
 103         switch (rdma_node_get_transport(device->node_type)) {
 104         case RDMA_TRANSPORT_IB:
 105                 return IB_LINK_LAYER_INFINIBAND;
 106         case RDMA_TRANSPORT_IWARP:
 107                 return IB_LINK_LAYER_ETHERNET;
 108         default:
 109                 return IB_LINK_LAYER_UNSPECIFIED;
 110         }
 111 }
 112 EXPORT_SYMBOL(rdma_port_get_link_layer);
 113
 114 /* Protection domains */
 115
 116 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 117 {
 118         struct ib_pd *pd;
 119
 120         pd = device->alloc_pd(device, NULL, NULL);
 121
 122         if (!IS_ERR(pd)) {
 123                 pd->device  = device;
 124                 pd->uobject = NULL;
 125                 atomic_set(&pd->usecnt, 0);
 126         }
 127
 128         return pd;
 129 }
 130 EXPORT_SYMBOL(ib_alloc_pd);
 131
 132 int ib_dealloc_pd(struct ib_pd *pd)
 133 {
 134         if (atomic_read(&pd->usecnt))
 135                 return -EBUSY;
 136
 137         return pd->device->dealloc_pd(pd);
 138 }
 139 EXPORT_SYMBOL(ib_dealloc_pd);
 140
 141 /* Address handles */
 142
 143 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 144 {
 145         struct ib_ah *ah;
 146
 147         ah = pd->device->create_ah(pd, ah_attr);
 148
 149         if (!IS_ERR(ah)) {
 150                 ah->device  = pd->device;
 151                 ah->pd      = pd;
 152                 ah->uobject = NULL;
 153                 atomic_inc(&pd->usecnt);
 154         }
 155
 156         return ah;
 157 }
 158 EXPORT_SYMBOL(ib_create_ah);
 159
 160 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
 161                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
 162 {
 163         u32 flow_class;
 164         u16 gid_index;
 165         int ret;
 166
 167         memset(ah_attr, 0, sizeof *ah_attr);
 168         ah_attr->dlid = wc->slid;
 169         ah_attr->sl = wc->sl;
 170         ah_attr->src_path_bits = wc->dlid_path_bits;
 171         ah_attr->port_num = port_num;
 172
 173         if (wc->wc_flags & IB_WC_GRH) {
 174                 ah_attr->ah_flags = IB_AH_GRH;
 175                 ah_attr->grh.dgid = grh->sgid;
 176
 177                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
 178                                          &gid_index);
 179                 if (ret)
 180                         return ret;
 181
 182                 ah_attr->grh.sgid_index = (u8) gid_index;
 183                 flow_class = be32_to_cpu(grh->version_tclass_flow);
 184                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
 185                 ah_attr->grh.hop_limit = 0xFF;
 186                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 187         }
 188         return 0;
 189 }
 190 EXPORT_SYMBOL(ib_init_ah_from_wc);
 191
 192 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
 193                                    struct ib_grh *grh, u8 port_num)
 194 {
 195         struct ib_ah_attr ah_attr;
 196         int ret;
 197
 198         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 199         if (ret)
 200                 return ERR_PTR(ret);
 201
 202         return ib_create_ah(pd, &ah_attr);
 203 }
 204 EXPORT_SYMBOL(ib_create_ah_from_wc);
 205
 206 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 207 {
 208         return ah->device->modify_ah ?
 209                 ah->device->modify_ah(ah, ah_attr) :
 210                 -ENOSYS;
 211 }
 212 EXPORT_SYMBOL(ib_modify_ah);
 213
 214 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 215 {
 216         return ah->device->query_ah ?
 217                 ah->device->query_ah(ah, ah_attr) :
 218                 -ENOSYS;
 219 }
 220 EXPORT_SYMBOL(ib_query_ah);
 221
 222 int ib_destroy_ah(struct ib_ah *ah)
 223 {
 224         struct ib_pd *pd;
 225         int ret;
 226
 227         pd = ah->pd;
 228         ret = ah->device->destroy_ah(ah);
 229         if (!ret)
 230                 atomic_dec(&pd->usecnt);
 231
 232         return ret;
 233 }
 234 EXPORT_SYMBOL(ib_destroy_ah);
 235
 236 /* Shared receive queues */
 237
 238 struct ib_srq *ib_create_srq(struct ib_pd *pd,
 239                              struct ib_srq_init_attr *srq_init_attr)
 240 {
 241         struct ib_srq *srq;
 242
 243         if (!pd->device->create_srq)
 244                 return ERR_PTR(-ENOSYS);
 245
 246         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
 247
 248         if (!IS_ERR(srq)) {
 249                 srq->device        = pd->device;
 250                 srq->pd            = pd;
 251                 srq->uobject       = NULL;
 252                 srq->event_handler = srq_init_attr->event_handler;
 253                 srq->srq_context   = srq_init_attr->srq_context;
 254                 srq->srq_type      = srq_init_attr->srq_type;
 255                 if (srq->srq_type == IB_SRQT_XRC) {
 256                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
 257                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 258                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
 259                         atomic_inc(&srq->ext.xrc.cq->usecnt);
 260                 }
 261                 atomic_inc(&pd->usecnt);
 262                 atomic_set(&srq->usecnt, 0);
 263         }
 264
 265         return srq;
 266 }
 267 EXPORT_SYMBOL(ib_create_srq);
 268
 269 int ib_modify_srq(struct ib_srq *srq,
 270                   struct ib_srq_attr *srq_attr,
 271                   enum ib_srq_attr_mask srq_attr_mask)
 272 {
 273         return srq->device->modify_srq ?
 274                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
 275                 -ENOSYS;
 276 }
 277 EXPORT_SYMBOL(ib_modify_srq);
 278
 279 int ib_query_srq(struct ib_srq *srq,
 280                  struct ib_srq_attr *srq_attr)
 281 {
 282         return srq->device->query_srq ?
 283                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
 284 }
 285 EXPORT_SYMBOL(ib_query_srq);
 286
 287 int ib_destroy_srq(struct ib_srq *srq)
 288 {
 289         struct ib_pd *pd;
 290         enum ib_srq_type srq_type;
 291         struct ib_xrcd *uninitialized_var(xrcd);
 292         struct ib_cq *uninitialized_var(cq);
 293         int ret;
 294
 295         if (atomic_read(&srq->usecnt))
 296                 return -EBUSY;
 297
 298         pd = srq->pd;
 299         srq_type = srq->srq_type;
 300         if (srq_type == IB_SRQT_XRC) {
 301                 xrcd = srq->ext.xrc.xrcd;
 302                 cq = srq->ext.xrc.cq;
 303         }
 304
 305         ret = srq->device->destroy_srq(srq);
 306         if (!ret) {
 307                 atomic_dec(&pd->usecnt);
 308                 if (srq_type == IB_SRQT_XRC) {
 309                         atomic_dec(&xrcd->usecnt);
 310                         atomic_dec(&cq->usecnt);
 311                 }
 312         }
 313
 314         return ret;
 315 }
 316 EXPORT_SYMBOL(ib_destroy_srq);
 317
 318 /* Queue pairs */
 319
 320 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
 321 {
 322         struct ib_qp *qp = context;
 323
 324         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
 325                 event->element.qp->event_handler(event, event->element.qp->qp_context);
 326 }
 327
 328 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
 329 {
 330         mutex_lock(&xrcd->tgt_qp_mutex);
 331         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
 332         mutex_unlock(&xrcd->tgt_qp_mutex);
 333 }
 334
 335 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
 336                                   void (*event_handler)(struct ib_event *, void *),
 337                                   void *qp_context)
 338 {
 339         struct ib_qp *qp;
 340         unsigned long flags;
 341
 342         qp = kzalloc(sizeof *qp, GFP_KERNEL);
 343         if (!qp)
 344                 return ERR_PTR(-ENOMEM);
 345
 346         qp->real_qp = real_qp;
 347         atomic_inc(&real_qp->usecnt);
 348         qp->device = real_qp->device;
 349         qp->event_handler = event_handler;
 350         qp->qp_context = qp_context;
 351         qp->qp_num = real_qp->qp_num;
 352         qp->qp_type = real_qp->qp_type;
 353
 354         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 355         list_add(&qp->open_list, &real_qp->open_list);
 356         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 357
 358         return qp;
 359 }
 360
 361 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 362                          struct ib_qp_open_attr *qp_open_attr)
 363 {
 364         struct ib_qp *qp, *real_qp;
 365
 366         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
 367                 return ERR_PTR(-EINVAL);
 368
 369         qp = ERR_PTR(-EINVAL);
 370         mutex_lock(&xrcd->tgt_qp_mutex);
 371         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
 372                 if (real_qp->qp_num == qp_open_attr->qp_num) {
 373                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
 374                                           qp_open_attr->qp_context);
 375                         break;
 376                 }
 377         }
 378         mutex_unlock(&xrcd->tgt_qp_mutex);
 379         return qp;
 380 }
 381 EXPORT_SYMBOL(ib_open_qp);
 382
 383 struct ib_qp *ib_create_qp(struct ib_pd *pd,
 384                            struct ib_qp_init_attr *qp_init_attr)
 385 {
 386         struct ib_qp *qp, *real_qp;
 387         struct ib_device *device;
 388
 389         device = pd ? pd->device : qp_init_attr->xrcd->device;
 390         qp = device->create_qp(pd, qp_init_attr, NULL);
 391
 392         if (!IS_ERR(qp)) {
 393                 qp->device     = device;
 394                 qp->real_qp    = qp;
 395                 qp->uobject    = NULL;
 396                 qp->qp_type    = qp_init_attr->qp_type;
 397
 398                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
 399                         qp->event_handler = __ib_shared_qp_event_handler;
 400                         qp->qp_context = qp;
 401                         qp->pd = NULL;
 402                         qp->send_cq = qp->recv_cq = NULL;
 403                         qp->srq = NULL;
 404                         qp->xrcd = qp_init_attr->xrcd;
 405                         atomic_inc(&qp_init_attr->xrcd->usecnt);
 406                         INIT_LIST_HEAD(&qp->open_list);
 407                         atomic_set(&qp->usecnt, 0);
 408
 409                         real_qp = qp;
 410                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
 411                                           qp_init_attr->qp_context);
 412                         if (!IS_ERR(qp))
 413                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
 414                         else
 415                                 real_qp->device->destroy_qp(real_qp);
 416                 } else {
 417                         qp->event_handler = qp_init_attr->event_handler;
 418                         qp->qp_context = qp_init_attr->qp_context;
 419                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
 420                                 qp->recv_cq = NULL;
 421                                 qp->srq = NULL;
 422                         } else {
 423                                 qp->recv_cq = qp_init_attr->recv_cq;
 424                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
 425                                 qp->srq = qp_init_attr->srq;
 426                                 if (qp->srq)
 427                                         atomic_inc(&qp_init_attr->srq->usecnt);
 428                         }
 429
 430                         qp->pd      = pd;
 431                         qp->send_cq = qp_init_attr->send_cq;
 432                         qp->xrcd    = NULL;
 433
 434                         atomic_inc(&pd->usecnt);
 435                         atomic_inc(&qp_init_attr->send_cq->usecnt);
 436                 }
 437         }
 438
 439         return qp;
 440 }
 441 EXPORT_SYMBOL(ib_create_qp);
 442
 443 static const struct {
 444         int                     valid;
 445         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
 446         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
 447 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
 448         [IB_QPS_RESET] = {
 449                 [IB_QPS_RESET] = { .valid = 1 },
 450                 [IB_QPS_INIT]  = {
 451                         .valid = 1,
 452                         .req_param = {
 453                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 454                                                 IB_QP_PORT                      |
 455                                                 IB_QP_QKEY),
 456                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 457                                                 IB_QP_PORT                      |
 458                                                 IB_QP_ACCESS_FLAGS),
 459                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 460                                                 IB_QP_PORT                      |
 461                                                 IB_QP_ACCESS_FLAGS),
 462                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 463                                                 IB_QP_PORT                      |
 464                                                 IB_QP_ACCESS_FLAGS),
 465                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 466                                                 IB_QP_PORT                      |
 467                                                 IB_QP_ACCESS_FLAGS),
 468                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 469                                                 IB_QP_QKEY),
 470                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 471                                                 IB_QP_QKEY),
 472                         }
 473                 },
 474         },
 475         [IB_QPS_INIT]  = {
 476                 [IB_QPS_RESET] = { .valid = 1 },
 477                 [IB_QPS_ERR] =   { .valid = 1 },
 478                 [IB_QPS_INIT]  = {
 479                         .valid = 1,
 480                         .opt_param = {
 481                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 482                                                 IB_QP_PORT                      |
 483                                                 IB_QP_QKEY),
 484                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 485                                                 IB_QP_PORT                      |
 486                                                 IB_QP_ACCESS_FLAGS),
 487                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 488                                                 IB_QP_PORT                      |
 489                                                 IB_QP_ACCESS_FLAGS),
 490                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 491                                                 IB_QP_PORT                      |
 492                                                 IB_QP_ACCESS_FLAGS),
 493                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 494                                                 IB_QP_PORT                      |
 495                                                 IB_QP_ACCESS_FLAGS),
 496                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 497                                                 IB_QP_QKEY),
 498                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 499                                                 IB_QP_QKEY),
 500                         }
 501                 },
 502                 [IB_QPS_RTR]   = {
 503                         .valid = 1,
 504                         .req_param = {
 505                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 506                                                 IB_QP_PATH_MTU                  |
 507                                                 IB_QP_DEST_QPN                  |
 508                                                 IB_QP_RQ_PSN),
 509                                 [IB_QPT_RC]  = (IB_QP_AV                        |
 510                                                 IB_QP_PATH_MTU                  |
 511                                                 IB_QP_DEST_QPN                  |
 512                                                 IB_QP_RQ_PSN                    |
 513                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 514                                                 IB_QP_MIN_RNR_TIMER),
 515                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
 516                                                 IB_QP_PATH_MTU                  |
 517                                                 IB_QP_DEST_QPN                  |
 518                                                 IB_QP_RQ_PSN),
 519                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
 520                                                 IB_QP_PATH_MTU                  |
 521                                                 IB_QP_DEST_QPN                  |
 522                                                 IB_QP_RQ_PSN                    |
 523                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 524                                                 IB_QP_MIN_RNR_TIMER),
 525                         },
 526                         .opt_param = {
 527                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
 528                                                  IB_QP_QKEY),
 529                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
 530                                                  IB_QP_ACCESS_FLAGS             |
 531                                                  IB_QP_PKEY_INDEX),
 532                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
 533                                                  IB_QP_ACCESS_FLAGS             |
 534                                                  IB_QP_PKEY_INDEX),
 535                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
 536                                                  IB_QP_ACCESS_FLAGS             |
 537                                                  IB_QP_PKEY_INDEX),
 538                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
 539                                                  IB_QP_ACCESS_FLAGS             |
 540                                                  IB_QP_PKEY_INDEX),
 541                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
 542                                                  IB_QP_QKEY),
 543                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
 544                                                  IB_QP_QKEY),
 545                          }
 546                 }
 547         },
 548         [IB_QPS_RTR]   = {
 549                 [IB_QPS_RESET] = { .valid = 1 },
 550                 [IB_QPS_ERR] =   { .valid = 1 },
 551                 [IB_QPS_RTS]   = {
 552                         .valid = 1,
 553                         .req_param = {
 554                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
 555                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
 556                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
 557                                                 IB_QP_RETRY_CNT                 |
 558                                                 IB_QP_RNR_RETRY                 |
 559                                                 IB_QP_SQ_PSN                    |
 560                                                 IB_QP_MAX_QP_RD_ATOMIC),
 561                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
 562                                                 IB_QP_RETRY_CNT                 |
 563                                                 IB_QP_RNR_RETRY                 |
 564                                                 IB_QP_SQ_PSN                    |
 565                                                 IB_QP_MAX_QP_RD_ATOMIC),
 566                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
 567                                                 IB_QP_SQ_PSN),
 568                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
 569                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
 570                         },
 571                         .opt_param = {
 572                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
 573                                                  IB_QP_QKEY),
 574                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
 575                                                  IB_QP_ALT_PATH                 |
 576                                                  IB_QP_ACCESS_FLAGS             |
 577                                                  IB_QP_PATH_MIG_STATE),
 578                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
 579                                                  IB_QP_ALT_PATH                 |
 580                                                  IB_QP_ACCESS_FLAGS             |
 581                                                  IB_QP_MIN_RNR_TIMER            |
 582                                                  IB_QP_PATH_MIG_STATE),
 583                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
 584                                                  IB_QP_ALT_PATH                 |
 585                                                  IB_QP_ACCESS_FLAGS             |
 586                                                  IB_QP_PATH_MIG_STATE),
 587                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
 588                                                  IB_QP_ALT_PATH                 |
 589                                                  IB_QP_ACCESS_FLAGS             |
 590                                                  IB_QP_MIN_RNR_TIMER            |
 591                                                  IB_QP_PATH_MIG_STATE),
 592                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
 593                                                  IB_QP_QKEY),
 594                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
 595                                                  IB_QP_QKEY),
 596                          }
 597                 }
 598         },
 599         [IB_QPS_RTS]   = {
 600                 [IB_QPS_RESET] = { .valid = 1 },
 601                 [IB_QPS_ERR] =   { .valid = 1 },
 602                 [IB_QPS_RTS]   = {
 603                         .valid = 1,
 604                         .opt_param = {
 605                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 606                                                 IB_QP_QKEY),
 607                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 608                                                 IB_QP_ACCESS_FLAGS              |
 609                                                 IB_QP_ALT_PATH                  |
 610                                                 IB_QP_PATH_MIG_STATE),
 611                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 612                                                 IB_QP_ACCESS_FLAGS              |
 613                                                 IB_QP_ALT_PATH                  |
 614                                                 IB_QP_PATH_MIG_STATE            |
 615                                                 IB_QP_MIN_RNR_TIMER),
 616                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 617                                                 IB_QP_ACCESS_FLAGS              |
 618                                                 IB_QP_ALT_PATH                  |
 619                                                 IB_QP_PATH_MIG_STATE),
 620                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 621                                                 IB_QP_ACCESS_FLAGS              |
 622                                                 IB_QP_ALT_PATH                  |
 623                                                 IB_QP_PATH_MIG_STATE            |
 624                                                 IB_QP_MIN_RNR_TIMER),
 625                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 626                                                 IB_QP_QKEY),
 627                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 628                                                 IB_QP_QKEY),
 629                         }
 630                 },
 631                 [IB_QPS_SQD]   = {
 632                         .valid = 1,
 633                         .opt_param = {
 634                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 635                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 636                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 637                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 638                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
 639                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 640                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
 641                         }
 642                 },
 643         },
 644         [IB_QPS_SQD]   = {
 645                 [IB_QPS_RESET] = { .valid = 1 },
 646                 [IB_QPS_ERR] =   { .valid = 1 },
 647                 [IB_QPS_RTS]   = {
 648                         .valid = 1,
 649                         .opt_param = {
 650                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 651                                                 IB_QP_QKEY),
 652                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 653                                                 IB_QP_ALT_PATH                  |
 654                                                 IB_QP_ACCESS_FLAGS              |
 655                                                 IB_QP_PATH_MIG_STATE),
 656                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 657                                                 IB_QP_ALT_PATH                  |
 658                                                 IB_QP_ACCESS_FLAGS              |
 659                                                 IB_QP_MIN_RNR_TIMER             |
 660                                                 IB_QP_PATH_MIG_STATE),
 661                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 662                                                 IB_QP_ALT_PATH                  |
 663                                                 IB_QP_ACCESS_FLAGS              |
 664                                                 IB_QP_PATH_MIG_STATE),
 665                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 666                                                 IB_QP_ALT_PATH                  |
 667                                                 IB_QP_ACCESS_FLAGS              |
 668                                                 IB_QP_MIN_RNR_TIMER             |
 669                                                 IB_QP_PATH_MIG_STATE),
 670                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 671                                                 IB_QP_QKEY),
 672                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 673                                                 IB_QP_QKEY),
 674                         }
 675                 },
 676                 [IB_QPS_SQD]   = {
 677                         .valid = 1,
 678                         .opt_param = {
 679                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 680                                                 IB_QP_QKEY),
 681                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 682                                                 IB_QP_ALT_PATH                  |
 683                                                 IB_QP_ACCESS_FLAGS              |
 684                                                 IB_QP_PKEY_INDEX                |
 685                                                 IB_QP_PATH_MIG_STATE),
 686                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
 687                                                 IB_QP_AV                        |
 688                                                 IB_QP_TIMEOUT                   |
 689                                                 IB_QP_RETRY_CNT                 |
 690                                                 IB_QP_RNR_RETRY                 |
 691                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 692                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 693                                                 IB_QP_ALT_PATH                  |
 694                                                 IB_QP_ACCESS_FLAGS              |
 695                                                 IB_QP_PKEY_INDEX                |
 696                                                 IB_QP_MIN_RNR_TIMER             |
 697                                                 IB_QP_PATH_MIG_STATE),
 698                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
 699                                                 IB_QP_AV                        |
 700                                                 IB_QP_TIMEOUT                   |
 701                                                 IB_QP_RETRY_CNT                 |
 702                                                 IB_QP_RNR_RETRY                 |
 703                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 704                                                 IB_QP_ALT_PATH                  |
 705                                                 IB_QP_ACCESS_FLAGS              |
 706                                                 IB_QP_PKEY_INDEX                |
 707                                                 IB_QP_PATH_MIG_STATE),
 708                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
 709                                                 IB_QP_AV                        |
 710                                                 IB_QP_TIMEOUT                   |
 711                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 712                                                 IB_QP_ALT_PATH                  |
 713                                                 IB_QP_ACCESS_FLAGS              |
 714                                                 IB_QP_PKEY_INDEX                |
 715                                                 IB_QP_MIN_RNR_TIMER             |
 716                                                 IB_QP_PATH_MIG_STATE),
 717                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 718                                                 IB_QP_QKEY),
 719                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 720                                                 IB_QP_QKEY),
 721                         }
 722                 }
 723         },
 724         [IB_QPS_SQE]   = {
 725                 [IB_QPS_RESET] = { .valid = 1 },
 726                 [IB_QPS_ERR] =   { .valid = 1 },
 727                 [IB_QPS_RTS]   = {
 728                         .valid = 1,
 729                         .opt_param = {
 730                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 731                                                 IB_QP_QKEY),
 732                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 733                                                 IB_QP_ACCESS_FLAGS),
 734                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 735                                                 IB_QP_QKEY),
 736                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 737                                                 IB_QP_QKEY),
 738                         }
 739                 }
 740         },
 741         [IB_QPS_ERR] = {
 742                 [IB_QPS_RESET] = { .valid = 1 },
 743                 [IB_QPS_ERR] =   { .valid = 1 }
 744         }
 745 };
 746
 747 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 748                        enum ib_qp_type type, enum ib_qp_attr_mask mask)
 749 {
 750         enum ib_qp_attr_mask req_param, opt_param;
 751
 752         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
 753             next_state < 0 || next_state > IB_QPS_ERR)
 754                 return 0;
 755
 756         if (mask & IB_QP_CUR_STATE  &&
 757             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
 758             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
 759                 return 0;
 760
 761         if (!qp_state_table[cur_state][next_state].valid)
 762                 return 0;
 763
 764         req_param = qp_state_table[cur_state][next_state].req_param[type];
 765         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
 766
 767         if ((mask & req_param) != req_param)
 768                 return 0;
 769
 770         if (mask & ~(req_param | opt_param | IB_QP_STATE))
 771                 return 0;
 772
 773         return 1;
 774 }
 775 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 776
 777 int ib_modify_qp(struct ib_qp *qp,
 778                  struct ib_qp_attr *qp_attr,
 779                  int qp_attr_mask)
 780 {
 781         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 782 }
 783 EXPORT_SYMBOL(ib_modify_qp);
 784
 785 int ib_query_qp(struct ib_qp *qp,
 786                 struct ib_qp_attr *qp_attr,
 787                 int qp_attr_mask,
 788                 struct ib_qp_init_attr *qp_init_attr)
 789 {
 790         return qp->device->query_qp ?
 791                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
 792                 -ENOSYS;
 793 }
 794 EXPORT_SYMBOL(ib_query_qp);
 795
 796 int ib_close_qp(struct ib_qp *qp)
 797 {
 798         struct ib_qp *real_qp;
 799         unsigned long flags;
 800
 801         real_qp = qp->real_qp;
 802         if (real_qp == qp)
 803                 return -EINVAL;
 804
 805         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 806         list_del(&qp->open_list);
 807         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 808
 809         atomic_dec(&real_qp->usecnt);
 810         kfree(qp);
 811
 812         return 0;
 813 }
 814 EXPORT_SYMBOL(ib_close_qp);
 815
 816 static int __ib_destroy_shared_qp(struct ib_qp *qp)
 817 {
 818         struct ib_xrcd *xrcd;
 819         struct ib_qp *real_qp;
 820         int ret;
 821
 822         real_qp = qp->real_qp;
 823         xrcd = real_qp->xrcd;
 824
 825         mutex_lock(&xrcd->tgt_qp_mutex);
 826         ib_close_qp(qp);
 827         if (atomic_read(&real_qp->usecnt) == 0)
 828                 list_del(&real_qp->xrcd_list);
 829         else
 830                 real_qp = NULL;
 831         mutex_unlock(&xrcd->tgt_qp_mutex);
 832
 833         if (real_qp) {
 834                 ret = ib_destroy_qp(real_qp);
 835                 if (!ret)
 836                         atomic_dec(&xrcd->usecnt);
 837                 else
 838                         __ib_insert_xrcd_qp(xrcd, real_qp);
 839         }
 840
 841         return 0;
 842 }
 843
 844 int ib_destroy_qp(struct ib_qp *qp)
 845 {
 846         struct ib_pd *pd;
 847         struct ib_cq *scq, *rcq;
 848         struct ib_srq *srq;
 849         int ret;
 850
 851         if (atomic_read(&qp->usecnt))
 852                 return -EBUSY;
 853
 854         if (qp->real_qp != qp)
 855                 return __ib_destroy_shared_qp(qp);
 856
 857         pd   = qp->pd;
 858         scq  = qp->send_cq;
 859         rcq  = qp->recv_cq;
 860         srq  = qp->srq;
 861
 862         ret = qp->device->destroy_qp(qp);
 863         if (!ret) {
 864                 if (pd)
 865                         atomic_dec(&pd->usecnt);
 866                 if (scq)
 867                         atomic_dec(&scq->usecnt);
 868                 if (rcq)
 869                         atomic_dec(&rcq->usecnt);
 870                 if (srq)
 871                         atomic_dec(&srq->usecnt);
 872         }
 873
 874         return ret;
 875 }
 876 EXPORT_SYMBOL(ib_destroy_qp);
 877
 878 /* Completion queues */
 879
 880 struct ib_cq *ib_create_cq(struct ib_device *device,
 881                            ib_comp_handler comp_handler,
 882                            void (*event_handler)(struct ib_event *, void *),
 883                            void *cq_context, int cqe, int comp_vector)
 884 {
 885         struct ib_cq *cq;
 886
 887         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
 888
 889         if (!IS_ERR(cq)) {
 890                 cq->device        = device;
 891                 cq->uobject       = NULL;
 892                 cq->comp_handler  = comp_handler;
 893                 cq->event_handler = event_handler;
 894                 cq->cq_context    = cq_context;
 895                 atomic_set(&cq->usecnt, 0);
 896         }
 897
 898         return cq;
 899 }
 900 EXPORT_SYMBOL(ib_create_cq);
 901
 902 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 903 {
 904         return cq->device->modify_cq ?
 905                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
 906 }
 907 EXPORT_SYMBOL(ib_modify_cq);
 908
 909 int ib_destroy_cq(struct ib_cq *cq)
 910 {
 911         if (atomic_read(&cq->usecnt))
 912                 return -EBUSY;
 913
 914         return cq->device->destroy_cq(cq);
 915 }
 916 EXPORT_SYMBOL(ib_destroy_cq);
 917
 918 int ib_resize_cq(struct ib_cq *cq, int cqe)
 919 {
 920         return cq->device->resize_cq ?
 921                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
 922 }
 923 EXPORT_SYMBOL(ib_resize_cq);
 924
 925 /* Memory regions */
 926
 927 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
 928 {
 929         struct ib_mr *mr;
 930
 931         mr = pd->device->get_dma_mr(pd, mr_access_flags);
 932
 933         if (!IS_ERR(mr)) {
 934                 mr->device  = pd->device;
 935                 mr->pd      = pd;
 936                 mr->uobject = NULL;
 937                 atomic_inc(&pd->usecnt);
 938                 atomic_set(&mr->usecnt, 0);
 939         }
 940
 941         return mr;
 942 }
 943 EXPORT_SYMBOL(ib_get_dma_mr);
 944
 945 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
 946                              struct ib_phys_buf *phys_buf_array,
 947                              int num_phys_buf,
 948                              int mr_access_flags,
 949                              u64 *iova_start)
 950 {
 951         struct ib_mr *mr;
 952
 953         if (!pd->device->reg_phys_mr)
 954                 return ERR_PTR(-ENOSYS);
 955
 956         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
 957                                      mr_access_flags, iova_start);
 958
 959         if (!IS_ERR(mr)) {
 960                 mr->device  = pd->device;
 961                 mr->pd      = pd;
 962                 mr->uobject = NULL;
 963                 atomic_inc(&pd->usecnt);
 964                 atomic_set(&mr->usecnt, 0);
 965         }
 966
 967         return mr;
 968 }
 969 EXPORT_SYMBOL(ib_reg_phys_mr);
 970
 971 int ib_rereg_phys_mr(struct ib_mr *mr,
 972                      int mr_rereg_mask,
 973                      struct ib_pd *pd,
 974                      struct ib_phys_buf *phys_buf_array,
 975                      int num_phys_buf,
 976                      int mr_access_flags,
 977                      u64 *iova_start)
 978 {
 979         struct ib_pd *old_pd;
 980         int ret;
 981
 982         if (!mr->device->rereg_phys_mr)
 983                 return -ENOSYS;
 984
 985         if (atomic_read(&mr->usecnt))
 986                 return -EBUSY;
 987
 988         old_pd = mr->pd;
 989
 990         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
 991                                         phys_buf_array, num_phys_buf,
 992                                         mr_access_flags, iova_start);
 993
 994         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
 995                 atomic_dec(&old_pd->usecnt);
 996                 atomic_inc(&pd->usecnt);
 997         }
 998
 999         return ret;
1000 }
1001 EXPORT_SYMBOL(ib_rereg_phys_mr);
1002
1003 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1004 {
1005         return mr->device->query_mr ?
1006                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1007 }
1008 EXPORT_SYMBOL(ib_query_mr);
1009
1010 int ib_dereg_mr(struct ib_mr *mr)
1011 {
1012         struct ib_pd *pd;
1013         int ret;
1014
1015         if (atomic_read(&mr->usecnt))
1016                 return -EBUSY;
1017
1018         pd = mr->pd;
1019         ret = mr->device->dereg_mr(mr);
1020         if (!ret)
1021                 atomic_dec(&pd->usecnt);
1022
1023         return ret;
1024 }
1025 EXPORT_SYMBOL(ib_dereg_mr);
1026
1027 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1028 {
1029         struct ib_mr *mr;
1030
1031         if (!pd->device->alloc_fast_reg_mr)
1032                 return ERR_PTR(-ENOSYS);
1033
1034         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1035
1036         if (!IS_ERR(mr)) {
1037                 mr->device  = pd->device;
1038                 mr->pd      = pd;
1039                 mr->uobject = NULL;
1040                 atomic_inc(&pd->usecnt);
1041                 atomic_set(&mr->usecnt, 0);
1042         }
1043
1044         return mr;
1045 }
1046 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1047
1048 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1049                                                           int max_page_list_len)
1050 {
1051         struct ib_fast_reg_page_list *page_list;
1052
1053         if (!device->alloc_fast_reg_page_list)
1054                 return ERR_PTR(-ENOSYS);
1055
1056         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1057
1058         if (!IS_ERR(page_list)) {
1059                 page_list->device = device;
1060                 page_list->max_page_list_len = max_page_list_len;
1061         }
1062
1063         return page_list;
1064 }
1065 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1066
1067 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1068 {
1069         page_list->device->free_fast_reg_page_list(page_list);
1070 }
1071 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1072
1073 /* Memory windows */
1074
1075 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
1076 {
1077         struct ib_mw *mw;
1078
1079         if (!pd->device->alloc_mw)
1080                 return ERR_PTR(-ENOSYS);
1081
1082         mw = pd->device->alloc_mw(pd);
1083         if (!IS_ERR(mw)) {
1084                 mw->device  = pd->device;
1085                 mw->pd      = pd;
1086                 mw->uobject = NULL;
1087                 atomic_inc(&pd->usecnt);
1088         }
1089
1090         return mw;
1091 }
1092 EXPORT_SYMBOL(ib_alloc_mw);
1093
1094 int ib_dealloc_mw(struct ib_mw *mw)
1095 {
1096         struct ib_pd *pd;
1097         int ret;
1098
1099         pd = mw->pd;
1100         ret = mw->device->dealloc_mw(mw);
1101         if (!ret)
1102                 atomic_dec(&pd->usecnt);
1103
1104         return ret;
1105 }
1106 EXPORT_SYMBOL(ib_dealloc_mw);
1107
1108 /* "Fast" memory regions */
1109
1110 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1111                             int mr_access_flags,
1112                             struct ib_fmr_attr *fmr_attr)
1113 {
1114         struct ib_fmr *fmr;
1115
1116         if (!pd->device->alloc_fmr)
1117                 return ERR_PTR(-ENOSYS);
1118
1119         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1120         if (!IS_ERR(fmr)) {
1121                 fmr->device = pd->device;
1122                 fmr->pd     = pd;
1123                 atomic_inc(&pd->usecnt);
1124         }
1125
1126         return fmr;
1127 }
1128 EXPORT_SYMBOL(ib_alloc_fmr);
1129
1130 int ib_unmap_fmr(struct list_head *fmr_list)
1131 {
1132         struct ib_fmr *fmr;
1133
1134         if (list_empty(fmr_list))
1135                 return 0;
1136
1137         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1138         return fmr->device->unmap_fmr(fmr_list);
1139 }
1140 EXPORT_SYMBOL(ib_unmap_fmr);
1141
1142 int ib_dealloc_fmr(struct ib_fmr *fmr)
1143 {
1144         struct ib_pd *pd;
1145         int ret;
1146
1147         pd = fmr->pd;
1148         ret = fmr->device->dealloc_fmr(fmr);
1149         if (!ret)
1150                 atomic_dec(&pd->usecnt);
1151
1152         return ret;
1153 }
1154 EXPORT_SYMBOL(ib_dealloc_fmr);
1155
1156 /* Multicast groups */
1157
1158 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1159 {
1160         if (!qp->device->attach_mcast)
1161                 return -ENOSYS;
1162         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1163                 return -EINVAL;
1164
1165         return qp->device->attach_mcast(qp, gid, lid);
1166 }
1167 EXPORT_SYMBOL(ib_attach_mcast);
1168
1169 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1170 {
1171         if (!qp->device->detach_mcast)
1172                 return -ENOSYS;
1173         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1174                 return -EINVAL;
1175
1176         return qp->device->detach_mcast(qp, gid, lid);
1177 }
1178 EXPORT_SYMBOL(ib_detach_mcast);
1179
1180 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1181 {
1182         struct ib_xrcd *xrcd;
1183
1184         if (!device->alloc_xrcd)
1185                 return ERR_PTR(-ENOSYS);
1186
1187         xrcd = device->alloc_xrcd(device, NULL, NULL);
1188         if (!IS_ERR(xrcd)) {
1189                 xrcd->device = device;
1190                 xrcd->inode = NULL;
1191                 atomic_set(&xrcd->usecnt, 0);
1192                 mutex_init(&xrcd->tgt_qp_mutex);
1193                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1194         }
1195
1196         return xrcd;
1197 }
1198 EXPORT_SYMBOL(ib_alloc_xrcd);
1199
1200 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1201 {
1202         struct ib_qp *qp;
1203         int ret;
1204
1205         if (atomic_read(&xrcd->usecnt))
1206                 return -EBUSY;
1207
1208         while (!list_empty(&xrcd->tgt_qp_list)) {
1209                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1210                 ret = ib_destroy_qp(qp);
1211                 if (ret)
1212                         return ret;
1213         }
1214
1215         return xrcd->device->dealloc_xrcd(xrcd);
1216 }
1217 EXPORT_SYMBOL(ib_dealloc_xrcd);