drivers/infiniband/core/verbs.c

   1 /*
   2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   9  *
  10  * This software is available to you under a choice of one of two
  11  * licenses.  You may choose to be licensed under the terms of the GNU
  12  * General Public License (GPL) Version 2, available from the file
  13  * COPYING in the main directory of this source tree, or the
  14  * OpenIB.org BSD license below:
  15  *
  16  *     Redistribution and use in source and binary forms, with or
  17  *     without modification, are permitted provided that the following
  18  *     conditions are met:
  19  *
  20  *      - Redistributions of source code must retain the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer.
  23  *
  24  *      - Redistributions in binary form must reproduce the above
  25  *        copyright notice, this list of conditions and the following
  26  *        disclaimer in the documentation and/or other materials
  27  *        provided with the distribution.
  28  *
  29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  36  * SOFTWARE.
  37  */
  38
  39 #include <linux/errno.h>
  40 #include <linux/err.h>
  41 #include <linux/export.h>
  42 #include <linux/string.h>
  43 #include <linux/slab.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_cache.h>
  47
  48 int ib_rate_to_mult(enum ib_rate rate)
  49 {
  50         switch (rate) {
  51         case IB_RATE_2_5_GBPS: return  1;
  52         case IB_RATE_5_GBPS:   return  2;
  53         case IB_RATE_10_GBPS:  return  4;
  54         case IB_RATE_20_GBPS:  return  8;
  55         case IB_RATE_30_GBPS:  return 12;
  56         case IB_RATE_40_GBPS:  return 16;
  57         case IB_RATE_60_GBPS:  return 24;
  58         case IB_RATE_80_GBPS:  return 32;
  59         case IB_RATE_120_GBPS: return 48;
  60         default:               return -1;
  61         }
  62 }
  63 EXPORT_SYMBOL(ib_rate_to_mult);
  64
  65 enum ib_rate mult_to_ib_rate(int mult)
  66 {
  67         switch (mult) {
  68         case 1:  return IB_RATE_2_5_GBPS;
  69         case 2:  return IB_RATE_5_GBPS;
  70         case 4:  return IB_RATE_10_GBPS;
  71         case 8:  return IB_RATE_20_GBPS;
  72         case 12: return IB_RATE_30_GBPS;
  73         case 16: return IB_RATE_40_GBPS;
  74         case 24: return IB_RATE_60_GBPS;
  75         case 32: return IB_RATE_80_GBPS;
  76         case 48: return IB_RATE_120_GBPS;
  77         default: return IB_RATE_PORT_CURRENT;
  78         }
  79 }
  80 EXPORT_SYMBOL(mult_to_ib_rate);
  81
  82 int ib_rate_to_mbps(enum ib_rate rate)
  83 {
  84         switch (rate) {
  85         case IB_RATE_2_5_GBPS: return 2500;
  86         case IB_RATE_5_GBPS:   return 5000;
  87         case IB_RATE_10_GBPS:  return 10000;
  88         case IB_RATE_20_GBPS:  return 20000;
  89         case IB_RATE_30_GBPS:  return 30000;
  90         case IB_RATE_40_GBPS:  return 40000;
  91         case IB_RATE_60_GBPS:  return 60000;
  92         case IB_RATE_80_GBPS:  return 80000;
  93         case IB_RATE_120_GBPS: return 120000;
  94         case IB_RATE_14_GBPS:  return 14062;
  95         case IB_RATE_56_GBPS:  return 56250;
  96         case IB_RATE_112_GBPS: return 112500;
  97         case IB_RATE_168_GBPS: return 168750;
  98         case IB_RATE_25_GBPS:  return 25781;
  99         case IB_RATE_100_GBPS: return 103125;
 100         case IB_RATE_200_GBPS: return 206250;
 101         case IB_RATE_300_GBPS: return 309375;
 102         default:               return -1;
 103         }
 104 }
 105 EXPORT_SYMBOL(ib_rate_to_mbps);
 106
 107 enum rdma_transport_type
 108 rdma_node_get_transport(enum rdma_node_type node_type)
 109 {
 110         switch (node_type) {
 111         case RDMA_NODE_IB_CA:
 112         case RDMA_NODE_IB_SWITCH:
 113         case RDMA_NODE_IB_ROUTER:
 114                 return RDMA_TRANSPORT_IB;
 115         case RDMA_NODE_RNIC:
 116                 return RDMA_TRANSPORT_IWARP;
 117         default:
 118                 BUG();
 119                 return 0;
 120         }
 121 }
 122 EXPORT_SYMBOL(rdma_node_get_transport);
 123
 124 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
 125 {
 126         if (device->get_link_layer)
 127                 return device->get_link_layer(device, port_num);
 128
 129         switch (rdma_node_get_transport(device->node_type)) {
 130         case RDMA_TRANSPORT_IB:
 131                 return IB_LINK_LAYER_INFINIBAND;
 132         case RDMA_TRANSPORT_IWARP:
 133                 return IB_LINK_LAYER_ETHERNET;
 134         default:
 135                 return IB_LINK_LAYER_UNSPECIFIED;
 136         }
 137 }
 138 EXPORT_SYMBOL(rdma_port_get_link_layer);
 139
 140 /* Protection domains */
 141
 142 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 143 {
 144         struct ib_pd *pd;
 145
 146         pd = device->alloc_pd(device, NULL, NULL);
 147
 148         if (!IS_ERR(pd)) {
 149                 pd->device  = device;
 150                 pd->uobject = NULL;
 151                 atomic_set(&pd->usecnt, 0);
 152         }
 153
 154         return pd;
 155 }
 156 EXPORT_SYMBOL(ib_alloc_pd);
 157
 158 int ib_dealloc_pd(struct ib_pd *pd)
 159 {
 160         if (atomic_read(&pd->usecnt))
 161                 return -EBUSY;
 162
 163         return pd->device->dealloc_pd(pd);
 164 }
 165 EXPORT_SYMBOL(ib_dealloc_pd);
 166
 167 /* Address handles */
 168
 169 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 170 {
 171         struct ib_ah *ah;
 172
 173         ah = pd->device->create_ah(pd, ah_attr);
 174
 175         if (!IS_ERR(ah)) {
 176                 ah->device  = pd->device;
 177                 ah->pd      = pd;
 178                 ah->uobject = NULL;
 179                 atomic_inc(&pd->usecnt);
 180         }
 181
 182         return ah;
 183 }
 184 EXPORT_SYMBOL(ib_create_ah);
 185
 186 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
 187                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
 188 {
 189         u32 flow_class;
 190         u16 gid_index;
 191         int ret;
 192
 193         memset(ah_attr, 0, sizeof *ah_attr);
 194         ah_attr->dlid = wc->slid;
 195         ah_attr->sl = wc->sl;
 196         ah_attr->src_path_bits = wc->dlid_path_bits;
 197         ah_attr->port_num = port_num;
 198
 199         if (wc->wc_flags & IB_WC_GRH) {
 200                 ah_attr->ah_flags = IB_AH_GRH;
 201                 ah_attr->grh.dgid = grh->sgid;
 202
 203                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
 204                                          &gid_index);
 205                 if (ret)
 206                         return ret;
 207
 208                 ah_attr->grh.sgid_index = (u8) gid_index;
 209                 flow_class = be32_to_cpu(grh->version_tclass_flow);
 210                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
 211                 ah_attr->grh.hop_limit = 0xFF;
 212                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 213         }
 214         return 0;
 215 }
 216 EXPORT_SYMBOL(ib_init_ah_from_wc);
 217
 218 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
 219                                    struct ib_grh *grh, u8 port_num)
 220 {
 221         struct ib_ah_attr ah_attr;
 222         int ret;
 223
 224         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 225         if (ret)
 226                 return ERR_PTR(ret);
 227
 228         return ib_create_ah(pd, &ah_attr);
 229 }
 230 EXPORT_SYMBOL(ib_create_ah_from_wc);
 231
 232 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 233 {
 234         return ah->device->modify_ah ?
 235                 ah->device->modify_ah(ah, ah_attr) :
 236                 -ENOSYS;
 237 }
 238 EXPORT_SYMBOL(ib_modify_ah);
 239
 240 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 241 {
 242         return ah->device->query_ah ?
 243                 ah->device->query_ah(ah, ah_attr) :
 244                 -ENOSYS;
 245 }
 246 EXPORT_SYMBOL(ib_query_ah);
 247
 248 int ib_destroy_ah(struct ib_ah *ah)
 249 {
 250         struct ib_pd *pd;
 251         int ret;
 252
 253         pd = ah->pd;
 254         ret = ah->device->destroy_ah(ah);
 255         if (!ret)
 256                 atomic_dec(&pd->usecnt);
 257
 258         return ret;
 259 }
 260 EXPORT_SYMBOL(ib_destroy_ah);
 261
 262 /* Shared receive queues */
 263
 264 struct ib_srq *ib_create_srq(struct ib_pd *pd,
 265                              struct ib_srq_init_attr *srq_init_attr)
 266 {
 267         struct ib_srq *srq;
 268
 269         if (!pd->device->create_srq)
 270                 return ERR_PTR(-ENOSYS);
 271
 272         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
 273
 274         if (!IS_ERR(srq)) {
 275                 srq->device        = pd->device;
 276                 srq->pd            = pd;
 277                 srq->uobject       = NULL;
 278                 srq->event_handler = srq_init_attr->event_handler;
 279                 srq->srq_context   = srq_init_attr->srq_context;
 280                 srq->srq_type      = srq_init_attr->srq_type;
 281                 if (srq->srq_type == IB_SRQT_XRC) {
 282                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
 283                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 284                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
 285                         atomic_inc(&srq->ext.xrc.cq->usecnt);
 286                 }
 287                 atomic_inc(&pd->usecnt);
 288                 atomic_set(&srq->usecnt, 0);
 289         }
 290
 291         return srq;
 292 }
 293 EXPORT_SYMBOL(ib_create_srq);
 294
 295 int ib_modify_srq(struct ib_srq *srq,
 296                   struct ib_srq_attr *srq_attr,
 297                   enum ib_srq_attr_mask srq_attr_mask)
 298 {
 299         return srq->device->modify_srq ?
 300                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
 301                 -ENOSYS;
 302 }
 303 EXPORT_SYMBOL(ib_modify_srq);
 304
 305 int ib_query_srq(struct ib_srq *srq,
 306                  struct ib_srq_attr *srq_attr)
 307 {
 308         return srq->device->query_srq ?
 309                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
 310 }
 311 EXPORT_SYMBOL(ib_query_srq);
 312
 313 int ib_destroy_srq(struct ib_srq *srq)
 314 {
 315         struct ib_pd *pd;
 316         enum ib_srq_type srq_type;
 317         struct ib_xrcd *uninitialized_var(xrcd);
 318         struct ib_cq *uninitialized_var(cq);
 319         int ret;
 320
 321         if (atomic_read(&srq->usecnt))
 322                 return -EBUSY;
 323
 324         pd = srq->pd;
 325         srq_type = srq->srq_type;
 326         if (srq_type == IB_SRQT_XRC) {
 327                 xrcd = srq->ext.xrc.xrcd;
 328                 cq = srq->ext.xrc.cq;
 329         }
 330
 331         ret = srq->device->destroy_srq(srq);
 332         if (!ret) {
 333                 atomic_dec(&pd->usecnt);
 334                 if (srq_type == IB_SRQT_XRC) {
 335                         atomic_dec(&xrcd->usecnt);
 336                         atomic_dec(&cq->usecnt);
 337                 }
 338         }
 339
 340         return ret;
 341 }
 342 EXPORT_SYMBOL(ib_destroy_srq);
 343
 344 /* Queue pairs */
 345
 346 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
 347 {
 348         struct ib_qp *qp = context;
 349
 350         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
 351                 event->element.qp->event_handler(event, event->element.qp->qp_context);
 352 }
 353
 354 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
 355 {
 356         mutex_lock(&xrcd->tgt_qp_mutex);
 357         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
 358         mutex_unlock(&xrcd->tgt_qp_mutex);
 359 }
 360
 361 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
 362                                   void (*event_handler)(struct ib_event *, void *),
 363                                   void *qp_context)
 364 {
 365         struct ib_qp *qp;
 366         unsigned long flags;
 367
 368         qp = kzalloc(sizeof *qp, GFP_KERNEL);
 369         if (!qp)
 370                 return ERR_PTR(-ENOMEM);
 371
 372         qp->real_qp = real_qp;
 373         atomic_inc(&real_qp->usecnt);
 374         qp->device = real_qp->device;
 375         qp->event_handler = event_handler;
 376         qp->qp_context = qp_context;
 377         qp->qp_num = real_qp->qp_num;
 378         qp->qp_type = real_qp->qp_type;
 379
 380         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 381         list_add(&qp->open_list, &real_qp->open_list);
 382         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 383
 384         return qp;
 385 }
 386
 387 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 388                          struct ib_qp_open_attr *qp_open_attr)
 389 {
 390         struct ib_qp *qp, *real_qp;
 391
 392         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
 393                 return ERR_PTR(-EINVAL);
 394
 395         qp = ERR_PTR(-EINVAL);
 396         mutex_lock(&xrcd->tgt_qp_mutex);
 397         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
 398                 if (real_qp->qp_num == qp_open_attr->qp_num) {
 399                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
 400                                           qp_open_attr->qp_context);
 401                         break;
 402                 }
 403         }
 404         mutex_unlock(&xrcd->tgt_qp_mutex);
 405         return qp;
 406 }
 407 EXPORT_SYMBOL(ib_open_qp);
 408
 409 struct ib_qp *ib_create_qp(struct ib_pd *pd,
 410                            struct ib_qp_init_attr *qp_init_attr)
 411 {
 412         struct ib_qp *qp, *real_qp;
 413         struct ib_device *device;
 414
 415         device = pd ? pd->device : qp_init_attr->xrcd->device;
 416         qp = device->create_qp(pd, qp_init_attr, NULL);
 417
 418         if (!IS_ERR(qp)) {
 419                 qp->device     = device;
 420                 qp->real_qp    = qp;
 421                 qp->uobject    = NULL;
 422                 qp->qp_type    = qp_init_attr->qp_type;
 423
 424                 atomic_set(&qp->usecnt, 0);
 425                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
 426                         qp->event_handler = __ib_shared_qp_event_handler;
 427                         qp->qp_context = qp;
 428                         qp->pd = NULL;
 429                         qp->send_cq = qp->recv_cq = NULL;
 430                         qp->srq = NULL;
 431                         qp->xrcd = qp_init_attr->xrcd;
 432                         atomic_inc(&qp_init_attr->xrcd->usecnt);
 433                         INIT_LIST_HEAD(&qp->open_list);
 434
 435                         real_qp = qp;
 436                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
 437                                           qp_init_attr->qp_context);
 438                         if (!IS_ERR(qp))
 439                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
 440                         else
 441                                 real_qp->device->destroy_qp(real_qp);
 442                 } else {
 443                         qp->event_handler = qp_init_attr->event_handler;
 444                         qp->qp_context = qp_init_attr->qp_context;
 445                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
 446                                 qp->recv_cq = NULL;
 447                                 qp->srq = NULL;
 448                         } else {
 449                                 qp->recv_cq = qp_init_attr->recv_cq;
 450                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
 451                                 qp->srq = qp_init_attr->srq;
 452                                 if (qp->srq)
 453                                         atomic_inc(&qp_init_attr->srq->usecnt);
 454                         }
 455
 456                         qp->pd      = pd;
 457                         qp->send_cq = qp_init_attr->send_cq;
 458                         qp->xrcd    = NULL;
 459
 460                         atomic_inc(&pd->usecnt);
 461                         atomic_inc(&qp_init_attr->send_cq->usecnt);
 462                 }
 463         }
 464
 465         return qp;
 466 }
 467 EXPORT_SYMBOL(ib_create_qp);
 468
 469 static const struct {
 470         int                     valid;
 471         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
 472         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
 473 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
 474         [IB_QPS_RESET] = {
 475                 [IB_QPS_RESET] = { .valid = 1 },
 476                 [IB_QPS_INIT]  = {
 477                         .valid = 1,
 478                         .req_param = {
 479                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 480                                                 IB_QP_PORT                      |
 481                                                 IB_QP_QKEY),
 482                                 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
 483                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 484                                                 IB_QP_PORT                      |
 485                                                 IB_QP_ACCESS_FLAGS),
 486                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 487                                                 IB_QP_PORT                      |
 488                                                 IB_QP_ACCESS_FLAGS),
 489                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 490                                                 IB_QP_PORT                      |
 491                                                 IB_QP_ACCESS_FLAGS),
 492                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 493                                                 IB_QP_PORT                      |
 494                                                 IB_QP_ACCESS_FLAGS),
 495                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 496                                                 IB_QP_QKEY),
 497                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 498                                                 IB_QP_QKEY),
 499                         }
 500                 },
 501         },
 502         [IB_QPS_INIT]  = {
 503                 [IB_QPS_RESET] = { .valid = 1 },
 504                 [IB_QPS_ERR] =   { .valid = 1 },
 505                 [IB_QPS_INIT]  = {
 506                         .valid = 1,
 507                         .opt_param = {
 508                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 509                                                 IB_QP_PORT                      |
 510                                                 IB_QP_QKEY),
 511                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 512                                                 IB_QP_PORT                      |
 513                                                 IB_QP_ACCESS_FLAGS),
 514                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 515                                                 IB_QP_PORT                      |
 516                                                 IB_QP_ACCESS_FLAGS),
 517                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 518                                                 IB_QP_PORT                      |
 519                                                 IB_QP_ACCESS_FLAGS),
 520                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 521                                                 IB_QP_PORT                      |
 522                                                 IB_QP_ACCESS_FLAGS),
 523                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 524                                                 IB_QP_QKEY),
 525                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 526                                                 IB_QP_QKEY),
 527                         }
 528                 },
 529                 [IB_QPS_RTR]   = {
 530                         .valid = 1,
 531                         .req_param = {
 532                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 533                                                 IB_QP_PATH_MTU                  |
 534                                                 IB_QP_DEST_QPN                  |
 535                                                 IB_QP_RQ_PSN),
 536                                 [IB_QPT_RC]  = (IB_QP_AV                        |
 537                                                 IB_QP_PATH_MTU                  |
 538                                                 IB_QP_DEST_QPN                  |
 539                                                 IB_QP_RQ_PSN                    |
 540                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 541                                                 IB_QP_MIN_RNR_TIMER),
 542                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
 543                                                 IB_QP_PATH_MTU                  |
 544                                                 IB_QP_DEST_QPN                  |
 545                                                 IB_QP_RQ_PSN),
 546                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
 547                                                 IB_QP_PATH_MTU                  |
 548                                                 IB_QP_DEST_QPN                  |
 549                                                 IB_QP_RQ_PSN                    |
 550                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 551                                                 IB_QP_MIN_RNR_TIMER),
 552                         },
 553                         .opt_param = {
 554                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
 555                                                  IB_QP_QKEY),
 556                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
 557                                                  IB_QP_ACCESS_FLAGS             |
 558                                                  IB_QP_PKEY_INDEX),
 559                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
 560                                                  IB_QP_ACCESS_FLAGS             |
 561                                                  IB_QP_PKEY_INDEX),
 562                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
 563                                                  IB_QP_ACCESS_FLAGS             |
 564                                                  IB_QP_PKEY_INDEX),
 565                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
 566                                                  IB_QP_ACCESS_FLAGS             |
 567                                                  IB_QP_PKEY_INDEX),
 568                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
 569                                                  IB_QP_QKEY),
 570                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
 571                                                  IB_QP_QKEY),
 572                          }
 573                 }
 574         },
 575         [IB_QPS_RTR]   = {
 576                 [IB_QPS_RESET] = { .valid = 1 },
 577                 [IB_QPS_ERR] =   { .valid = 1 },
 578                 [IB_QPS_RTS]   = {
 579                         .valid = 1,
 580                         .req_param = {
 581                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
 582                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
 583                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
 584                                                 IB_QP_RETRY_CNT                 |
 585                                                 IB_QP_RNR_RETRY                 |
 586                                                 IB_QP_SQ_PSN                    |
 587                                                 IB_QP_MAX_QP_RD_ATOMIC),
 588                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
 589                                                 IB_QP_RETRY_CNT                 |
 590                                                 IB_QP_RNR_RETRY                 |
 591                                                 IB_QP_SQ_PSN                    |
 592                                                 IB_QP_MAX_QP_RD_ATOMIC),
 593                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
 594                                                 IB_QP_SQ_PSN),
 595                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
 596                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
 597                         },
 598                         .opt_param = {
 599                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
 600                                                  IB_QP_QKEY),
 601                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
 602                                                  IB_QP_ALT_PATH                 |
 603                                                  IB_QP_ACCESS_FLAGS             |
 604                                                  IB_QP_PATH_MIG_STATE),
 605                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
 606                                                  IB_QP_ALT_PATH                 |
 607                                                  IB_QP_ACCESS_FLAGS             |
 608                                                  IB_QP_MIN_RNR_TIMER            |
 609                                                  IB_QP_PATH_MIG_STATE),
 610                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
 611                                                  IB_QP_ALT_PATH                 |
 612                                                  IB_QP_ACCESS_FLAGS             |
 613                                                  IB_QP_PATH_MIG_STATE),
 614                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
 615                                                  IB_QP_ALT_PATH                 |
 616                                                  IB_QP_ACCESS_FLAGS             |
 617                                                  IB_QP_MIN_RNR_TIMER            |
 618                                                  IB_QP_PATH_MIG_STATE),
 619                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
 620                                                  IB_QP_QKEY),
 621                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
 622                                                  IB_QP_QKEY),
 623                          }
 624                 }
 625         },
 626         [IB_QPS_RTS]   = {
 627                 [IB_QPS_RESET] = { .valid = 1 },
 628                 [IB_QPS_ERR] =   { .valid = 1 },
 629                 [IB_QPS_RTS]   = {
 630                         .valid = 1,
 631                         .opt_param = {
 632                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 633                                                 IB_QP_QKEY),
 634                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 635                                                 IB_QP_ACCESS_FLAGS              |
 636                                                 IB_QP_ALT_PATH                  |
 637                                                 IB_QP_PATH_MIG_STATE),
 638                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 639                                                 IB_QP_ACCESS_FLAGS              |
 640                                                 IB_QP_ALT_PATH                  |
 641                                                 IB_QP_PATH_MIG_STATE            |
 642                                                 IB_QP_MIN_RNR_TIMER),
 643                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 644                                                 IB_QP_ACCESS_FLAGS              |
 645                                                 IB_QP_ALT_PATH                  |
 646                                                 IB_QP_PATH_MIG_STATE),
 647                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 648                                                 IB_QP_ACCESS_FLAGS              |
 649                                                 IB_QP_ALT_PATH                  |
 650                                                 IB_QP_PATH_MIG_STATE            |
 651                                                 IB_QP_MIN_RNR_TIMER),
 652                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 653                                                 IB_QP_QKEY),
 654                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 655                                                 IB_QP_QKEY),
 656                         }
 657                 },
 658                 [IB_QPS_SQD]   = {
 659                         .valid = 1,
 660                         .opt_param = {
 661                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 662                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 663                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 664                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 665                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
 666                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 667                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
 668                         }
 669                 },
 670         },
 671         [IB_QPS_SQD]   = {
 672                 [IB_QPS_RESET] = { .valid = 1 },
 673                 [IB_QPS_ERR] =   { .valid = 1 },
 674                 [IB_QPS_RTS]   = {
 675                         .valid = 1,
 676                         .opt_param = {
 677                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 678                                                 IB_QP_QKEY),
 679                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 680                                                 IB_QP_ALT_PATH                  |
 681                                                 IB_QP_ACCESS_FLAGS              |
 682                                                 IB_QP_PATH_MIG_STATE),
 683                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 684                                                 IB_QP_ALT_PATH                  |
 685                                                 IB_QP_ACCESS_FLAGS              |
 686                                                 IB_QP_MIN_RNR_TIMER             |
 687                                                 IB_QP_PATH_MIG_STATE),
 688                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 689                                                 IB_QP_ALT_PATH                  |
 690                                                 IB_QP_ACCESS_FLAGS              |
 691                                                 IB_QP_PATH_MIG_STATE),
 692                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 693                                                 IB_QP_ALT_PATH                  |
 694                                                 IB_QP_ACCESS_FLAGS              |
 695                                                 IB_QP_MIN_RNR_TIMER             |
 696                                                 IB_QP_PATH_MIG_STATE),
 697                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 698                                                 IB_QP_QKEY),
 699                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 700                                                 IB_QP_QKEY),
 701                         }
 702                 },
 703                 [IB_QPS_SQD]   = {
 704                         .valid = 1,
 705                         .opt_param = {
 706                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 707                                                 IB_QP_QKEY),
 708                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 709                                                 IB_QP_ALT_PATH                  |
 710                                                 IB_QP_ACCESS_FLAGS              |
 711                                                 IB_QP_PKEY_INDEX                |
 712                                                 IB_QP_PATH_MIG_STATE),
 713                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
 714                                                 IB_QP_AV                        |
 715                                                 IB_QP_TIMEOUT                   |
 716                                                 IB_QP_RETRY_CNT                 |
 717                                                 IB_QP_RNR_RETRY                 |
 718                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 719                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 720                                                 IB_QP_ALT_PATH                  |
 721                                                 IB_QP_ACCESS_FLAGS              |
 722                                                 IB_QP_PKEY_INDEX                |
 723                                                 IB_QP_MIN_RNR_TIMER             |
 724                                                 IB_QP_PATH_MIG_STATE),
 725                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
 726                                                 IB_QP_AV                        |
 727                                                 IB_QP_TIMEOUT                   |
 728                                                 IB_QP_RETRY_CNT                 |
 729                                                 IB_QP_RNR_RETRY                 |
 730                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 731                                                 IB_QP_ALT_PATH                  |
 732                                                 IB_QP_ACCESS_FLAGS              |
 733                                                 IB_QP_PKEY_INDEX                |
 734                                                 IB_QP_PATH_MIG_STATE),
 735                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
 736                                                 IB_QP_AV                        |
 737                                                 IB_QP_TIMEOUT                   |
 738                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 739                                                 IB_QP_ALT_PATH                  |
 740                                                 IB_QP_ACCESS_FLAGS              |
 741                                                 IB_QP_PKEY_INDEX                |
 742                                                 IB_QP_MIN_RNR_TIMER             |
 743                                                 IB_QP_PATH_MIG_STATE),
 744                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 745                                                 IB_QP_QKEY),
 746                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 747                                                 IB_QP_QKEY),
 748                         }
 749                 }
 750         },
 751         [IB_QPS_SQE]   = {
 752                 [IB_QPS_RESET] = { .valid = 1 },
 753                 [IB_QPS_ERR] =   { .valid = 1 },
 754                 [IB_QPS_RTS]   = {
 755                         .valid = 1,
 756                         .opt_param = {
 757                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 758                                                 IB_QP_QKEY),
 759                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 760                                                 IB_QP_ACCESS_FLAGS),
 761                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 762                                                 IB_QP_QKEY),
 763                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 764                                                 IB_QP_QKEY),
 765                         }
 766                 }
 767         },
 768         [IB_QPS_ERR] = {
 769                 [IB_QPS_RESET] = { .valid = 1 },
 770                 [IB_QPS_ERR] =   { .valid = 1 }
 771         }
 772 };
 773
 774 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 775                        enum ib_qp_type type, enum ib_qp_attr_mask mask)
 776 {
 777         enum ib_qp_attr_mask req_param, opt_param;
 778
 779         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
 780             next_state < 0 || next_state > IB_QPS_ERR)
 781                 return 0;
 782
 783         if (mask & IB_QP_CUR_STATE  &&
 784             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
 785             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
 786                 return 0;
 787
 788         if (!qp_state_table[cur_state][next_state].valid)
 789                 return 0;
 790
 791         req_param = qp_state_table[cur_state][next_state].req_param[type];
 792         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
 793
 794         if ((mask & req_param) != req_param)
 795                 return 0;
 796
 797         if (mask & ~(req_param | opt_param | IB_QP_STATE))
 798                 return 0;
 799
 800         return 1;
 801 }
 802 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 803
 804 int ib_modify_qp(struct ib_qp *qp,
 805                  struct ib_qp_attr *qp_attr,
 806                  int qp_attr_mask)
 807 {
 808         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 809 }
 810 EXPORT_SYMBOL(ib_modify_qp);
 811
 812 int ib_query_qp(struct ib_qp *qp,
 813                 struct ib_qp_attr *qp_attr,
 814                 int qp_attr_mask,
 815                 struct ib_qp_init_attr *qp_init_attr)
 816 {
 817         return qp->device->query_qp ?
 818                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
 819                 -ENOSYS;
 820 }
 821 EXPORT_SYMBOL(ib_query_qp);
 822
 823 int ib_close_qp(struct ib_qp *qp)
 824 {
 825         struct ib_qp *real_qp;
 826         unsigned long flags;
 827
 828         real_qp = qp->real_qp;
 829         if (real_qp == qp)
 830                 return -EINVAL;
 831
 832         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 833         list_del(&qp->open_list);
 834         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 835
 836         atomic_dec(&real_qp->usecnt);
 837         kfree(qp);
 838
 839         return 0;
 840 }
 841 EXPORT_SYMBOL(ib_close_qp);
 842
 843 static int __ib_destroy_shared_qp(struct ib_qp *qp)
 844 {
 845         struct ib_xrcd *xrcd;
 846         struct ib_qp *real_qp;
 847         int ret;
 848
 849         real_qp = qp->real_qp;
 850         xrcd = real_qp->xrcd;
 851
 852         mutex_lock(&xrcd->tgt_qp_mutex);
 853         ib_close_qp(qp);
 854         if (atomic_read(&real_qp->usecnt) == 0)
 855                 list_del(&real_qp->xrcd_list);
 856         else
 857                 real_qp = NULL;
 858         mutex_unlock(&xrcd->tgt_qp_mutex);
 859
 860         if (real_qp) {
 861                 ret = ib_destroy_qp(real_qp);
 862                 if (!ret)
 863                         atomic_dec(&xrcd->usecnt);
 864                 else
 865                         __ib_insert_xrcd_qp(xrcd, real_qp);
 866         }
 867
 868         return 0;
 869 }
 870
 871 int ib_destroy_qp(struct ib_qp *qp)
 872 {
 873         struct ib_pd *pd;
 874         struct ib_cq *scq, *rcq;
 875         struct ib_srq *srq;
 876         int ret;
 877
 878         if (atomic_read(&qp->usecnt))
 879                 return -EBUSY;
 880
 881         if (qp->real_qp != qp)
 882                 return __ib_destroy_shared_qp(qp);
 883
 884         pd   = qp->pd;
 885         scq  = qp->send_cq;
 886         rcq  = qp->recv_cq;
 887         srq  = qp->srq;
 888
 889         ret = qp->device->destroy_qp(qp);
 890         if (!ret) {
 891                 if (pd)
 892                         atomic_dec(&pd->usecnt);
 893                 if (scq)
 894                         atomic_dec(&scq->usecnt);
 895                 if (rcq)
 896                         atomic_dec(&rcq->usecnt);
 897                 if (srq)
 898                         atomic_dec(&srq->usecnt);
 899         }
 900
 901         return ret;
 902 }
 903 EXPORT_SYMBOL(ib_destroy_qp);
 904
 905 /* Completion queues */
 906
 907 struct ib_cq *ib_create_cq(struct ib_device *device,
 908                            ib_comp_handler comp_handler,
 909                            void (*event_handler)(struct ib_event *, void *),
 910                            void *cq_context, int cqe, int comp_vector)
 911 {
 912         struct ib_cq *cq;
 913
 914         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
 915
 916         if (!IS_ERR(cq)) {
 917                 cq->device        = device;
 918                 cq->uobject       = NULL;
 919                 cq->comp_handler  = comp_handler;
 920                 cq->event_handler = event_handler;
 921                 cq->cq_context    = cq_context;
 922                 atomic_set(&cq->usecnt, 0);
 923         }
 924
 925         return cq;
 926 }
 927 EXPORT_SYMBOL(ib_create_cq);
 928
 929 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
 930 {
 931         return cq->device->modify_cq ?
 932                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
 933 }
 934 EXPORT_SYMBOL(ib_modify_cq);
 935
 936 int ib_destroy_cq(struct ib_cq *cq)
 937 {
 938         if (atomic_read(&cq->usecnt))
 939                 return -EBUSY;
 940
 941         return cq->device->destroy_cq(cq);
 942 }
 943 EXPORT_SYMBOL(ib_destroy_cq);
 944
 945 int ib_resize_cq(struct ib_cq *cq, int cqe)
 946 {
 947         return cq->device->resize_cq ?
 948                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
 949 }
 950 EXPORT_SYMBOL(ib_resize_cq);
 951
 952 /* Memory regions */
 953
 954 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
 955 {
 956         struct ib_mr *mr;
 957
 958         mr = pd->device->get_dma_mr(pd, mr_access_flags);
 959
 960         if (!IS_ERR(mr)) {
 961                 mr->device  = pd->device;
 962                 mr->pd      = pd;
 963                 mr->uobject = NULL;
 964                 atomic_inc(&pd->usecnt);
 965                 atomic_set(&mr->usecnt, 0);
 966         }
 967
 968         return mr;
 969 }
 970 EXPORT_SYMBOL(ib_get_dma_mr);
 971
 972 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
 973                              struct ib_phys_buf *phys_buf_array,
 974                              int num_phys_buf,
 975                              int mr_access_flags,
 976                              u64 *iova_start)
 977 {
 978         struct ib_mr *mr;
 979
 980         if (!pd->device->reg_phys_mr)
 981                 return ERR_PTR(-ENOSYS);
 982
 983         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
 984                                      mr_access_flags, iova_start);
 985
 986         if (!IS_ERR(mr)) {
 987                 mr->device  = pd->device;
 988                 mr->pd      = pd;
 989                 mr->uobject = NULL;
 990                 atomic_inc(&pd->usecnt);
 991                 atomic_set(&mr->usecnt, 0);
 992         }
 993
 994         return mr;
 995 }
 996 EXPORT_SYMBOL(ib_reg_phys_mr);
 997
 998 int ib_rereg_phys_mr(struct ib_mr *mr,
 999                      int mr_rereg_mask,
1000                      struct ib_pd *pd,
1001                      struct ib_phys_buf *phys_buf_array,
1002                      int num_phys_buf,
1003                      int mr_access_flags,
1004                      u64 *iova_start)
1005 {
1006         struct ib_pd *old_pd;
1007         int ret;
1008
1009         if (!mr->device->rereg_phys_mr)
1010                 return -ENOSYS;
1011
1012         if (atomic_read(&mr->usecnt))
1013                 return -EBUSY;
1014
1015         old_pd = mr->pd;
1016
1017         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1018                                         phys_buf_array, num_phys_buf,
1019                                         mr_access_flags, iova_start);
1020
1021         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1022                 atomic_dec(&old_pd->usecnt);
1023                 atomic_inc(&pd->usecnt);
1024         }
1025
1026         return ret;
1027 }
1028 EXPORT_SYMBOL(ib_rereg_phys_mr);
1029
1030 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1031 {
1032         return mr->device->query_mr ?
1033                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1034 }
1035 EXPORT_SYMBOL(ib_query_mr);
1036
1037 int ib_dereg_mr(struct ib_mr *mr)
1038 {
1039         struct ib_pd *pd;
1040         int ret;
1041
1042         if (atomic_read(&mr->usecnt))
1043                 return -EBUSY;
1044
1045         pd = mr->pd;
1046         ret = mr->device->dereg_mr(mr);
1047         if (!ret)
1048                 atomic_dec(&pd->usecnt);
1049
1050         return ret;
1051 }
1052 EXPORT_SYMBOL(ib_dereg_mr);
1053
1054 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1055 {
1056         struct ib_mr *mr;
1057
1058         if (!pd->device->alloc_fast_reg_mr)
1059                 return ERR_PTR(-ENOSYS);
1060
1061         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1062
1063         if (!IS_ERR(mr)) {
1064                 mr->device  = pd->device;
1065                 mr->pd      = pd;
1066                 mr->uobject = NULL;
1067                 atomic_inc(&pd->usecnt);
1068                 atomic_set(&mr->usecnt, 0);
1069         }
1070
1071         return mr;
1072 }
1073 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1074
1075 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1076                                                           int max_page_list_len)
1077 {
1078         struct ib_fast_reg_page_list *page_list;
1079
1080         if (!device->alloc_fast_reg_page_list)
1081                 return ERR_PTR(-ENOSYS);
1082
1083         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1084
1085         if (!IS_ERR(page_list)) {
1086                 page_list->device = device;
1087                 page_list->max_page_list_len = max_page_list_len;
1088         }
1089
1090         return page_list;
1091 }
1092 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1093
1094 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1095 {
1096         page_list->device->free_fast_reg_page_list(page_list);
1097 }
1098 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1099
1100 /* Memory windows */
1101
1102 struct ib_mw *ib_alloc_mw(struct ib_pd *pd)
1103 {
1104         struct ib_mw *mw;
1105
1106         if (!pd->device->alloc_mw)
1107                 return ERR_PTR(-ENOSYS);
1108
1109         mw = pd->device->alloc_mw(pd);
1110         if (!IS_ERR(mw)) {
1111                 mw->device  = pd->device;
1112                 mw->pd      = pd;
1113                 mw->uobject = NULL;
1114                 atomic_inc(&pd->usecnt);
1115         }
1116
1117         return mw;
1118 }
1119 EXPORT_SYMBOL(ib_alloc_mw);
1120
1121 int ib_dealloc_mw(struct ib_mw *mw)
1122 {
1123         struct ib_pd *pd;
1124         int ret;
1125
1126         pd = mw->pd;
1127         ret = mw->device->dealloc_mw(mw);
1128         if (!ret)
1129                 atomic_dec(&pd->usecnt);
1130
1131         return ret;
1132 }
1133 EXPORT_SYMBOL(ib_dealloc_mw);
1134
1135 /* "Fast" memory regions */
1136
1137 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1138                             int mr_access_flags,
1139                             struct ib_fmr_attr *fmr_attr)
1140 {
1141         struct ib_fmr *fmr;
1142
1143         if (!pd->device->alloc_fmr)
1144                 return ERR_PTR(-ENOSYS);
1145
1146         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1147         if (!IS_ERR(fmr)) {
1148                 fmr->device = pd->device;
1149                 fmr->pd     = pd;
1150                 atomic_inc(&pd->usecnt);
1151         }
1152
1153         return fmr;
1154 }
1155 EXPORT_SYMBOL(ib_alloc_fmr);
1156
1157 int ib_unmap_fmr(struct list_head *fmr_list)
1158 {
1159         struct ib_fmr *fmr;
1160
1161         if (list_empty(fmr_list))
1162                 return 0;
1163
1164         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1165         return fmr->device->unmap_fmr(fmr_list);
1166 }
1167 EXPORT_SYMBOL(ib_unmap_fmr);
1168
1169 int ib_dealloc_fmr(struct ib_fmr *fmr)
1170 {
1171         struct ib_pd *pd;
1172         int ret;
1173
1174         pd = fmr->pd;
1175         ret = fmr->device->dealloc_fmr(fmr);
1176         if (!ret)
1177                 atomic_dec(&pd->usecnt);
1178
1179         return ret;
1180 }
1181 EXPORT_SYMBOL(ib_dealloc_fmr);
1182
1183 /* Multicast groups */
1184
1185 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1186 {
1187         if (!qp->device->attach_mcast)
1188                 return -ENOSYS;
1189         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1190                 return -EINVAL;
1191
1192         return qp->device->attach_mcast(qp, gid, lid);
1193 }
1194 EXPORT_SYMBOL(ib_attach_mcast);
1195
1196 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1197 {
1198         if (!qp->device->detach_mcast)
1199                 return -ENOSYS;
1200         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1201                 return -EINVAL;
1202
1203         return qp->device->detach_mcast(qp, gid, lid);
1204 }
1205 EXPORT_SYMBOL(ib_detach_mcast);
1206
1207 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1208 {
1209         struct ib_xrcd *xrcd;
1210
1211         if (!device->alloc_xrcd)
1212                 return ERR_PTR(-ENOSYS);
1213
1214         xrcd = device->alloc_xrcd(device, NULL, NULL);
1215         if (!IS_ERR(xrcd)) {
1216                 xrcd->device = device;
1217                 xrcd->inode = NULL;
1218                 atomic_set(&xrcd->usecnt, 0);
1219                 mutex_init(&xrcd->tgt_qp_mutex);
1220                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1221         }
1222
1223         return xrcd;
1224 }
1225 EXPORT_SYMBOL(ib_alloc_xrcd);
1226
1227 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1228 {
1229         struct ib_qp *qp;
1230         int ret;
1231
1232         if (atomic_read(&xrcd->usecnt))
1233                 return -EBUSY;
1234
1235         while (!list_empty(&xrcd->tgt_qp_list)) {
1236                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1237                 ret = ib_destroy_qp(qp);
1238                 if (ret)
1239                         return ret;
1240         }
1241
1242         return xrcd->device->dealloc_xrcd(xrcd);
1243 }
1244 EXPORT_SYMBOL(ib_dealloc_xrcd);