drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44 #include <linux/module.h>
  45
  46 #include <net/tcp.h>
  47 #include <net/ipv6.h>
  48
  49 #include <rdma/rdma_cm.h>
  50 #include <rdma/rdma_cm_ib.h>
  51 #include <rdma/rdma_netlink.h>
  52 #include <rdma/ib_cache.h>
  53 #include <rdma/ib_cm.h>
  54 #include <rdma/ib_sa.h>
  55 #include <rdma/iw_cm.h>
  56
  57 MODULE_AUTHOR("Sean Hefty");
  58 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  59 MODULE_LICENSE("Dual BSD/GPL");
  60
  61 #define CMA_CM_RESPONSE_TIMEOUT 20
  62 #define CMA_MAX_CM_RETRIES 15
  63 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  64 #define CMA_IBOE_PACKET_LIFETIME 18
  65
  66 static void cma_add_one(struct ib_device *device);
  67 static void cma_remove_one(struct ib_device *device);
  68
  69 static struct ib_client cma_client = {
  70         .name   = "cma",
  71         .add    = cma_add_one,
  72         .remove = cma_remove_one
  73 };
  74
  75 static struct ib_sa_client sa_client;
  76 static struct rdma_addr_client addr_client;
  77 static LIST_HEAD(dev_list);
  78 static LIST_HEAD(listen_any_list);
  79 static DEFINE_MUTEX(lock);
  80 static struct workqueue_struct *cma_wq;
  81 static DEFINE_IDR(sdp_ps);
  82 static DEFINE_IDR(tcp_ps);
  83 static DEFINE_IDR(udp_ps);
  84 static DEFINE_IDR(ipoib_ps);
  85 static DEFINE_IDR(ib_ps);
  86
  87 struct cma_device {
  88         struct list_head        list;
  89         struct ib_device        *device;
  90         struct completion       comp;
  91         atomic_t                refcount;
  92         struct list_head        id_list;
  93 };
  94
  95 struct rdma_bind_list {
  96         struct idr              *ps;
  97         struct hlist_head       owners;
  98         unsigned short          port;
  99 };
 100
 101 /*
 102  * Device removal can occur at anytime, so we need extra handling to
 103  * serialize notifying the user of device removal with other callbacks.
 104  * We do this by disabling removal notification while a callback is in process,
 105  * and reporting it after the callback completes.
 106  */
 107 struct rdma_id_private {
 108         struct rdma_cm_id       id;
 109
 110         struct rdma_bind_list   *bind_list;
 111         struct hlist_node       node;
 112         struct list_head        list; /* listen_any_list or cma_device.list */
 113         struct list_head        listen_list; /* per device listens */
 114         struct cma_device       *cma_dev;
 115         struct list_head        mc_list;
 116
 117         int                     internal_id;
 118         enum rdma_cm_state      state;
 119         spinlock_t              lock;
 120         struct mutex            qp_mutex;
 121
 122         struct completion       comp;
 123         atomic_t                refcount;
 124         struct mutex            handler_mutex;
 125
 126         int                     backlog;
 127         int                     timeout_ms;
 128         struct ib_sa_query      *query;
 129         int                     query_id;
 130         union {
 131                 struct ib_cm_id *ib;
 132                 struct iw_cm_id *iw;
 133         } cm_id;
 134
 135         u32                     seq_num;
 136         u32                     qkey;
 137         u32                     qp_num;
 138         pid_t                   owner;
 139         u8                      srq;
 140         u8                      tos;
 141         u8                      reuseaddr;
 142 };
 143
 144 struct cma_multicast {
 145         struct rdma_id_private *id_priv;
 146         union {
 147                 struct ib_sa_multicast *ib;
 148         } multicast;
 149         struct list_head        list;
 150         void                    *context;
 151         struct sockaddr_storage addr;
 152         struct kref             mcref;
 153 };
 154
 155 struct cma_work {
 156         struct work_struct      work;
 157         struct rdma_id_private  *id;
 158         enum rdma_cm_state      old_state;
 159         enum rdma_cm_state      new_state;
 160         struct rdma_cm_event    event;
 161 };
 162
 163 struct cma_ndev_work {
 164         struct work_struct      work;
 165         struct rdma_id_private  *id;
 166         struct rdma_cm_event    event;
 167 };
 168
 169 struct iboe_mcast_work {
 170         struct work_struct       work;
 171         struct rdma_id_private  *id;
 172         struct cma_multicast    *mc;
 173 };
 174
 175 union cma_ip_addr {
 176         struct in6_addr ip6;
 177         struct {
 178                 __be32 pad[3];
 179                 __be32 addr;
 180         } ip4;
 181 };
 182
 183 struct cma_hdr {
 184         u8 cma_version;
 185         u8 ip_version;  /* IP version: 7:4 */
 186         __be16 port;
 187         union cma_ip_addr src_addr;
 188         union cma_ip_addr dst_addr;
 189 };
 190
 191 struct sdp_hh {
 192         u8 bsdh[16];
 193         u8 sdp_version; /* Major version: 7:4 */
 194         u8 ip_version;  /* IP version: 7:4 */
 195         u8 sdp_specific1[10];
 196         __be16 port;
 197         __be16 sdp_specific2;
 198         union cma_ip_addr src_addr;
 199         union cma_ip_addr dst_addr;
 200 };
 201
 202 struct sdp_hah {
 203         u8 bsdh[16];
 204         u8 sdp_version;
 205 };
 206
 207 #define CMA_VERSION 0x00
 208 #define SDP_MAJ_VERSION 0x2
 209
 210 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 211 {
 212         unsigned long flags;
 213         int ret;
 214
 215         spin_lock_irqsave(&id_priv->lock, flags);
 216         ret = (id_priv->state == comp);
 217         spin_unlock_irqrestore(&id_priv->lock, flags);
 218         return ret;
 219 }
 220
 221 static int cma_comp_exch(struct rdma_id_private *id_priv,
 222                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 223 {
 224         unsigned long flags;
 225         int ret;
 226
 227         spin_lock_irqsave(&id_priv->lock, flags);
 228         if ((ret = (id_priv->state == comp)))
 229                 id_priv->state = exch;
 230         spin_unlock_irqrestore(&id_priv->lock, flags);
 231         return ret;
 232 }
 233
 234 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 235                                    enum rdma_cm_state exch)
 236 {
 237         unsigned long flags;
 238         enum rdma_cm_state old;
 239
 240         spin_lock_irqsave(&id_priv->lock, flags);
 241         old = id_priv->state;
 242         id_priv->state = exch;
 243         spin_unlock_irqrestore(&id_priv->lock, flags);
 244         return old;
 245 }
 246
 247 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 248 {
 249         return hdr->ip_version >> 4;
 250 }
 251
 252 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 253 {
 254         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 255 }
 256
 257 static inline u8 sdp_get_majv(u8 sdp_version)
 258 {
 259         return sdp_version >> 4;
 260 }
 261
 262 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 263 {
 264         return hh->ip_version >> 4;
 265 }
 266
 267 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 268 {
 269         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 270 }
 271
 272 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 273                               struct cma_device *cma_dev)
 274 {
 275         atomic_inc(&cma_dev->refcount);
 276         id_priv->cma_dev = cma_dev;
 277         id_priv->id.device = cma_dev->device;
 278         id_priv->id.route.addr.dev_addr.transport =
 279                 rdma_node_get_transport(cma_dev->device->node_type);
 280         list_add_tail(&id_priv->list, &cma_dev->id_list);
 281 }
 282
 283 static inline void cma_deref_dev(struct cma_device *cma_dev)
 284 {
 285         if (atomic_dec_and_test(&cma_dev->refcount))
 286                 complete(&cma_dev->comp);
 287 }
 288
 289 static inline void release_mc(struct kref *kref)
 290 {
 291         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 292
 293         kfree(mc->multicast.ib);
 294         kfree(mc);
 295 }
 296
 297 static void cma_release_dev(struct rdma_id_private *id_priv)
 298 {
 299         mutex_lock(&lock);
 300         list_del(&id_priv->list);
 301         cma_deref_dev(id_priv->cma_dev);
 302         id_priv->cma_dev = NULL;
 303         mutex_unlock(&lock);
 304 }
 305
 306 static int cma_set_qkey(struct rdma_id_private *id_priv)
 307 {
 308         struct ib_sa_mcmember_rec rec;
 309         int ret = 0;
 310
 311         if (id_priv->qkey)
 312                 return 0;
 313
 314         switch (id_priv->id.ps) {
 315         case RDMA_PS_UDP:
 316                 id_priv->qkey = RDMA_UDP_QKEY;
 317                 break;
 318         case RDMA_PS_IPOIB:
 319                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 320                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 321                                              id_priv->id.port_num, &rec.mgid,
 322                                              &rec);
 323                 if (!ret)
 324                         id_priv->qkey = be32_to_cpu(rec.qkey);
 325                 break;
 326         default:
 327                 break;
 328         }
 329         return ret;
 330 }
 331
 332 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 333 {
 334         int i;
 335         int err;
 336         struct ib_port_attr props;
 337         union ib_gid tmp;
 338
 339         err = ib_query_port(device, port_num, &props);
 340         if (err)
 341                 return 1;
 342
 343         for (i = 0; i < props.gid_tbl_len; ++i) {
 344                 err = ib_query_gid(device, port_num, i, &tmp);
 345                 if (err)
 346                         return 1;
 347                 if (!memcmp(&tmp, gid, sizeof tmp))
 348                         return 0;
 349         }
 350
 351         return -EAGAIN;
 352 }
 353
 354 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 355 {
 356         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 357         struct cma_device *cma_dev;
 358         union ib_gid gid, iboe_gid;
 359         int ret = -ENODEV;
 360         u8 port;
 361         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 362                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 363
 364         if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
 365             id_priv->id.ps == RDMA_PS_IPOIB)
 366                 return -EINVAL;
 367
 368         mutex_lock(&lock);
 369         iboe_addr_get_sgid(dev_addr, &iboe_gid);
 370         memcpy(&gid, dev_addr->src_dev_addr +
 371                rdma_addr_gid_offset(dev_addr), sizeof gid);
 372         list_for_each_entry(cma_dev, &dev_list, list) {
 373                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 374                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 375                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 376                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 377                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 378                                 else
 379                                         ret = find_gid_port(cma_dev->device, &gid, port);
 380
 381                                 if (!ret) {
 382                                         id_priv->id.port_num = port;
 383                                         goto out;
 384                                 } else if (ret == 1)
 385                                         break;
 386                         }
 387                 }
 388         }
 389
 390 out:
 391         if (!ret)
 392                 cma_attach_to_dev(id_priv, cma_dev);
 393
 394         mutex_unlock(&lock);
 395         return ret;
 396 }
 397
 398 static void cma_deref_id(struct rdma_id_private *id_priv)
 399 {
 400         if (atomic_dec_and_test(&id_priv->refcount))
 401                 complete(&id_priv->comp);
 402 }
 403
 404 static int cma_disable_callback(struct rdma_id_private *id_priv,
 405                                 enum rdma_cm_state state)
 406 {
 407         mutex_lock(&id_priv->handler_mutex);
 408         if (id_priv->state != state) {
 409                 mutex_unlock(&id_priv->handler_mutex);
 410                 return -EINVAL;
 411         }
 412         return 0;
 413 }
 414
 415 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 416                                   void *context, enum rdma_port_space ps,
 417                                   enum ib_qp_type qp_type)
 418 {
 419         struct rdma_id_private *id_priv;
 420
 421         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 422         if (!id_priv)
 423                 return ERR_PTR(-ENOMEM);
 424
 425         id_priv->owner = task_pid_nr(current);
 426         id_priv->state = RDMA_CM_IDLE;
 427         id_priv->id.context = context;
 428         id_priv->id.event_handler = event_handler;
 429         id_priv->id.ps = ps;
 430         id_priv->id.qp_type = qp_type;
 431         spin_lock_init(&id_priv->lock);
 432         mutex_init(&id_priv->qp_mutex);
 433         init_completion(&id_priv->comp);
 434         atomic_set(&id_priv->refcount, 1);
 435         mutex_init(&id_priv->handler_mutex);
 436         INIT_LIST_HEAD(&id_priv->listen_list);
 437         INIT_LIST_HEAD(&id_priv->mc_list);
 438         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 439
 440         return &id_priv->id;
 441 }
 442 EXPORT_SYMBOL(rdma_create_id);
 443
 444 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 445 {
 446         struct ib_qp_attr qp_attr;
 447         int qp_attr_mask, ret;
 448
 449         qp_attr.qp_state = IB_QPS_INIT;
 450         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 451         if (ret)
 452                 return ret;
 453
 454         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 455         if (ret)
 456                 return ret;
 457
 458         qp_attr.qp_state = IB_QPS_RTR;
 459         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 460         if (ret)
 461                 return ret;
 462
 463         qp_attr.qp_state = IB_QPS_RTS;
 464         qp_attr.sq_psn = 0;
 465         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 466
 467         return ret;
 468 }
 469
 470 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 471 {
 472         struct ib_qp_attr qp_attr;
 473         int qp_attr_mask, ret;
 474
 475         qp_attr.qp_state = IB_QPS_INIT;
 476         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 477         if (ret)
 478                 return ret;
 479
 480         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 481 }
 482
 483 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 484                    struct ib_qp_init_attr *qp_init_attr)
 485 {
 486         struct rdma_id_private *id_priv;
 487         struct ib_qp *qp;
 488         int ret;
 489
 490         id_priv = container_of(id, struct rdma_id_private, id);
 491         if (id->device != pd->device)
 492                 return -EINVAL;
 493
 494         qp = ib_create_qp(pd, qp_init_attr);
 495         if (IS_ERR(qp))
 496                 return PTR_ERR(qp);
 497
 498         if (id->qp_type == IB_QPT_UD)
 499                 ret = cma_init_ud_qp(id_priv, qp);
 500         else
 501                 ret = cma_init_conn_qp(id_priv, qp);
 502         if (ret)
 503                 goto err;
 504
 505         id->qp = qp;
 506         id_priv->qp_num = qp->qp_num;
 507         id_priv->srq = (qp->srq != NULL);
 508         return 0;
 509 err:
 510         ib_destroy_qp(qp);
 511         return ret;
 512 }
 513 EXPORT_SYMBOL(rdma_create_qp);
 514
 515 void rdma_destroy_qp(struct rdma_cm_id *id)
 516 {
 517         struct rdma_id_private *id_priv;
 518
 519         id_priv = container_of(id, struct rdma_id_private, id);
 520         mutex_lock(&id_priv->qp_mutex);
 521         ib_destroy_qp(id_priv->id.qp);
 522         id_priv->id.qp = NULL;
 523         mutex_unlock(&id_priv->qp_mutex);
 524 }
 525 EXPORT_SYMBOL(rdma_destroy_qp);
 526
 527 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 528                              struct rdma_conn_param *conn_param)
 529 {
 530         struct ib_qp_attr qp_attr;
 531         int qp_attr_mask, ret;
 532
 533         mutex_lock(&id_priv->qp_mutex);
 534         if (!id_priv->id.qp) {
 535                 ret = 0;
 536                 goto out;
 537         }
 538
 539         /* Need to update QP attributes from default values. */
 540         qp_attr.qp_state = IB_QPS_INIT;
 541         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 542         if (ret)
 543                 goto out;
 544
 545         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 546         if (ret)
 547                 goto out;
 548
 549         qp_attr.qp_state = IB_QPS_RTR;
 550         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 551         if (ret)
 552                 goto out;
 553
 554         if (conn_param)
 555                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 556         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 557 out:
 558         mutex_unlock(&id_priv->qp_mutex);
 559         return ret;
 560 }
 561
 562 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 563                              struct rdma_conn_param *conn_param)
 564 {
 565         struct ib_qp_attr qp_attr;
 566         int qp_attr_mask, ret;
 567
 568         mutex_lock(&id_priv->qp_mutex);
 569         if (!id_priv->id.qp) {
 570                 ret = 0;
 571                 goto out;
 572         }
 573
 574         qp_attr.qp_state = IB_QPS_RTS;
 575         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 576         if (ret)
 577                 goto out;
 578
 579         if (conn_param)
 580                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 581         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 582 out:
 583         mutex_unlock(&id_priv->qp_mutex);
 584         return ret;
 585 }
 586
 587 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 588 {
 589         struct ib_qp_attr qp_attr;
 590         int ret;
 591
 592         mutex_lock(&id_priv->qp_mutex);
 593         if (!id_priv->id.qp) {
 594                 ret = 0;
 595                 goto out;
 596         }
 597
 598         qp_attr.qp_state = IB_QPS_ERR;
 599         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 600 out:
 601         mutex_unlock(&id_priv->qp_mutex);
 602         return ret;
 603 }
 604
 605 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 606                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 607 {
 608         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 609         int ret;
 610         u16 pkey;
 611
 612         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 613             IB_LINK_LAYER_INFINIBAND)
 614                 pkey = ib_addr_get_pkey(dev_addr);
 615         else
 616                 pkey = 0xffff;
 617
 618         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 619                                   pkey, &qp_attr->pkey_index);
 620         if (ret)
 621                 return ret;
 622
 623         qp_attr->port_num = id_priv->id.port_num;
 624         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 625
 626         if (id_priv->id.qp_type == IB_QPT_UD) {
 627                 ret = cma_set_qkey(id_priv);
 628                 if (ret)
 629                         return ret;
 630
 631                 qp_attr->qkey = id_priv->qkey;
 632                 *qp_attr_mask |= IB_QP_QKEY;
 633         } else {
 634                 qp_attr->qp_access_flags = 0;
 635                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 636         }
 637         return 0;
 638 }
 639
 640 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 641                        int *qp_attr_mask)
 642 {
 643         struct rdma_id_private *id_priv;
 644         int ret = 0;
 645
 646         id_priv = container_of(id, struct rdma_id_private, id);
 647         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 648         case RDMA_TRANSPORT_IB:
 649                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
 650                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 651                 else
 652                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 653                                                  qp_attr_mask);
 654                 if (qp_attr->qp_state == IB_QPS_RTR)
 655                         qp_attr->rq_psn = id_priv->seq_num;
 656                 break;
 657         case RDMA_TRANSPORT_IWARP:
 658                 if (!id_priv->cm_id.iw) {
 659                         qp_attr->qp_access_flags = 0;
 660                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 661                 } else
 662                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 663                                                  qp_attr_mask);
 664                 break;
 665         default:
 666                 ret = -ENOSYS;
 667                 break;
 668         }
 669
 670         return ret;
 671 }
 672 EXPORT_SYMBOL(rdma_init_qp_attr);
 673
 674 static inline int cma_zero_addr(struct sockaddr *addr)
 675 {
 676         struct in6_addr *ip6;
 677
 678         if (addr->sa_family == AF_INET)
 679                 return ipv4_is_zeronet(
 680                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
 681         else {
 682                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 683                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 684                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 685         }
 686 }
 687
 688 static inline int cma_loopback_addr(struct sockaddr *addr)
 689 {
 690         if (addr->sa_family == AF_INET)
 691                 return ipv4_is_loopback(
 692                         ((struct sockaddr_in *) addr)->sin_addr.s_addr);
 693         else
 694                 return ipv6_addr_loopback(
 695                         &((struct sockaddr_in6 *) addr)->sin6_addr);
 696 }
 697
 698 static inline int cma_any_addr(struct sockaddr *addr)
 699 {
 700         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 701 }
 702
 703 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 704 {
 705         if (src->sa_family != dst->sa_family)
 706                 return -1;
 707
 708         switch (src->sa_family) {
 709         case AF_INET:
 710                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 711                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 712         default:
 713                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 714                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 715         }
 716 }
 717
 718 static inline __be16 cma_port(struct sockaddr *addr)
 719 {
 720         if (addr->sa_family == AF_INET)
 721                 return ((struct sockaddr_in *) addr)->sin_port;
 722         else
 723                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 724 }
 725
 726 static inline int cma_any_port(struct sockaddr *addr)
 727 {
 728         return !cma_port(addr);
 729 }
 730
 731 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 732                             u8 *ip_ver, __be16 *port,
 733                             union cma_ip_addr **src, union cma_ip_addr **dst)
 734 {
 735         switch (ps) {
 736         case RDMA_PS_SDP:
 737                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 738                     SDP_MAJ_VERSION)
 739                         return -EINVAL;
 740
 741                 *ip_ver = sdp_get_ip_ver(hdr);
 742                 *port   = ((struct sdp_hh *) hdr)->port;
 743                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 744                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 745                 break;
 746         default:
 747                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 748                         return -EINVAL;
 749
 750                 *ip_ver = cma_get_ip_ver(hdr);
 751                 *port   = ((struct cma_hdr *) hdr)->port;
 752                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 753                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 754                 break;
 755         }
 756
 757         if (*ip_ver != 4 && *ip_ver != 6)
 758                 return -EINVAL;
 759         return 0;
 760 }
 761
 762 static void cma_save_net_info(struct rdma_addr *addr,
 763                               struct rdma_addr *listen_addr,
 764                               u8 ip_ver, __be16 port,
 765                               union cma_ip_addr *src, union cma_ip_addr *dst)
 766 {
 767         struct sockaddr_in *listen4, *ip4;
 768         struct sockaddr_in6 *listen6, *ip6;
 769
 770         switch (ip_ver) {
 771         case 4:
 772                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 773                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 774                 ip4->sin_family = listen4->sin_family;
 775                 ip4->sin_addr.s_addr = dst->ip4.addr;
 776                 ip4->sin_port = listen4->sin_port;
 777
 778                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 779                 ip4->sin_family = listen4->sin_family;
 780                 ip4->sin_addr.s_addr = src->ip4.addr;
 781                 ip4->sin_port = port;
 782                 break;
 783         case 6:
 784                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 785                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 786                 ip6->sin6_family = listen6->sin6_family;
 787                 ip6->sin6_addr = dst->ip6;
 788                 ip6->sin6_port = listen6->sin6_port;
 789
 790                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 791                 ip6->sin6_family = listen6->sin6_family;
 792                 ip6->sin6_addr = src->ip6;
 793                 ip6->sin6_port = port;
 794                 break;
 795         default:
 796                 break;
 797         }
 798 }
 799
 800 static inline int cma_user_data_offset(enum rdma_port_space ps)
 801 {
 802         switch (ps) {
 803         case RDMA_PS_SDP:
 804                 return 0;
 805         default:
 806                 return sizeof(struct cma_hdr);
 807         }
 808 }
 809
 810 static void cma_cancel_route(struct rdma_id_private *id_priv)
 811 {
 812         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 813         case IB_LINK_LAYER_INFINIBAND:
 814                 if (id_priv->query)
 815                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 816                 break;
 817         default:
 818                 break;
 819         }
 820 }
 821
 822 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 823 {
 824         struct rdma_id_private *dev_id_priv;
 825
 826         /*
 827          * Remove from listen_any_list to prevent added devices from spawning
 828          * additional listen requests.
 829          */
 830         mutex_lock(&lock);
 831         list_del(&id_priv->list);
 832
 833         while (!list_empty(&id_priv->listen_list)) {
 834                 dev_id_priv = list_entry(id_priv->listen_list.next,
 835                                          struct rdma_id_private, listen_list);
 836                 /* sync with device removal to avoid duplicate destruction */
 837                 list_del_init(&dev_id_priv->list);
 838                 list_del(&dev_id_priv->listen_list);
 839                 mutex_unlock(&lock);
 840
 841                 rdma_destroy_id(&dev_id_priv->id);
 842                 mutex_lock(&lock);
 843         }
 844         mutex_unlock(&lock);
 845 }
 846
 847 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 848                                  enum rdma_cm_state state)
 849 {
 850         switch (state) {
 851         case RDMA_CM_ADDR_QUERY:
 852                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 853                 break;
 854         case RDMA_CM_ROUTE_QUERY:
 855                 cma_cancel_route(id_priv);
 856                 break;
 857         case RDMA_CM_LISTEN:
 858                 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
 859                                 && !id_priv->cma_dev)
 860                         cma_cancel_listens(id_priv);
 861                 break;
 862         default:
 863                 break;
 864         }
 865 }
 866
 867 static void cma_release_port(struct rdma_id_private *id_priv)
 868 {
 869         struct rdma_bind_list *bind_list = id_priv->bind_list;
 870
 871         if (!bind_list)
 872                 return;
 873
 874         mutex_lock(&lock);
 875         hlist_del(&id_priv->node);
 876         if (hlist_empty(&bind_list->owners)) {
 877                 idr_remove(bind_list->ps, bind_list->port);
 878                 kfree(bind_list);
 879         }
 880         mutex_unlock(&lock);
 881 }
 882
 883 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 884 {
 885         struct cma_multicast *mc;
 886
 887         while (!list_empty(&id_priv->mc_list)) {
 888                 mc = container_of(id_priv->mc_list.next,
 889                                   struct cma_multicast, list);
 890                 list_del(&mc->list);
 891                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 892                 case IB_LINK_LAYER_INFINIBAND:
 893                         ib_sa_free_multicast(mc->multicast.ib);
 894                         kfree(mc);
 895                         break;
 896                 case IB_LINK_LAYER_ETHERNET:
 897                         kref_put(&mc->mcref, release_mc);
 898                         break;
 899                 default:
 900                         break;
 901                 }
 902         }
 903 }
 904
 905 void rdma_destroy_id(struct rdma_cm_id *id)
 906 {
 907         struct rdma_id_private *id_priv;
 908         enum rdma_cm_state state;
 909
 910         id_priv = container_of(id, struct rdma_id_private, id);
 911         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
 912         cma_cancel_operation(id_priv, state);
 913
 914         /*
 915          * Wait for any active callback to finish.  New callbacks will find
 916          * the id_priv state set to destroying and abort.
 917          */
 918         mutex_lock(&id_priv->handler_mutex);
 919         mutex_unlock(&id_priv->handler_mutex);
 920
 921         if (id_priv->cma_dev) {
 922                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 923                 case RDMA_TRANSPORT_IB:
 924                         if (id_priv->cm_id.ib)
 925                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 926                         break;
 927                 case RDMA_TRANSPORT_IWARP:
 928                         if (id_priv->cm_id.iw)
 929                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 930                         break;
 931                 default:
 932                         break;
 933                 }
 934                 cma_leave_mc_groups(id_priv);
 935                 cma_release_dev(id_priv);
 936         }
 937
 938         cma_release_port(id_priv);
 939         cma_deref_id(id_priv);
 940         wait_for_completion(&id_priv->comp);
 941
 942         if (id_priv->internal_id)
 943                 cma_deref_id(id_priv->id.context);
 944
 945         kfree(id_priv->id.route.path_rec);
 946         kfree(id_priv);
 947 }
 948 EXPORT_SYMBOL(rdma_destroy_id);
 949
 950 static int cma_rep_recv(struct rdma_id_private *id_priv)
 951 {
 952         int ret;
 953
 954         ret = cma_modify_qp_rtr(id_priv, NULL);
 955         if (ret)
 956                 goto reject;
 957
 958         ret = cma_modify_qp_rts(id_priv, NULL);
 959         if (ret)
 960                 goto reject;
 961
 962         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
 963         if (ret)
 964                 goto reject;
 965
 966         return 0;
 967 reject:
 968         cma_modify_qp_err(id_priv);
 969         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
 970                        NULL, 0, NULL, 0);
 971         return ret;
 972 }
 973
 974 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
 975 {
 976         if (id_priv->id.ps == RDMA_PS_SDP &&
 977             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
 978             SDP_MAJ_VERSION)
 979                 return -EINVAL;
 980
 981         return 0;
 982 }
 983
 984 static void cma_set_rep_event_data(struct rdma_cm_event *event,
 985                                    struct ib_cm_rep_event_param *rep_data,
 986                                    void *private_data)
 987 {
 988         event->param.conn.private_data = private_data;
 989         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
 990         event->param.conn.responder_resources = rep_data->responder_resources;
 991         event->param.conn.initiator_depth = rep_data->initiator_depth;
 992         event->param.conn.flow_control = rep_data->flow_control;
 993         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
 994         event->param.conn.srq = rep_data->srq;
 995         event->param.conn.qp_num = rep_data->remote_qpn;
 996 }
 997
 998 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
 999 {
1000         struct rdma_id_private *id_priv = cm_id->context;
1001         struct rdma_cm_event event;
1002         int ret = 0;
1003
1004         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1005                 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
1006             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1007                 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
1008                 return 0;
1009
1010         memset(&event, 0, sizeof event);
1011         switch (ib_event->event) {
1012         case IB_CM_REQ_ERROR:
1013         case IB_CM_REP_ERROR:
1014                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1015                 event.status = -ETIMEDOUT;
1016                 break;
1017         case IB_CM_REP_RECEIVED:
1018                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1019                 if (event.status)
1020                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1021                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1022                         event.status = cma_rep_recv(id_priv);
1023                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1024                                                      RDMA_CM_EVENT_ESTABLISHED;
1025                 } else
1026                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1027                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1028                                        ib_event->private_data);
1029                 break;
1030         case IB_CM_RTU_RECEIVED:
1031         case IB_CM_USER_ESTABLISHED:
1032                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1033                 break;
1034         case IB_CM_DREQ_ERROR:
1035                 event.status = -ETIMEDOUT; /* fall through */
1036         case IB_CM_DREQ_RECEIVED:
1037         case IB_CM_DREP_RECEIVED:
1038                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1039                                    RDMA_CM_DISCONNECT))
1040                         goto out;
1041                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1042                 break;
1043         case IB_CM_TIMEWAIT_EXIT:
1044                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1045                 break;
1046         case IB_CM_MRA_RECEIVED:
1047                 /* ignore event */
1048                 goto out;
1049         case IB_CM_REJ_RECEIVED:
1050                 cma_modify_qp_err(id_priv);
1051                 event.status = ib_event->param.rej_rcvd.reason;
1052                 event.event = RDMA_CM_EVENT_REJECTED;
1053                 event.param.conn.private_data = ib_event->private_data;
1054                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1055                 break;
1056         default:
1057                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1058                        ib_event->event);
1059                 goto out;
1060         }
1061
1062         ret = id_priv->id.event_handler(&id_priv->id, &event);
1063         if (ret) {
1064                 /* Destroy the CM ID by returning a non-zero value. */
1065                 id_priv->cm_id.ib = NULL;
1066                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1067                 mutex_unlock(&id_priv->handler_mutex);
1068                 rdma_destroy_id(&id_priv->id);
1069                 return ret;
1070         }
1071 out:
1072         mutex_unlock(&id_priv->handler_mutex);
1073         return ret;
1074 }
1075
1076 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1077                                                struct ib_cm_event *ib_event)
1078 {
1079         struct rdma_id_private *id_priv;
1080         struct rdma_cm_id *id;
1081         struct rdma_route *rt;
1082         union cma_ip_addr *src, *dst;
1083         __be16 port;
1084         u8 ip_ver;
1085         int ret;
1086
1087         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1088                              &ip_ver, &port, &src, &dst))
1089                 return NULL;
1090
1091         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1092                             listen_id->ps, ib_event->param.req_rcvd.qp_type);
1093         if (IS_ERR(id))
1094                 return NULL;
1095
1096         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1097                           ip_ver, port, src, dst);
1098
1099         rt = &id->route;
1100         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1101         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1102                                GFP_KERNEL);
1103         if (!rt->path_rec)
1104                 goto err;
1105
1106         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1107         if (rt->num_paths == 2)
1108                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1109
1110         if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1111                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1112                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1113                 ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey);
1114         } else {
1115                 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
1116                                         &rt->addr.dev_addr);
1117                 if (ret)
1118                         goto err;
1119         }
1120         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1121
1122         id_priv = container_of(id, struct rdma_id_private, id);
1123         id_priv->state = RDMA_CM_CONNECT;
1124         return id_priv;
1125
1126 err:
1127         rdma_destroy_id(id);
1128         return NULL;
1129 }
1130
1131 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1132                                               struct ib_cm_event *ib_event)
1133 {
1134         struct rdma_id_private *id_priv;
1135         struct rdma_cm_id *id;
1136         union cma_ip_addr *src, *dst;
1137         __be16 port;
1138         u8 ip_ver;
1139         int ret;
1140
1141         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1142                             listen_id->ps, IB_QPT_UD);
1143         if (IS_ERR(id))
1144                 return NULL;
1145
1146
1147         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1148                              &ip_ver, &port, &src, &dst))
1149                 goto err;
1150
1151         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1152                           ip_ver, port, src, dst);
1153
1154         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1155                 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1156                                         &id->route.addr.dev_addr);
1157                 if (ret)
1158                         goto err;
1159         }
1160
1161         id_priv = container_of(id, struct rdma_id_private, id);
1162         id_priv->state = RDMA_CM_CONNECT;
1163         return id_priv;
1164 err:
1165         rdma_destroy_id(id);
1166         return NULL;
1167 }
1168
1169 static void cma_set_req_event_data(struct rdma_cm_event *event,
1170                                    struct ib_cm_req_event_param *req_data,
1171                                    void *private_data, int offset)
1172 {
1173         event->param.conn.private_data = private_data + offset;
1174         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1175         event->param.conn.responder_resources = req_data->responder_resources;
1176         event->param.conn.initiator_depth = req_data->initiator_depth;
1177         event->param.conn.flow_control = req_data->flow_control;
1178         event->param.conn.retry_count = req_data->retry_count;
1179         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1180         event->param.conn.srq = req_data->srq;
1181         event->param.conn.qp_num = req_data->remote_qpn;
1182 }
1183
1184 static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1185 {
1186         return (((ib_event->event == IB_CM_REQ_RECEIVED) ||
1187                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1188                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1189                  (id->qp_type == IB_QPT_UD)) ||
1190                 (!id->qp_type));
1191 }
1192
1193 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1194 {
1195         struct rdma_id_private *listen_id, *conn_id;
1196         struct rdma_cm_event event;
1197         int offset, ret;
1198
1199         listen_id = cm_id->context;
1200         if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1201                 return -EINVAL;
1202
1203         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1204                 return -ECONNABORTED;
1205
1206         memset(&event, 0, sizeof event);
1207         offset = cma_user_data_offset(listen_id->id.ps);
1208         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1209         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
1210                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1211                 event.param.ud.private_data = ib_event->private_data + offset;
1212                 event.param.ud.private_data_len =
1213                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1214         } else {
1215                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1216                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1217                                        ib_event->private_data, offset);
1218         }
1219         if (!conn_id) {
1220                 ret = -ENOMEM;
1221                 goto out;
1222         }
1223
1224         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1225         ret = cma_acquire_dev(conn_id);
1226         if (ret)
1227                 goto release_conn_id;
1228
1229         conn_id->cm_id.ib = cm_id;
1230         cm_id->context = conn_id;
1231         cm_id->cm_handler = cma_ib_handler;
1232
1233         /*
1234          * Protect against the user destroying conn_id from another thread
1235          * until we're done accessing it.
1236          */
1237         atomic_inc(&conn_id->refcount);
1238         ret = conn_id->id.event_handler(&conn_id->id, &event);
1239         if (!ret) {
1240                 /*
1241                  * Acquire mutex to prevent user executing rdma_destroy_id()
1242                  * while we're accessing the cm_id.
1243                  */
1244                 mutex_lock(&lock);
1245                 if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
1246                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1247                 mutex_unlock(&lock);
1248                 mutex_unlock(&conn_id->handler_mutex);
1249                 cma_deref_id(conn_id);
1250                 goto out;
1251         }
1252         cma_deref_id(conn_id);
1253
1254         /* Destroy the CM ID by returning a non-zero value. */
1255         conn_id->cm_id.ib = NULL;
1256
1257 release_conn_id:
1258         cma_exch(conn_id, RDMA_CM_DESTROYING);
1259         mutex_unlock(&conn_id->handler_mutex);
1260         rdma_destroy_id(&conn_id->id);
1261
1262 out:
1263         mutex_unlock(&listen_id->handler_mutex);
1264         return ret;
1265 }
1266
1267 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1268 {
1269         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1270 }
1271
1272 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1273                                  struct ib_cm_compare_data *compare)
1274 {
1275         struct cma_hdr *cma_data, *cma_mask;
1276         struct sdp_hh *sdp_data, *sdp_mask;
1277         __be32 ip4_addr;
1278         struct in6_addr ip6_addr;
1279
1280         memset(compare, 0, sizeof *compare);
1281         cma_data = (void *) compare->data;
1282         cma_mask = (void *) compare->mask;
1283         sdp_data = (void *) compare->data;
1284         sdp_mask = (void *) compare->mask;
1285
1286         switch (addr->sa_family) {
1287         case AF_INET:
1288                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1289                 if (ps == RDMA_PS_SDP) {
1290                         sdp_set_ip_ver(sdp_data, 4);
1291                         sdp_set_ip_ver(sdp_mask, 0xF);
1292                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1293                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1294                 } else {
1295                         cma_set_ip_ver(cma_data, 4);
1296                         cma_set_ip_ver(cma_mask, 0xF);
1297                         cma_data->dst_addr.ip4.addr = ip4_addr;
1298                         cma_mask->dst_addr.ip4.addr = htonl(~0);
1299                 }
1300                 break;
1301         case AF_INET6:
1302                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1303                 if (ps == RDMA_PS_SDP) {
1304                         sdp_set_ip_ver(sdp_data, 6);
1305                         sdp_set_ip_ver(sdp_mask, 0xF);
1306                         sdp_data->dst_addr.ip6 = ip6_addr;
1307                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1308                                sizeof sdp_mask->dst_addr.ip6);
1309                 } else {
1310                         cma_set_ip_ver(cma_data, 6);
1311                         cma_set_ip_ver(cma_mask, 0xF);
1312                         cma_data->dst_addr.ip6 = ip6_addr;
1313                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1314                                sizeof cma_mask->dst_addr.ip6);
1315                 }
1316                 break;
1317         default:
1318                 break;
1319         }
1320 }
1321
1322 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1323 {
1324         struct rdma_id_private *id_priv = iw_id->context;
1325         struct rdma_cm_event event;
1326         struct sockaddr_in *sin;
1327         int ret = 0;
1328
1329         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
1330                 return 0;
1331
1332         memset(&event, 0, sizeof event);
1333         switch (iw_event->event) {
1334         case IW_CM_EVENT_CLOSE:
1335                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1336                 break;
1337         case IW_CM_EVENT_CONNECT_REPLY:
1338                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1339                 *sin = iw_event->local_addr;
1340                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1341                 *sin = iw_event->remote_addr;
1342                 switch (iw_event->status) {
1343                 case 0:
1344                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1345                         event.param.conn.initiator_depth = iw_event->ird;
1346                         event.param.conn.responder_resources = iw_event->ord;
1347                         break;
1348                 case -ECONNRESET:
1349                 case -ECONNREFUSED:
1350                         event.event = RDMA_CM_EVENT_REJECTED;
1351                         break;
1352                 case -ETIMEDOUT:
1353                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1354                         break;
1355                 default:
1356                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1357                         break;
1358                 }
1359                 break;
1360         case IW_CM_EVENT_ESTABLISHED:
1361                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1362                 event.param.conn.initiator_depth = iw_event->ird;
1363                 event.param.conn.responder_resources = iw_event->ord;
1364                 break;
1365         default:
1366                 BUG_ON(1);
1367         }
1368
1369         event.status = iw_event->status;
1370         event.param.conn.private_data = iw_event->private_data;
1371         event.param.conn.private_data_len = iw_event->private_data_len;
1372         ret = id_priv->id.event_handler(&id_priv->id, &event);
1373         if (ret) {
1374                 /* Destroy the CM ID by returning a non-zero value. */
1375                 id_priv->cm_id.iw = NULL;
1376                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1377                 mutex_unlock(&id_priv->handler_mutex);
1378                 rdma_destroy_id(&id_priv->id);
1379                 return ret;
1380         }
1381
1382         mutex_unlock(&id_priv->handler_mutex);
1383         return ret;
1384 }
1385
1386 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1387                                struct iw_cm_event *iw_event)
1388 {
1389         struct rdma_cm_id *new_cm_id;
1390         struct rdma_id_private *listen_id, *conn_id;
1391         struct sockaddr_in *sin;
1392         struct net_device *dev = NULL;
1393         struct rdma_cm_event event;
1394         int ret;
1395         struct ib_device_attr attr;
1396
1397         listen_id = cm_id->context;
1398         if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
1399                 return -ECONNABORTED;
1400
1401         /* Create a new RDMA id for the new IW CM ID */
1402         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1403                                    listen_id->id.context,
1404                                    RDMA_PS_TCP, IB_QPT_RC);
1405         if (IS_ERR(new_cm_id)) {
1406                 ret = -ENOMEM;
1407                 goto out;
1408         }
1409         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1410         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1411         conn_id->state = RDMA_CM_CONNECT;
1412
1413         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1414         if (!dev) {
1415                 ret = -EADDRNOTAVAIL;
1416                 mutex_unlock(&conn_id->handler_mutex);
1417                 rdma_destroy_id(new_cm_id);
1418                 goto out;
1419         }
1420         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1421         if (ret) {
1422                 mutex_unlock(&conn_id->handler_mutex);
1423                 rdma_destroy_id(new_cm_id);
1424                 goto out;
1425         }
1426
1427         ret = cma_acquire_dev(conn_id);
1428         if (ret) {
1429                 mutex_unlock(&conn_id->handler_mutex);
1430                 rdma_destroy_id(new_cm_id);
1431                 goto out;
1432         }
1433
1434         conn_id->cm_id.iw = cm_id;
1435         cm_id->context = conn_id;
1436         cm_id->cm_handler = cma_iw_handler;
1437
1438         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1439         *sin = iw_event->local_addr;
1440         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1441         *sin = iw_event->remote_addr;
1442
1443         ret = ib_query_device(conn_id->id.device, &attr);
1444         if (ret) {
1445                 mutex_unlock(&conn_id->handler_mutex);
1446                 rdma_destroy_id(new_cm_id);
1447                 goto out;
1448         }
1449
1450         memset(&event, 0, sizeof event);
1451         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1452         event.param.conn.private_data = iw_event->private_data;
1453         event.param.conn.private_data_len = iw_event->private_data_len;
1454         event.param.conn.initiator_depth = iw_event->ird;
1455         event.param.conn.responder_resources = iw_event->ord;
1456
1457         /*
1458          * Protect against the user destroying conn_id from another thread
1459          * until we're done accessing it.
1460          */
1461         atomic_inc(&conn_id->refcount);
1462         ret = conn_id->id.event_handler(&conn_id->id, &event);
1463         if (ret) {
1464                 /* User wants to destroy the CM ID */
1465                 conn_id->cm_id.iw = NULL;
1466                 cma_exch(conn_id, RDMA_CM_DESTROYING);
1467                 mutex_unlock(&conn_id->handler_mutex);
1468                 cma_deref_id(conn_id);
1469                 rdma_destroy_id(&conn_id->id);
1470                 goto out;
1471         }
1472
1473         mutex_unlock(&conn_id->handler_mutex);
1474         cma_deref_id(conn_id);
1475
1476 out:
1477         if (dev)
1478                 dev_put(dev);
1479         mutex_unlock(&listen_id->handler_mutex);
1480         return ret;
1481 }
1482
1483 static int cma_ib_listen(struct rdma_id_private *id_priv)
1484 {
1485         struct ib_cm_compare_data compare_data;
1486         struct sockaddr *addr;
1487         struct ib_cm_id *id;
1488         __be64 svc_id;
1489         int ret;
1490
1491         id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1492         if (IS_ERR(id))
1493                 return PTR_ERR(id);
1494
1495         id_priv->cm_id.ib = id;
1496
1497         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1498         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1499         if (cma_any_addr(addr))
1500                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1501         else {
1502                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1503                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1504         }
1505
1506         if (ret) {
1507                 ib_destroy_cm_id(id_priv->cm_id.ib);
1508                 id_priv->cm_id.ib = NULL;
1509         }
1510
1511         return ret;
1512 }
1513
1514 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1515 {
1516         int ret;
1517         struct sockaddr_in *sin;
1518         struct iw_cm_id *id;
1519
1520         id = iw_create_cm_id(id_priv->id.device,
1521                              iw_conn_req_handler,
1522                              id_priv);
1523         if (IS_ERR(id))
1524                 return PTR_ERR(id);
1525
1526         id_priv->cm_id.iw = id;
1527
1528         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1529         id_priv->cm_id.iw->local_addr = *sin;
1530
1531         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1532
1533         if (ret) {
1534                 iw_destroy_cm_id(id_priv->cm_id.iw);
1535                 id_priv->cm_id.iw = NULL;
1536         }
1537
1538         return ret;
1539 }
1540
1541 static int cma_listen_handler(struct rdma_cm_id *id,
1542                               struct rdma_cm_event *event)
1543 {
1544         struct rdma_id_private *id_priv = id->context;
1545
1546         id->context = id_priv->id.context;
1547         id->event_handler = id_priv->id.event_handler;
1548         return id_priv->id.event_handler(id, event);
1549 }
1550
1551 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1552                               struct cma_device *cma_dev)
1553 {
1554         struct rdma_id_private *dev_id_priv;
1555         struct rdma_cm_id *id;
1556         int ret;
1557
1558         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1559                             id_priv->id.qp_type);
1560         if (IS_ERR(id))
1561                 return;
1562
1563         dev_id_priv = container_of(id, struct rdma_id_private, id);
1564
1565         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
1566         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1567                ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1568
1569         cma_attach_to_dev(dev_id_priv, cma_dev);
1570         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1571         atomic_inc(&id_priv->refcount);
1572         dev_id_priv->internal_id = 1;
1573
1574         ret = rdma_listen(id, id_priv->backlog);
1575         if (ret)
1576                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1577                        "listening on device %s\n", ret, cma_dev->device->name);
1578 }
1579
1580 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1581 {
1582         struct cma_device *cma_dev;
1583
1584         mutex_lock(&lock);
1585         list_add_tail(&id_priv->list, &listen_any_list);
1586         list_for_each_entry(cma_dev, &dev_list, list)
1587                 cma_listen_on_dev(id_priv, cma_dev);
1588         mutex_unlock(&lock);
1589 }
1590
1591 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1592 {
1593         struct rdma_id_private *id_priv;
1594
1595         id_priv = container_of(id, struct rdma_id_private, id);
1596         id_priv->tos = (u8) tos;
1597 }
1598 EXPORT_SYMBOL(rdma_set_service_type);
1599
1600 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1601                               void *context)
1602 {
1603         struct cma_work *work = context;
1604         struct rdma_route *route;
1605
1606         route = &work->id->id.route;
1607
1608         if (!status) {
1609                 route->num_paths = 1;
1610                 *route->path_rec = *path_rec;
1611         } else {
1612                 work->old_state = RDMA_CM_ROUTE_QUERY;
1613                 work->new_state = RDMA_CM_ADDR_RESOLVED;
1614                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1615                 work->event.status = status;
1616         }
1617
1618         queue_work(cma_wq, &work->work);
1619 }
1620
1621 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1622                               struct cma_work *work)
1623 {
1624         struct rdma_addr *addr = &id_priv->id.route.addr;
1625         struct ib_sa_path_rec path_rec;
1626         ib_sa_comp_mask comp_mask;
1627         struct sockaddr_in6 *sin6;
1628
1629         memset(&path_rec, 0, sizeof path_rec);
1630         rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1631         rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1632         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1633         path_rec.numb_path = 1;
1634         path_rec.reversible = 1;
1635         path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1636                                                         (struct sockaddr *) &addr->dst_addr);
1637
1638         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1639                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1640                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1641
1642         if (addr->src_addr.ss_family == AF_INET) {
1643                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1644                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1645         } else {
1646                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1647                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1648                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1649         }
1650
1651         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1652                                                id_priv->id.port_num, &path_rec,
1653                                                comp_mask, timeout_ms,
1654                                                GFP_KERNEL, cma_query_handler,
1655                                                work, &id_priv->query);
1656
1657         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1658 }
1659
1660 static void cma_work_handler(struct work_struct *_work)
1661 {
1662         struct cma_work *work = container_of(_work, struct cma_work, work);
1663         struct rdma_id_private *id_priv = work->id;
1664         int destroy = 0;
1665
1666         mutex_lock(&id_priv->handler_mutex);
1667         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1668                 goto out;
1669
1670         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1671                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1672                 destroy = 1;
1673         }
1674 out:
1675         mutex_unlock(&id_priv->handler_mutex);
1676         cma_deref_id(id_priv);
1677         if (destroy)
1678                 rdma_destroy_id(&id_priv->id);
1679         kfree(work);
1680 }
1681
1682 static void cma_ndev_work_handler(struct work_struct *_work)
1683 {
1684         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1685         struct rdma_id_private *id_priv = work->id;
1686         int destroy = 0;
1687
1688         mutex_lock(&id_priv->handler_mutex);
1689         if (id_priv->state == RDMA_CM_DESTROYING ||
1690             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
1691                 goto out;
1692
1693         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1694                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1695                 destroy = 1;
1696         }
1697
1698 out:
1699         mutex_unlock(&id_priv->handler_mutex);
1700         cma_deref_id(id_priv);
1701         if (destroy)
1702                 rdma_destroy_id(&id_priv->id);
1703         kfree(work);
1704 }
1705
1706 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1707 {
1708         struct rdma_route *route = &id_priv->id.route;
1709         struct cma_work *work;
1710         int ret;
1711
1712         work = kzalloc(sizeof *work, GFP_KERNEL);
1713         if (!work)
1714                 return -ENOMEM;
1715
1716         work->id = id_priv;
1717         INIT_WORK(&work->work, cma_work_handler);
1718         work->old_state = RDMA_CM_ROUTE_QUERY;
1719         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1720         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1721
1722         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1723         if (!route->path_rec) {
1724                 ret = -ENOMEM;
1725                 goto err1;
1726         }
1727
1728         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1729         if (ret)
1730                 goto err2;
1731
1732         return 0;
1733 err2:
1734         kfree(route->path_rec);
1735         route->path_rec = NULL;
1736 err1:
1737         kfree(work);
1738         return ret;
1739 }
1740
1741 int rdma_set_ib_paths(struct rdma_cm_id *id,
1742                       struct ib_sa_path_rec *path_rec, int num_paths)
1743 {
1744         struct rdma_id_private *id_priv;
1745         int ret;
1746
1747         id_priv = container_of(id, struct rdma_id_private, id);
1748         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1749                            RDMA_CM_ROUTE_RESOLVED))
1750                 return -EINVAL;
1751
1752         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1753                                      GFP_KERNEL);
1754         if (!id->route.path_rec) {
1755                 ret = -ENOMEM;
1756                 goto err;
1757         }
1758
1759         id->route.num_paths = num_paths;
1760         return 0;
1761 err:
1762         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
1763         return ret;
1764 }
1765 EXPORT_SYMBOL(rdma_set_ib_paths);
1766
1767 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1768 {
1769         struct cma_work *work;
1770
1771         work = kzalloc(sizeof *work, GFP_KERNEL);
1772         if (!work)
1773                 return -ENOMEM;
1774
1775         work->id = id_priv;
1776         INIT_WORK(&work->work, cma_work_handler);
1777         work->old_state = RDMA_CM_ROUTE_QUERY;
1778         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1779         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1780         queue_work(cma_wq, &work->work);
1781         return 0;
1782 }
1783
1784 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1785 {
1786         struct rdma_route *route = &id_priv->id.route;
1787         struct rdma_addr *addr = &route->addr;
1788         struct cma_work *work;
1789         int ret;
1790         struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1791         struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1792         struct net_device *ndev = NULL;
1793         u16 vid;
1794
1795         if (src_addr->sin_family != dst_addr->sin_family)
1796                 return -EINVAL;
1797
1798         work = kzalloc(sizeof *work, GFP_KERNEL);
1799         if (!work)
1800                 return -ENOMEM;
1801
1802         work->id = id_priv;
1803         INIT_WORK(&work->work, cma_work_handler);
1804
1805         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1806         if (!route->path_rec) {
1807                 ret = -ENOMEM;
1808                 goto err1;
1809         }
1810
1811         route->num_paths = 1;
1812
1813         if (addr->dev_addr.bound_dev_if)
1814                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1815         if (!ndev) {
1816                 ret = -ENODEV;
1817                 goto err2;
1818         }
1819
1820         vid = rdma_vlan_dev_vlan_id(ndev);
1821
1822         iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1823         iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1824
1825         route->path_rec->hop_limit = 1;
1826         route->path_rec->reversible = 1;
1827         route->path_rec->pkey = cpu_to_be16(0xffff);
1828         route->path_rec->mtu_selector = IB_SA_EQ;
1829         route->path_rec->sl = id_priv->tos >> 5;
1830
1831         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1832         route->path_rec->rate_selector = IB_SA_EQ;
1833         route->path_rec->rate = iboe_get_rate(ndev);
1834         dev_put(ndev);
1835         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1836         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1837         if (!route->path_rec->mtu) {
1838                 ret = -EINVAL;
1839                 goto err2;
1840         }
1841
1842         work->old_state = RDMA_CM_ROUTE_QUERY;
1843         work->new_state = RDMA_CM_ROUTE_RESOLVED;
1844         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1845         work->event.status = 0;
1846
1847         queue_work(cma_wq, &work->work);
1848
1849         return 0;
1850
1851 err2:
1852         kfree(route->path_rec);
1853         route->path_rec = NULL;
1854 err1:
1855         kfree(work);
1856         return ret;
1857 }
1858
1859 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1860 {
1861         struct rdma_id_private *id_priv;
1862         int ret;
1863
1864         id_priv = container_of(id, struct rdma_id_private, id);
1865         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
1866                 return -EINVAL;
1867
1868         atomic_inc(&id_priv->refcount);
1869         switch (rdma_node_get_transport(id->device->node_type)) {
1870         case RDMA_TRANSPORT_IB:
1871                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1872                 case IB_LINK_LAYER_INFINIBAND:
1873                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
1874                         break;
1875                 case IB_LINK_LAYER_ETHERNET:
1876                         ret = cma_resolve_iboe_route(id_priv);
1877                         break;
1878                 default:
1879                         ret = -ENOSYS;
1880                 }
1881                 break;
1882         case RDMA_TRANSPORT_IWARP:
1883                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1884                 break;
1885         default:
1886                 ret = -ENOSYS;
1887                 break;
1888         }
1889         if (ret)
1890                 goto err;
1891
1892         return 0;
1893 err:
1894         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
1895         cma_deref_id(id_priv);
1896         return ret;
1897 }
1898 EXPORT_SYMBOL(rdma_resolve_route);
1899
1900 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1901 {
1902         struct cma_device *cma_dev;
1903         struct ib_port_attr port_attr;
1904         union ib_gid gid;
1905         u16 pkey;
1906         int ret;
1907         u8 p;
1908
1909         mutex_lock(&lock);
1910         if (list_empty(&dev_list)) {
1911                 ret = -ENODEV;
1912                 goto out;
1913         }
1914         list_for_each_entry(cma_dev, &dev_list, list)
1915                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1916                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1917                             port_attr.state == IB_PORT_ACTIVE)
1918                                 goto port_found;
1919
1920         p = 1;
1921         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1922
1923 port_found:
1924         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1925         if (ret)
1926                 goto out;
1927
1928         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1929         if (ret)
1930                 goto out;
1931
1932         id_priv->id.route.addr.dev_addr.dev_type =
1933                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
1934                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
1935
1936         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1937         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1938         id_priv->id.port_num = p;
1939         cma_attach_to_dev(id_priv, cma_dev);
1940 out:
1941         mutex_unlock(&lock);
1942         return ret;
1943 }
1944
1945 static void addr_handler(int status, struct sockaddr *src_addr,
1946                          struct rdma_dev_addr *dev_addr, void *context)
1947 {
1948         struct rdma_id_private *id_priv = context;
1949         struct rdma_cm_event event;
1950
1951         memset(&event, 0, sizeof event);
1952         mutex_lock(&id_priv->handler_mutex);
1953         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
1954                            RDMA_CM_ADDR_RESOLVED))
1955                 goto out;
1956
1957         if (!status && !id_priv->cma_dev)
1958                 status = cma_acquire_dev(id_priv);
1959
1960         if (status) {
1961                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1962                                    RDMA_CM_ADDR_BOUND))
1963                         goto out;
1964                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1965                 event.status = status;
1966         } else {
1967                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1968                        ip_addr_size(src_addr));
1969                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1970         }
1971
1972         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1973                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1974                 mutex_unlock(&id_priv->handler_mutex);
1975                 cma_deref_id(id_priv);
1976                 rdma_destroy_id(&id_priv->id);
1977                 return;
1978         }
1979 out:
1980         mutex_unlock(&id_priv->handler_mutex);
1981         cma_deref_id(id_priv);
1982 }
1983
1984 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1985 {
1986         struct cma_work *work;
1987         struct sockaddr *src, *dst;
1988         union ib_gid gid;
1989         int ret;
1990
1991         work = kzalloc(sizeof *work, GFP_KERNEL);
1992         if (!work)
1993                 return -ENOMEM;
1994
1995         if (!id_priv->cma_dev) {
1996                 ret = cma_bind_loopback(id_priv);
1997                 if (ret)
1998                         goto err;
1999         }
2000
2001         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2002         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2003
2004         src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2005         if (cma_zero_addr(src)) {
2006                 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2007                 if ((src->sa_family = dst->sa_family) == AF_INET) {
2008                         ((struct sockaddr_in *) src)->sin_addr.s_addr =
2009                                 ((struct sockaddr_in *) dst)->sin_addr.s_addr;
2010                 } else {
2011                         ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr,
2012                                        &((struct sockaddr_in6 *) dst)->sin6_addr);
2013                 }
2014         }
2015
2016         work->id = id_priv;
2017         INIT_WORK(&work->work, cma_work_handler);
2018         work->old_state = RDMA_CM_ADDR_QUERY;
2019         work->new_state = RDMA_CM_ADDR_RESOLVED;
2020         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2021         queue_work(cma_wq, &work->work);
2022         return 0;
2023 err:
2024         kfree(work);
2025         return ret;
2026 }
2027
2028 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2029                          struct sockaddr *dst_addr)
2030 {
2031         if (!src_addr || !src_addr->sa_family) {
2032                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2033                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2034                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2035                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2036                 }
2037         }
2038         return rdma_bind_addr(id, src_addr);
2039 }
2040
2041 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2042                       struct sockaddr *dst_addr, int timeout_ms)
2043 {
2044         struct rdma_id_private *id_priv;
2045         int ret;
2046
2047         id_priv = container_of(id, struct rdma_id_private, id);
2048         if (id_priv->state == RDMA_CM_IDLE) {
2049                 ret = cma_bind_addr(id, src_addr, dst_addr);
2050                 if (ret)
2051                         return ret;
2052         }
2053
2054         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
2055                 return -EINVAL;
2056
2057         atomic_inc(&id_priv->refcount);
2058         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
2059         if (cma_any_addr(dst_addr))
2060                 ret = cma_resolve_loopback(id_priv);
2061         else
2062                 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
2063                                       dst_addr, &id->route.addr.dev_addr,
2064                                       timeout_ms, addr_handler, id_priv);
2065         if (ret)
2066                 goto err;
2067
2068         return 0;
2069 err:
2070         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
2071         cma_deref_id(id_priv);
2072         return ret;
2073 }
2074 EXPORT_SYMBOL(rdma_resolve_addr);
2075
2076 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2077 {
2078         struct rdma_id_private *id_priv;
2079         unsigned long flags;
2080         int ret;
2081
2082         id_priv = container_of(id, struct rdma_id_private, id);
2083         spin_lock_irqsave(&id_priv->lock, flags);
2084         if (id_priv->state == RDMA_CM_IDLE) {
2085                 id_priv->reuseaddr = reuse;
2086                 ret = 0;
2087         } else {
2088                 ret = -EINVAL;
2089         }
2090         spin_unlock_irqrestore(&id_priv->lock, flags);
2091         return ret;
2092 }
2093 EXPORT_SYMBOL(rdma_set_reuseaddr);
2094
2095 static void cma_bind_port(struct rdma_bind_list *bind_list,
2096                           struct rdma_id_private *id_priv)
2097 {
2098         struct sockaddr_in *sin;
2099
2100         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2101         sin->sin_port = htons(bind_list->port);
2102         id_priv->bind_list = bind_list;
2103         hlist_add_head(&id_priv->node, &bind_list->owners);
2104 }
2105
2106 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2107                           unsigned short snum)
2108 {
2109         struct rdma_bind_list *bind_list;
2110         int port, ret;
2111
2112         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2113         if (!bind_list)
2114                 return -ENOMEM;
2115
2116         do {
2117                 ret = idr_get_new_above(ps, bind_list, snum, &port);
2118         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
2119
2120         if (ret)
2121                 goto err1;
2122
2123         if (port != snum) {
2124                 ret = -EADDRNOTAVAIL;
2125                 goto err2;
2126         }
2127
2128         bind_list->ps = ps;
2129         bind_list->port = (unsigned short) port;
2130         cma_bind_port(bind_list, id_priv);
2131         return 0;
2132 err2:
2133         idr_remove(ps, port);
2134 err1:
2135         kfree(bind_list);
2136         return ret;
2137 }
2138
2139 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2140 {
2141         static unsigned int last_used_port;
2142         int low, high, remaining;
2143         unsigned int rover;
2144
2145         inet_get_local_port_range(&low, &high);
2146         remaining = (high - low) + 1;
2147         rover = net_random() % remaining + low;
2148 retry:
2149         if (last_used_port != rover &&
2150             !idr_find(ps, (unsigned short) rover)) {
2151                 int ret = cma_alloc_port(ps, id_priv, rover);
2152                 /*
2153                  * Remember previously used port number in order to avoid
2154                  * re-using same port immediately after it is closed.
2155                  */
2156                 if (!ret)
2157                         last_used_port = rover;
2158                 if (ret != -EADDRNOTAVAIL)
2159                         return ret;
2160         }
2161         if (--remaining) {
2162                 rover++;
2163                 if ((rover < low) || (rover > high))
2164                         rover = low;
2165                 goto retry;
2166         }
2167         return -EADDRNOTAVAIL;
2168 }
2169
2170 /*
2171  * Check that the requested port is available.  This is called when trying to
2172  * bind to a specific port, or when trying to listen on a bound port.  In
2173  * the latter case, the provided id_priv may already be on the bind_list, but
2174  * we still need to check that it's okay to start listening.
2175  */
2176 static int cma_check_port(struct rdma_bind_list *bind_list,
2177                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
2178 {
2179         struct rdma_id_private *cur_id;
2180         struct sockaddr *addr, *cur_addr;
2181         struct hlist_node *node;
2182
2183         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2184         if (cma_any_addr(addr) && !reuseaddr)
2185                 return -EADDRNOTAVAIL;
2186
2187         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2188                 if (id_priv == cur_id)
2189                         continue;
2190
2191                 if ((cur_id->state == RDMA_CM_LISTEN) ||
2192                     !reuseaddr || !cur_id->reuseaddr) {
2193                         cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2194                         if (cma_any_addr(cur_addr))
2195                                 return -EADDRNOTAVAIL;
2196
2197                         if (!cma_addr_cmp(addr, cur_addr))
2198                                 return -EADDRINUSE;
2199                 }
2200         }
2201         return 0;
2202 }
2203
2204 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2205 {
2206         struct rdma_bind_list *bind_list;
2207         unsigned short snum;
2208         int ret;
2209
2210         snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2211         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2212                 return -EACCES;
2213
2214         bind_list = idr_find(ps, snum);
2215         if (!bind_list) {
2216                 ret = cma_alloc_port(ps, id_priv, snum);
2217         } else {
2218                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2219                 if (!ret)
2220                         cma_bind_port(bind_list, id_priv);
2221         }
2222         return ret;
2223 }
2224
2225 static int cma_bind_listen(struct rdma_id_private *id_priv)
2226 {
2227         struct rdma_bind_list *bind_list = id_priv->bind_list;
2228         int ret = 0;
2229
2230         mutex_lock(&lock);
2231         if (bind_list->owners.first->next)
2232                 ret = cma_check_port(bind_list, id_priv, 0);
2233         mutex_unlock(&lock);
2234         return ret;
2235 }
2236
2237 static int cma_get_port(struct rdma_id_private *id_priv)
2238 {
2239         struct idr *ps;
2240         int ret;
2241
2242         switch (id_priv->id.ps) {
2243         case RDMA_PS_SDP:
2244                 ps = &sdp_ps;
2245                 break;
2246         case RDMA_PS_TCP:
2247                 ps = &tcp_ps;
2248                 break;
2249         case RDMA_PS_UDP:
2250                 ps = &udp_ps;
2251                 break;
2252         case RDMA_PS_IPOIB:
2253                 ps = &ipoib_ps;
2254                 break;
2255         case RDMA_PS_IB:
2256                 ps = &ib_ps;
2257                 break;
2258         default:
2259                 return -EPROTONOSUPPORT;
2260         }
2261
2262         mutex_lock(&lock);
2263         if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2264                 ret = cma_alloc_any_port(ps, id_priv);
2265         else
2266                 ret = cma_use_port(ps, id_priv);
2267         mutex_unlock(&lock);
2268
2269         return ret;
2270 }
2271
2272 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2273                                struct sockaddr *addr)
2274 {
2275 #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
2276         struct sockaddr_in6 *sin6;
2277
2278         if (addr->sa_family != AF_INET6)
2279                 return 0;
2280
2281         sin6 = (struct sockaddr_in6 *) addr;
2282         if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2283             !sin6->sin6_scope_id)
2284                         return -EINVAL;
2285
2286         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2287 #endif
2288         return 0;
2289 }
2290
2291 int rdma_listen(struct rdma_cm_id *id, int backlog)
2292 {
2293         struct rdma_id_private *id_priv;
2294         int ret;
2295
2296         id_priv = container_of(id, struct rdma_id_private, id);
2297         if (id_priv->state == RDMA_CM_IDLE) {
2298                 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
2299                 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
2300                 if (ret)
2301                         return ret;
2302         }
2303
2304         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
2305                 return -EINVAL;
2306
2307         if (id_priv->reuseaddr) {
2308                 ret = cma_bind_listen(id_priv);
2309                 if (ret)
2310                         goto err;
2311         }
2312
2313         id_priv->backlog = backlog;
2314         if (id->device) {
2315                 switch (rdma_node_get_transport(id->device->node_type)) {
2316                 case RDMA_TRANSPORT_IB:
2317                         ret = cma_ib_listen(id_priv);
2318                         if (ret)
2319                                 goto err;
2320                         break;
2321                 case RDMA_TRANSPORT_IWARP:
2322                         ret = cma_iw_listen(id_priv, backlog);
2323                         if (ret)
2324                                 goto err;
2325                         break;
2326                 default:
2327                         ret = -ENOSYS;
2328                         goto err;
2329                 }
2330         } else
2331                 cma_listen_on_all(id_priv);
2332
2333         return 0;
2334 err:
2335         id_priv->backlog = 0;
2336         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
2337         return ret;
2338 }
2339 EXPORT_SYMBOL(rdma_listen);
2340
2341 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2342 {
2343         struct rdma_id_private *id_priv;
2344         int ret;
2345
2346         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2347                 return -EAFNOSUPPORT;
2348
2349         id_priv = container_of(id, struct rdma_id_private, id);
2350         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
2351                 return -EINVAL;
2352
2353         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2354         if (ret)
2355                 goto err1;
2356
2357         if (!cma_any_addr(addr)) {
2358                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2359                 if (ret)
2360                         goto err1;
2361
2362                 ret = cma_acquire_dev(id_priv);
2363                 if (ret)
2364                         goto err1;
2365         }
2366
2367         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2368         ret = cma_get_port(id_priv);
2369         if (ret)
2370                 goto err2;
2371
2372         return 0;
2373 err2:
2374         if (id_priv->cma_dev)
2375                 cma_release_dev(id_priv);
2376 err1:
2377         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
2378         return ret;
2379 }
2380 EXPORT_SYMBOL(rdma_bind_addr);
2381
2382 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2383                           struct rdma_route *route)
2384 {
2385         struct cma_hdr *cma_hdr;
2386         struct sdp_hh *sdp_hdr;
2387
2388         if (route->addr.src_addr.ss_family == AF_INET) {
2389                 struct sockaddr_in *src4, *dst4;
2390
2391                 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2392                 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2393
2394                 switch (ps) {
2395                 case RDMA_PS_SDP:
2396                         sdp_hdr = hdr;
2397                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2398                                 return -EINVAL;
2399                         sdp_set_ip_ver(sdp_hdr, 4);
2400                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2401                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2402                         sdp_hdr->port = src4->sin_port;
2403                         break;
2404                 default:
2405                         cma_hdr = hdr;
2406                         cma_hdr->cma_version = CMA_VERSION;
2407                         cma_set_ip_ver(cma_hdr, 4);
2408                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2409                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2410                         cma_hdr->port = src4->sin_port;
2411                         break;
2412                 }
2413         } else {
2414                 struct sockaddr_in6 *src6, *dst6;
2415
2416                 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2417                 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2418
2419                 switch (ps) {
2420                 case RDMA_PS_SDP:
2421                         sdp_hdr = hdr;
2422                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2423                                 return -EINVAL;
2424                         sdp_set_ip_ver(sdp_hdr, 6);
2425                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2426                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2427                         sdp_hdr->port = src6->sin6_port;
2428                         break;
2429                 default:
2430                         cma_hdr = hdr;
2431                         cma_hdr->cma_version = CMA_VERSION;
2432                         cma_set_ip_ver(cma_hdr, 6);
2433                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2434                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2435                         cma_hdr->port = src6->sin6_port;
2436                         break;
2437                 }
2438         }
2439         return 0;
2440 }
2441
2442 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2443                                 struct ib_cm_event *ib_event)
2444 {
2445         struct rdma_id_private *id_priv = cm_id->context;
2446         struct rdma_cm_event event;
2447         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2448         int ret = 0;
2449
2450         if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
2451                 return 0;
2452
2453         memset(&event, 0, sizeof event);
2454         switch (ib_event->event) {
2455         case IB_CM_SIDR_REQ_ERROR:
2456                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2457                 event.status = -ETIMEDOUT;
2458                 break;
2459         case IB_CM_SIDR_REP_RECEIVED:
2460                 event.param.ud.private_data = ib_event->private_data;
2461                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2462                 if (rep->status != IB_SIDR_SUCCESS) {
2463                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2464                         event.status = ib_event->param.sidr_rep_rcvd.status;
2465                         break;
2466                 }
2467                 ret = cma_set_qkey(id_priv);
2468                 if (ret) {
2469                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2470                         event.status = -EINVAL;
2471                         break;
2472                 }
2473                 if (id_priv->qkey != rep->qkey) {
2474                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2475                         event.status = -EINVAL;
2476                         break;
2477                 }
2478                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2479                                      id_priv->id.route.path_rec,
2480                                      &event.param.ud.ah_attr);
2481                 event.param.ud.qp_num = rep->qpn;
2482                 event.param.ud.qkey = rep->qkey;
2483                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2484                 event.status = 0;
2485                 break;
2486         default:
2487                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2488                        ib_event->event);
2489                 goto out;
2490         }
2491
2492         ret = id_priv->id.event_handler(&id_priv->id, &event);
2493         if (ret) {
2494                 /* Destroy the CM ID by returning a non-zero value. */
2495                 id_priv->cm_id.ib = NULL;
2496                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2497                 mutex_unlock(&id_priv->handler_mutex);
2498                 rdma_destroy_id(&id_priv->id);
2499                 return ret;
2500         }
2501 out:
2502         mutex_unlock(&id_priv->handler_mutex);
2503         return ret;
2504 }
2505
2506 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2507                               struct rdma_conn_param *conn_param)
2508 {
2509         struct ib_cm_sidr_req_param req;
2510         struct rdma_route *route;
2511         struct ib_cm_id *id;
2512         int ret;
2513
2514         req.private_data_len = sizeof(struct cma_hdr) +
2515                                conn_param->private_data_len;
2516         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2517         if (!req.private_data)
2518                 return -ENOMEM;
2519
2520         if (conn_param->private_data && conn_param->private_data_len)
2521                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2522                        conn_param->private_data, conn_param->private_data_len);
2523
2524         route = &id_priv->id.route;
2525         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2526         if (ret)
2527                 goto out;
2528
2529         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2530                              id_priv);
2531         if (IS_ERR(id)) {
2532                 ret = PTR_ERR(id);
2533                 goto out;
2534         }
2535         id_priv->cm_id.ib = id;
2536
2537         req.path = route->path_rec;
2538         req.service_id = cma_get_service_id(id_priv->id.ps,
2539                                             (struct sockaddr *) &route->addr.dst_addr);
2540         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2541         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2542
2543         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2544         if (ret) {
2545                 ib_destroy_cm_id(id_priv->cm_id.ib);
2546                 id_priv->cm_id.ib = NULL;
2547         }
2548 out:
2549         kfree(req.private_data);
2550         return ret;
2551 }
2552
2553 static int cma_connect_ib(struct rdma_id_private *id_priv,
2554                           struct rdma_conn_param *conn_param)
2555 {
2556         struct ib_cm_req_param req;
2557         struct rdma_route *route;
2558         void *private_data;
2559         struct ib_cm_id *id;
2560         int offset, ret;
2561
2562         memset(&req, 0, sizeof req);
2563         offset = cma_user_data_offset(id_priv->id.ps);
2564         req.private_data_len = offset + conn_param->private_data_len;
2565         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2566         if (!private_data)
2567                 return -ENOMEM;
2568
2569         if (conn_param->private_data && conn_param->private_data_len)
2570                 memcpy(private_data + offset, conn_param->private_data,
2571                        conn_param->private_data_len);
2572
2573         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2574         if (IS_ERR(id)) {
2575                 ret = PTR_ERR(id);
2576                 goto out;
2577         }
2578         id_priv->cm_id.ib = id;
2579
2580         route = &id_priv->id.route;
2581         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2582         if (ret)
2583                 goto out;
2584         req.private_data = private_data;
2585
2586         req.primary_path = &route->path_rec[0];
2587         if (route->num_paths == 2)
2588                 req.alternate_path = &route->path_rec[1];
2589
2590         req.service_id = cma_get_service_id(id_priv->id.ps,
2591                                             (struct sockaddr *) &route->addr.dst_addr);
2592         req.qp_num = id_priv->qp_num;
2593         req.qp_type = id_priv->id.qp_type;
2594         req.starting_psn = id_priv->seq_num;
2595         req.responder_resources = conn_param->responder_resources;
2596         req.initiator_depth = conn_param->initiator_depth;
2597         req.flow_control = conn_param->flow_control;
2598         req.retry_count = conn_param->retry_count;
2599         req.rnr_retry_count = conn_param->rnr_retry_count;
2600         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2601         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2602         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2603         req.srq = id_priv->srq ? 1 : 0;
2604
2605         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2606 out:
2607         if (ret && !IS_ERR(id)) {
2608                 ib_destroy_cm_id(id);
2609                 id_priv->cm_id.ib = NULL;
2610         }
2611
2612         kfree(private_data);
2613         return ret;
2614 }
2615
2616 static int cma_connect_iw(struct rdma_id_private *id_priv,
2617                           struct rdma_conn_param *conn_param)
2618 {
2619         struct iw_cm_id *cm_id;
2620         struct sockaddr_in* sin;
2621         int ret;
2622         struct iw_cm_conn_param iw_param;
2623
2624         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2625         if (IS_ERR(cm_id))
2626                 return PTR_ERR(cm_id);
2627
2628         id_priv->cm_id.iw = cm_id;
2629
2630         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2631         cm_id->local_addr = *sin;
2632
2633         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2634         cm_id->remote_addr = *sin;
2635
2636         ret = cma_modify_qp_rtr(id_priv, conn_param);
2637         if (ret)
2638                 goto out;
2639
2640         if (conn_param) {
2641                 iw_param.ord = conn_param->initiator_depth;
2642                 iw_param.ird = conn_param->responder_resources;
2643                 iw_param.private_data = conn_param->private_data;
2644                 iw_param.private_data_len = conn_param->private_data_len;
2645                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2646         } else {
2647                 memset(&iw_param, 0, sizeof iw_param);
2648                 iw_param.qpn = id_priv->qp_num;
2649         }
2650         ret = iw_cm_connect(cm_id, &iw_param);
2651 out:
2652         if (ret) {
2653                 iw_destroy_cm_id(cm_id);
2654                 id_priv->cm_id.iw = NULL;
2655         }
2656         return ret;
2657 }
2658
2659 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2660 {
2661         struct rdma_id_private *id_priv;
2662         int ret;
2663
2664         id_priv = container_of(id, struct rdma_id_private, id);
2665         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
2666                 return -EINVAL;
2667
2668         if (!id->qp) {
2669                 id_priv->qp_num = conn_param->qp_num;
2670                 id_priv->srq = conn_param->srq;
2671         }
2672
2673         switch (rdma_node_get_transport(id->device->node_type)) {
2674         case RDMA_TRANSPORT_IB:
2675                 if (id->qp_type == IB_QPT_UD)
2676                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2677                 else
2678                         ret = cma_connect_ib(id_priv, conn_param);
2679                 break;
2680         case RDMA_TRANSPORT_IWARP:
2681                 ret = cma_connect_iw(id_priv, conn_param);
2682                 break;
2683         default:
2684                 ret = -ENOSYS;
2685                 break;
2686         }
2687         if (ret)
2688                 goto err;
2689
2690         return 0;
2691 err:
2692         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
2693         return ret;
2694 }
2695 EXPORT_SYMBOL(rdma_connect);
2696
2697 static int cma_accept_ib(struct rdma_id_private *id_priv,
2698                          struct rdma_conn_param *conn_param)
2699 {
2700         struct ib_cm_rep_param rep;
2701         int ret;
2702
2703         ret = cma_modify_qp_rtr(id_priv, conn_param);
2704         if (ret)
2705                 goto out;
2706
2707         ret = cma_modify_qp_rts(id_priv, conn_param);
2708         if (ret)
2709                 goto out;
2710
2711         memset(&rep, 0, sizeof rep);
2712         rep.qp_num = id_priv->qp_num;
2713         rep.starting_psn = id_priv->seq_num;
2714         rep.private_data = conn_param->private_data;
2715         rep.private_data_len = conn_param->private_data_len;
2716         rep.responder_resources = conn_param->responder_resources;
2717         rep.initiator_depth = conn_param->initiator_depth;
2718         rep.failover_accepted = 0;
2719         rep.flow_control = conn_param->flow_control;
2720         rep.rnr_retry_count = conn_param->rnr_retry_count;
2721         rep.srq = id_priv->srq ? 1 : 0;
2722
2723         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2724 out:
2725         return ret;
2726 }
2727
2728 static int cma_accept_iw(struct rdma_id_private *id_priv,
2729                   struct rdma_conn_param *conn_param)
2730 {
2731         struct iw_cm_conn_param iw_param;
2732         int ret;
2733
2734         ret = cma_modify_qp_rtr(id_priv, conn_param);
2735         if (ret)
2736                 return ret;
2737
2738         iw_param.ord = conn_param->initiator_depth;
2739         iw_param.ird = conn_param->responder_resources;
2740         iw_param.private_data = conn_param->private_data;
2741         iw_param.private_data_len = conn_param->private_data_len;
2742         if (id_priv->id.qp) {
2743                 iw_param.qpn = id_priv->qp_num;
2744         } else
2745                 iw_param.qpn = conn_param->qp_num;
2746
2747         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2748 }
2749
2750 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2751                              enum ib_cm_sidr_status status,
2752                              const void *private_data, int private_data_len)
2753 {
2754         struct ib_cm_sidr_rep_param rep;
2755         int ret;
2756
2757         memset(&rep, 0, sizeof rep);
2758         rep.status = status;
2759         if (status == IB_SIDR_SUCCESS) {
2760                 ret = cma_set_qkey(id_priv);
2761                 if (ret)
2762                         return ret;
2763                 rep.qp_num = id_priv->qp_num;
2764                 rep.qkey = id_priv->qkey;
2765         }
2766         rep.private_data = private_data;
2767         rep.private_data_len = private_data_len;
2768
2769         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2770 }
2771
2772 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2773 {
2774         struct rdma_id_private *id_priv;
2775         int ret;
2776
2777         id_priv = container_of(id, struct rdma_id_private, id);
2778
2779         id_priv->owner = task_pid_nr(current);
2780
2781         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
2782                 return -EINVAL;
2783
2784         if (!id->qp && conn_param) {
2785                 id_priv->qp_num = conn_param->qp_num;
2786                 id_priv->srq = conn_param->srq;
2787         }
2788
2789         switch (rdma_node_get_transport(id->device->node_type)) {
2790         case RDMA_TRANSPORT_IB:
2791                 if (id->qp_type == IB_QPT_UD) {
2792                         if (conn_param)
2793                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2794                                                         conn_param->private_data,
2795                                                         conn_param->private_data_len);
2796                         else
2797                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2798                                                         NULL, 0);
2799                 } else {
2800                         if (conn_param)
2801                                 ret = cma_accept_ib(id_priv, conn_param);
2802                         else
2803                                 ret = cma_rep_recv(id_priv);
2804                 }
2805                 break;
2806         case RDMA_TRANSPORT_IWARP:
2807                 ret = cma_accept_iw(id_priv, conn_param);
2808                 break;
2809         default:
2810                 ret = -ENOSYS;
2811                 break;
2812         }
2813
2814         if (ret)
2815                 goto reject;
2816
2817         return 0;
2818 reject:
2819         cma_modify_qp_err(id_priv);
2820         rdma_reject(id, NULL, 0);
2821         return ret;
2822 }
2823 EXPORT_SYMBOL(rdma_accept);
2824
2825 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2826 {
2827         struct rdma_id_private *id_priv;
2828         int ret;
2829
2830         id_priv = container_of(id, struct rdma_id_private, id);
2831         if (!id_priv->cm_id.ib)
2832                 return -EINVAL;
2833
2834         switch (id->device->node_type) {
2835         case RDMA_NODE_IB_CA:
2836                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2837                 break;
2838         default:
2839                 ret = 0;
2840                 break;
2841         }
2842         return ret;
2843 }
2844 EXPORT_SYMBOL(rdma_notify);
2845
2846 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2847                 u8 private_data_len)
2848 {
2849         struct rdma_id_private *id_priv;
2850         int ret;
2851
2852         id_priv = container_of(id, struct rdma_id_private, id);
2853         if (!id_priv->cm_id.ib)
2854                 return -EINVAL;
2855
2856         switch (rdma_node_get_transport(id->device->node_type)) {
2857         case RDMA_TRANSPORT_IB:
2858                 if (id->qp_type == IB_QPT_UD)
2859                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2860                                                 private_data, private_data_len);
2861                 else
2862                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2863                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2864                                              0, private_data, private_data_len);
2865                 break;
2866         case RDMA_TRANSPORT_IWARP:
2867                 ret = iw_cm_reject(id_priv->cm_id.iw,
2868                                    private_data, private_data_len);
2869                 break;
2870         default:
2871                 ret = -ENOSYS;
2872                 break;
2873         }
2874         return ret;
2875 }
2876 EXPORT_SYMBOL(rdma_reject);
2877
2878 int rdma_disconnect(struct rdma_cm_id *id)
2879 {
2880         struct rdma_id_private *id_priv;
2881         int ret;
2882
2883         id_priv = container_of(id, struct rdma_id_private, id);
2884         if (!id_priv->cm_id.ib)
2885                 return -EINVAL;
2886
2887         switch (rdma_node_get_transport(id->device->node_type)) {
2888         case RDMA_TRANSPORT_IB:
2889                 ret = cma_modify_qp_err(id_priv);
2890                 if (ret)
2891                         goto out;
2892                 /* Initiate or respond to a disconnect. */
2893                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2894                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2895                 break;
2896         case RDMA_TRANSPORT_IWARP:
2897                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2898                 break;
2899         default:
2900                 ret = -EINVAL;
2901                 break;
2902         }
2903 out:
2904         return ret;
2905 }
2906 EXPORT_SYMBOL(rdma_disconnect);
2907
2908 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2909 {
2910         struct rdma_id_private *id_priv;
2911         struct cma_multicast *mc = multicast->context;
2912         struct rdma_cm_event event;
2913         int ret;
2914
2915         id_priv = mc->id_priv;
2916         if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
2917             cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
2918                 return 0;
2919
2920         mutex_lock(&id_priv->qp_mutex);
2921         if (!status && id_priv->id.qp)
2922                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2923                                          multicast->rec.mlid);
2924         mutex_unlock(&id_priv->qp_mutex);
2925
2926         memset(&event, 0, sizeof event);
2927         event.status = status;
2928         event.param.ud.private_data = mc->context;
2929         if (!status) {
2930                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2931                 ib_init_ah_from_mcmember(id_priv->id.device,
2932                                          id_priv->id.port_num, &multicast->rec,
2933                                          &event.param.ud.ah_attr);
2934                 event.param.ud.qp_num = 0xFFFFFF;
2935                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2936         } else
2937                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2938
2939         ret = id_priv->id.event_handler(&id_priv->id, &event);
2940         if (ret) {
2941                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2942                 mutex_unlock(&id_priv->handler_mutex);
2943                 rdma_destroy_id(&id_priv->id);
2944                 return 0;
2945         }
2946
2947         mutex_unlock(&id_priv->handler_mutex);
2948         return 0;
2949 }
2950
2951 static void cma_set_mgid(struct rdma_id_private *id_priv,
2952                          struct sockaddr *addr, union ib_gid *mgid)
2953 {
2954         unsigned char mc_map[MAX_ADDR_LEN];
2955         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2956         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2957         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2958
2959         if (cma_any_addr(addr)) {
2960                 memset(mgid, 0, sizeof *mgid);
2961         } else if ((addr->sa_family == AF_INET6) &&
2962                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
2963                                                                  0xFF10A01B)) {
2964                 /* IPv6 address is an SA assigned MGID. */
2965                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2966         } else if ((addr->sa_family == AF_INET6)) {
2967                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
2968                 if (id_priv->id.ps == RDMA_PS_UDP)
2969                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2970                 *mgid = *(union ib_gid *) (mc_map + 4);
2971         } else {
2972                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
2973                 if (id_priv->id.ps == RDMA_PS_UDP)
2974                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2975                 *mgid = *(union ib_gid *) (mc_map + 4);
2976         }
2977 }
2978
2979 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2980                                  struct cma_multicast *mc)
2981 {
2982         struct ib_sa_mcmember_rec rec;
2983         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2984         ib_sa_comp_mask comp_mask;
2985         int ret;
2986
2987         ib_addr_get_mgid(dev_addr, &rec.mgid);
2988         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2989                                      &rec.mgid, &rec);
2990         if (ret)
2991                 return ret;
2992
2993         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
2994         if (id_priv->id.ps == RDMA_PS_UDP)
2995                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2996         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
2997         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2998         rec.join_state = 1;
2999
3000         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3001                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3002                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3003                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
3004                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3005
3006         if (id_priv->id.ps == RDMA_PS_IPOIB)
3007                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
3008                              IB_SA_MCMEMBER_REC_RATE_SELECTOR;
3009
3010         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3011                                                 id_priv->id.port_num, &rec,
3012                                                 comp_mask, GFP_KERNEL,
3013                                                 cma_ib_mc_handler, mc);
3014         if (IS_ERR(mc->multicast.ib))
3015                 return PTR_ERR(mc->multicast.ib);
3016
3017         return 0;
3018 }
3019
3020 static void iboe_mcast_work_handler(struct work_struct *work)
3021 {
3022         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3023         struct cma_multicast *mc = mw->mc;
3024         struct ib_sa_multicast *m = mc->multicast.ib;
3025
3026         mc->multicast.ib->context = mc;
3027         cma_ib_mc_handler(0, m);
3028         kref_put(&mc->mcref, release_mc);
3029         kfree(mw);
3030 }
3031
3032 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3033 {
3034         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3035         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3036
3037         if (cma_any_addr(addr)) {
3038                 memset(mgid, 0, sizeof *mgid);
3039         } else if (addr->sa_family == AF_INET6) {
3040                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3041         } else {
3042                 mgid->raw[0] = 0xff;
3043                 mgid->raw[1] = 0x0e;
3044                 mgid->raw[2] = 0;
3045                 mgid->raw[3] = 0;
3046                 mgid->raw[4] = 0;
3047                 mgid->raw[5] = 0;
3048                 mgid->raw[6] = 0;
3049                 mgid->raw[7] = 0;
3050                 mgid->raw[8] = 0;
3051                 mgid->raw[9] = 0;
3052                 mgid->raw[10] = 0xff;
3053                 mgid->raw[11] = 0xff;
3054                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3055         }
3056 }
3057
3058 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3059                                    struct cma_multicast *mc)
3060 {
3061         struct iboe_mcast_work *work;
3062         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3063         int err;
3064         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3065         struct net_device *ndev = NULL;
3066
3067         if (cma_zero_addr((struct sockaddr *)&mc->addr))
3068                 return -EINVAL;
3069
3070         work = kzalloc(sizeof *work, GFP_KERNEL);
3071         if (!work)
3072                 return -ENOMEM;
3073
3074         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3075         if (!mc->multicast.ib) {
3076                 err = -ENOMEM;
3077                 goto out1;
3078         }
3079
3080         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3081
3082         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3083         if (id_priv->id.ps == RDMA_PS_UDP)
3084                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3085
3086         if (dev_addr->bound_dev_if)
3087                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3088         if (!ndev) {
3089                 err = -ENODEV;
3090                 goto out2;
3091         }
3092         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3093         mc->multicast.ib->rec.hop_limit = 1;
3094         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3095         dev_put(ndev);
3096         if (!mc->multicast.ib->rec.mtu) {
3097                 err = -EINVAL;
3098                 goto out2;
3099         }
3100         iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3101         work->id = id_priv;
3102         work->mc = mc;
3103         INIT_WORK(&work->work, iboe_mcast_work_handler);
3104         kref_get(&mc->mcref);
3105         queue_work(cma_wq, &work->work);
3106
3107         return 0;
3108
3109 out2:
3110         kfree(mc->multicast.ib);
3111 out1:
3112         kfree(work);
3113         return err;
3114 }
3115
3116 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3117                         void *context)
3118 {
3119         struct rdma_id_private *id_priv;
3120         struct cma_multicast *mc;
3121         int ret;
3122
3123         id_priv = container_of(id, struct rdma_id_private, id);
3124         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3125             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
3126                 return -EINVAL;
3127
3128         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3129         if (!mc)
3130                 return -ENOMEM;
3131
3132         memcpy(&mc->addr, addr, ip_addr_size(addr));
3133         mc->context = context;
3134         mc->id_priv = id_priv;
3135
3136         spin_lock(&id_priv->lock);
3137         list_add(&mc->list, &id_priv->mc_list);
3138         spin_unlock(&id_priv->lock);
3139
3140         switch (rdma_node_get_transport(id->device->node_type)) {
3141         case RDMA_TRANSPORT_IB:
3142                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3143                 case IB_LINK_LAYER_INFINIBAND:
3144                         ret = cma_join_ib_multicast(id_priv, mc);
3145                         break;
3146                 case IB_LINK_LAYER_ETHERNET:
3147                         kref_init(&mc->mcref);
3148                         ret = cma_iboe_join_multicast(id_priv, mc);
3149                         break;
3150                 default:
3151                         ret = -EINVAL;
3152                 }
3153                 break;
3154         default:
3155                 ret = -ENOSYS;
3156                 break;
3157         }
3158
3159         if (ret) {
3160                 spin_lock_irq(&id_priv->lock);
3161                 list_del(&mc->list);
3162                 spin_unlock_irq(&id_priv->lock);
3163                 kfree(mc);
3164         }
3165         return ret;
3166 }
3167 EXPORT_SYMBOL(rdma_join_multicast);
3168
3169 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3170 {
3171         struct rdma_id_private *id_priv;
3172         struct cma_multicast *mc;
3173
3174         id_priv = container_of(id, struct rdma_id_private, id);
3175         spin_lock_irq(&id_priv->lock);
3176         list_for_each_entry(mc, &id_priv->mc_list, list) {
3177                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
3178                         list_del(&mc->list);
3179                         spin_unlock_irq(&id_priv->lock);
3180
3181                         if (id->qp)
3182                                 ib_detach_mcast(id->qp,
3183                                                 &mc->multicast.ib->rec.mgid,
3184                                                 mc->multicast.ib->rec.mlid);
3185                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3186                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3187                                 case IB_LINK_LAYER_INFINIBAND:
3188                                         ib_sa_free_multicast(mc->multicast.ib);
3189                                         kfree(mc);
3190                                         break;
3191                                 case IB_LINK_LAYER_ETHERNET:
3192                                         kref_put(&mc->mcref, release_mc);
3193                                         break;
3194                                 default:
3195                                         break;
3196                                 }
3197                         }
3198                         return;
3199                 }
3200         }
3201         spin_unlock_irq(&id_priv->lock);
3202 }
3203 EXPORT_SYMBOL(rdma_leave_multicast);
3204
3205 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3206 {
3207         struct rdma_dev_addr *dev_addr;
3208         struct cma_ndev_work *work;
3209
3210         dev_addr = &id_priv->id.route.addr.dev_addr;
3211
3212         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3213             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3214                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3215                        ndev->name, &id_priv->id);
3216                 work = kzalloc(sizeof *work, GFP_KERNEL);
3217                 if (!work)
3218                         return -ENOMEM;
3219
3220                 INIT_WORK(&work->work, cma_ndev_work_handler);
3221                 work->id = id_priv;
3222                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3223                 atomic_inc(&id_priv->refcount);
3224                 queue_work(cma_wq, &work->work);
3225         }
3226
3227         return 0;
3228 }
3229
3230 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3231                                void *ctx)
3232 {
3233         struct net_device *ndev = (struct net_device *)ctx;
3234         struct cma_device *cma_dev;
3235         struct rdma_id_private *id_priv;
3236         int ret = NOTIFY_DONE;
3237
3238         if (dev_net(ndev) != &init_net)
3239                 return NOTIFY_DONE;
3240
3241         if (event != NETDEV_BONDING_FAILOVER)
3242                 return NOTIFY_DONE;
3243
3244         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3245                 return NOTIFY_DONE;
3246
3247         mutex_lock(&lock);
3248         list_for_each_entry(cma_dev, &dev_list, list)
3249                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3250                         ret = cma_netdev_change(ndev, id_priv);
3251                         if (ret)
3252                                 goto out;
3253                 }
3254
3255 out:
3256         mutex_unlock(&lock);
3257         return ret;
3258 }
3259
3260 static struct notifier_block cma_nb = {
3261         .notifier_call = cma_netdev_callback
3262 };
3263
3264 static void cma_add_one(struct ib_device *device)
3265 {
3266         struct cma_device *cma_dev;
3267         struct rdma_id_private *id_priv;
3268
3269         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3270         if (!cma_dev)
3271                 return;
3272
3273         cma_dev->device = device;
3274
3275         init_completion(&cma_dev->comp);
3276         atomic_set(&cma_dev->refcount, 1);
3277         INIT_LIST_HEAD(&cma_dev->id_list);
3278         ib_set_client_data(device, &cma_client, cma_dev);
3279
3280         mutex_lock(&lock);
3281         list_add_tail(&cma_dev->list, &dev_list);
3282         list_for_each_entry(id_priv, &listen_any_list, list)
3283                 cma_listen_on_dev(id_priv, cma_dev);
3284         mutex_unlock(&lock);
3285 }
3286
3287 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3288 {
3289         struct rdma_cm_event event;
3290         enum rdma_cm_state state;
3291         int ret = 0;
3292
3293         /* Record that we want to remove the device */
3294         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3295         if (state == RDMA_CM_DESTROYING)
3296                 return 0;
3297
3298         cma_cancel_operation(id_priv, state);
3299         mutex_lock(&id_priv->handler_mutex);
3300
3301         /* Check for destruction from another callback. */
3302         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
3303                 goto out;
3304
3305         memset(&event, 0, sizeof event);
3306         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3307         ret = id_priv->id.event_handler(&id_priv->id, &event);
3308 out:
3309         mutex_unlock(&id_priv->handler_mutex);
3310         return ret;
3311 }
3312
3313 static void cma_process_remove(struct cma_device *cma_dev)
3314 {
3315         struct rdma_id_private *id_priv;
3316         int ret;
3317
3318         mutex_lock(&lock);
3319         while (!list_empty(&cma_dev->id_list)) {
3320                 id_priv = list_entry(cma_dev->id_list.next,
3321                                      struct rdma_id_private, list);
3322
3323                 list_del(&id_priv->listen_list);
3324                 list_del_init(&id_priv->list);
3325                 atomic_inc(&id_priv->refcount);
3326                 mutex_unlock(&lock);
3327
3328                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3329                 cma_deref_id(id_priv);
3330                 if (ret)
3331                         rdma_destroy_id(&id_priv->id);
3332
3333                 mutex_lock(&lock);
3334         }
3335         mutex_unlock(&lock);
3336
3337         cma_deref_dev(cma_dev);
3338         wait_for_completion(&cma_dev->comp);
3339 }
3340
3341 static void cma_remove_one(struct ib_device *device)
3342 {
3343         struct cma_device *cma_dev;
3344
3345         cma_dev = ib_get_client_data(device, &cma_client);
3346         if (!cma_dev)
3347                 return;
3348
3349         mutex_lock(&lock);
3350         list_del(&cma_dev->list);
3351         mutex_unlock(&lock);
3352
3353         cma_process_remove(cma_dev);
3354         kfree(cma_dev);
3355 }
3356
3357 static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
3358 {
3359         struct nlmsghdr *nlh;
3360         struct rdma_cm_id_stats *id_stats;
3361         struct rdma_id_private *id_priv;
3362         struct rdma_cm_id *id = NULL;
3363         struct cma_device *cma_dev;
3364         int i_dev = 0, i_id = 0;
3365
3366         /*
3367          * We export all of the IDs as a sequence of messages.  Each
3368          * ID gets its own netlink message.
3369          */
3370         mutex_lock(&lock);
3371
3372         list_for_each_entry(cma_dev, &dev_list, list) {
3373                 if (i_dev < cb->args[0]) {
3374                         i_dev++;
3375                         continue;
3376                 }
3377
3378                 i_id = 0;
3379                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3380                         if (i_id < cb->args[1]) {
3381                                 i_id++;
3382                                 continue;
3383                         }
3384
3385                         id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
3386                                                 sizeof *id_stats, RDMA_NL_RDMA_CM,
3387                                                 RDMA_NL_RDMA_CM_ID_STATS);
3388                         if (!id_stats)
3389                                 goto out;
3390
3391                         memset(id_stats, 0, sizeof *id_stats);
3392                         id = &id_priv->id;
3393                         id_stats->node_type = id->route.addr.dev_addr.dev_type;
3394                         id_stats->port_num = id->port_num;
3395                         id_stats->bound_dev_if =
3396                                 id->route.addr.dev_addr.bound_dev_if;
3397
3398                         if (id->route.addr.src_addr.ss_family == AF_INET) {
3399                                 if (ibnl_put_attr(skb, nlh,
3400                                                   sizeof(struct sockaddr_in),
3401                                                   &id->route.addr.src_addr,
3402                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3403                                         goto out;
3404                                 }
3405                                 if (ibnl_put_attr(skb, nlh,
3406                                                   sizeof(struct sockaddr_in),
3407                                                   &id->route.addr.dst_addr,
3408                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3409                                         goto out;
3410                                 }
3411                         } else if (id->route.addr.src_addr.ss_family == AF_INET6) {
3412                                 if (ibnl_put_attr(skb, nlh,
3413                                                   sizeof(struct sockaddr_in6),
3414                                                   &id->route.addr.src_addr,
3415                                                   RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3416                                         goto out;
3417                                 }
3418                                 if (ibnl_put_attr(skb, nlh,
3419                                                   sizeof(struct sockaddr_in6),
3420                                                   &id->route.addr.dst_addr,
3421                                                   RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3422                                         goto out;
3423                                 }
3424                         }
3425
3426                         id_stats->pid           = id_priv->owner;
3427                         id_stats->port_space    = id->ps;
3428                         id_stats->cm_state      = id_priv->state;
3429                         id_stats->qp_num        = id_priv->qp_num;
3430                         id_stats->qp_type       = id->qp_type;
3431
3432                         i_id++;
3433                 }
3434
3435                 cb->args[1] = 0;
3436                 i_dev++;
3437         }
3438
3439 out:
3440         mutex_unlock(&lock);
3441         cb->args[0] = i_dev;
3442         cb->args[1] = i_id;
3443
3444         return skb->len;
3445 }
3446
3447 static const struct ibnl_client_cbs cma_cb_table[] = {
3448         [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats },
3449 };
3450
3451 static int __init cma_init(void)
3452 {
3453         int ret;
3454
3455         cma_wq = create_singlethread_workqueue("rdma_cm");
3456         if (!cma_wq)
3457                 return -ENOMEM;
3458
3459         ib_sa_register_client(&sa_client);
3460         rdma_addr_register_client(&addr_client);
3461         register_netdevice_notifier(&cma_nb);
3462
3463         ret = ib_register_client(&cma_client);
3464         if (ret)
3465                 goto err;
3466
3467         if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
3468                 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
3469
3470         return 0;
3471
3472 err:
3473         unregister_netdevice_notifier(&cma_nb);
3474         rdma_addr_unregister_client(&addr_client);
3475         ib_sa_unregister_client(&sa_client);
3476         destroy_workqueue(cma_wq);
3477         return ret;
3478 }
3479
3480 static void __exit cma_cleanup(void)
3481 {
3482         ibnl_remove_client(RDMA_NL_RDMA_CM);
3483         ib_unregister_client(&cma_client);
3484         unregister_netdevice_notifier(&cma_nb);
3485         rdma_addr_unregister_client(&addr_client);
3486         ib_sa_unregister_client(&sa_client);
3487         destroy_workqueue(cma_wq);
3488         idr_destroy(&sdp_ps);
3489         idr_destroy(&tcp_ps);
3490         idr_destroy(&udp_ps);
3491         idr_destroy(&ipoib_ps);
3492         idr_destroy(&ib_ps);
3493 }
3494
3495 module_init(cma_init);
3496 module_exit(cma_cleanup);