drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43 #include <linux/slab.h>
  44
  45 #include <net/tcp.h>
  46 #include <net/ipv6.h>
  47
  48 #include <rdma/rdma_cm.h>
  49 #include <rdma/rdma_cm_ib.h>
  50 #include <rdma/ib_cache.h>
  51 #include <rdma/ib_cm.h>
  52 #include <rdma/ib_sa.h>
  53 #include <rdma/iw_cm.h>
  54
  55 MODULE_AUTHOR("Sean Hefty");
  56 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  57 MODULE_LICENSE("Dual BSD/GPL");
  58
  59 #define CMA_CM_RESPONSE_TIMEOUT 20
  60 #define CMA_MAX_CM_RETRIES 15
  61 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  62 #define CMA_IBOE_PACKET_LIFETIME 18
  63
  64 static void cma_add_one(struct ib_device *device);
  65 static void cma_remove_one(struct ib_device *device);
  66
  67 static struct ib_client cma_client = {
  68         .name   = "cma",
  69         .add    = cma_add_one,
  70         .remove = cma_remove_one
  71 };
  72
  73 static struct ib_sa_client sa_client;
  74 static struct rdma_addr_client addr_client;
  75 static LIST_HEAD(dev_list);
  76 static LIST_HEAD(listen_any_list);
  77 static DEFINE_MUTEX(lock);
  78 static struct workqueue_struct *cma_wq;
  79 static DEFINE_IDR(sdp_ps);
  80 static DEFINE_IDR(tcp_ps);
  81 static DEFINE_IDR(udp_ps);
  82 static DEFINE_IDR(ipoib_ps);
  83
  84 struct cma_device {
  85         struct list_head        list;
  86         struct ib_device        *device;
  87         struct completion       comp;
  88         atomic_t                refcount;
  89         struct list_head        id_list;
  90 };
  91
  92 enum cma_state {
  93         CMA_IDLE,
  94         CMA_ADDR_QUERY,
  95         CMA_ADDR_RESOLVED,
  96         CMA_ROUTE_QUERY,
  97         CMA_ROUTE_RESOLVED,
  98         CMA_CONNECT,
  99         CMA_DISCONNECT,
 100         CMA_ADDR_BOUND,
 101         CMA_LISTEN,
 102         CMA_DEVICE_REMOVAL,
 103         CMA_DESTROYING
 104 };
 105
 106 struct rdma_bind_list {
 107         struct idr              *ps;
 108         struct hlist_head       owners;
 109         unsigned short          port;
 110 };
 111
 112 /*
 113  * Device removal can occur at anytime, so we need extra handling to
 114  * serialize notifying the user of device removal with other callbacks.
 115  * We do this by disabling removal notification while a callback is in process,
 116  * and reporting it after the callback completes.
 117  */
 118 struct rdma_id_private {
 119         struct rdma_cm_id       id;
 120
 121         struct rdma_bind_list   *bind_list;
 122         struct hlist_node       node;
 123         struct list_head        list; /* listen_any_list or cma_device.list */
 124         struct list_head        listen_list; /* per device listens */
 125         struct cma_device       *cma_dev;
 126         struct list_head        mc_list;
 127
 128         int                     internal_id;
 129         enum cma_state          state;
 130         spinlock_t              lock;
 131         struct mutex            qp_mutex;
 132
 133         struct completion       comp;
 134         atomic_t                refcount;
 135         struct mutex            handler_mutex;
 136
 137         int                     backlog;
 138         int                     timeout_ms;
 139         struct ib_sa_query      *query;
 140         int                     query_id;
 141         union {
 142                 struct ib_cm_id *ib;
 143                 struct iw_cm_id *iw;
 144         } cm_id;
 145
 146         u32                     seq_num;
 147         u32                     qkey;
 148         u32                     qp_num;
 149         u8                      srq;
 150         u8                      tos;
 151 };
 152
 153 struct cma_multicast {
 154         struct rdma_id_private *id_priv;
 155         union {
 156                 struct ib_sa_multicast *ib;
 157         } multicast;
 158         struct list_head        list;
 159         void                    *context;
 160         struct sockaddr_storage addr;
 161         struct kref             mcref;
 162 };
 163
 164 struct cma_work {
 165         struct work_struct      work;
 166         struct rdma_id_private  *id;
 167         enum cma_state          old_state;
 168         enum cma_state          new_state;
 169         struct rdma_cm_event    event;
 170 };
 171
 172 struct cma_ndev_work {
 173         struct work_struct      work;
 174         struct rdma_id_private  *id;
 175         struct rdma_cm_event    event;
 176 };
 177
 178 struct iboe_mcast_work {
 179         struct work_struct       work;
 180         struct rdma_id_private  *id;
 181         struct cma_multicast    *mc;
 182 };
 183
 184 union cma_ip_addr {
 185         struct in6_addr ip6;
 186         struct {
 187                 __be32 pad[3];
 188                 __be32 addr;
 189         } ip4;
 190 };
 191
 192 struct cma_hdr {
 193         u8 cma_version;
 194         u8 ip_version;  /* IP version: 7:4 */
 195         __be16 port;
 196         union cma_ip_addr src_addr;
 197         union cma_ip_addr dst_addr;
 198 };
 199
 200 struct sdp_hh {
 201         u8 bsdh[16];
 202         u8 sdp_version; /* Major version: 7:4 */
 203         u8 ip_version;  /* IP version: 7:4 */
 204         u8 sdp_specific1[10];
 205         __be16 port;
 206         __be16 sdp_specific2;
 207         union cma_ip_addr src_addr;
 208         union cma_ip_addr dst_addr;
 209 };
 210
 211 struct sdp_hah {
 212         u8 bsdh[16];
 213         u8 sdp_version;
 214 };
 215
 216 #define CMA_VERSION 0x00
 217 #define SDP_MAJ_VERSION 0x2
 218
 219 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
 220 {
 221         unsigned long flags;
 222         int ret;
 223
 224         spin_lock_irqsave(&id_priv->lock, flags);
 225         ret = (id_priv->state == comp);
 226         spin_unlock_irqrestore(&id_priv->lock, flags);
 227         return ret;
 228 }
 229
 230 static int cma_comp_exch(struct rdma_id_private *id_priv,
 231                          enum cma_state comp, enum cma_state exch)
 232 {
 233         unsigned long flags;
 234         int ret;
 235
 236         spin_lock_irqsave(&id_priv->lock, flags);
 237         if ((ret = (id_priv->state == comp)))
 238                 id_priv->state = exch;
 239         spin_unlock_irqrestore(&id_priv->lock, flags);
 240         return ret;
 241 }
 242
 243 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
 244                                enum cma_state exch)
 245 {
 246         unsigned long flags;
 247         enum cma_state old;
 248
 249         spin_lock_irqsave(&id_priv->lock, flags);
 250         old = id_priv->state;
 251         id_priv->state = exch;
 252         spin_unlock_irqrestore(&id_priv->lock, flags);
 253         return old;
 254 }
 255
 256 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 257 {
 258         return hdr->ip_version >> 4;
 259 }
 260
 261 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 262 {
 263         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 264 }
 265
 266 static inline u8 sdp_get_majv(u8 sdp_version)
 267 {
 268         return sdp_version >> 4;
 269 }
 270
 271 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 272 {
 273         return hh->ip_version >> 4;
 274 }
 275
 276 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 277 {
 278         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 279 }
 280
 281 static inline int cma_is_ud_ps(enum rdma_port_space ps)
 282 {
 283         return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
 284 }
 285
 286 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 287                               struct cma_device *cma_dev)
 288 {
 289         atomic_inc(&cma_dev->refcount);
 290         id_priv->cma_dev = cma_dev;
 291         id_priv->id.device = cma_dev->device;
 292         id_priv->id.route.addr.dev_addr.transport =
 293                 rdma_node_get_transport(cma_dev->device->node_type);
 294         list_add_tail(&id_priv->list, &cma_dev->id_list);
 295 }
 296
 297 static inline void cma_deref_dev(struct cma_device *cma_dev)
 298 {
 299         if (atomic_dec_and_test(&cma_dev->refcount))
 300                 complete(&cma_dev->comp);
 301 }
 302
 303 static inline void release_mc(struct kref *kref)
 304 {
 305         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 306
 307         kfree(mc->multicast.ib);
 308         kfree(mc);
 309 }
 310
 311 static void cma_release_dev(struct rdma_id_private *id_priv)
 312 {
 313         mutex_lock(&lock);
 314         list_del(&id_priv->list);
 315         cma_deref_dev(id_priv->cma_dev);
 316         id_priv->cma_dev = NULL;
 317         mutex_unlock(&lock);
 318 }
 319
 320 static int cma_set_qkey(struct rdma_id_private *id_priv)
 321 {
 322         struct ib_sa_mcmember_rec rec;
 323         int ret = 0;
 324
 325         if (id_priv->qkey)
 326                 return 0;
 327
 328         switch (id_priv->id.ps) {
 329         case RDMA_PS_UDP:
 330                 id_priv->qkey = RDMA_UDP_QKEY;
 331                 break;
 332         case RDMA_PS_IPOIB:
 333                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 334                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 335                                              id_priv->id.port_num, &rec.mgid,
 336                                              &rec);
 337                 if (!ret)
 338                         id_priv->qkey = be32_to_cpu(rec.qkey);
 339                 break;
 340         default:
 341                 break;
 342         }
 343         return ret;
 344 }
 345
 346 static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
 347 {
 348         int i;
 349         int err;
 350         struct ib_port_attr props;
 351         union ib_gid tmp;
 352
 353         err = ib_query_port(device, port_num, &props);
 354         if (err)
 355                 return 1;
 356
 357         for (i = 0; i < props.gid_tbl_len; ++i) {
 358                 err = ib_query_gid(device, port_num, i, &tmp);
 359                 if (err)
 360                         return 1;
 361                 if (!memcmp(&tmp, gid, sizeof tmp))
 362                         return 0;
 363         }
 364
 365         return -EAGAIN;
 366 }
 367
 368 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 369 {
 370         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 371         struct cma_device *cma_dev;
 372         union ib_gid gid, iboe_gid;
 373         int ret = -ENODEV;
 374         u8 port;
 375         enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
 376                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 377
 378         mutex_lock(&lock);
 379         iboe_addr_get_sgid(dev_addr, &iboe_gid);
 380         memcpy(&gid, dev_addr->src_dev_addr +
 381                rdma_addr_gid_offset(dev_addr), sizeof gid);
 382         list_for_each_entry(cma_dev, &dev_list, list) {
 383                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 384                         if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
 385                                 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
 386                                     rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
 387                                         ret = find_gid_port(cma_dev->device, &iboe_gid, port);
 388                                 else
 389                                         ret = find_gid_port(cma_dev->device, &gid, port);
 390
 391                                 if (!ret) {
 392                                         id_priv->id.port_num = port;
 393                                         goto out;
 394                                 } else if (ret == 1)
 395                                         break;
 396                         }
 397                 }
 398         }
 399
 400 out:
 401         if (!ret)
 402                 cma_attach_to_dev(id_priv, cma_dev);
 403
 404         mutex_unlock(&lock);
 405         return ret;
 406 }
 407
 408 static void cma_deref_id(struct rdma_id_private *id_priv)
 409 {
 410         if (atomic_dec_and_test(&id_priv->refcount))
 411                 complete(&id_priv->comp);
 412 }
 413
 414 static int cma_disable_callback(struct rdma_id_private *id_priv,
 415                               enum cma_state state)
 416 {
 417         mutex_lock(&id_priv->handler_mutex);
 418         if (id_priv->state != state) {
 419                 mutex_unlock(&id_priv->handler_mutex);
 420                 return -EINVAL;
 421         }
 422         return 0;
 423 }
 424
 425 static int cma_has_cm_dev(struct rdma_id_private *id_priv)
 426 {
 427         return (id_priv->id.device && id_priv->cm_id.ib);
 428 }
 429
 430 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 431                                   void *context, enum rdma_port_space ps)
 432 {
 433         struct rdma_id_private *id_priv;
 434
 435         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 436         if (!id_priv)
 437                 return ERR_PTR(-ENOMEM);
 438
 439         id_priv->state = CMA_IDLE;
 440         id_priv->id.context = context;
 441         id_priv->id.event_handler = event_handler;
 442         id_priv->id.ps = ps;
 443         spin_lock_init(&id_priv->lock);
 444         mutex_init(&id_priv->qp_mutex);
 445         init_completion(&id_priv->comp);
 446         atomic_set(&id_priv->refcount, 1);
 447         mutex_init(&id_priv->handler_mutex);
 448         INIT_LIST_HEAD(&id_priv->listen_list);
 449         INIT_LIST_HEAD(&id_priv->mc_list);
 450         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 451
 452         return &id_priv->id;
 453 }
 454 EXPORT_SYMBOL(rdma_create_id);
 455
 456 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 457 {
 458         struct ib_qp_attr qp_attr;
 459         int qp_attr_mask, ret;
 460
 461         qp_attr.qp_state = IB_QPS_INIT;
 462         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 463         if (ret)
 464                 return ret;
 465
 466         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 467         if (ret)
 468                 return ret;
 469
 470         qp_attr.qp_state = IB_QPS_RTR;
 471         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 472         if (ret)
 473                 return ret;
 474
 475         qp_attr.qp_state = IB_QPS_RTS;
 476         qp_attr.sq_psn = 0;
 477         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 478
 479         return ret;
 480 }
 481
 482 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 483 {
 484         struct ib_qp_attr qp_attr;
 485         int qp_attr_mask, ret;
 486
 487         qp_attr.qp_state = IB_QPS_INIT;
 488         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 489         if (ret)
 490                 return ret;
 491
 492         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 493 }
 494
 495 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 496                    struct ib_qp_init_attr *qp_init_attr)
 497 {
 498         struct rdma_id_private *id_priv;
 499         struct ib_qp *qp;
 500         int ret;
 501
 502         id_priv = container_of(id, struct rdma_id_private, id);
 503         if (id->device != pd->device)
 504                 return -EINVAL;
 505
 506         qp = ib_create_qp(pd, qp_init_attr);
 507         if (IS_ERR(qp))
 508                 return PTR_ERR(qp);
 509
 510         if (cma_is_ud_ps(id_priv->id.ps))
 511                 ret = cma_init_ud_qp(id_priv, qp);
 512         else
 513                 ret = cma_init_conn_qp(id_priv, qp);
 514         if (ret)
 515                 goto err;
 516
 517         id->qp = qp;
 518         id_priv->qp_num = qp->qp_num;
 519         id_priv->srq = (qp->srq != NULL);
 520         return 0;
 521 err:
 522         ib_destroy_qp(qp);
 523         return ret;
 524 }
 525 EXPORT_SYMBOL(rdma_create_qp);
 526
 527 void rdma_destroy_qp(struct rdma_cm_id *id)
 528 {
 529         struct rdma_id_private *id_priv;
 530
 531         id_priv = container_of(id, struct rdma_id_private, id);
 532         mutex_lock(&id_priv->qp_mutex);
 533         ib_destroy_qp(id_priv->id.qp);
 534         id_priv->id.qp = NULL;
 535         mutex_unlock(&id_priv->qp_mutex);
 536 }
 537 EXPORT_SYMBOL(rdma_destroy_qp);
 538
 539 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 540                              struct rdma_conn_param *conn_param)
 541 {
 542         struct ib_qp_attr qp_attr;
 543         int qp_attr_mask, ret;
 544
 545         mutex_lock(&id_priv->qp_mutex);
 546         if (!id_priv->id.qp) {
 547                 ret = 0;
 548                 goto out;
 549         }
 550
 551         /* Need to update QP attributes from default values. */
 552         qp_attr.qp_state = IB_QPS_INIT;
 553         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 554         if (ret)
 555                 goto out;
 556
 557         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 558         if (ret)
 559                 goto out;
 560
 561         qp_attr.qp_state = IB_QPS_RTR;
 562         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 563         if (ret)
 564                 goto out;
 565
 566         if (conn_param)
 567                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 568         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 569 out:
 570         mutex_unlock(&id_priv->qp_mutex);
 571         return ret;
 572 }
 573
 574 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 575                              struct rdma_conn_param *conn_param)
 576 {
 577         struct ib_qp_attr qp_attr;
 578         int qp_attr_mask, ret;
 579
 580         mutex_lock(&id_priv->qp_mutex);
 581         if (!id_priv->id.qp) {
 582                 ret = 0;
 583                 goto out;
 584         }
 585
 586         qp_attr.qp_state = IB_QPS_RTS;
 587         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 588         if (ret)
 589                 goto out;
 590
 591         if (conn_param)
 592                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 593         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 594 out:
 595         mutex_unlock(&id_priv->qp_mutex);
 596         return ret;
 597 }
 598
 599 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 600 {
 601         struct ib_qp_attr qp_attr;
 602         int ret;
 603
 604         mutex_lock(&id_priv->qp_mutex);
 605         if (!id_priv->id.qp) {
 606                 ret = 0;
 607                 goto out;
 608         }
 609
 610         qp_attr.qp_state = IB_QPS_ERR;
 611         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 612 out:
 613         mutex_unlock(&id_priv->qp_mutex);
 614         return ret;
 615 }
 616
 617 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 618                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 619 {
 620         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 621         int ret;
 622         u16 pkey;
 623
 624         if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
 625             IB_LINK_LAYER_INFINIBAND)
 626                 pkey = ib_addr_get_pkey(dev_addr);
 627         else
 628                 pkey = 0xffff;
 629
 630         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 631                                   pkey, &qp_attr->pkey_index);
 632         if (ret)
 633                 return ret;
 634
 635         qp_attr->port_num = id_priv->id.port_num;
 636         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 637
 638         if (cma_is_ud_ps(id_priv->id.ps)) {
 639                 ret = cma_set_qkey(id_priv);
 640                 if (ret)
 641                         return ret;
 642
 643                 qp_attr->qkey = id_priv->qkey;
 644                 *qp_attr_mask |= IB_QP_QKEY;
 645         } else {
 646                 qp_attr->qp_access_flags = 0;
 647                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 648         }
 649         return 0;
 650 }
 651
 652 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 653                        int *qp_attr_mask)
 654 {
 655         struct rdma_id_private *id_priv;
 656         int ret = 0;
 657
 658         id_priv = container_of(id, struct rdma_id_private, id);
 659         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 660         case RDMA_TRANSPORT_IB:
 661                 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
 662                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 663                 else
 664                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 665                                                  qp_attr_mask);
 666                 if (qp_attr->qp_state == IB_QPS_RTR)
 667                         qp_attr->rq_psn = id_priv->seq_num;
 668                 break;
 669         case RDMA_TRANSPORT_IWARP:
 670                 if (!id_priv->cm_id.iw) {
 671                         qp_attr->qp_access_flags = 0;
 672                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 673                 } else
 674                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 675                                                  qp_attr_mask);
 676                 break;
 677         default:
 678                 ret = -ENOSYS;
 679                 break;
 680         }
 681
 682         return ret;
 683 }
 684 EXPORT_SYMBOL(rdma_init_qp_attr);
 685
 686 static inline int cma_zero_addr(struct sockaddr *addr)
 687 {
 688         struct in6_addr *ip6;
 689
 690         if (addr->sa_family == AF_INET)
 691                 return ipv4_is_zeronet(
 692                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
 693         else {
 694                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 695                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 696                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 697         }
 698 }
 699
 700 static inline int cma_loopback_addr(struct sockaddr *addr)
 701 {
 702         if (addr->sa_family == AF_INET)
 703                 return ipv4_is_loopback(
 704                         ((struct sockaddr_in *) addr)->sin_addr.s_addr);
 705         else
 706                 return ipv6_addr_loopback(
 707                         &((struct sockaddr_in6 *) addr)->sin6_addr);
 708 }
 709
 710 static inline int cma_any_addr(struct sockaddr *addr)
 711 {
 712         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 713 }
 714
 715 static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
 716 {
 717         if (src->sa_family != dst->sa_family)
 718                 return -1;
 719
 720         switch (src->sa_family) {
 721         case AF_INET:
 722                 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
 723                        ((struct sockaddr_in *) dst)->sin_addr.s_addr;
 724         default:
 725                 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
 726                                      &((struct sockaddr_in6 *) dst)->sin6_addr);
 727         }
 728 }
 729
 730 static inline __be16 cma_port(struct sockaddr *addr)
 731 {
 732         if (addr->sa_family == AF_INET)
 733                 return ((struct sockaddr_in *) addr)->sin_port;
 734         else
 735                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 736 }
 737
 738 static inline int cma_any_port(struct sockaddr *addr)
 739 {
 740         return !cma_port(addr);
 741 }
 742
 743 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 744                             u8 *ip_ver, __be16 *port,
 745                             union cma_ip_addr **src, union cma_ip_addr **dst)
 746 {
 747         switch (ps) {
 748         case RDMA_PS_SDP:
 749                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 750                     SDP_MAJ_VERSION)
 751                         return -EINVAL;
 752
 753                 *ip_ver = sdp_get_ip_ver(hdr);
 754                 *port   = ((struct sdp_hh *) hdr)->port;
 755                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 756                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 757                 break;
 758         default:
 759                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 760                         return -EINVAL;
 761
 762                 *ip_ver = cma_get_ip_ver(hdr);
 763                 *port   = ((struct cma_hdr *) hdr)->port;
 764                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 765                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 766                 break;
 767         }
 768
 769         if (*ip_ver != 4 && *ip_ver != 6)
 770                 return -EINVAL;
 771         return 0;
 772 }
 773
 774 static void cma_save_net_info(struct rdma_addr *addr,
 775                               struct rdma_addr *listen_addr,
 776                               u8 ip_ver, __be16 port,
 777                               union cma_ip_addr *src, union cma_ip_addr *dst)
 778 {
 779         struct sockaddr_in *listen4, *ip4;
 780         struct sockaddr_in6 *listen6, *ip6;
 781
 782         switch (ip_ver) {
 783         case 4:
 784                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 785                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 786                 ip4->sin_family = listen4->sin_family;
 787                 ip4->sin_addr.s_addr = dst->ip4.addr;
 788                 ip4->sin_port = listen4->sin_port;
 789
 790                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 791                 ip4->sin_family = listen4->sin_family;
 792                 ip4->sin_addr.s_addr = src->ip4.addr;
 793                 ip4->sin_port = port;
 794                 break;
 795         case 6:
 796                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 797                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 798                 ip6->sin6_family = listen6->sin6_family;
 799                 ip6->sin6_addr = dst->ip6;
 800                 ip6->sin6_port = listen6->sin6_port;
 801
 802                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 803                 ip6->sin6_family = listen6->sin6_family;
 804                 ip6->sin6_addr = src->ip6;
 805                 ip6->sin6_port = port;
 806                 break;
 807         default:
 808                 break;
 809         }
 810 }
 811
 812 static inline int cma_user_data_offset(enum rdma_port_space ps)
 813 {
 814         switch (ps) {
 815         case RDMA_PS_SDP:
 816                 return 0;
 817         default:
 818                 return sizeof(struct cma_hdr);
 819         }
 820 }
 821
 822 static void cma_cancel_route(struct rdma_id_private *id_priv)
 823 {
 824         switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
 825         case IB_LINK_LAYER_INFINIBAND:
 826                 if (id_priv->query)
 827                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 828                 break;
 829         default:
 830                 break;
 831         }
 832 }
 833
 834 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 835 {
 836         struct rdma_id_private *dev_id_priv;
 837
 838         /*
 839          * Remove from listen_any_list to prevent added devices from spawning
 840          * additional listen requests.
 841          */
 842         mutex_lock(&lock);
 843         list_del(&id_priv->list);
 844
 845         while (!list_empty(&id_priv->listen_list)) {
 846                 dev_id_priv = list_entry(id_priv->listen_list.next,
 847                                          struct rdma_id_private, listen_list);
 848                 /* sync with device removal to avoid duplicate destruction */
 849                 list_del_init(&dev_id_priv->list);
 850                 list_del(&dev_id_priv->listen_list);
 851                 mutex_unlock(&lock);
 852
 853                 rdma_destroy_id(&dev_id_priv->id);
 854                 mutex_lock(&lock);
 855         }
 856         mutex_unlock(&lock);
 857 }
 858
 859 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 860                                  enum cma_state state)
 861 {
 862         switch (state) {
 863         case CMA_ADDR_QUERY:
 864                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 865                 break;
 866         case CMA_ROUTE_QUERY:
 867                 cma_cancel_route(id_priv);
 868                 break;
 869         case CMA_LISTEN:
 870                 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
 871                                 && !id_priv->cma_dev)
 872                         cma_cancel_listens(id_priv);
 873                 break;
 874         default:
 875                 break;
 876         }
 877 }
 878
 879 static void cma_release_port(struct rdma_id_private *id_priv)
 880 {
 881         struct rdma_bind_list *bind_list = id_priv->bind_list;
 882
 883         if (!bind_list)
 884                 return;
 885
 886         mutex_lock(&lock);
 887         hlist_del(&id_priv->node);
 888         if (hlist_empty(&bind_list->owners)) {
 889                 idr_remove(bind_list->ps, bind_list->port);
 890                 kfree(bind_list);
 891         }
 892         mutex_unlock(&lock);
 893 }
 894
 895 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 896 {
 897         struct cma_multicast *mc;
 898
 899         while (!list_empty(&id_priv->mc_list)) {
 900                 mc = container_of(id_priv->mc_list.next,
 901                                   struct cma_multicast, list);
 902                 list_del(&mc->list);
 903                 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
 904                 case IB_LINK_LAYER_INFINIBAND:
 905                         ib_sa_free_multicast(mc->multicast.ib);
 906                         kfree(mc);
 907                         break;
 908                 case IB_LINK_LAYER_ETHERNET:
 909                         kref_put(&mc->mcref, release_mc);
 910                         break;
 911                 default:
 912                         break;
 913                 }
 914         }
 915 }
 916
 917 void rdma_destroy_id(struct rdma_cm_id *id)
 918 {
 919         struct rdma_id_private *id_priv;
 920         enum cma_state state;
 921
 922         id_priv = container_of(id, struct rdma_id_private, id);
 923         state = cma_exch(id_priv, CMA_DESTROYING);
 924         cma_cancel_operation(id_priv, state);
 925
 926         /*
 927          * Wait for any active callback to finish.  New callbacks will find
 928          * the id_priv state set to destroying and abort.
 929          */
 930         mutex_lock(&id_priv->handler_mutex);
 931         mutex_unlock(&id_priv->handler_mutex);
 932
 933         if (id_priv->cma_dev) {
 934                 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 935                 case RDMA_TRANSPORT_IB:
 936                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 937                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 938                         break;
 939                 case RDMA_TRANSPORT_IWARP:
 940                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
 941                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 942                         break;
 943                 default:
 944                         break;
 945                 }
 946                 cma_leave_mc_groups(id_priv);
 947                 cma_release_dev(id_priv);
 948         }
 949
 950         cma_release_port(id_priv);
 951         cma_deref_id(id_priv);
 952         wait_for_completion(&id_priv->comp);
 953
 954         if (id_priv->internal_id)
 955                 cma_deref_id(id_priv->id.context);
 956
 957         kfree(id_priv->id.route.path_rec);
 958         kfree(id_priv);
 959 }
 960 EXPORT_SYMBOL(rdma_destroy_id);
 961
 962 static int cma_rep_recv(struct rdma_id_private *id_priv)
 963 {
 964         int ret;
 965
 966         ret = cma_modify_qp_rtr(id_priv, NULL);
 967         if (ret)
 968                 goto reject;
 969
 970         ret = cma_modify_qp_rts(id_priv, NULL);
 971         if (ret)
 972                 goto reject;
 973
 974         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
 975         if (ret)
 976                 goto reject;
 977
 978         return 0;
 979 reject:
 980         cma_modify_qp_err(id_priv);
 981         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
 982                        NULL, 0, NULL, 0);
 983         return ret;
 984 }
 985
 986 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
 987 {
 988         if (id_priv->id.ps == RDMA_PS_SDP &&
 989             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
 990             SDP_MAJ_VERSION)
 991                 return -EINVAL;
 992
 993         return 0;
 994 }
 995
 996 static void cma_set_rep_event_data(struct rdma_cm_event *event,
 997                                    struct ib_cm_rep_event_param *rep_data,
 998                                    void *private_data)
 999 {
1000         event->param.conn.private_data = private_data;
1001         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1002         event->param.conn.responder_resources = rep_data->responder_resources;
1003         event->param.conn.initiator_depth = rep_data->initiator_depth;
1004         event->param.conn.flow_control = rep_data->flow_control;
1005         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1006         event->param.conn.srq = rep_data->srq;
1007         event->param.conn.qp_num = rep_data->remote_qpn;
1008 }
1009
1010 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1011 {
1012         struct rdma_id_private *id_priv = cm_id->context;
1013         struct rdma_cm_event event;
1014         int ret = 0;
1015
1016         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1017                 cma_disable_callback(id_priv, CMA_CONNECT)) ||
1018             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1019                 cma_disable_callback(id_priv, CMA_DISCONNECT)))
1020                 return 0;
1021
1022         memset(&event, 0, sizeof event);
1023         switch (ib_event->event) {
1024         case IB_CM_REQ_ERROR:
1025         case IB_CM_REP_ERROR:
1026                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1027                 event.status = -ETIMEDOUT;
1028                 break;
1029         case IB_CM_REP_RECEIVED:
1030                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1031                 if (event.status)
1032                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1033                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
1034                         event.status = cma_rep_recv(id_priv);
1035                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1036                                                      RDMA_CM_EVENT_ESTABLISHED;
1037                 } else
1038                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1039                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1040                                        ib_event->private_data);
1041                 break;
1042         case IB_CM_RTU_RECEIVED:
1043         case IB_CM_USER_ESTABLISHED:
1044                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1045                 break;
1046         case IB_CM_DREQ_ERROR:
1047                 event.status = -ETIMEDOUT; /* fall through */
1048         case IB_CM_DREQ_RECEIVED:
1049         case IB_CM_DREP_RECEIVED:
1050                 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
1051                         goto out;
1052                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1053                 break;
1054         case IB_CM_TIMEWAIT_EXIT:
1055                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1056                 break;
1057         case IB_CM_MRA_RECEIVED:
1058                 /* ignore event */
1059                 goto out;
1060         case IB_CM_REJ_RECEIVED:
1061                 cma_modify_qp_err(id_priv);
1062                 event.status = ib_event->param.rej_rcvd.reason;
1063                 event.event = RDMA_CM_EVENT_REJECTED;
1064                 event.param.conn.private_data = ib_event->private_data;
1065                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1066                 break;
1067         default:
1068                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
1069                        ib_event->event);
1070                 goto out;
1071         }
1072
1073         ret = id_priv->id.event_handler(&id_priv->id, &event);
1074         if (ret) {
1075                 /* Destroy the CM ID by returning a non-zero value. */
1076                 id_priv->cm_id.ib = NULL;
1077                 cma_exch(id_priv, CMA_DESTROYING);
1078                 mutex_unlock(&id_priv->handler_mutex);
1079                 rdma_destroy_id(&id_priv->id);
1080                 return ret;
1081         }
1082 out:
1083         mutex_unlock(&id_priv->handler_mutex);
1084         return ret;
1085 }
1086
1087 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1088                                                struct ib_cm_event *ib_event)
1089 {
1090         struct rdma_id_private *id_priv;
1091         struct rdma_cm_id *id;
1092         struct rdma_route *rt;
1093         union cma_ip_addr *src, *dst;
1094         __be16 port;
1095         u8 ip_ver;
1096         int ret;
1097
1098         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1099                              &ip_ver, &port, &src, &dst))
1100                 goto err;
1101
1102         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1103                             listen_id->ps);
1104         if (IS_ERR(id))
1105                 goto err;
1106
1107         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1108                           ip_ver, port, src, dst);
1109
1110         rt = &id->route;
1111         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1112         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1113                                GFP_KERNEL);
1114         if (!rt->path_rec)
1115                 goto destroy_id;
1116
1117         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1118         if (rt->num_paths == 2)
1119                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1120
1121         if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1122                 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1123                 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1124                 ib_addr_set_pkey(&rt->addr.dev_addr, rt->path_rec[0].pkey);
1125         } else {
1126                 ret = rdma_translate_ip((struct sockaddr *) &rt->addr.src_addr,
1127                                         &rt->addr.dev_addr);
1128                 if (ret)
1129                         goto destroy_id;
1130         }
1131         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1132
1133         id_priv = container_of(id, struct rdma_id_private, id);
1134         id_priv->state = CMA_CONNECT;
1135         return id_priv;
1136
1137 destroy_id:
1138         rdma_destroy_id(id);
1139 err:
1140         return NULL;
1141 }
1142
1143 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1144                                               struct ib_cm_event *ib_event)
1145 {
1146         struct rdma_id_private *id_priv;
1147         struct rdma_cm_id *id;
1148         union cma_ip_addr *src, *dst;
1149         __be16 port;
1150         u8 ip_ver;
1151         int ret;
1152
1153         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1154                             listen_id->ps);
1155         if (IS_ERR(id))
1156                 return NULL;
1157
1158
1159         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1160                              &ip_ver, &port, &src, &dst))
1161                 goto err;
1162
1163         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1164                           ip_ver, port, src, dst);
1165
1166         if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
1167                 ret = rdma_translate_ip((struct sockaddr *) &id->route.addr.src_addr,
1168                                         &id->route.addr.dev_addr);
1169                 if (ret)
1170                         goto err;
1171         }
1172
1173         id_priv = container_of(id, struct rdma_id_private, id);
1174         id_priv->state = CMA_CONNECT;
1175         return id_priv;
1176 err:
1177         rdma_destroy_id(id);
1178         return NULL;
1179 }
1180
1181 static void cma_set_req_event_data(struct rdma_cm_event *event,
1182                                    struct ib_cm_req_event_param *req_data,
1183                                    void *private_data, int offset)
1184 {
1185         event->param.conn.private_data = private_data + offset;
1186         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1187         event->param.conn.responder_resources = req_data->responder_resources;
1188         event->param.conn.initiator_depth = req_data->initiator_depth;
1189         event->param.conn.flow_control = req_data->flow_control;
1190         event->param.conn.retry_count = req_data->retry_count;
1191         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1192         event->param.conn.srq = req_data->srq;
1193         event->param.conn.qp_num = req_data->remote_qpn;
1194 }
1195
1196 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1197 {
1198         struct rdma_id_private *listen_id, *conn_id;
1199         struct rdma_cm_event event;
1200         int offset, ret;
1201
1202         listen_id = cm_id->context;
1203         if (cma_disable_callback(listen_id, CMA_LISTEN))
1204                 return -ECONNABORTED;
1205
1206         memset(&event, 0, sizeof event);
1207         offset = cma_user_data_offset(listen_id->id.ps);
1208         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1209         if (cma_is_ud_ps(listen_id->id.ps)) {
1210                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1211                 event.param.ud.private_data = ib_event->private_data + offset;
1212                 event.param.ud.private_data_len =
1213                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1214         } else {
1215                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1216                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1217                                        ib_event->private_data, offset);
1218         }
1219         if (!conn_id) {
1220                 ret = -ENOMEM;
1221                 goto out;
1222         }
1223
1224         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1225         ret = cma_acquire_dev(conn_id);
1226         if (ret)
1227                 goto release_conn_id;
1228
1229         conn_id->cm_id.ib = cm_id;
1230         cm_id->context = conn_id;
1231         cm_id->cm_handler = cma_ib_handler;
1232
1233         /*
1234          * Protect against the user destroying conn_id from another thread
1235          * until we're done accessing it.
1236          */
1237         atomic_inc(&conn_id->refcount);
1238         ret = conn_id->id.event_handler(&conn_id->id, &event);
1239         if (!ret) {
1240                 /*
1241                  * Acquire mutex to prevent user executing rdma_destroy_id()
1242                  * while we're accessing the cm_id.
1243                  */
1244                 mutex_lock(&lock);
1245                 if (cma_comp(conn_id, CMA_CONNECT) &&
1246                     !cma_is_ud_ps(conn_id->id.ps))
1247                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1248                 mutex_unlock(&lock);
1249                 mutex_unlock(&conn_id->handler_mutex);
1250                 cma_deref_id(conn_id);
1251                 goto out;
1252         }
1253         cma_deref_id(conn_id);
1254
1255         /* Destroy the CM ID by returning a non-zero value. */
1256         conn_id->cm_id.ib = NULL;
1257
1258 release_conn_id:
1259         cma_exch(conn_id, CMA_DESTROYING);
1260         mutex_unlock(&conn_id->handler_mutex);
1261         rdma_destroy_id(&conn_id->id);
1262
1263 out:
1264         mutex_unlock(&listen_id->handler_mutex);
1265         return ret;
1266 }
1267
1268 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1269 {
1270         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1271 }
1272
1273 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1274                                  struct ib_cm_compare_data *compare)
1275 {
1276         struct cma_hdr *cma_data, *cma_mask;
1277         struct sdp_hh *sdp_data, *sdp_mask;
1278         __be32 ip4_addr;
1279         struct in6_addr ip6_addr;
1280
1281         memset(compare, 0, sizeof *compare);
1282         cma_data = (void *) compare->data;
1283         cma_mask = (void *) compare->mask;
1284         sdp_data = (void *) compare->data;
1285         sdp_mask = (void *) compare->mask;
1286
1287         switch (addr->sa_family) {
1288         case AF_INET:
1289                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1290                 if (ps == RDMA_PS_SDP) {
1291                         sdp_set_ip_ver(sdp_data, 4);
1292                         sdp_set_ip_ver(sdp_mask, 0xF);
1293                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1294                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1295                 } else {
1296                         cma_set_ip_ver(cma_data, 4);
1297                         cma_set_ip_ver(cma_mask, 0xF);
1298                         cma_data->dst_addr.ip4.addr = ip4_addr;
1299                         cma_mask->dst_addr.ip4.addr = htonl(~0);
1300                 }
1301                 break;
1302         case AF_INET6:
1303                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1304                 if (ps == RDMA_PS_SDP) {
1305                         sdp_set_ip_ver(sdp_data, 6);
1306                         sdp_set_ip_ver(sdp_mask, 0xF);
1307                         sdp_data->dst_addr.ip6 = ip6_addr;
1308                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1309                                sizeof sdp_mask->dst_addr.ip6);
1310                 } else {
1311                         cma_set_ip_ver(cma_data, 6);
1312                         cma_set_ip_ver(cma_mask, 0xF);
1313                         cma_data->dst_addr.ip6 = ip6_addr;
1314                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1315                                sizeof cma_mask->dst_addr.ip6);
1316                 }
1317                 break;
1318         default:
1319                 break;
1320         }
1321 }
1322
1323 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1324 {
1325         struct rdma_id_private *id_priv = iw_id->context;
1326         struct rdma_cm_event event;
1327         struct sockaddr_in *sin;
1328         int ret = 0;
1329
1330         if (cma_disable_callback(id_priv, CMA_CONNECT))
1331                 return 0;
1332
1333         memset(&event, 0, sizeof event);
1334         switch (iw_event->event) {
1335         case IW_CM_EVENT_CLOSE:
1336                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1337                 break;
1338         case IW_CM_EVENT_CONNECT_REPLY:
1339                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1340                 *sin = iw_event->local_addr;
1341                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1342                 *sin = iw_event->remote_addr;
1343                 switch (iw_event->status) {
1344                 case 0:
1345                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1346                         break;
1347                 case -ECONNRESET:
1348                 case -ECONNREFUSED:
1349                         event.event = RDMA_CM_EVENT_REJECTED;
1350                         break;
1351                 case -ETIMEDOUT:
1352                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1353                         break;
1354                 default:
1355                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1356                         break;
1357                 }
1358                 break;
1359         case IW_CM_EVENT_ESTABLISHED:
1360                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1361                 break;
1362         default:
1363                 BUG_ON(1);
1364         }
1365
1366         event.status = iw_event->status;
1367         event.param.conn.private_data = iw_event->private_data;
1368         event.param.conn.private_data_len = iw_event->private_data_len;
1369         ret = id_priv->id.event_handler(&id_priv->id, &event);
1370         if (ret) {
1371                 /* Destroy the CM ID by returning a non-zero value. */
1372                 id_priv->cm_id.iw = NULL;
1373                 cma_exch(id_priv, CMA_DESTROYING);
1374                 mutex_unlock(&id_priv->handler_mutex);
1375                 rdma_destroy_id(&id_priv->id);
1376                 return ret;
1377         }
1378
1379         mutex_unlock(&id_priv->handler_mutex);
1380         return ret;
1381 }
1382
1383 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1384                                struct iw_cm_event *iw_event)
1385 {
1386         struct rdma_cm_id *new_cm_id;
1387         struct rdma_id_private *listen_id, *conn_id;
1388         struct sockaddr_in *sin;
1389         struct net_device *dev = NULL;
1390         struct rdma_cm_event event;
1391         int ret;
1392         struct ib_device_attr attr;
1393
1394         listen_id = cm_id->context;
1395         if (cma_disable_callback(listen_id, CMA_LISTEN))
1396                 return -ECONNABORTED;
1397
1398         /* Create a new RDMA id for the new IW CM ID */
1399         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1400                                    listen_id->id.context,
1401                                    RDMA_PS_TCP);
1402         if (IS_ERR(new_cm_id)) {
1403                 ret = -ENOMEM;
1404                 goto out;
1405         }
1406         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1407         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
1408         conn_id->state = CMA_CONNECT;
1409
1410         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1411         if (!dev) {
1412                 ret = -EADDRNOTAVAIL;
1413                 mutex_unlock(&conn_id->handler_mutex);
1414                 rdma_destroy_id(new_cm_id);
1415                 goto out;
1416         }
1417         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1418         if (ret) {
1419                 mutex_unlock(&conn_id->handler_mutex);
1420                 rdma_destroy_id(new_cm_id);
1421                 goto out;
1422         }
1423
1424         ret = cma_acquire_dev(conn_id);
1425         if (ret) {
1426                 mutex_unlock(&conn_id->handler_mutex);
1427                 rdma_destroy_id(new_cm_id);
1428                 goto out;
1429         }
1430
1431         conn_id->cm_id.iw = cm_id;
1432         cm_id->context = conn_id;
1433         cm_id->cm_handler = cma_iw_handler;
1434
1435         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1436         *sin = iw_event->local_addr;
1437         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1438         *sin = iw_event->remote_addr;
1439
1440         ret = ib_query_device(conn_id->id.device, &attr);
1441         if (ret) {
1442                 mutex_unlock(&conn_id->handler_mutex);
1443                 rdma_destroy_id(new_cm_id);
1444                 goto out;
1445         }
1446
1447         memset(&event, 0, sizeof event);
1448         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1449         event.param.conn.private_data = iw_event->private_data;
1450         event.param.conn.private_data_len = iw_event->private_data_len;
1451         event.param.conn.initiator_depth = attr.max_qp_init_rd_atom;
1452         event.param.conn.responder_resources = attr.max_qp_rd_atom;
1453
1454         /*
1455          * Protect against the user destroying conn_id from another thread
1456          * until we're done accessing it.
1457          */
1458         atomic_inc(&conn_id->refcount);
1459         ret = conn_id->id.event_handler(&conn_id->id, &event);
1460         if (ret) {
1461                 /* User wants to destroy the CM ID */
1462                 conn_id->cm_id.iw = NULL;
1463                 cma_exch(conn_id, CMA_DESTROYING);
1464                 mutex_unlock(&conn_id->handler_mutex);
1465                 cma_deref_id(conn_id);
1466                 rdma_destroy_id(&conn_id->id);
1467                 goto out;
1468         }
1469
1470         mutex_unlock(&conn_id->handler_mutex);
1471         cma_deref_id(conn_id);
1472
1473 out:
1474         if (dev)
1475                 dev_put(dev);
1476         mutex_unlock(&listen_id->handler_mutex);
1477         return ret;
1478 }
1479
1480 static int cma_ib_listen(struct rdma_id_private *id_priv)
1481 {
1482         struct ib_cm_compare_data compare_data;
1483         struct sockaddr *addr;
1484         __be64 svc_id;
1485         int ret;
1486
1487         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1488                                             id_priv);
1489         if (IS_ERR(id_priv->cm_id.ib))
1490                 return PTR_ERR(id_priv->cm_id.ib);
1491
1492         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
1493         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1494         if (cma_any_addr(addr))
1495                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1496         else {
1497                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1498                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1499         }
1500
1501         if (ret) {
1502                 ib_destroy_cm_id(id_priv->cm_id.ib);
1503                 id_priv->cm_id.ib = NULL;
1504         }
1505
1506         return ret;
1507 }
1508
1509 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1510 {
1511         int ret;
1512         struct sockaddr_in *sin;
1513
1514         id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1515                                             iw_conn_req_handler,
1516                                             id_priv);
1517         if (IS_ERR(id_priv->cm_id.iw))
1518                 return PTR_ERR(id_priv->cm_id.iw);
1519
1520         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1521         id_priv->cm_id.iw->local_addr = *sin;
1522
1523         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1524
1525         if (ret) {
1526                 iw_destroy_cm_id(id_priv->cm_id.iw);
1527                 id_priv->cm_id.iw = NULL;
1528         }
1529
1530         return ret;
1531 }
1532
1533 static int cma_listen_handler(struct rdma_cm_id *id,
1534                               struct rdma_cm_event *event)
1535 {
1536         struct rdma_id_private *id_priv = id->context;
1537
1538         id->context = id_priv->id.context;
1539         id->event_handler = id_priv->id.event_handler;
1540         return id_priv->id.event_handler(id, event);
1541 }
1542
1543 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1544                               struct cma_device *cma_dev)
1545 {
1546         struct rdma_id_private *dev_id_priv;
1547         struct rdma_cm_id *id;
1548         int ret;
1549
1550         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1551         if (IS_ERR(id))
1552                 return;
1553
1554         dev_id_priv = container_of(id, struct rdma_id_private, id);
1555
1556         dev_id_priv->state = CMA_ADDR_BOUND;
1557         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1558                ip_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
1559
1560         cma_attach_to_dev(dev_id_priv, cma_dev);
1561         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1562         atomic_inc(&id_priv->refcount);
1563         dev_id_priv->internal_id = 1;
1564
1565         ret = rdma_listen(id, id_priv->backlog);
1566         if (ret)
1567                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1568                        "listening on device %s\n", ret, cma_dev->device->name);
1569 }
1570
1571 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1572 {
1573         struct cma_device *cma_dev;
1574
1575         mutex_lock(&lock);
1576         list_add_tail(&id_priv->list, &listen_any_list);
1577         list_for_each_entry(cma_dev, &dev_list, list)
1578                 cma_listen_on_dev(id_priv, cma_dev);
1579         mutex_unlock(&lock);
1580 }
1581
1582 int rdma_listen(struct rdma_cm_id *id, int backlog)
1583 {
1584         struct rdma_id_private *id_priv;
1585         int ret;
1586
1587         id_priv = container_of(id, struct rdma_id_private, id);
1588         if (id_priv->state == CMA_IDLE) {
1589                 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
1590                 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
1591                 if (ret)
1592                         return ret;
1593         }
1594
1595         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1596                 return -EINVAL;
1597
1598         id_priv->backlog = backlog;
1599         if (id->device) {
1600                 switch (rdma_node_get_transport(id->device->node_type)) {
1601                 case RDMA_TRANSPORT_IB:
1602                         ret = cma_ib_listen(id_priv);
1603                         if (ret)
1604                                 goto err;
1605                         break;
1606                 case RDMA_TRANSPORT_IWARP:
1607                         ret = cma_iw_listen(id_priv, backlog);
1608                         if (ret)
1609                                 goto err;
1610                         break;
1611                 default:
1612                         ret = -ENOSYS;
1613                         goto err;
1614                 }
1615         } else
1616                 cma_listen_on_all(id_priv);
1617
1618         return 0;
1619 err:
1620         id_priv->backlog = 0;
1621         cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1622         return ret;
1623 }
1624 EXPORT_SYMBOL(rdma_listen);
1625
1626 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1627 {
1628         struct rdma_id_private *id_priv;
1629
1630         id_priv = container_of(id, struct rdma_id_private, id);
1631         id_priv->tos = (u8) tos;
1632 }
1633 EXPORT_SYMBOL(rdma_set_service_type);
1634
1635 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1636                               void *context)
1637 {
1638         struct cma_work *work = context;
1639         struct rdma_route *route;
1640
1641         route = &work->id->id.route;
1642
1643         if (!status) {
1644                 route->num_paths = 1;
1645                 *route->path_rec = *path_rec;
1646         } else {
1647                 work->old_state = CMA_ROUTE_QUERY;
1648                 work->new_state = CMA_ADDR_RESOLVED;
1649                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1650                 work->event.status = status;
1651         }
1652
1653         queue_work(cma_wq, &work->work);
1654 }
1655
1656 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1657                               struct cma_work *work)
1658 {
1659         struct rdma_addr *addr = &id_priv->id.route.addr;
1660         struct ib_sa_path_rec path_rec;
1661         ib_sa_comp_mask comp_mask;
1662         struct sockaddr_in6 *sin6;
1663
1664         memset(&path_rec, 0, sizeof path_rec);
1665         rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1666         rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1667         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1668         path_rec.numb_path = 1;
1669         path_rec.reversible = 1;
1670         path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1671                                                         (struct sockaddr *) &addr->dst_addr);
1672
1673         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1674                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1675                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1676
1677         if (addr->src_addr.ss_family == AF_INET) {
1678                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1679                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1680         } else {
1681                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1682                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1683                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1684         }
1685
1686         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1687                                                id_priv->id.port_num, &path_rec,
1688                                                comp_mask, timeout_ms,
1689                                                GFP_KERNEL, cma_query_handler,
1690                                                work, &id_priv->query);
1691
1692         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1693 }
1694
1695 static void cma_work_handler(struct work_struct *_work)
1696 {
1697         struct cma_work *work = container_of(_work, struct cma_work, work);
1698         struct rdma_id_private *id_priv = work->id;
1699         int destroy = 0;
1700
1701         mutex_lock(&id_priv->handler_mutex);
1702         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1703                 goto out;
1704
1705         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1706                 cma_exch(id_priv, CMA_DESTROYING);
1707                 destroy = 1;
1708         }
1709 out:
1710         mutex_unlock(&id_priv->handler_mutex);
1711         cma_deref_id(id_priv);
1712         if (destroy)
1713                 rdma_destroy_id(&id_priv->id);
1714         kfree(work);
1715 }
1716
1717 static void cma_ndev_work_handler(struct work_struct *_work)
1718 {
1719         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1720         struct rdma_id_private *id_priv = work->id;
1721         int destroy = 0;
1722
1723         mutex_lock(&id_priv->handler_mutex);
1724         if (id_priv->state == CMA_DESTROYING ||
1725             id_priv->state == CMA_DEVICE_REMOVAL)
1726                 goto out;
1727
1728         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1729                 cma_exch(id_priv, CMA_DESTROYING);
1730                 destroy = 1;
1731         }
1732
1733 out:
1734         mutex_unlock(&id_priv->handler_mutex);
1735         cma_deref_id(id_priv);
1736         if (destroy)
1737                 rdma_destroy_id(&id_priv->id);
1738         kfree(work);
1739 }
1740
1741 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1742 {
1743         struct rdma_route *route = &id_priv->id.route;
1744         struct cma_work *work;
1745         int ret;
1746
1747         work = kzalloc(sizeof *work, GFP_KERNEL);
1748         if (!work)
1749                 return -ENOMEM;
1750
1751         work->id = id_priv;
1752         INIT_WORK(&work->work, cma_work_handler);
1753         work->old_state = CMA_ROUTE_QUERY;
1754         work->new_state = CMA_ROUTE_RESOLVED;
1755         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1756
1757         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1758         if (!route->path_rec) {
1759                 ret = -ENOMEM;
1760                 goto err1;
1761         }
1762
1763         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1764         if (ret)
1765                 goto err2;
1766
1767         return 0;
1768 err2:
1769         kfree(route->path_rec);
1770         route->path_rec = NULL;
1771 err1:
1772         kfree(work);
1773         return ret;
1774 }
1775
1776 int rdma_set_ib_paths(struct rdma_cm_id *id,
1777                       struct ib_sa_path_rec *path_rec, int num_paths)
1778 {
1779         struct rdma_id_private *id_priv;
1780         int ret;
1781
1782         id_priv = container_of(id, struct rdma_id_private, id);
1783         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1784                 return -EINVAL;
1785
1786         id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1787                                      GFP_KERNEL);
1788         if (!id->route.path_rec) {
1789                 ret = -ENOMEM;
1790                 goto err;
1791         }
1792
1793         id->route.num_paths = num_paths;
1794         return 0;
1795 err:
1796         cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1797         return ret;
1798 }
1799 EXPORT_SYMBOL(rdma_set_ib_paths);
1800
1801 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1802 {
1803         struct cma_work *work;
1804
1805         work = kzalloc(sizeof *work, GFP_KERNEL);
1806         if (!work)
1807                 return -ENOMEM;
1808
1809         work->id = id_priv;
1810         INIT_WORK(&work->work, cma_work_handler);
1811         work->old_state = CMA_ROUTE_QUERY;
1812         work->new_state = CMA_ROUTE_RESOLVED;
1813         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1814         queue_work(cma_wq, &work->work);
1815         return 0;
1816 }
1817
1818 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1819 {
1820         struct rdma_route *route = &id_priv->id.route;
1821         struct rdma_addr *addr = &route->addr;
1822         struct cma_work *work;
1823         int ret;
1824         struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1825         struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1826         struct net_device *ndev = NULL;
1827         u16 vid;
1828
1829         if (src_addr->sin_family != dst_addr->sin_family)
1830                 return -EINVAL;
1831
1832         work = kzalloc(sizeof *work, GFP_KERNEL);
1833         if (!work)
1834                 return -ENOMEM;
1835
1836         work->id = id_priv;
1837         INIT_WORK(&work->work, cma_work_handler);
1838
1839         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1840         if (!route->path_rec) {
1841                 ret = -ENOMEM;
1842                 goto err1;
1843         }
1844
1845         route->num_paths = 1;
1846
1847         if (addr->dev_addr.bound_dev_if)
1848                 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1849         if (!ndev) {
1850                 ret = -ENODEV;
1851                 goto err2;
1852         }
1853
1854         vid = rdma_vlan_dev_vlan_id(ndev);
1855
1856         iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1857         iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1858
1859         route->path_rec->hop_limit = 1;
1860         route->path_rec->reversible = 1;
1861         route->path_rec->pkey = cpu_to_be16(0xffff);
1862         route->path_rec->mtu_selector = IB_SA_EQ;
1863         route->path_rec->sl = id_priv->tos >> 5;
1864
1865         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1866         route->path_rec->rate_selector = IB_SA_EQ;
1867         route->path_rec->rate = iboe_get_rate(ndev);
1868         dev_put(ndev);
1869         route->path_rec->packet_life_time_selector = IB_SA_EQ;
1870         route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1871         if (!route->path_rec->mtu) {
1872                 ret = -EINVAL;
1873                 goto err2;
1874         }
1875
1876         work->old_state = CMA_ROUTE_QUERY;
1877         work->new_state = CMA_ROUTE_RESOLVED;
1878         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1879         work->event.status = 0;
1880
1881         queue_work(cma_wq, &work->work);
1882
1883         return 0;
1884
1885 err2:
1886         kfree(route->path_rec);
1887         route->path_rec = NULL;
1888 err1:
1889         kfree(work);
1890         return ret;
1891 }
1892
1893 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1894 {
1895         struct rdma_id_private *id_priv;
1896         int ret;
1897
1898         id_priv = container_of(id, struct rdma_id_private, id);
1899         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1900                 return -EINVAL;
1901
1902         atomic_inc(&id_priv->refcount);
1903         switch (rdma_node_get_transport(id->device->node_type)) {
1904         case RDMA_TRANSPORT_IB:
1905                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1906                 case IB_LINK_LAYER_INFINIBAND:
1907                         ret = cma_resolve_ib_route(id_priv, timeout_ms);
1908                         break;
1909                 case IB_LINK_LAYER_ETHERNET:
1910                         ret = cma_resolve_iboe_route(id_priv);
1911                         break;
1912                 default:
1913                         ret = -ENOSYS;
1914                 }
1915                 break;
1916         case RDMA_TRANSPORT_IWARP:
1917                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1918                 break;
1919         default:
1920                 ret = -ENOSYS;
1921                 break;
1922         }
1923         if (ret)
1924                 goto err;
1925
1926         return 0;
1927 err:
1928         cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1929         cma_deref_id(id_priv);
1930         return ret;
1931 }
1932 EXPORT_SYMBOL(rdma_resolve_route);
1933
1934 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1935 {
1936         struct cma_device *cma_dev;
1937         struct ib_port_attr port_attr;
1938         union ib_gid gid;
1939         u16 pkey;
1940         int ret;
1941         u8 p;
1942
1943         mutex_lock(&lock);
1944         if (list_empty(&dev_list)) {
1945                 ret = -ENODEV;
1946                 goto out;
1947         }
1948         list_for_each_entry(cma_dev, &dev_list, list)
1949                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1950                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1951                             port_attr.state == IB_PORT_ACTIVE)
1952                                 goto port_found;
1953
1954         p = 1;
1955         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1956
1957 port_found:
1958         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1959         if (ret)
1960                 goto out;
1961
1962         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1963         if (ret)
1964                 goto out;
1965
1966         id_priv->id.route.addr.dev_addr.dev_type =
1967                 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
1968                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
1969
1970         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1971         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1972         id_priv->id.port_num = p;
1973         cma_attach_to_dev(id_priv, cma_dev);
1974 out:
1975         mutex_unlock(&lock);
1976         return ret;
1977 }
1978
1979 static void addr_handler(int status, struct sockaddr *src_addr,
1980                          struct rdma_dev_addr *dev_addr, void *context)
1981 {
1982         struct rdma_id_private *id_priv = context;
1983         struct rdma_cm_event event;
1984
1985         memset(&event, 0, sizeof event);
1986         mutex_lock(&id_priv->handler_mutex);
1987         if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED))
1988                 goto out;
1989
1990         if (!status && !id_priv->cma_dev)
1991                 status = cma_acquire_dev(id_priv);
1992
1993         if (status) {
1994                 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1995                         goto out;
1996                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1997                 event.status = status;
1998         } else {
1999                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
2000                        ip_addr_size(src_addr));
2001                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2002         }
2003
2004         if (id_priv->id.event_handler(&id_priv->id, &event)) {
2005                 cma_exch(id_priv, CMA_DESTROYING);
2006                 mutex_unlock(&id_priv->handler_mutex);
2007                 cma_deref_id(id_priv);
2008                 rdma_destroy_id(&id_priv->id);
2009                 return;
2010         }
2011 out:
2012         mutex_unlock(&id_priv->handler_mutex);
2013         cma_deref_id(id_priv);
2014 }
2015
2016 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2017 {
2018         struct cma_work *work;
2019         struct sockaddr *src, *dst;
2020         union ib_gid gid;
2021         int ret;
2022
2023         work = kzalloc(sizeof *work, GFP_KERNEL);
2024         if (!work)
2025                 return -ENOMEM;
2026
2027         if (!id_priv->cma_dev) {
2028                 ret = cma_bind_loopback(id_priv);
2029                 if (ret)
2030                         goto err;
2031         }
2032
2033         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2034         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
2035
2036         src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2037         if (cma_zero_addr(src)) {
2038                 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2039                 if ((src->sa_family = dst->sa_family) == AF_INET) {
2040                         ((struct sockaddr_in *) src)->sin_addr.s_addr =
2041                                 ((struct sockaddr_in *) dst)->sin_addr.s_addr;
2042                 } else {
2043                         ipv6_addr_copy(&((struct sockaddr_in6 *) src)->sin6_addr,
2044                                        &((struct sockaddr_in6 *) dst)->sin6_addr);
2045                 }
2046         }
2047
2048         work->id = id_priv;
2049         INIT_WORK(&work->work, cma_work_handler);
2050         work->old_state = CMA_ADDR_QUERY;
2051         work->new_state = CMA_ADDR_RESOLVED;
2052         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2053         queue_work(cma_wq, &work->work);
2054         return 0;
2055 err:
2056         kfree(work);
2057         return ret;
2058 }
2059
2060 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2061                          struct sockaddr *dst_addr)
2062 {
2063         if (!src_addr || !src_addr->sa_family) {
2064                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2065                 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2066                         ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2067                                 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2068                 }
2069         }
2070         return rdma_bind_addr(id, src_addr);
2071 }
2072
2073 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2074                       struct sockaddr *dst_addr, int timeout_ms)
2075 {
2076         struct rdma_id_private *id_priv;
2077         int ret;
2078
2079         id_priv = container_of(id, struct rdma_id_private, id);
2080         if (id_priv->state == CMA_IDLE) {
2081                 ret = cma_bind_addr(id, src_addr, dst_addr);
2082                 if (ret)
2083                         return ret;
2084         }
2085
2086         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
2087                 return -EINVAL;
2088
2089         atomic_inc(&id_priv->refcount);
2090         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
2091         if (cma_any_addr(dst_addr))
2092                 ret = cma_resolve_loopback(id_priv);
2093         else
2094                 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
2095                                       dst_addr, &id->route.addr.dev_addr,
2096                                       timeout_ms, addr_handler, id_priv);
2097         if (ret)
2098                 goto err;
2099
2100         return 0;
2101 err:
2102         cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
2103         cma_deref_id(id_priv);
2104         return ret;
2105 }
2106 EXPORT_SYMBOL(rdma_resolve_addr);
2107
2108 static void cma_bind_port(struct rdma_bind_list *bind_list,
2109                           struct rdma_id_private *id_priv)
2110 {
2111         struct sockaddr_in *sin;
2112
2113         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
2114         sin->sin_port = htons(bind_list->port);
2115         id_priv->bind_list = bind_list;
2116         hlist_add_head(&id_priv->node, &bind_list->owners);
2117 }
2118
2119 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2120                           unsigned short snum)
2121 {
2122         struct rdma_bind_list *bind_list;
2123         int port, ret;
2124
2125         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
2126         if (!bind_list)
2127                 return -ENOMEM;
2128
2129         do {
2130                 ret = idr_get_new_above(ps, bind_list, snum, &port);
2131         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
2132
2133         if (ret)
2134                 goto err1;
2135
2136         if (port != snum) {
2137                 ret = -EADDRNOTAVAIL;
2138                 goto err2;
2139         }
2140
2141         bind_list->ps = ps;
2142         bind_list->port = (unsigned short) port;
2143         cma_bind_port(bind_list, id_priv);
2144         return 0;
2145 err2:
2146         idr_remove(ps, port);
2147 err1:
2148         kfree(bind_list);
2149         return ret;
2150 }
2151
2152 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2153 {
2154         static unsigned int last_used_port;
2155         int low, high, remaining;
2156         unsigned int rover;
2157
2158         inet_get_local_port_range(&low, &high);
2159         remaining = (high - low) + 1;
2160         rover = net_random() % remaining + low;
2161 retry:
2162         if (last_used_port != rover &&
2163             !idr_find(ps, (unsigned short) rover)) {
2164                 int ret = cma_alloc_port(ps, id_priv, rover);
2165                 /*
2166                  * Remember previously used port number in order to avoid
2167                  * re-using same port immediately after it is closed.
2168                  */
2169                 if (!ret)
2170                         last_used_port = rover;
2171                 if (ret != -EADDRNOTAVAIL)
2172                         return ret;
2173         }
2174         if (--remaining) {
2175                 rover++;
2176                 if ((rover < low) || (rover > high))
2177                         rover = low;
2178                 goto retry;
2179         }
2180         return -EADDRNOTAVAIL;
2181 }
2182
2183 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2184 {
2185         struct rdma_id_private *cur_id;
2186         struct sockaddr *addr, *cur_addr;
2187         struct rdma_bind_list *bind_list;
2188         struct hlist_node *node;
2189         unsigned short snum;
2190
2191         addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2192         snum = ntohs(cma_port(addr));
2193         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2194                 return -EACCES;
2195
2196         bind_list = idr_find(ps, snum);
2197         if (!bind_list)
2198                 return cma_alloc_port(ps, id_priv, snum);
2199
2200         /*
2201          * We don't support binding to any address if anyone is bound to
2202          * a specific address on the same port.
2203          */
2204         if (cma_any_addr(addr))
2205                 return -EADDRNOTAVAIL;
2206
2207         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2208                 cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2209                 if (cma_any_addr(cur_addr))
2210                         return -EADDRNOTAVAIL;
2211
2212                 if (!cma_addr_cmp(addr, cur_addr))
2213                         return -EADDRINUSE;
2214         }
2215
2216         cma_bind_port(bind_list, id_priv);
2217         return 0;
2218 }
2219
2220 static int cma_get_port(struct rdma_id_private *id_priv)
2221 {
2222         struct idr *ps;
2223         int ret;
2224
2225         switch (id_priv->id.ps) {
2226         case RDMA_PS_SDP:
2227                 ps = &sdp_ps;
2228                 break;
2229         case RDMA_PS_TCP:
2230                 ps = &tcp_ps;
2231                 break;
2232         case RDMA_PS_UDP:
2233                 ps = &udp_ps;
2234                 break;
2235         case RDMA_PS_IPOIB:
2236                 ps = &ipoib_ps;
2237                 break;
2238         default:
2239                 return -EPROTONOSUPPORT;
2240         }
2241
2242         mutex_lock(&lock);
2243         if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
2244                 ret = cma_alloc_any_port(ps, id_priv);
2245         else
2246                 ret = cma_use_port(ps, id_priv);
2247         mutex_unlock(&lock);
2248
2249         return ret;
2250 }
2251
2252 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2253                                struct sockaddr *addr)
2254 {
2255 #if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
2256         struct sockaddr_in6 *sin6;
2257
2258         if (addr->sa_family != AF_INET6)
2259                 return 0;
2260
2261         sin6 = (struct sockaddr_in6 *) addr;
2262         if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2263             !sin6->sin6_scope_id)
2264                         return -EINVAL;
2265
2266         dev_addr->bound_dev_if = sin6->sin6_scope_id;
2267 #endif
2268         return 0;
2269 }
2270
2271 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2272 {
2273         struct rdma_id_private *id_priv;
2274         int ret;
2275
2276         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
2277                 return -EAFNOSUPPORT;
2278
2279         id_priv = container_of(id, struct rdma_id_private, id);
2280         if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
2281                 return -EINVAL;
2282
2283         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2284         if (ret)
2285                 goto err1;
2286
2287         if (!cma_any_addr(addr)) {
2288                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2289                 if (ret)
2290                         goto err1;
2291
2292                 ret = cma_acquire_dev(id_priv);
2293                 if (ret)
2294                         goto err1;
2295         }
2296
2297         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2298         ret = cma_get_port(id_priv);
2299         if (ret)
2300                 goto err2;
2301
2302         return 0;
2303 err2:
2304         if (id_priv->cma_dev)
2305                 cma_release_dev(id_priv);
2306 err1:
2307         cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2308         return ret;
2309 }
2310 EXPORT_SYMBOL(rdma_bind_addr);
2311
2312 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2313                           struct rdma_route *route)
2314 {
2315         struct cma_hdr *cma_hdr;
2316         struct sdp_hh *sdp_hdr;
2317
2318         if (route->addr.src_addr.ss_family == AF_INET) {
2319                 struct sockaddr_in *src4, *dst4;
2320
2321                 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2322                 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2323
2324                 switch (ps) {
2325                 case RDMA_PS_SDP:
2326                         sdp_hdr = hdr;
2327                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2328                                 return -EINVAL;
2329                         sdp_set_ip_ver(sdp_hdr, 4);
2330                         sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2331                         sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2332                         sdp_hdr->port = src4->sin_port;
2333                         break;
2334                 default:
2335                         cma_hdr = hdr;
2336                         cma_hdr->cma_version = CMA_VERSION;
2337                         cma_set_ip_ver(cma_hdr, 4);
2338                         cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2339                         cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2340                         cma_hdr->port = src4->sin_port;
2341                         break;
2342                 }
2343         } else {
2344                 struct sockaddr_in6 *src6, *dst6;
2345
2346                 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2347                 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2348
2349                 switch (ps) {
2350                 case RDMA_PS_SDP:
2351                         sdp_hdr = hdr;
2352                         if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2353                                 return -EINVAL;
2354                         sdp_set_ip_ver(sdp_hdr, 6);
2355                         sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2356                         sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2357                         sdp_hdr->port = src6->sin6_port;
2358                         break;
2359                 default:
2360                         cma_hdr = hdr;
2361                         cma_hdr->cma_version = CMA_VERSION;
2362                         cma_set_ip_ver(cma_hdr, 6);
2363                         cma_hdr->src_addr.ip6 = src6->sin6_addr;
2364                         cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2365                         cma_hdr->port = src6->sin6_port;
2366                         break;
2367                 }
2368         }
2369         return 0;
2370 }
2371
2372 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2373                                 struct ib_cm_event *ib_event)
2374 {
2375         struct rdma_id_private *id_priv = cm_id->context;
2376         struct rdma_cm_event event;
2377         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2378         int ret = 0;
2379
2380         if (cma_disable_callback(id_priv, CMA_CONNECT))
2381                 return 0;
2382
2383         memset(&event, 0, sizeof event);
2384         switch (ib_event->event) {
2385         case IB_CM_SIDR_REQ_ERROR:
2386                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2387                 event.status = -ETIMEDOUT;
2388                 break;
2389         case IB_CM_SIDR_REP_RECEIVED:
2390                 event.param.ud.private_data = ib_event->private_data;
2391                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2392                 if (rep->status != IB_SIDR_SUCCESS) {
2393                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2394                         event.status = ib_event->param.sidr_rep_rcvd.status;
2395                         break;
2396                 }
2397                 ret = cma_set_qkey(id_priv);
2398                 if (ret) {
2399                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
2400                         event.status = -EINVAL;
2401                         break;
2402                 }
2403                 if (id_priv->qkey != rep->qkey) {
2404                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2405                         event.status = -EINVAL;
2406                         break;
2407                 }
2408                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2409                                      id_priv->id.route.path_rec,
2410                                      &event.param.ud.ah_attr);
2411                 event.param.ud.qp_num = rep->qpn;
2412                 event.param.ud.qkey = rep->qkey;
2413                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2414                 event.status = 0;
2415                 break;
2416         default:
2417                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
2418                        ib_event->event);
2419                 goto out;
2420         }
2421
2422         ret = id_priv->id.event_handler(&id_priv->id, &event);
2423         if (ret) {
2424                 /* Destroy the CM ID by returning a non-zero value. */
2425                 id_priv->cm_id.ib = NULL;
2426                 cma_exch(id_priv, CMA_DESTROYING);
2427                 mutex_unlock(&id_priv->handler_mutex);
2428                 rdma_destroy_id(&id_priv->id);
2429                 return ret;
2430         }
2431 out:
2432         mutex_unlock(&id_priv->handler_mutex);
2433         return ret;
2434 }
2435
2436 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2437                               struct rdma_conn_param *conn_param)
2438 {
2439         struct ib_cm_sidr_req_param req;
2440         struct rdma_route *route;
2441         int ret;
2442
2443         req.private_data_len = sizeof(struct cma_hdr) +
2444                                conn_param->private_data_len;
2445         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2446         if (!req.private_data)
2447                 return -ENOMEM;
2448
2449         if (conn_param->private_data && conn_param->private_data_len)
2450                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2451                        conn_param->private_data, conn_param->private_data_len);
2452
2453         route = &id_priv->id.route;
2454         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2455         if (ret)
2456                 goto out;
2457
2458         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2459                                             cma_sidr_rep_handler, id_priv);
2460         if (IS_ERR(id_priv->cm_id.ib)) {
2461                 ret = PTR_ERR(id_priv->cm_id.ib);
2462                 goto out;
2463         }
2464
2465         req.path = route->path_rec;
2466         req.service_id = cma_get_service_id(id_priv->id.ps,
2467                                             (struct sockaddr *) &route->addr.dst_addr);
2468         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2469         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2470
2471         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2472         if (ret) {
2473                 ib_destroy_cm_id(id_priv->cm_id.ib);
2474                 id_priv->cm_id.ib = NULL;
2475         }
2476 out:
2477         kfree(req.private_data);
2478         return ret;
2479 }
2480
2481 static int cma_connect_ib(struct rdma_id_private *id_priv,
2482                           struct rdma_conn_param *conn_param)
2483 {
2484         struct ib_cm_req_param req;
2485         struct rdma_route *route;
2486         void *private_data;
2487         int offset, ret;
2488
2489         memset(&req, 0, sizeof req);
2490         offset = cma_user_data_offset(id_priv->id.ps);
2491         req.private_data_len = offset + conn_param->private_data_len;
2492         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2493         if (!private_data)
2494                 return -ENOMEM;
2495
2496         if (conn_param->private_data && conn_param->private_data_len)
2497                 memcpy(private_data + offset, conn_param->private_data,
2498                        conn_param->private_data_len);
2499
2500         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2501                                             id_priv);
2502         if (IS_ERR(id_priv->cm_id.ib)) {
2503                 ret = PTR_ERR(id_priv->cm_id.ib);
2504                 goto out;
2505         }
2506
2507         route = &id_priv->id.route;
2508         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2509         if (ret)
2510                 goto out;
2511         req.private_data = private_data;
2512
2513         req.primary_path = &route->path_rec[0];
2514         if (route->num_paths == 2)
2515                 req.alternate_path = &route->path_rec[1];
2516
2517         req.service_id = cma_get_service_id(id_priv->id.ps,
2518                                             (struct sockaddr *) &route->addr.dst_addr);
2519         req.qp_num = id_priv->qp_num;
2520         req.qp_type = IB_QPT_RC;
2521         req.starting_psn = id_priv->seq_num;
2522         req.responder_resources = conn_param->responder_resources;
2523         req.initiator_depth = conn_param->initiator_depth;
2524         req.flow_control = conn_param->flow_control;
2525         req.retry_count = conn_param->retry_count;
2526         req.rnr_retry_count = conn_param->rnr_retry_count;
2527         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2528         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2529         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2530         req.srq = id_priv->srq ? 1 : 0;
2531
2532         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2533 out:
2534         if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2535                 ib_destroy_cm_id(id_priv->cm_id.ib);
2536                 id_priv->cm_id.ib = NULL;
2537         }
2538
2539         kfree(private_data);
2540         return ret;
2541 }
2542
2543 static int cma_connect_iw(struct rdma_id_private *id_priv,
2544                           struct rdma_conn_param *conn_param)
2545 {
2546         struct iw_cm_id *cm_id;
2547         struct sockaddr_in* sin;
2548         int ret;
2549         struct iw_cm_conn_param iw_param;
2550
2551         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2552         if (IS_ERR(cm_id)) {
2553                 ret = PTR_ERR(cm_id);
2554                 goto out;
2555         }
2556
2557         id_priv->cm_id.iw = cm_id;
2558
2559         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2560         cm_id->local_addr = *sin;
2561
2562         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2563         cm_id->remote_addr = *sin;
2564
2565         ret = cma_modify_qp_rtr(id_priv, conn_param);
2566         if (ret)
2567                 goto out;
2568
2569         iw_param.ord = conn_param->initiator_depth;
2570         iw_param.ird = conn_param->responder_resources;
2571         iw_param.private_data = conn_param->private_data;
2572         iw_param.private_data_len = conn_param->private_data_len;
2573         if (id_priv->id.qp)
2574                 iw_param.qpn = id_priv->qp_num;
2575         else
2576                 iw_param.qpn = conn_param->qp_num;
2577         ret = iw_cm_connect(cm_id, &iw_param);
2578 out:
2579         if (ret && !IS_ERR(cm_id)) {
2580                 iw_destroy_cm_id(cm_id);
2581                 id_priv->cm_id.iw = NULL;
2582         }
2583         return ret;
2584 }
2585
2586 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2587 {
2588         struct rdma_id_private *id_priv;
2589         int ret;
2590
2591         id_priv = container_of(id, struct rdma_id_private, id);
2592         if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2593                 return -EINVAL;
2594
2595         if (!id->qp) {
2596                 id_priv->qp_num = conn_param->qp_num;
2597                 id_priv->srq = conn_param->srq;
2598         }
2599
2600         switch (rdma_node_get_transport(id->device->node_type)) {
2601         case RDMA_TRANSPORT_IB:
2602                 if (cma_is_ud_ps(id->ps))
2603                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2604                 else
2605                         ret = cma_connect_ib(id_priv, conn_param);
2606                 break;
2607         case RDMA_TRANSPORT_IWARP:
2608                 ret = cma_connect_iw(id_priv, conn_param);
2609                 break;
2610         default:
2611                 ret = -ENOSYS;
2612                 break;
2613         }
2614         if (ret)
2615                 goto err;
2616
2617         return 0;
2618 err:
2619         cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2620         return ret;
2621 }
2622 EXPORT_SYMBOL(rdma_connect);
2623
2624 static int cma_accept_ib(struct rdma_id_private *id_priv,
2625                          struct rdma_conn_param *conn_param)
2626 {
2627         struct ib_cm_rep_param rep;
2628         int ret;
2629
2630         ret = cma_modify_qp_rtr(id_priv, conn_param);
2631         if (ret)
2632                 goto out;
2633
2634         ret = cma_modify_qp_rts(id_priv, conn_param);
2635         if (ret)
2636                 goto out;
2637
2638         memset(&rep, 0, sizeof rep);
2639         rep.qp_num = id_priv->qp_num;
2640         rep.starting_psn = id_priv->seq_num;
2641         rep.private_data = conn_param->private_data;
2642         rep.private_data_len = conn_param->private_data_len;
2643         rep.responder_resources = conn_param->responder_resources;
2644         rep.initiator_depth = conn_param->initiator_depth;
2645         rep.failover_accepted = 0;
2646         rep.flow_control = conn_param->flow_control;
2647         rep.rnr_retry_count = conn_param->rnr_retry_count;
2648         rep.srq = id_priv->srq ? 1 : 0;
2649
2650         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2651 out:
2652         return ret;
2653 }
2654
2655 static int cma_accept_iw(struct rdma_id_private *id_priv,
2656                   struct rdma_conn_param *conn_param)
2657 {
2658         struct iw_cm_conn_param iw_param;
2659         int ret;
2660
2661         ret = cma_modify_qp_rtr(id_priv, conn_param);
2662         if (ret)
2663                 return ret;
2664
2665         iw_param.ord = conn_param->initiator_depth;
2666         iw_param.ird = conn_param->responder_resources;
2667         iw_param.private_data = conn_param->private_data;
2668         iw_param.private_data_len = conn_param->private_data_len;
2669         if (id_priv->id.qp) {
2670                 iw_param.qpn = id_priv->qp_num;
2671         } else
2672                 iw_param.qpn = conn_param->qp_num;
2673
2674         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2675 }
2676
2677 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2678                              enum ib_cm_sidr_status status,
2679                              const void *private_data, int private_data_len)
2680 {
2681         struct ib_cm_sidr_rep_param rep;
2682         int ret;
2683
2684         memset(&rep, 0, sizeof rep);
2685         rep.status = status;
2686         if (status == IB_SIDR_SUCCESS) {
2687                 ret = cma_set_qkey(id_priv);
2688                 if (ret)
2689                         return ret;
2690                 rep.qp_num = id_priv->qp_num;
2691                 rep.qkey = id_priv->qkey;
2692         }
2693         rep.private_data = private_data;
2694         rep.private_data_len = private_data_len;
2695
2696         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2697 }
2698
2699 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2700 {
2701         struct rdma_id_private *id_priv;
2702         int ret;
2703
2704         id_priv = container_of(id, struct rdma_id_private, id);
2705         if (!cma_comp(id_priv, CMA_CONNECT))
2706                 return -EINVAL;
2707
2708         if (!id->qp && conn_param) {
2709                 id_priv->qp_num = conn_param->qp_num;
2710                 id_priv->srq = conn_param->srq;
2711         }
2712
2713         switch (rdma_node_get_transport(id->device->node_type)) {
2714         case RDMA_TRANSPORT_IB:
2715                 if (cma_is_ud_ps(id->ps))
2716                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2717                                                 conn_param->private_data,
2718                                                 conn_param->private_data_len);
2719                 else if (conn_param)
2720                         ret = cma_accept_ib(id_priv, conn_param);
2721                 else
2722                         ret = cma_rep_recv(id_priv);
2723                 break;
2724         case RDMA_TRANSPORT_IWARP:
2725                 ret = cma_accept_iw(id_priv, conn_param);
2726                 break;
2727         default:
2728                 ret = -ENOSYS;
2729                 break;
2730         }
2731
2732         if (ret)
2733                 goto reject;
2734
2735         return 0;
2736 reject:
2737         cma_modify_qp_err(id_priv);
2738         rdma_reject(id, NULL, 0);
2739         return ret;
2740 }
2741 EXPORT_SYMBOL(rdma_accept);
2742
2743 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2744 {
2745         struct rdma_id_private *id_priv;
2746         int ret;
2747
2748         id_priv = container_of(id, struct rdma_id_private, id);
2749         if (!cma_has_cm_dev(id_priv))
2750                 return -EINVAL;
2751
2752         switch (id->device->node_type) {
2753         case RDMA_NODE_IB_CA:
2754                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2755                 break;
2756         default:
2757                 ret = 0;
2758                 break;
2759         }
2760         return ret;
2761 }
2762 EXPORT_SYMBOL(rdma_notify);
2763
2764 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2765                 u8 private_data_len)
2766 {
2767         struct rdma_id_private *id_priv;
2768         int ret;
2769
2770         id_priv = container_of(id, struct rdma_id_private, id);
2771         if (!cma_has_cm_dev(id_priv))
2772                 return -EINVAL;
2773
2774         switch (rdma_node_get_transport(id->device->node_type)) {
2775         case RDMA_TRANSPORT_IB:
2776                 if (cma_is_ud_ps(id->ps))
2777                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2778                                                 private_data, private_data_len);
2779                 else
2780                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2781                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2782                                              0, private_data, private_data_len);
2783                 break;
2784         case RDMA_TRANSPORT_IWARP:
2785                 ret = iw_cm_reject(id_priv->cm_id.iw,
2786                                    private_data, private_data_len);
2787                 break;
2788         default:
2789                 ret = -ENOSYS;
2790                 break;
2791         }
2792         return ret;
2793 }
2794 EXPORT_SYMBOL(rdma_reject);
2795
2796 int rdma_disconnect(struct rdma_cm_id *id)
2797 {
2798         struct rdma_id_private *id_priv;
2799         int ret;
2800
2801         id_priv = container_of(id, struct rdma_id_private, id);
2802         if (!cma_has_cm_dev(id_priv))
2803                 return -EINVAL;
2804
2805         switch (rdma_node_get_transport(id->device->node_type)) {
2806         case RDMA_TRANSPORT_IB:
2807                 ret = cma_modify_qp_err(id_priv);
2808                 if (ret)
2809                         goto out;
2810                 /* Initiate or respond to a disconnect. */
2811                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2812                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2813                 break;
2814         case RDMA_TRANSPORT_IWARP:
2815                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2816                 break;
2817         default:
2818                 ret = -EINVAL;
2819                 break;
2820         }
2821 out:
2822         return ret;
2823 }
2824 EXPORT_SYMBOL(rdma_disconnect);
2825
2826 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2827 {
2828         struct rdma_id_private *id_priv;
2829         struct cma_multicast *mc = multicast->context;
2830         struct rdma_cm_event event;
2831         int ret;
2832
2833         id_priv = mc->id_priv;
2834         if (cma_disable_callback(id_priv, CMA_ADDR_BOUND) &&
2835             cma_disable_callback(id_priv, CMA_ADDR_RESOLVED))
2836                 return 0;
2837
2838         mutex_lock(&id_priv->qp_mutex);
2839         if (!status && id_priv->id.qp)
2840                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2841                                          multicast->rec.mlid);
2842         mutex_unlock(&id_priv->qp_mutex);
2843
2844         memset(&event, 0, sizeof event);
2845         event.status = status;
2846         event.param.ud.private_data = mc->context;
2847         if (!status) {
2848                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2849                 ib_init_ah_from_mcmember(id_priv->id.device,
2850                                          id_priv->id.port_num, &multicast->rec,
2851                                          &event.param.ud.ah_attr);
2852                 event.param.ud.qp_num = 0xFFFFFF;
2853                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2854         } else
2855                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2856
2857         ret = id_priv->id.event_handler(&id_priv->id, &event);
2858         if (ret) {
2859                 cma_exch(id_priv, CMA_DESTROYING);
2860                 mutex_unlock(&id_priv->handler_mutex);
2861                 rdma_destroy_id(&id_priv->id);
2862                 return 0;
2863         }
2864
2865         mutex_unlock(&id_priv->handler_mutex);
2866         return 0;
2867 }
2868
2869 static void cma_set_mgid(struct rdma_id_private *id_priv,
2870                          struct sockaddr *addr, union ib_gid *mgid)
2871 {
2872         unsigned char mc_map[MAX_ADDR_LEN];
2873         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2874         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2875         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2876
2877         if (cma_any_addr(addr)) {
2878                 memset(mgid, 0, sizeof *mgid);
2879         } else if ((addr->sa_family == AF_INET6) &&
2880                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
2881                                                                  0xFF10A01B)) {
2882                 /* IPv6 address is an SA assigned MGID. */
2883                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2884         } else if ((addr->sa_family == AF_INET6)) {
2885                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
2886                 if (id_priv->id.ps == RDMA_PS_UDP)
2887                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2888                 *mgid = *(union ib_gid *) (mc_map + 4);
2889         } else {
2890                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
2891                 if (id_priv->id.ps == RDMA_PS_UDP)
2892                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2893                 *mgid = *(union ib_gid *) (mc_map + 4);
2894         }
2895 }
2896
2897 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2898                                  struct cma_multicast *mc)
2899 {
2900         struct ib_sa_mcmember_rec rec;
2901         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2902         ib_sa_comp_mask comp_mask;
2903         int ret;
2904
2905         ib_addr_get_mgid(dev_addr, &rec.mgid);
2906         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2907                                      &rec.mgid, &rec);
2908         if (ret)
2909                 return ret;
2910
2911         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
2912         if (id_priv->id.ps == RDMA_PS_UDP)
2913                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2914         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
2915         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2916         rec.join_state = 1;
2917
2918         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2919                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2920                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2921                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
2922                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2923
2924         if (id_priv->id.ps == RDMA_PS_IPOIB)
2925                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
2926                              IB_SA_MCMEMBER_REC_RATE_SELECTOR;
2927
2928         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2929                                                 id_priv->id.port_num, &rec,
2930                                                 comp_mask, GFP_KERNEL,
2931                                                 cma_ib_mc_handler, mc);
2932         if (IS_ERR(mc->multicast.ib))
2933                 return PTR_ERR(mc->multicast.ib);
2934
2935         return 0;
2936 }
2937
2938 static void iboe_mcast_work_handler(struct work_struct *work)
2939 {
2940         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
2941         struct cma_multicast *mc = mw->mc;
2942         struct ib_sa_multicast *m = mc->multicast.ib;
2943
2944         mc->multicast.ib->context = mc;
2945         cma_ib_mc_handler(0, m);
2946         kref_put(&mc->mcref, release_mc);
2947         kfree(mw);
2948 }
2949
2950 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
2951 {
2952         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
2953         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
2954
2955         if (cma_any_addr(addr)) {
2956                 memset(mgid, 0, sizeof *mgid);
2957         } else if (addr->sa_family == AF_INET6) {
2958                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2959         } else {
2960                 mgid->raw[0] = 0xff;
2961                 mgid->raw[1] = 0x0e;
2962                 mgid->raw[2] = 0;
2963                 mgid->raw[3] = 0;
2964                 mgid->raw[4] = 0;
2965                 mgid->raw[5] = 0;
2966                 mgid->raw[6] = 0;
2967                 mgid->raw[7] = 0;
2968                 mgid->raw[8] = 0;
2969                 mgid->raw[9] = 0;
2970                 mgid->raw[10] = 0xff;
2971                 mgid->raw[11] = 0xff;
2972                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
2973         }
2974 }
2975
2976 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
2977                                    struct cma_multicast *mc)
2978 {
2979         struct iboe_mcast_work *work;
2980         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2981         int err;
2982         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
2983         struct net_device *ndev = NULL;
2984
2985         if (cma_zero_addr((struct sockaddr *)&mc->addr))
2986                 return -EINVAL;
2987
2988         work = kzalloc(sizeof *work, GFP_KERNEL);
2989         if (!work)
2990                 return -ENOMEM;
2991
2992         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
2993         if (!mc->multicast.ib) {
2994                 err = -ENOMEM;
2995                 goto out1;
2996         }
2997
2998         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
2999
3000         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3001         if (id_priv->id.ps == RDMA_PS_UDP)
3002                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3003
3004         if (dev_addr->bound_dev_if)
3005                 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3006         if (!ndev) {
3007                 err = -ENODEV;
3008                 goto out2;
3009         }
3010         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3011         mc->multicast.ib->rec.hop_limit = 1;
3012         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3013         dev_put(ndev);
3014         if (!mc->multicast.ib->rec.mtu) {
3015                 err = -EINVAL;
3016                 goto out2;
3017         }
3018         iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3019         work->id = id_priv;
3020         work->mc = mc;
3021         INIT_WORK(&work->work, iboe_mcast_work_handler);
3022         kref_get(&mc->mcref);
3023         queue_work(cma_wq, &work->work);
3024
3025         return 0;
3026
3027 out2:
3028         kfree(mc->multicast.ib);
3029 out1:
3030         kfree(work);
3031         return err;
3032 }
3033
3034 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3035                         void *context)
3036 {
3037         struct rdma_id_private *id_priv;
3038         struct cma_multicast *mc;
3039         int ret;
3040
3041         id_priv = container_of(id, struct rdma_id_private, id);
3042         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
3043             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
3044                 return -EINVAL;
3045
3046         mc = kmalloc(sizeof *mc, GFP_KERNEL);
3047         if (!mc)
3048                 return -ENOMEM;
3049
3050         memcpy(&mc->addr, addr, ip_addr_size(addr));
3051         mc->context = context;
3052         mc->id_priv = id_priv;
3053
3054         spin_lock(&id_priv->lock);
3055         list_add(&mc->list, &id_priv->mc_list);
3056         spin_unlock(&id_priv->lock);
3057
3058         switch (rdma_node_get_transport(id->device->node_type)) {
3059         case RDMA_TRANSPORT_IB:
3060                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3061                 case IB_LINK_LAYER_INFINIBAND:
3062                         ret = cma_join_ib_multicast(id_priv, mc);
3063                         break;
3064                 case IB_LINK_LAYER_ETHERNET:
3065                         kref_init(&mc->mcref);
3066                         ret = cma_iboe_join_multicast(id_priv, mc);
3067                         break;
3068                 default:
3069                         ret = -EINVAL;
3070                 }
3071                 break;
3072         default:
3073                 ret = -ENOSYS;
3074                 break;
3075         }
3076
3077         if (ret) {
3078                 spin_lock_irq(&id_priv->lock);
3079                 list_del(&mc->list);
3080                 spin_unlock_irq(&id_priv->lock);
3081                 kfree(mc);
3082         }
3083         return ret;
3084 }
3085 EXPORT_SYMBOL(rdma_join_multicast);
3086
3087 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3088 {
3089         struct rdma_id_private *id_priv;
3090         struct cma_multicast *mc;
3091
3092         id_priv = container_of(id, struct rdma_id_private, id);
3093         spin_lock_irq(&id_priv->lock);
3094         list_for_each_entry(mc, &id_priv->mc_list, list) {
3095                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
3096                         list_del(&mc->list);
3097                         spin_unlock_irq(&id_priv->lock);
3098
3099                         if (id->qp)
3100                                 ib_detach_mcast(id->qp,
3101                                                 &mc->multicast.ib->rec.mgid,
3102                                                 mc->multicast.ib->rec.mlid);
3103                         if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3104                                 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3105                                 case IB_LINK_LAYER_INFINIBAND:
3106                                         ib_sa_free_multicast(mc->multicast.ib);
3107                                         kfree(mc);
3108                                         break;
3109                                 case IB_LINK_LAYER_ETHERNET:
3110                                         kref_put(&mc->mcref, release_mc);
3111                                         break;
3112                                 default:
3113                                         break;
3114                                 }
3115                         }
3116                         return;
3117                 }
3118         }
3119         spin_unlock_irq(&id_priv->lock);
3120 }
3121 EXPORT_SYMBOL(rdma_leave_multicast);
3122
3123 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3124 {
3125         struct rdma_dev_addr *dev_addr;
3126         struct cma_ndev_work *work;
3127
3128         dev_addr = &id_priv->id.route.addr.dev_addr;
3129
3130         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
3131             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3132                 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3133                        ndev->name, &id_priv->id);
3134                 work = kzalloc(sizeof *work, GFP_KERNEL);
3135                 if (!work)
3136                         return -ENOMEM;
3137
3138                 INIT_WORK(&work->work, cma_ndev_work_handler);
3139                 work->id = id_priv;
3140                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3141                 atomic_inc(&id_priv->refcount);
3142                 queue_work(cma_wq, &work->work);
3143         }
3144
3145         return 0;
3146 }
3147
3148 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
3149                                void *ctx)
3150 {
3151         struct net_device *ndev = (struct net_device *)ctx;
3152         struct cma_device *cma_dev;
3153         struct rdma_id_private *id_priv;
3154         int ret = NOTIFY_DONE;
3155
3156         if (dev_net(ndev) != &init_net)
3157                 return NOTIFY_DONE;
3158
3159         if (event != NETDEV_BONDING_FAILOVER)
3160                 return NOTIFY_DONE;
3161
3162         if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3163                 return NOTIFY_DONE;
3164
3165         mutex_lock(&lock);
3166         list_for_each_entry(cma_dev, &dev_list, list)
3167                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3168                         ret = cma_netdev_change(ndev, id_priv);
3169                         if (ret)
3170                                 goto out;
3171                 }
3172
3173 out:
3174         mutex_unlock(&lock);
3175         return ret;
3176 }
3177
3178 static struct notifier_block cma_nb = {
3179         .notifier_call = cma_netdev_callback
3180 };
3181
3182 static void cma_add_one(struct ib_device *device)
3183 {
3184         struct cma_device *cma_dev;
3185         struct rdma_id_private *id_priv;
3186
3187         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3188         if (!cma_dev)
3189                 return;
3190
3191         cma_dev->device = device;
3192
3193         init_completion(&cma_dev->comp);
3194         atomic_set(&cma_dev->refcount, 1);
3195         INIT_LIST_HEAD(&cma_dev->id_list);
3196         ib_set_client_data(device, &cma_client, cma_dev);
3197
3198         mutex_lock(&lock);
3199         list_add_tail(&cma_dev->list, &dev_list);
3200         list_for_each_entry(id_priv, &listen_any_list, list)
3201                 cma_listen_on_dev(id_priv, cma_dev);
3202         mutex_unlock(&lock);
3203 }
3204
3205 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3206 {
3207         struct rdma_cm_event event;
3208         enum cma_state state;
3209         int ret = 0;
3210
3211         /* Record that we want to remove the device */
3212         state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
3213         if (state == CMA_DESTROYING)
3214                 return 0;
3215
3216         cma_cancel_operation(id_priv, state);
3217         mutex_lock(&id_priv->handler_mutex);
3218
3219         /* Check for destruction from another callback. */
3220         if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
3221                 goto out;
3222
3223         memset(&event, 0, sizeof event);
3224         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
3225         ret = id_priv->id.event_handler(&id_priv->id, &event);
3226 out:
3227         mutex_unlock(&id_priv->handler_mutex);
3228         return ret;
3229 }
3230
3231 static void cma_process_remove(struct cma_device *cma_dev)
3232 {
3233         struct rdma_id_private *id_priv;
3234         int ret;
3235
3236         mutex_lock(&lock);
3237         while (!list_empty(&cma_dev->id_list)) {
3238                 id_priv = list_entry(cma_dev->id_list.next,
3239                                      struct rdma_id_private, list);
3240
3241                 list_del(&id_priv->listen_list);
3242                 list_del_init(&id_priv->list);
3243                 atomic_inc(&id_priv->refcount);
3244                 mutex_unlock(&lock);
3245
3246                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
3247                 cma_deref_id(id_priv);
3248                 if (ret)
3249                         rdma_destroy_id(&id_priv->id);
3250
3251                 mutex_lock(&lock);
3252         }
3253         mutex_unlock(&lock);
3254
3255         cma_deref_dev(cma_dev);
3256         wait_for_completion(&cma_dev->comp);
3257 }
3258
3259 static void cma_remove_one(struct ib_device *device)
3260 {
3261         struct cma_device *cma_dev;
3262
3263         cma_dev = ib_get_client_data(device, &cma_client);
3264         if (!cma_dev)
3265                 return;
3266
3267         mutex_lock(&lock);
3268         list_del(&cma_dev->list);
3269         mutex_unlock(&lock);
3270
3271         cma_process_remove(cma_dev);
3272         kfree(cma_dev);
3273 }
3274
3275 static int __init cma_init(void)
3276 {
3277         int ret;
3278
3279         cma_wq = create_singlethread_workqueue("rdma_cm");
3280         if (!cma_wq)
3281                 return -ENOMEM;
3282
3283         ib_sa_register_client(&sa_client);
3284         rdma_addr_register_client(&addr_client);
3285         register_netdevice_notifier(&cma_nb);
3286
3287         ret = ib_register_client(&cma_client);
3288         if (ret)
3289                 goto err;
3290         return 0;
3291
3292 err:
3293         unregister_netdevice_notifier(&cma_nb);
3294         rdma_addr_unregister_client(&addr_client);
3295         ib_sa_unregister_client(&sa_client);
3296         destroy_workqueue(cma_wq);
3297         return ret;
3298 }
3299
3300 static void __exit cma_cleanup(void)
3301 {
3302         ib_unregister_client(&cma_client);
3303         unregister_netdevice_notifier(&cma_nb);
3304         rdma_addr_unregister_client(&addr_client);
3305         ib_sa_unregister_client(&sa_client);
3306         destroy_workqueue(cma_wq);
3307         idr_destroy(&sdp_ps);
3308         idr_destroy(&tcp_ps);
3309         idr_destroy(&udp_ps);
3310         idr_destroy(&ipoib_ps);
3311 }
3312
3313 module_init(cma_init);
3314 module_exit(cma_cleanup);