drivers/infiniband/core/cma.c

   1 /*
   2  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   3  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   4  * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
   5  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/in.h>
  38 #include <linux/in6.h>
  39 #include <linux/mutex.h>
  40 #include <linux/random.h>
  41 #include <linux/idr.h>
  42 #include <linux/inetdevice.h>
  43
  44 #include <net/tcp.h>
  45
  46 #include <rdma/rdma_cm.h>
  47 #include <rdma/rdma_cm_ib.h>
  48 #include <rdma/ib_cache.h>
  49 #include <rdma/ib_cm.h>
  50 #include <rdma/ib_sa.h>
  51 #include <rdma/iw_cm.h>
  52
  53 MODULE_AUTHOR("Sean Hefty");
  54 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  55 MODULE_LICENSE("Dual BSD/GPL");
  56
  57 #define CMA_CM_RESPONSE_TIMEOUT 20
  58 #define CMA_MAX_CM_RETRIES 15
  59 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  60
  61 static void cma_add_one(struct ib_device *device);
  62 static void cma_remove_one(struct ib_device *device);
  63
  64 static struct ib_client cma_client = {
  65         .name   = "cma",
  66         .add    = cma_add_one,
  67         .remove = cma_remove_one
  68 };
  69
  70 static struct ib_sa_client sa_client;
  71 static struct rdma_addr_client addr_client;
  72 static LIST_HEAD(dev_list);
  73 static LIST_HEAD(listen_any_list);
  74 static DEFINE_MUTEX(lock);
  75 static struct workqueue_struct *cma_wq;
  76 static DEFINE_IDR(sdp_ps);
  77 static DEFINE_IDR(tcp_ps);
  78 static DEFINE_IDR(udp_ps);
  79 static DEFINE_IDR(ipoib_ps);
  80 static int next_port;
  81
  82 struct cma_device {
  83         struct list_head        list;
  84         struct ib_device        *device;
  85         struct completion       comp;
  86         atomic_t                refcount;
  87         struct list_head        id_list;
  88 };
  89
  90 enum cma_state {
  91         CMA_IDLE,
  92         CMA_ADDR_QUERY,
  93         CMA_ADDR_RESOLVED,
  94         CMA_ROUTE_QUERY,
  95         CMA_ROUTE_RESOLVED,
  96         CMA_CONNECT,
  97         CMA_DISCONNECT,
  98         CMA_ADDR_BOUND,
  99         CMA_LISTEN,
 100         CMA_DEVICE_REMOVAL,
 101         CMA_DESTROYING
 102 };
 103
 104 struct rdma_bind_list {
 105         struct idr              *ps;
 106         struct hlist_head       owners;
 107         unsigned short          port;
 108 };
 109
 110 /*
 111  * Device removal can occur at anytime, so we need extra handling to
 112  * serialize notifying the user of device removal with other callbacks.
 113  * We do this by disabling removal notification while a callback is in process,
 114  * and reporting it after the callback completes.
 115  */
 116 struct rdma_id_private {
 117         struct rdma_cm_id       id;
 118
 119         struct rdma_bind_list   *bind_list;
 120         struct hlist_node       node;
 121         struct list_head        list; /* listen_any_list or cma_device.list */
 122         struct list_head        listen_list; /* per device listens */
 123         struct cma_device       *cma_dev;
 124         struct list_head        mc_list;
 125
 126         int                     internal_id;
 127         enum cma_state          state;
 128         spinlock_t              lock;
 129         struct mutex            qp_mutex;
 130
 131         struct completion       comp;
 132         atomic_t                refcount;
 133         wait_queue_head_t       wait_remove;
 134         atomic_t                dev_remove;
 135
 136         int                     backlog;
 137         int                     timeout_ms;
 138         struct ib_sa_query      *query;
 139         int                     query_id;
 140         union {
 141                 struct ib_cm_id *ib;
 142                 struct iw_cm_id *iw;
 143         } cm_id;
 144
 145         u32                     seq_num;
 146         u32                     qkey;
 147         u32                     qp_num;
 148         u8                      srq;
 149         u8                      tos;
 150 };
 151
 152 struct cma_multicast {
 153         struct rdma_id_private *id_priv;
 154         union {
 155                 struct ib_sa_multicast *ib;
 156         } multicast;
 157         struct list_head        list;
 158         void                    *context;
 159         struct sockaddr         addr;
 160         u8                      pad[sizeof(struct sockaddr_in6) -
 161                                     sizeof(struct sockaddr)];
 162 };
 163
 164 struct cma_work {
 165         struct work_struct      work;
 166         struct rdma_id_private  *id;
 167         enum cma_state          old_state;
 168         enum cma_state          new_state;
 169         struct rdma_cm_event    event;
 170 };
 171
 172 union cma_ip_addr {
 173         struct in6_addr ip6;
 174         struct {
 175                 __be32 pad[3];
 176                 __be32 addr;
 177         } ip4;
 178 };
 179
 180 struct cma_hdr {
 181         u8 cma_version;
 182         u8 ip_version;  /* IP version: 7:4 */
 183         __be16 port;
 184         union cma_ip_addr src_addr;
 185         union cma_ip_addr dst_addr;
 186 };
 187
 188 struct sdp_hh {
 189         u8 bsdh[16];
 190         u8 sdp_version; /* Major version: 7:4 */
 191         u8 ip_version;  /* IP version: 7:4 */
 192         u8 sdp_specific1[10];
 193         __be16 port;
 194         __be16 sdp_specific2;
 195         union cma_ip_addr src_addr;
 196         union cma_ip_addr dst_addr;
 197 };
 198
 199 struct sdp_hah {
 200         u8 bsdh[16];
 201         u8 sdp_version;
 202 };
 203
 204 #define CMA_VERSION 0x00
 205 #define SDP_MAJ_VERSION 0x2
 206
 207 static int cma_comp(struct rdma_id_private *id_priv, enum cma_state comp)
 208 {
 209         unsigned long flags;
 210         int ret;
 211
 212         spin_lock_irqsave(&id_priv->lock, flags);
 213         ret = (id_priv->state == comp);
 214         spin_unlock_irqrestore(&id_priv->lock, flags);
 215         return ret;
 216 }
 217
 218 static int cma_comp_exch(struct rdma_id_private *id_priv,
 219                          enum cma_state comp, enum cma_state exch)
 220 {
 221         unsigned long flags;
 222         int ret;
 223
 224         spin_lock_irqsave(&id_priv->lock, flags);
 225         if ((ret = (id_priv->state == comp)))
 226                 id_priv->state = exch;
 227         spin_unlock_irqrestore(&id_priv->lock, flags);
 228         return ret;
 229 }
 230
 231 static enum cma_state cma_exch(struct rdma_id_private *id_priv,
 232                                enum cma_state exch)
 233 {
 234         unsigned long flags;
 235         enum cma_state old;
 236
 237         spin_lock_irqsave(&id_priv->lock, flags);
 238         old = id_priv->state;
 239         id_priv->state = exch;
 240         spin_unlock_irqrestore(&id_priv->lock, flags);
 241         return old;
 242 }
 243
 244 static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
 245 {
 246         return hdr->ip_version >> 4;
 247 }
 248
 249 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 250 {
 251         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 252 }
 253
 254 static inline u8 sdp_get_majv(u8 sdp_version)
 255 {
 256         return sdp_version >> 4;
 257 }
 258
 259 static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
 260 {
 261         return hh->ip_version >> 4;
 262 }
 263
 264 static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
 265 {
 266         hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
 267 }
 268
 269 static inline int cma_is_ud_ps(enum rdma_port_space ps)
 270 {
 271         return (ps == RDMA_PS_UDP || ps == RDMA_PS_IPOIB);
 272 }
 273
 274 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 275                               struct cma_device *cma_dev)
 276 {
 277         atomic_inc(&cma_dev->refcount);
 278         id_priv->cma_dev = cma_dev;
 279         id_priv->id.device = cma_dev->device;
 280         list_add_tail(&id_priv->list, &cma_dev->id_list);
 281 }
 282
 283 static inline void cma_deref_dev(struct cma_device *cma_dev)
 284 {
 285         if (atomic_dec_and_test(&cma_dev->refcount))
 286                 complete(&cma_dev->comp);
 287 }
 288
 289 static void cma_detach_from_dev(struct rdma_id_private *id_priv)
 290 {
 291         list_del(&id_priv->list);
 292         cma_deref_dev(id_priv->cma_dev);
 293         id_priv->cma_dev = NULL;
 294 }
 295
 296 static int cma_set_qkey(struct ib_device *device, u8 port_num,
 297                         enum rdma_port_space ps,
 298                         struct rdma_dev_addr *dev_addr, u32 *qkey)
 299 {
 300         struct ib_sa_mcmember_rec rec;
 301         int ret = 0;
 302
 303         switch (ps) {
 304         case RDMA_PS_UDP:
 305                 *qkey = RDMA_UDP_QKEY;
 306                 break;
 307         case RDMA_PS_IPOIB:
 308                 ib_addr_get_mgid(dev_addr, &rec.mgid);
 309                 ret = ib_sa_get_mcmember_rec(device, port_num, &rec.mgid, &rec);
 310                 *qkey = be32_to_cpu(rec.qkey);
 311                 break;
 312         default:
 313                 break;
 314         }
 315         return ret;
 316 }
 317
 318 static int cma_acquire_dev(struct rdma_id_private *id_priv)
 319 {
 320         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 321         struct cma_device *cma_dev;
 322         union ib_gid gid;
 323         int ret = -ENODEV;
 324
 325         switch (rdma_node_get_transport(dev_addr->dev_type)) {
 326         case RDMA_TRANSPORT_IB:
 327                 ib_addr_get_sgid(dev_addr, &gid);
 328                 break;
 329         case RDMA_TRANSPORT_IWARP:
 330                 iw_addr_get_sgid(dev_addr, &gid);
 331                 break;
 332         default:
 333                 return -ENODEV;
 334         }
 335
 336         list_for_each_entry(cma_dev, &dev_list, list) {
 337                 ret = ib_find_cached_gid(cma_dev->device, &gid,
 338                                          &id_priv->id.port_num, NULL);
 339                 if (!ret) {
 340                         ret = cma_set_qkey(cma_dev->device,
 341                                            id_priv->id.port_num,
 342                                            id_priv->id.ps, dev_addr,
 343                                            &id_priv->qkey);
 344                         if (!ret)
 345                                 cma_attach_to_dev(id_priv, cma_dev);
 346                         break;
 347                 }
 348         }
 349         return ret;
 350 }
 351
 352 static void cma_deref_id(struct rdma_id_private *id_priv)
 353 {
 354         if (atomic_dec_and_test(&id_priv->refcount))
 355                 complete(&id_priv->comp);
 356 }
 357
 358 static int cma_disable_remove(struct rdma_id_private *id_priv,
 359                               enum cma_state state)
 360 {
 361         unsigned long flags;
 362         int ret;
 363
 364         spin_lock_irqsave(&id_priv->lock, flags);
 365         if (id_priv->state == state) {
 366                 atomic_inc(&id_priv->dev_remove);
 367                 ret = 0;
 368         } else
 369                 ret = -EINVAL;
 370         spin_unlock_irqrestore(&id_priv->lock, flags);
 371         return ret;
 372 }
 373
 374 static void cma_enable_remove(struct rdma_id_private *id_priv)
 375 {
 376         if (atomic_dec_and_test(&id_priv->dev_remove))
 377                 wake_up(&id_priv->wait_remove);
 378 }
 379
 380 static int cma_has_cm_dev(struct rdma_id_private *id_priv)
 381 {
 382         return (id_priv->id.device && id_priv->cm_id.ib);
 383 }
 384
 385 struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
 386                                   void *context, enum rdma_port_space ps)
 387 {
 388         struct rdma_id_private *id_priv;
 389
 390         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 391         if (!id_priv)
 392                 return ERR_PTR(-ENOMEM);
 393
 394         id_priv->state = CMA_IDLE;
 395         id_priv->id.context = context;
 396         id_priv->id.event_handler = event_handler;
 397         id_priv->id.ps = ps;
 398         spin_lock_init(&id_priv->lock);
 399         mutex_init(&id_priv->qp_mutex);
 400         init_completion(&id_priv->comp);
 401         atomic_set(&id_priv->refcount, 1);
 402         init_waitqueue_head(&id_priv->wait_remove);
 403         atomic_set(&id_priv->dev_remove, 0);
 404         INIT_LIST_HEAD(&id_priv->listen_list);
 405         INIT_LIST_HEAD(&id_priv->mc_list);
 406         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 407
 408         return &id_priv->id;
 409 }
 410 EXPORT_SYMBOL(rdma_create_id);
 411
 412 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 413 {
 414         struct ib_qp_attr qp_attr;
 415         int qp_attr_mask, ret;
 416
 417         qp_attr.qp_state = IB_QPS_INIT;
 418         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 419         if (ret)
 420                 return ret;
 421
 422         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 423         if (ret)
 424                 return ret;
 425
 426         qp_attr.qp_state = IB_QPS_RTR;
 427         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 428         if (ret)
 429                 return ret;
 430
 431         qp_attr.qp_state = IB_QPS_RTS;
 432         qp_attr.sq_psn = 0;
 433         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 434
 435         return ret;
 436 }
 437
 438 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 439 {
 440         struct ib_qp_attr qp_attr;
 441         int qp_attr_mask, ret;
 442
 443         qp_attr.qp_state = IB_QPS_INIT;
 444         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 445         if (ret)
 446                 return ret;
 447
 448         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 449 }
 450
 451 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 452                    struct ib_qp_init_attr *qp_init_attr)
 453 {
 454         struct rdma_id_private *id_priv;
 455         struct ib_qp *qp;
 456         int ret;
 457
 458         id_priv = container_of(id, struct rdma_id_private, id);
 459         if (id->device != pd->device)
 460                 return -EINVAL;
 461
 462         qp = ib_create_qp(pd, qp_init_attr);
 463         if (IS_ERR(qp))
 464                 return PTR_ERR(qp);
 465
 466         if (cma_is_ud_ps(id_priv->id.ps))
 467                 ret = cma_init_ud_qp(id_priv, qp);
 468         else
 469                 ret = cma_init_conn_qp(id_priv, qp);
 470         if (ret)
 471                 goto err;
 472
 473         id->qp = qp;
 474         id_priv->qp_num = qp->qp_num;
 475         id_priv->srq = (qp->srq != NULL);
 476         return 0;
 477 err:
 478         ib_destroy_qp(qp);
 479         return ret;
 480 }
 481 EXPORT_SYMBOL(rdma_create_qp);
 482
 483 void rdma_destroy_qp(struct rdma_cm_id *id)
 484 {
 485         struct rdma_id_private *id_priv;
 486
 487         id_priv = container_of(id, struct rdma_id_private, id);
 488         mutex_lock(&id_priv->qp_mutex);
 489         ib_destroy_qp(id_priv->id.qp);
 490         id_priv->id.qp = NULL;
 491         mutex_unlock(&id_priv->qp_mutex);
 492 }
 493 EXPORT_SYMBOL(rdma_destroy_qp);
 494
 495 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 496                              struct rdma_conn_param *conn_param)
 497 {
 498         struct ib_qp_attr qp_attr;
 499         int qp_attr_mask, ret;
 500
 501         mutex_lock(&id_priv->qp_mutex);
 502         if (!id_priv->id.qp) {
 503                 ret = 0;
 504                 goto out;
 505         }
 506
 507         /* Need to update QP attributes from default values. */
 508         qp_attr.qp_state = IB_QPS_INIT;
 509         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 510         if (ret)
 511                 goto out;
 512
 513         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 514         if (ret)
 515                 goto out;
 516
 517         qp_attr.qp_state = IB_QPS_RTR;
 518         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 519         if (ret)
 520                 goto out;
 521
 522         if (conn_param)
 523                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
 524         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 525 out:
 526         mutex_unlock(&id_priv->qp_mutex);
 527         return ret;
 528 }
 529
 530 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
 531                              struct rdma_conn_param *conn_param)
 532 {
 533         struct ib_qp_attr qp_attr;
 534         int qp_attr_mask, ret;
 535
 536         mutex_lock(&id_priv->qp_mutex);
 537         if (!id_priv->id.qp) {
 538                 ret = 0;
 539                 goto out;
 540         }
 541
 542         qp_attr.qp_state = IB_QPS_RTS;
 543         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 544         if (ret)
 545                 goto out;
 546
 547         if (conn_param)
 548                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
 549         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 550 out:
 551         mutex_unlock(&id_priv->qp_mutex);
 552         return ret;
 553 }
 554
 555 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
 556 {
 557         struct ib_qp_attr qp_attr;
 558         int ret;
 559
 560         mutex_lock(&id_priv->qp_mutex);
 561         if (!id_priv->id.qp) {
 562                 ret = 0;
 563                 goto out;
 564         }
 565
 566         qp_attr.qp_state = IB_QPS_ERR;
 567         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
 568 out:
 569         mutex_unlock(&id_priv->qp_mutex);
 570         return ret;
 571 }
 572
 573 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
 574                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 575 {
 576         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 577         int ret;
 578
 579         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
 580                                   ib_addr_get_pkey(dev_addr),
 581                                   &qp_attr->pkey_index);
 582         if (ret)
 583                 return ret;
 584
 585         qp_attr->port_num = id_priv->id.port_num;
 586         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
 587
 588         if (cma_is_ud_ps(id_priv->id.ps)) {
 589                 qp_attr->qkey = id_priv->qkey;
 590                 *qp_attr_mask |= IB_QP_QKEY;
 591         } else {
 592                 qp_attr->qp_access_flags = 0;
 593                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
 594         }
 595         return 0;
 596 }
 597
 598 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 599                        int *qp_attr_mask)
 600 {
 601         struct rdma_id_private *id_priv;
 602         int ret = 0;
 603
 604         id_priv = container_of(id, struct rdma_id_private, id);
 605         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 606         case RDMA_TRANSPORT_IB:
 607                 if (!id_priv->cm_id.ib || cma_is_ud_ps(id_priv->id.ps))
 608                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
 609                 else
 610                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 611                                                  qp_attr_mask);
 612                 if (qp_attr->qp_state == IB_QPS_RTR)
 613                         qp_attr->rq_psn = id_priv->seq_num;
 614                 break;
 615         case RDMA_TRANSPORT_IWARP:
 616                 if (!id_priv->cm_id.iw) {
 617                         qp_attr->qp_access_flags = 0;
 618                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
 619                 } else
 620                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
 621                                                  qp_attr_mask);
 622                 break;
 623         default:
 624                 ret = -ENOSYS;
 625                 break;
 626         }
 627
 628         return ret;
 629 }
 630 EXPORT_SYMBOL(rdma_init_qp_attr);
 631
 632 static inline int cma_zero_addr(struct sockaddr *addr)
 633 {
 634         struct in6_addr *ip6;
 635
 636         if (addr->sa_family == AF_INET)
 637                 return ipv4_is_zeronet(
 638                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
 639         else {
 640                 ip6 = &((struct sockaddr_in6 *) addr)->sin6_addr;
 641                 return (ip6->s6_addr32[0] | ip6->s6_addr32[1] |
 642                         ip6->s6_addr32[2] | ip6->s6_addr32[3]) == 0;
 643         }
 644 }
 645
 646 static inline int cma_loopback_addr(struct sockaddr *addr)
 647 {
 648         return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
 649 }
 650
 651 static inline int cma_any_addr(struct sockaddr *addr)
 652 {
 653         return cma_zero_addr(addr) || cma_loopback_addr(addr);
 654 }
 655
 656 static inline __be16 cma_port(struct sockaddr *addr)
 657 {
 658         if (addr->sa_family == AF_INET)
 659                 return ((struct sockaddr_in *) addr)->sin_port;
 660         else
 661                 return ((struct sockaddr_in6 *) addr)->sin6_port;
 662 }
 663
 664 static inline int cma_any_port(struct sockaddr *addr)
 665 {
 666         return !cma_port(addr);
 667 }
 668
 669 static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
 670                             u8 *ip_ver, __be16 *port,
 671                             union cma_ip_addr **src, union cma_ip_addr **dst)
 672 {
 673         switch (ps) {
 674         case RDMA_PS_SDP:
 675                 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
 676                     SDP_MAJ_VERSION)
 677                         return -EINVAL;
 678
 679                 *ip_ver = sdp_get_ip_ver(hdr);
 680                 *port   = ((struct sdp_hh *) hdr)->port;
 681                 *src    = &((struct sdp_hh *) hdr)->src_addr;
 682                 *dst    = &((struct sdp_hh *) hdr)->dst_addr;
 683                 break;
 684         default:
 685                 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
 686                         return -EINVAL;
 687
 688                 *ip_ver = cma_get_ip_ver(hdr);
 689                 *port   = ((struct cma_hdr *) hdr)->port;
 690                 *src    = &((struct cma_hdr *) hdr)->src_addr;
 691                 *dst    = &((struct cma_hdr *) hdr)->dst_addr;
 692                 break;
 693         }
 694
 695         if (*ip_ver != 4 && *ip_ver != 6)
 696                 return -EINVAL;
 697         return 0;
 698 }
 699
 700 static void cma_save_net_info(struct rdma_addr *addr,
 701                               struct rdma_addr *listen_addr,
 702                               u8 ip_ver, __be16 port,
 703                               union cma_ip_addr *src, union cma_ip_addr *dst)
 704 {
 705         struct sockaddr_in *listen4, *ip4;
 706         struct sockaddr_in6 *listen6, *ip6;
 707
 708         switch (ip_ver) {
 709         case 4:
 710                 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
 711                 ip4 = (struct sockaddr_in *) &addr->src_addr;
 712                 ip4->sin_family = listen4->sin_family;
 713                 ip4->sin_addr.s_addr = dst->ip4.addr;
 714                 ip4->sin_port = listen4->sin_port;
 715
 716                 ip4 = (struct sockaddr_in *) &addr->dst_addr;
 717                 ip4->sin_family = listen4->sin_family;
 718                 ip4->sin_addr.s_addr = src->ip4.addr;
 719                 ip4->sin_port = port;
 720                 break;
 721         case 6:
 722                 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
 723                 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
 724                 ip6->sin6_family = listen6->sin6_family;
 725                 ip6->sin6_addr = dst->ip6;
 726                 ip6->sin6_port = listen6->sin6_port;
 727
 728                 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
 729                 ip6->sin6_family = listen6->sin6_family;
 730                 ip6->sin6_addr = src->ip6;
 731                 ip6->sin6_port = port;
 732                 break;
 733         default:
 734                 break;
 735         }
 736 }
 737
 738 static inline int cma_user_data_offset(enum rdma_port_space ps)
 739 {
 740         switch (ps) {
 741         case RDMA_PS_SDP:
 742                 return 0;
 743         default:
 744                 return sizeof(struct cma_hdr);
 745         }
 746 }
 747
 748 static void cma_cancel_route(struct rdma_id_private *id_priv)
 749 {
 750         switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
 751         case RDMA_TRANSPORT_IB:
 752                 if (id_priv->query)
 753                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 754                 break;
 755         default:
 756                 break;
 757         }
 758 }
 759
 760 static void cma_cancel_listens(struct rdma_id_private *id_priv)
 761 {
 762         struct rdma_id_private *dev_id_priv;
 763
 764         /*
 765          * Remove from listen_any_list to prevent added devices from spawning
 766          * additional listen requests.
 767          */
 768         mutex_lock(&lock);
 769         list_del(&id_priv->list);
 770
 771         while (!list_empty(&id_priv->listen_list)) {
 772                 dev_id_priv = list_entry(id_priv->listen_list.next,
 773                                          struct rdma_id_private, listen_list);
 774                 /* sync with device removal to avoid duplicate destruction */
 775                 list_del_init(&dev_id_priv->list);
 776                 list_del(&dev_id_priv->listen_list);
 777                 mutex_unlock(&lock);
 778
 779                 rdma_destroy_id(&dev_id_priv->id);
 780                 mutex_lock(&lock);
 781         }
 782         mutex_unlock(&lock);
 783 }
 784
 785 static void cma_cancel_operation(struct rdma_id_private *id_priv,
 786                                  enum cma_state state)
 787 {
 788         switch (state) {
 789         case CMA_ADDR_QUERY:
 790                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
 791                 break;
 792         case CMA_ROUTE_QUERY:
 793                 cma_cancel_route(id_priv);
 794                 break;
 795         case CMA_LISTEN:
 796                 if (cma_any_addr(&id_priv->id.route.addr.src_addr) &&
 797                     !id_priv->cma_dev)
 798                         cma_cancel_listens(id_priv);
 799                 break;
 800         default:
 801                 break;
 802         }
 803 }
 804
 805 static void cma_release_port(struct rdma_id_private *id_priv)
 806 {
 807         struct rdma_bind_list *bind_list = id_priv->bind_list;
 808
 809         if (!bind_list)
 810                 return;
 811
 812         mutex_lock(&lock);
 813         hlist_del(&id_priv->node);
 814         if (hlist_empty(&bind_list->owners)) {
 815                 idr_remove(bind_list->ps, bind_list->port);
 816                 kfree(bind_list);
 817         }
 818         mutex_unlock(&lock);
 819 }
 820
 821 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
 822 {
 823         struct cma_multicast *mc;
 824
 825         while (!list_empty(&id_priv->mc_list)) {
 826                 mc = container_of(id_priv->mc_list.next,
 827                                   struct cma_multicast, list);
 828                 list_del(&mc->list);
 829                 ib_sa_free_multicast(mc->multicast.ib);
 830                 kfree(mc);
 831         }
 832 }
 833
 834 void rdma_destroy_id(struct rdma_cm_id *id)
 835 {
 836         struct rdma_id_private *id_priv;
 837         enum cma_state state;
 838
 839         id_priv = container_of(id, struct rdma_id_private, id);
 840         state = cma_exch(id_priv, CMA_DESTROYING);
 841         cma_cancel_operation(id_priv, state);
 842
 843         mutex_lock(&lock);
 844         if (id_priv->cma_dev) {
 845                 mutex_unlock(&lock);
 846                 switch (rdma_node_get_transport(id->device->node_type)) {
 847                 case RDMA_TRANSPORT_IB:
 848                         if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 849                                 ib_destroy_cm_id(id_priv->cm_id.ib);
 850                         break;
 851                 case RDMA_TRANSPORT_IWARP:
 852                         if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
 853                                 iw_destroy_cm_id(id_priv->cm_id.iw);
 854                         break;
 855                 default:
 856                         break;
 857                 }
 858                 cma_leave_mc_groups(id_priv);
 859                 mutex_lock(&lock);
 860                 cma_detach_from_dev(id_priv);
 861         }
 862         mutex_unlock(&lock);
 863
 864         cma_release_port(id_priv);
 865         cma_deref_id(id_priv);
 866         wait_for_completion(&id_priv->comp);
 867
 868         if (id_priv->internal_id)
 869                 cma_deref_id(id_priv->id.context);
 870
 871         kfree(id_priv->id.route.path_rec);
 872         kfree(id_priv);
 873 }
 874 EXPORT_SYMBOL(rdma_destroy_id);
 875
 876 static int cma_rep_recv(struct rdma_id_private *id_priv)
 877 {
 878         int ret;
 879
 880         ret = cma_modify_qp_rtr(id_priv, NULL);
 881         if (ret)
 882                 goto reject;
 883
 884         ret = cma_modify_qp_rts(id_priv, NULL);
 885         if (ret)
 886                 goto reject;
 887
 888         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
 889         if (ret)
 890                 goto reject;
 891
 892         return 0;
 893 reject:
 894         cma_modify_qp_err(id_priv);
 895         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
 896                        NULL, 0, NULL, 0);
 897         return ret;
 898 }
 899
 900 static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
 901 {
 902         if (id_priv->id.ps == RDMA_PS_SDP &&
 903             sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
 904             SDP_MAJ_VERSION)
 905                 return -EINVAL;
 906
 907         return 0;
 908 }
 909
 910 static void cma_set_rep_event_data(struct rdma_cm_event *event,
 911                                    struct ib_cm_rep_event_param *rep_data,
 912                                    void *private_data)
 913 {
 914         event->param.conn.private_data = private_data;
 915         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
 916         event->param.conn.responder_resources = rep_data->responder_resources;
 917         event->param.conn.initiator_depth = rep_data->initiator_depth;
 918         event->param.conn.flow_control = rep_data->flow_control;
 919         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
 920         event->param.conn.srq = rep_data->srq;
 921         event->param.conn.qp_num = rep_data->remote_qpn;
 922 }
 923
 924 static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
 925 {
 926         struct rdma_id_private *id_priv = cm_id->context;
 927         struct rdma_cm_event event;
 928         int ret = 0;
 929
 930         if (cma_disable_remove(id_priv, CMA_CONNECT))
 931                 return 0;
 932
 933         memset(&event, 0, sizeof event);
 934         switch (ib_event->event) {
 935         case IB_CM_REQ_ERROR:
 936         case IB_CM_REP_ERROR:
 937                 event.event = RDMA_CM_EVENT_UNREACHABLE;
 938                 event.status = -ETIMEDOUT;
 939                 break;
 940         case IB_CM_REP_RECEIVED:
 941                 event.status = cma_verify_rep(id_priv, ib_event->private_data);
 942                 if (event.status)
 943                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
 944                 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
 945                         event.status = cma_rep_recv(id_priv);
 946                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
 947                                                      RDMA_CM_EVENT_ESTABLISHED;
 948                 } else
 949                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
 950                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
 951                                        ib_event->private_data);
 952                 break;
 953         case IB_CM_RTU_RECEIVED:
 954         case IB_CM_USER_ESTABLISHED:
 955                 event.event = RDMA_CM_EVENT_ESTABLISHED;
 956                 break;
 957         case IB_CM_DREQ_ERROR:
 958                 event.status = -ETIMEDOUT; /* fall through */
 959         case IB_CM_DREQ_RECEIVED:
 960         case IB_CM_DREP_RECEIVED:
 961                 if (!cma_comp_exch(id_priv, CMA_CONNECT, CMA_DISCONNECT))
 962                         goto out;
 963                 event.event = RDMA_CM_EVENT_DISCONNECTED;
 964                 break;
 965         case IB_CM_TIMEWAIT_EXIT:
 966         case IB_CM_MRA_RECEIVED:
 967                 /* ignore event */
 968                 goto out;
 969         case IB_CM_REJ_RECEIVED:
 970                 cma_modify_qp_err(id_priv);
 971                 event.status = ib_event->param.rej_rcvd.reason;
 972                 event.event = RDMA_CM_EVENT_REJECTED;
 973                 event.param.conn.private_data = ib_event->private_data;
 974                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
 975                 break;
 976         default:
 977                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
 978                        ib_event->event);
 979                 goto out;
 980         }
 981
 982         ret = id_priv->id.event_handler(&id_priv->id, &event);
 983         if (ret) {
 984                 /* Destroy the CM ID by returning a non-zero value. */
 985                 id_priv->cm_id.ib = NULL;
 986                 cma_exch(id_priv, CMA_DESTROYING);
 987                 cma_enable_remove(id_priv);
 988                 rdma_destroy_id(&id_priv->id);
 989                 return ret;
 990         }
 991 out:
 992         cma_enable_remove(id_priv);
 993         return ret;
 994 }
 995
 996 static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
 997                                                struct ib_cm_event *ib_event)
 998 {
 999         struct rdma_id_private *id_priv;
1000         struct rdma_cm_id *id;
1001         struct rdma_route *rt;
1002         union cma_ip_addr *src, *dst;
1003         __be16 port;
1004         u8 ip_ver;
1005
1006         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1007                              &ip_ver, &port, &src, &dst))
1008                 goto err;
1009
1010         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1011                             listen_id->ps);
1012         if (IS_ERR(id))
1013                 goto err;
1014
1015         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1016                           ip_ver, port, src, dst);
1017
1018         rt = &id->route;
1019         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
1020         rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1021                                GFP_KERNEL);
1022         if (!rt->path_rec)
1023                 goto destroy_id;
1024
1025         rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1026         if (rt->num_paths == 2)
1027                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1028
1029         ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
1030         ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
1031         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
1032         rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA;
1033
1034         id_priv = container_of(id, struct rdma_id_private, id);
1035         id_priv->state = CMA_CONNECT;
1036         return id_priv;
1037
1038 destroy_id:
1039         rdma_destroy_id(id);
1040 err:
1041         return NULL;
1042 }
1043
1044 static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1045                                               struct ib_cm_event *ib_event)
1046 {
1047         struct rdma_id_private *id_priv;
1048         struct rdma_cm_id *id;
1049         union cma_ip_addr *src, *dst;
1050         __be16 port;
1051         u8 ip_ver;
1052         int ret;
1053
1054         id = rdma_create_id(listen_id->event_handler, listen_id->context,
1055                             listen_id->ps);
1056         if (IS_ERR(id))
1057                 return NULL;
1058
1059
1060         if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1061                              &ip_ver, &port, &src, &dst))
1062                 goto err;
1063
1064         cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1065                           ip_ver, port, src, dst);
1066
1067         ret = rdma_translate_ip(&id->route.addr.src_addr,
1068                                 &id->route.addr.dev_addr);
1069         if (ret)
1070                 goto err;
1071
1072         id_priv = container_of(id, struct rdma_id_private, id);
1073         id_priv->state = CMA_CONNECT;
1074         return id_priv;
1075 err:
1076         rdma_destroy_id(id);
1077         return NULL;
1078 }
1079
1080 static void cma_set_req_event_data(struct rdma_cm_event *event,
1081                                    struct ib_cm_req_event_param *req_data,
1082                                    void *private_data, int offset)
1083 {
1084         event->param.conn.private_data = private_data + offset;
1085         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1086         event->param.conn.responder_resources = req_data->responder_resources;
1087         event->param.conn.initiator_depth = req_data->initiator_depth;
1088         event->param.conn.flow_control = req_data->flow_control;
1089         event->param.conn.retry_count = req_data->retry_count;
1090         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1091         event->param.conn.srq = req_data->srq;
1092         event->param.conn.qp_num = req_data->remote_qpn;
1093 }
1094
1095 static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1096 {
1097         struct rdma_id_private *listen_id, *conn_id;
1098         struct rdma_cm_event event;
1099         int offset, ret;
1100
1101         listen_id = cm_id->context;
1102         if (cma_disable_remove(listen_id, CMA_LISTEN))
1103                 return -ECONNABORTED;
1104
1105         memset(&event, 0, sizeof event);
1106         offset = cma_user_data_offset(listen_id->id.ps);
1107         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1108         if (cma_is_ud_ps(listen_id->id.ps)) {
1109                 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1110                 event.param.ud.private_data = ib_event->private_data + offset;
1111                 event.param.ud.private_data_len =
1112                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1113         } else {
1114                 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1115                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1116                                        ib_event->private_data, offset);
1117         }
1118         if (!conn_id) {
1119                 ret = -ENOMEM;
1120                 goto out;
1121         }
1122
1123         atomic_inc(&conn_id->dev_remove);
1124         mutex_lock(&lock);
1125         ret = cma_acquire_dev(conn_id);
1126         mutex_unlock(&lock);
1127         if (ret)
1128                 goto release_conn_id;
1129
1130         conn_id->cm_id.ib = cm_id;
1131         cm_id->context = conn_id;
1132         cm_id->cm_handler = cma_ib_handler;
1133
1134         ret = conn_id->id.event_handler(&conn_id->id, &event);
1135         if (!ret) {
1136                 /*
1137                  * Acquire mutex to prevent user executing rdma_destroy_id()
1138                  * while we're accessing the cm_id.
1139                  */
1140                 mutex_lock(&lock);
1141                 if (cma_comp(conn_id, CMA_CONNECT) &&
1142                     !cma_is_ud_ps(conn_id->id.ps))
1143                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1144                 mutex_unlock(&lock);
1145                 cma_enable_remove(conn_id);
1146                 goto out;
1147         }
1148
1149         /* Destroy the CM ID by returning a non-zero value. */
1150         conn_id->cm_id.ib = NULL;
1151
1152 release_conn_id:
1153         cma_exch(conn_id, CMA_DESTROYING);
1154         cma_enable_remove(conn_id);
1155         rdma_destroy_id(&conn_id->id);
1156
1157 out:
1158         cma_enable_remove(listen_id);
1159         return ret;
1160 }
1161
1162 static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1163 {
1164         return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
1165 }
1166
1167 static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1168                                  struct ib_cm_compare_data *compare)
1169 {
1170         struct cma_hdr *cma_data, *cma_mask;
1171         struct sdp_hh *sdp_data, *sdp_mask;
1172         __be32 ip4_addr;
1173         struct in6_addr ip6_addr;
1174
1175         memset(compare, 0, sizeof *compare);
1176         cma_data = (void *) compare->data;
1177         cma_mask = (void *) compare->mask;
1178         sdp_data = (void *) compare->data;
1179         sdp_mask = (void *) compare->mask;
1180
1181         switch (addr->sa_family) {
1182         case AF_INET:
1183                 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1184                 if (ps == RDMA_PS_SDP) {
1185                         sdp_set_ip_ver(sdp_data, 4);
1186                         sdp_set_ip_ver(sdp_mask, 0xF);
1187                         sdp_data->dst_addr.ip4.addr = ip4_addr;
1188                         sdp_mask->dst_addr.ip4.addr = htonl(~0);
1189                 } else {
1190                         cma_set_ip_ver(cma_data, 4);
1191                         cma_set_ip_ver(cma_mask, 0xF);
1192                         cma_data->dst_addr.ip4.addr = ip4_addr;
1193                         cma_mask->dst_addr.ip4.addr = htonl(~0);
1194                 }
1195                 break;
1196         case AF_INET6:
1197                 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1198                 if (ps == RDMA_PS_SDP) {
1199                         sdp_set_ip_ver(sdp_data, 6);
1200                         sdp_set_ip_ver(sdp_mask, 0xF);
1201                         sdp_data->dst_addr.ip6 = ip6_addr;
1202                         memset(&sdp_mask->dst_addr.ip6, 0xFF,
1203                                sizeof sdp_mask->dst_addr.ip6);
1204                 } else {
1205                         cma_set_ip_ver(cma_data, 6);
1206                         cma_set_ip_ver(cma_mask, 0xF);
1207                         cma_data->dst_addr.ip6 = ip6_addr;
1208                         memset(&cma_mask->dst_addr.ip6, 0xFF,
1209                                sizeof cma_mask->dst_addr.ip6);
1210                 }
1211                 break;
1212         default:
1213                 break;
1214         }
1215 }
1216
1217 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1218 {
1219         struct rdma_id_private *id_priv = iw_id->context;
1220         struct rdma_cm_event event;
1221         struct sockaddr_in *sin;
1222         int ret = 0;
1223
1224         if (cma_disable_remove(id_priv, CMA_CONNECT))
1225                 return 0;
1226
1227         memset(&event, 0, sizeof event);
1228         switch (iw_event->event) {
1229         case IW_CM_EVENT_CLOSE:
1230                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1231                 break;
1232         case IW_CM_EVENT_CONNECT_REPLY:
1233                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1234                 *sin = iw_event->local_addr;
1235                 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1236                 *sin = iw_event->remote_addr;
1237                 switch (iw_event->status) {
1238                 case 0:
1239                         event.event = RDMA_CM_EVENT_ESTABLISHED;
1240                         break;
1241                 case -ECONNRESET:
1242                 case -ECONNREFUSED:
1243                         event.event = RDMA_CM_EVENT_REJECTED;
1244                         break;
1245                 case -ETIMEDOUT:
1246                         event.event = RDMA_CM_EVENT_UNREACHABLE;
1247                         break;
1248                 default:
1249                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1250                         break;
1251                 }
1252                 break;
1253         case IW_CM_EVENT_ESTABLISHED:
1254                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1255                 break;
1256         default:
1257                 BUG_ON(1);
1258         }
1259
1260         event.status = iw_event->status;
1261         event.param.conn.private_data = iw_event->private_data;
1262         event.param.conn.private_data_len = iw_event->private_data_len;
1263         ret = id_priv->id.event_handler(&id_priv->id, &event);
1264         if (ret) {
1265                 /* Destroy the CM ID by returning a non-zero value. */
1266                 id_priv->cm_id.iw = NULL;
1267                 cma_exch(id_priv, CMA_DESTROYING);
1268                 cma_enable_remove(id_priv);
1269                 rdma_destroy_id(&id_priv->id);
1270                 return ret;
1271         }
1272
1273         cma_enable_remove(id_priv);
1274         return ret;
1275 }
1276
1277 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1278                                struct iw_cm_event *iw_event)
1279 {
1280         struct rdma_cm_id *new_cm_id;
1281         struct rdma_id_private *listen_id, *conn_id;
1282         struct sockaddr_in *sin;
1283         struct net_device *dev = NULL;
1284         struct rdma_cm_event event;
1285         int ret;
1286         struct ib_device_attr attr;
1287
1288         listen_id = cm_id->context;
1289         if (cma_disable_remove(listen_id, CMA_LISTEN))
1290                 return -ECONNABORTED;
1291
1292         /* Create a new RDMA id for the new IW CM ID */
1293         new_cm_id = rdma_create_id(listen_id->id.event_handler,
1294                                    listen_id->id.context,
1295                                    RDMA_PS_TCP);
1296         if (IS_ERR(new_cm_id)) {
1297                 ret = -ENOMEM;
1298                 goto out;
1299         }
1300         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
1301         atomic_inc(&conn_id->dev_remove);
1302         conn_id->state = CMA_CONNECT;
1303
1304         dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
1305         if (!dev) {
1306                 ret = -EADDRNOTAVAIL;
1307                 cma_enable_remove(conn_id);
1308                 rdma_destroy_id(new_cm_id);
1309                 goto out;
1310         }
1311         ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1312         if (ret) {
1313                 cma_enable_remove(conn_id);
1314                 rdma_destroy_id(new_cm_id);
1315                 goto out;
1316         }
1317
1318         mutex_lock(&lock);
1319         ret = cma_acquire_dev(conn_id);
1320         mutex_unlock(&lock);
1321         if (ret) {
1322                 cma_enable_remove(conn_id);
1323                 rdma_destroy_id(new_cm_id);
1324                 goto out;
1325         }
1326
1327         conn_id->cm_id.iw = cm_id;
1328         cm_id->context = conn_id;
1329         cm_id->cm_handler = cma_iw_handler;
1330
1331         sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1332         *sin = iw_event->local_addr;
1333         sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1334         *sin = iw_event->remote_addr;
1335
1336         ret = ib_query_device(conn_id->id.device, &attr);
1337         if (ret) {
1338                 cma_enable_remove(conn_id);
1339                 rdma_destroy_id(new_cm_id);
1340                 goto out;
1341         }
1342
1343         memset(&event, 0, sizeof event);
1344         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1345         event.param.conn.private_data = iw_event->private_data;
1346         event.param.conn.private_data_len = iw_event->private_data_len;
1347         event.param.conn.initiator_depth = attr.max_qp_init_rd_atom;
1348         event.param.conn.responder_resources = attr.max_qp_rd_atom;
1349         ret = conn_id->id.event_handler(&conn_id->id, &event);
1350         if (ret) {
1351                 /* User wants to destroy the CM ID */
1352                 conn_id->cm_id.iw = NULL;
1353                 cma_exch(conn_id, CMA_DESTROYING);
1354                 cma_enable_remove(conn_id);
1355                 rdma_destroy_id(&conn_id->id);
1356         }
1357
1358 out:
1359         if (dev)
1360                 dev_put(dev);
1361         cma_enable_remove(listen_id);
1362         return ret;
1363 }
1364
1365 static int cma_ib_listen(struct rdma_id_private *id_priv)
1366 {
1367         struct ib_cm_compare_data compare_data;
1368         struct sockaddr *addr;
1369         __be64 svc_id;
1370         int ret;
1371
1372         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_req_handler,
1373                                             id_priv);
1374         if (IS_ERR(id_priv->cm_id.ib))
1375                 return PTR_ERR(id_priv->cm_id.ib);
1376
1377         addr = &id_priv->id.route.addr.src_addr;
1378         svc_id = cma_get_service_id(id_priv->id.ps, addr);
1379         if (cma_any_addr(addr))
1380                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1381         else {
1382                 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1383                 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1384         }
1385
1386         if (ret) {
1387                 ib_destroy_cm_id(id_priv->cm_id.ib);
1388                 id_priv->cm_id.ib = NULL;
1389         }
1390
1391         return ret;
1392 }
1393
1394 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1395 {
1396         int ret;
1397         struct sockaddr_in *sin;
1398
1399         id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
1400                                             iw_conn_req_handler,
1401                                             id_priv);
1402         if (IS_ERR(id_priv->cm_id.iw))
1403                 return PTR_ERR(id_priv->cm_id.iw);
1404
1405         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1406         id_priv->cm_id.iw->local_addr = *sin;
1407
1408         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1409
1410         if (ret) {
1411                 iw_destroy_cm_id(id_priv->cm_id.iw);
1412                 id_priv->cm_id.iw = NULL;
1413         }
1414
1415         return ret;
1416 }
1417
1418 static int cma_listen_handler(struct rdma_cm_id *id,
1419                               struct rdma_cm_event *event)
1420 {
1421         struct rdma_id_private *id_priv = id->context;
1422
1423         id->context = id_priv->id.context;
1424         id->event_handler = id_priv->id.event_handler;
1425         return id_priv->id.event_handler(id, event);
1426 }
1427
1428 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1429                               struct cma_device *cma_dev)
1430 {
1431         struct rdma_id_private *dev_id_priv;
1432         struct rdma_cm_id *id;
1433         int ret;
1434
1435         id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps);
1436         if (IS_ERR(id))
1437                 return;
1438
1439         dev_id_priv = container_of(id, struct rdma_id_private, id);
1440
1441         dev_id_priv->state = CMA_ADDR_BOUND;
1442         memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
1443                ip_addr_size(&id_priv->id.route.addr.src_addr));
1444
1445         cma_attach_to_dev(dev_id_priv, cma_dev);
1446         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
1447         atomic_inc(&id_priv->refcount);
1448         dev_id_priv->internal_id = 1;
1449
1450         ret = rdma_listen(id, id_priv->backlog);
1451         if (ret)
1452                 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
1453                        "listening on device %s", ret, cma_dev->device->name);
1454 }
1455
1456 static void cma_listen_on_all(struct rdma_id_private *id_priv)
1457 {
1458         struct cma_device *cma_dev;
1459
1460         mutex_lock(&lock);
1461         list_add_tail(&id_priv->list, &listen_any_list);
1462         list_for_each_entry(cma_dev, &dev_list, list)
1463                 cma_listen_on_dev(id_priv, cma_dev);
1464         mutex_unlock(&lock);
1465 }
1466
1467 static int cma_bind_any(struct rdma_cm_id *id, sa_family_t af)
1468 {
1469         struct sockaddr_in addr_in;
1470
1471         memset(&addr_in, 0, sizeof addr_in);
1472         addr_in.sin_family = af;
1473         return rdma_bind_addr(id, (struct sockaddr *) &addr_in);
1474 }
1475
1476 int rdma_listen(struct rdma_cm_id *id, int backlog)
1477 {
1478         struct rdma_id_private *id_priv;
1479         int ret;
1480
1481         id_priv = container_of(id, struct rdma_id_private, id);
1482         if (id_priv->state == CMA_IDLE) {
1483                 ret = cma_bind_any(id, AF_INET);
1484                 if (ret)
1485                         return ret;
1486         }
1487
1488         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_LISTEN))
1489                 return -EINVAL;
1490
1491         id_priv->backlog = backlog;
1492         if (id->device) {
1493                 switch (rdma_node_get_transport(id->device->node_type)) {
1494                 case RDMA_TRANSPORT_IB:
1495                         ret = cma_ib_listen(id_priv);
1496                         if (ret)
1497                                 goto err;
1498                         break;
1499                 case RDMA_TRANSPORT_IWARP:
1500                         ret = cma_iw_listen(id_priv, backlog);
1501                         if (ret)
1502                                 goto err;
1503                         break;
1504                 default:
1505                         ret = -ENOSYS;
1506                         goto err;
1507                 }
1508         } else
1509                 cma_listen_on_all(id_priv);
1510
1511         return 0;
1512 err:
1513         id_priv->backlog = 0;
1514         cma_comp_exch(id_priv, CMA_LISTEN, CMA_ADDR_BOUND);
1515         return ret;
1516 }
1517 EXPORT_SYMBOL(rdma_listen);
1518
1519 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1520 {
1521         struct rdma_id_private *id_priv;
1522
1523         id_priv = container_of(id, struct rdma_id_private, id);
1524         id_priv->tos = (u8) tos;
1525 }
1526 EXPORT_SYMBOL(rdma_set_service_type);
1527
1528 static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1529                               void *context)
1530 {
1531         struct cma_work *work = context;
1532         struct rdma_route *route;
1533
1534         route = &work->id->id.route;
1535
1536         if (!status) {
1537                 route->num_paths = 1;
1538                 *route->path_rec = *path_rec;
1539         } else {
1540                 work->old_state = CMA_ROUTE_QUERY;
1541                 work->new_state = CMA_ADDR_RESOLVED;
1542                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
1543                 work->event.status = status;
1544         }
1545
1546         queue_work(cma_wq, &work->work);
1547 }
1548
1549 static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1550                               struct cma_work *work)
1551 {
1552         struct rdma_addr *addr = &id_priv->id.route.addr;
1553         struct ib_sa_path_rec path_rec;
1554         ib_sa_comp_mask comp_mask;
1555         struct sockaddr_in6 *sin6;
1556
1557         memset(&path_rec, 0, sizeof path_rec);
1558         ib_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1559         ib_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
1560         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
1561         path_rec.numb_path = 1;
1562         path_rec.reversible = 1;
1563         path_rec.service_id = cma_get_service_id(id_priv->id.ps, &addr->dst_addr);
1564
1565         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1566                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1567                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1568
1569         if (addr->src_addr.sa_family == AF_INET) {
1570                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1571                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1572         } else {
1573                 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1574                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1575                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1576         }
1577
1578         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
1579                                                id_priv->id.port_num, &path_rec,
1580                                                comp_mask, timeout_ms,
1581                                                GFP_KERNEL, cma_query_handler,
1582                                                work, &id_priv->query);
1583
1584         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1585 }
1586
1587 static void cma_work_handler(struct work_struct *_work)
1588 {
1589         struct cma_work *work = container_of(_work, struct cma_work, work);
1590         struct rdma_id_private *id_priv = work->id;
1591         int destroy = 0;
1592
1593         atomic_inc(&id_priv->dev_remove);
1594         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1595                 goto out;
1596
1597         if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
1598                 cma_exch(id_priv, CMA_DESTROYING);
1599                 destroy = 1;
1600         }
1601 out:
1602         cma_enable_remove(id_priv);
1603         cma_deref_id(id_priv);
1604         if (destroy)
1605                 rdma_destroy_id(&id_priv->id);
1606         kfree(work);
1607 }
1608
1609 static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1610 {
1611         struct rdma_route *route = &id_priv->id.route;
1612         struct cma_work *work;
1613         int ret;
1614
1615         work = kzalloc(sizeof *work, GFP_KERNEL);
1616         if (!work)
1617                 return -ENOMEM;
1618
1619         work->id = id_priv;
1620         INIT_WORK(&work->work, cma_work_handler);
1621         work->old_state = CMA_ROUTE_QUERY;
1622         work->new_state = CMA_ROUTE_RESOLVED;
1623         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1624
1625         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1626         if (!route->path_rec) {
1627                 ret = -ENOMEM;
1628                 goto err1;
1629         }
1630
1631         ret = cma_query_ib_route(id_priv, timeout_ms, work);
1632         if (ret)
1633                 goto err2;
1634
1635         return 0;
1636 err2:
1637         kfree(route->path_rec);
1638         route->path_rec = NULL;
1639 err1:
1640         kfree(work);
1641         return ret;
1642 }
1643
1644 int rdma_set_ib_paths(struct rdma_cm_id *id,
1645                       struct ib_sa_path_rec *path_rec, int num_paths)
1646 {
1647         struct rdma_id_private *id_priv;
1648         int ret;
1649
1650         id_priv = container_of(id, struct rdma_id_private, id);
1651         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_RESOLVED))
1652                 return -EINVAL;
1653
1654         id->route.path_rec = kmalloc(sizeof *path_rec * num_paths, GFP_KERNEL);
1655         if (!id->route.path_rec) {
1656                 ret = -ENOMEM;
1657                 goto err;
1658         }
1659
1660         memcpy(id->route.path_rec, path_rec, sizeof *path_rec * num_paths);
1661         return 0;
1662 err:
1663         cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_ADDR_RESOLVED);
1664         return ret;
1665 }
1666 EXPORT_SYMBOL(rdma_set_ib_paths);
1667
1668 static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1669 {
1670         struct cma_work *work;
1671
1672         work = kzalloc(sizeof *work, GFP_KERNEL);
1673         if (!work)
1674                 return -ENOMEM;
1675
1676         work->id = id_priv;
1677         INIT_WORK(&work->work, cma_work_handler);
1678         work->old_state = CMA_ROUTE_QUERY;
1679         work->new_state = CMA_ROUTE_RESOLVED;
1680         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1681         queue_work(cma_wq, &work->work);
1682         return 0;
1683 }
1684
1685 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1686 {
1687         struct rdma_id_private *id_priv;
1688         int ret;
1689
1690         id_priv = container_of(id, struct rdma_id_private, id);
1691         if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
1692                 return -EINVAL;
1693
1694         atomic_inc(&id_priv->refcount);
1695         switch (rdma_node_get_transport(id->device->node_type)) {
1696         case RDMA_TRANSPORT_IB:
1697                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1698                 break;
1699         case RDMA_TRANSPORT_IWARP:
1700                 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1701                 break;
1702         default:
1703                 ret = -ENOSYS;
1704                 break;
1705         }
1706         if (ret)
1707                 goto err;
1708
1709         return 0;
1710 err:
1711         cma_comp_exch(id_priv, CMA_ROUTE_QUERY, CMA_ADDR_RESOLVED);
1712         cma_deref_id(id_priv);
1713         return ret;
1714 }
1715 EXPORT_SYMBOL(rdma_resolve_route);
1716
1717 static int cma_bind_loopback(struct rdma_id_private *id_priv)
1718 {
1719         struct cma_device *cma_dev;
1720         struct ib_port_attr port_attr;
1721         union ib_gid gid;
1722         u16 pkey;
1723         int ret;
1724         u8 p;
1725
1726         mutex_lock(&lock);
1727         if (list_empty(&dev_list)) {
1728                 ret = -ENODEV;
1729                 goto out;
1730         }
1731         list_for_each_entry(cma_dev, &dev_list, list)
1732                 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
1733                         if (!ib_query_port(cma_dev->device, p, &port_attr) &&
1734                             port_attr.state == IB_PORT_ACTIVE)
1735                                 goto port_found;
1736
1737         p = 1;
1738         cma_dev = list_entry(dev_list.next, struct cma_device, list);
1739
1740 port_found:
1741         ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
1742         if (ret)
1743                 goto out;
1744
1745         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1746         if (ret)
1747                 goto out;
1748
1749         ib_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1750         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1751         id_priv->id.port_num = p;
1752         cma_attach_to_dev(id_priv, cma_dev);
1753 out:
1754         mutex_unlock(&lock);
1755         return ret;
1756 }
1757
1758 static void addr_handler(int status, struct sockaddr *src_addr,
1759                          struct rdma_dev_addr *dev_addr, void *context)
1760 {
1761         struct rdma_id_private *id_priv = context;
1762         struct rdma_cm_event event;
1763
1764         memset(&event, 0, sizeof event);
1765         atomic_inc(&id_priv->dev_remove);
1766
1767         /*
1768          * Grab mutex to block rdma_destroy_id() from removing the device while
1769          * we're trying to acquire it.
1770          */
1771         mutex_lock(&lock);
1772         if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
1773                 mutex_unlock(&lock);
1774                 goto out;
1775         }
1776
1777         if (!status && !id_priv->cma_dev)
1778                 status = cma_acquire_dev(id_priv);
1779         mutex_unlock(&lock);
1780
1781         if (status) {
1782                 if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
1783                         goto out;
1784                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
1785                 event.status = status;
1786         } else {
1787                 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
1788                        ip_addr_size(src_addr));
1789                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1790         }
1791
1792         if (id_priv->id.event_handler(&id_priv->id, &event)) {
1793                 cma_exch(id_priv, CMA_DESTROYING);
1794                 cma_enable_remove(id_priv);
1795                 cma_deref_id(id_priv);
1796                 rdma_destroy_id(&id_priv->id);
1797                 return;
1798         }
1799 out:
1800         cma_enable_remove(id_priv);
1801         cma_deref_id(id_priv);
1802 }
1803
1804 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
1805 {
1806         struct cma_work *work;
1807         struct sockaddr_in *src_in, *dst_in;
1808         union ib_gid gid;
1809         int ret;
1810
1811         work = kzalloc(sizeof *work, GFP_KERNEL);
1812         if (!work)
1813                 return -ENOMEM;
1814
1815         if (!id_priv->cma_dev) {
1816                 ret = cma_bind_loopback(id_priv);
1817                 if (ret)
1818                         goto err;
1819         }
1820
1821         ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
1822         ib_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
1823
1824         if (cma_zero_addr(&id_priv->id.route.addr.src_addr)) {
1825                 src_in = (struct sockaddr_in *)&id_priv->id.route.addr.src_addr;
1826                 dst_in = (struct sockaddr_in *)&id_priv->id.route.addr.dst_addr;
1827                 src_in->sin_family = dst_in->sin_family;
1828                 src_in->sin_addr.s_addr = dst_in->sin_addr.s_addr;
1829         }
1830
1831         work->id = id_priv;
1832         INIT_WORK(&work->work, cma_work_handler);
1833         work->old_state = CMA_ADDR_QUERY;
1834         work->new_state = CMA_ADDR_RESOLVED;
1835         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
1836         queue_work(cma_wq, &work->work);
1837         return 0;
1838 err:
1839         kfree(work);
1840         return ret;
1841 }
1842
1843 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1844                          struct sockaddr *dst_addr)
1845 {
1846         if (src_addr && src_addr->sa_family)
1847                 return rdma_bind_addr(id, src_addr);
1848         else
1849                 return cma_bind_any(id, dst_addr->sa_family);
1850 }
1851
1852 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
1853                       struct sockaddr *dst_addr, int timeout_ms)
1854 {
1855         struct rdma_id_private *id_priv;
1856         int ret;
1857
1858         id_priv = container_of(id, struct rdma_id_private, id);
1859         if (id_priv->state == CMA_IDLE) {
1860                 ret = cma_bind_addr(id, src_addr, dst_addr);
1861                 if (ret)
1862                         return ret;
1863         }
1864
1865         if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
1866                 return -EINVAL;
1867
1868         atomic_inc(&id_priv->refcount);
1869         memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
1870         if (cma_any_addr(dst_addr))
1871                 ret = cma_resolve_loopback(id_priv);
1872         else
1873                 ret = rdma_resolve_ip(&addr_client, &id->route.addr.src_addr,
1874                                       dst_addr, &id->route.addr.dev_addr,
1875                                       timeout_ms, addr_handler, id_priv);
1876         if (ret)
1877                 goto err;
1878
1879         return 0;
1880 err:
1881         cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND);
1882         cma_deref_id(id_priv);
1883         return ret;
1884 }
1885 EXPORT_SYMBOL(rdma_resolve_addr);
1886
1887 static void cma_bind_port(struct rdma_bind_list *bind_list,
1888                           struct rdma_id_private *id_priv)
1889 {
1890         struct sockaddr_in *sin;
1891
1892         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1893         sin->sin_port = htons(bind_list->port);
1894         id_priv->bind_list = bind_list;
1895         hlist_add_head(&id_priv->node, &bind_list->owners);
1896 }
1897
1898 static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
1899                           unsigned short snum)
1900 {
1901         struct rdma_bind_list *bind_list;
1902         int port, ret;
1903
1904         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1905         if (!bind_list)
1906                 return -ENOMEM;
1907
1908         do {
1909                 ret = idr_get_new_above(ps, bind_list, snum, &port);
1910         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1911
1912         if (ret)
1913                 goto err1;
1914
1915         if (port != snum) {
1916                 ret = -EADDRNOTAVAIL;
1917                 goto err2;
1918         }
1919
1920         bind_list->ps = ps;
1921         bind_list->port = (unsigned short) port;
1922         cma_bind_port(bind_list, id_priv);
1923         return 0;
1924 err2:
1925         idr_remove(ps, port);
1926 err1:
1927         kfree(bind_list);
1928         return ret;
1929 }
1930
1931 static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
1932 {
1933         struct rdma_bind_list *bind_list;
1934         int port, ret, low, high;
1935
1936         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
1937         if (!bind_list)
1938                 return -ENOMEM;
1939
1940 retry:
1941         /* FIXME: add proper port randomization per like inet_csk_get_port */
1942         do {
1943                 ret = idr_get_new_above(ps, bind_list, next_port, &port);
1944         } while ((ret == -EAGAIN) && idr_pre_get(ps, GFP_KERNEL));
1945
1946         if (ret)
1947                 goto err1;
1948
1949         inet_get_local_port_range(&low, &high);
1950         if (port > high) {
1951                 if (next_port != low) {
1952                         idr_remove(ps, port);
1953                         next_port = low;
1954                         goto retry;
1955                 }
1956                 ret = -EADDRNOTAVAIL;
1957                 goto err2;
1958         }
1959
1960         if (port == high)
1961                 next_port = low;
1962         else
1963                 next_port = port + 1;
1964
1965         bind_list->ps = ps;
1966         bind_list->port = (unsigned short) port;
1967         cma_bind_port(bind_list, id_priv);
1968         return 0;
1969 err2:
1970         idr_remove(ps, port);
1971 err1:
1972         kfree(bind_list);
1973         return ret;
1974 }
1975
1976 static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
1977 {
1978         struct rdma_id_private *cur_id;
1979         struct sockaddr_in *sin, *cur_sin;
1980         struct rdma_bind_list *bind_list;
1981         struct hlist_node *node;
1982         unsigned short snum;
1983
1984         sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1985         snum = ntohs(sin->sin_port);
1986         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
1987                 return -EACCES;
1988
1989         bind_list = idr_find(ps, snum);
1990         if (!bind_list)
1991                 return cma_alloc_port(ps, id_priv, snum);
1992
1993         /*
1994          * We don't support binding to any address if anyone is bound to
1995          * a specific address on the same port.
1996          */
1997         if (cma_any_addr(&id_priv->id.route.addr.src_addr))
1998                 return -EADDRNOTAVAIL;
1999
2000         hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
2001                 if (cma_any_addr(&cur_id->id.route.addr.src_addr))
2002                         return -EADDRNOTAVAIL;
2003
2004                 cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
2005                 if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
2006                         return -EADDRINUSE;
2007         }
2008
2009         cma_bind_port(bind_list, id_priv);
2010         return 0;
2011 }
2012
2013 static int cma_get_port(struct rdma_id_private *id_priv)
2014 {
2015         struct idr *ps;
2016         int ret;
2017
2018         switch (id_priv->id.ps) {
2019         case RDMA_PS_SDP:
2020                 ps = &sdp_ps;
2021                 break;
2022         case RDMA_PS_TCP:
2023                 ps = &tcp_ps;
2024                 break;
2025         case RDMA_PS_UDP:
2026                 ps = &udp_ps;
2027                 break;
2028         case RDMA_PS_IPOIB:
2029                 ps = &ipoib_ps;
2030                 break;
2031         default:
2032                 return -EPROTONOSUPPORT;
2033         }
2034
2035         mutex_lock(&lock);
2036         if (cma_any_port(&id_priv->id.route.addr.src_addr))
2037                 ret = cma_alloc_any_port(ps, id_priv);
2038         else
2039                 ret = cma_use_port(ps, id_priv);
2040         mutex_unlock(&lock);
2041
2042         return ret;
2043 }
2044
2045 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2046 {
2047         struct rdma_id_private *id_priv;
2048         int ret;
2049
2050         if (addr->sa_family != AF_INET)
2051                 return -EAFNOSUPPORT;
2052
2053         id_priv = container_of(id, struct rdma_id_private, id);
2054         if (!cma_comp_exch(id_priv, CMA_IDLE, CMA_ADDR_BOUND))
2055                 return -EINVAL;
2056
2057         if (!cma_any_addr(addr)) {
2058                 ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
2059                 if (ret)
2060                         goto err1;
2061
2062                 mutex_lock(&lock);
2063                 ret = cma_acquire_dev(id_priv);
2064                 mutex_unlock(&lock);
2065                 if (ret)
2066                         goto err1;
2067         }
2068
2069         memcpy(&id->route.addr.src_addr, addr, ip_addr_size(addr));
2070         ret = cma_get_port(id_priv);
2071         if (ret)
2072                 goto err2;
2073
2074         return 0;
2075 err2:
2076         if (!cma_any_addr(addr)) {
2077                 mutex_lock(&lock);
2078                 cma_detach_from_dev(id_priv);
2079                 mutex_unlock(&lock);
2080         }
2081 err1:
2082         cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_IDLE);
2083         return ret;
2084 }
2085 EXPORT_SYMBOL(rdma_bind_addr);
2086
2087 static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2088                           struct rdma_route *route)
2089 {
2090         struct sockaddr_in *src4, *dst4;
2091         struct cma_hdr *cma_hdr;
2092         struct sdp_hh *sdp_hdr;
2093
2094         src4 = (struct sockaddr_in *) &route->addr.src_addr;
2095         dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2096
2097         switch (ps) {
2098         case RDMA_PS_SDP:
2099                 sdp_hdr = hdr;
2100                 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2101                         return -EINVAL;
2102                 sdp_set_ip_ver(sdp_hdr, 4);
2103                 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2104                 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2105                 sdp_hdr->port = src4->sin_port;
2106                 break;
2107         default:
2108                 cma_hdr = hdr;
2109                 cma_hdr->cma_version = CMA_VERSION;
2110                 cma_set_ip_ver(cma_hdr, 4);
2111                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2112                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2113                 cma_hdr->port = src4->sin_port;
2114                 break;
2115         }
2116         return 0;
2117 }
2118
2119 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2120                                 struct ib_cm_event *ib_event)
2121 {
2122         struct rdma_id_private *id_priv = cm_id->context;
2123         struct rdma_cm_event event;
2124         struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2125         int ret = 0;
2126
2127         if (cma_disable_remove(id_priv, CMA_CONNECT))
2128                 return 0;
2129
2130         memset(&event, 0, sizeof event);
2131         switch (ib_event->event) {
2132         case IB_CM_SIDR_REQ_ERROR:
2133                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2134                 event.status = -ETIMEDOUT;
2135                 break;
2136         case IB_CM_SIDR_REP_RECEIVED:
2137                 event.param.ud.private_data = ib_event->private_data;
2138                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2139                 if (rep->status != IB_SIDR_SUCCESS) {
2140                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2141                         event.status = ib_event->param.sidr_rep_rcvd.status;
2142                         break;
2143                 }
2144                 if (id_priv->qkey != rep->qkey) {
2145                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2146                         event.status = -EINVAL;
2147                         break;
2148                 }
2149                 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2150                                      id_priv->id.route.path_rec,
2151                                      &event.param.ud.ah_attr);
2152                 event.param.ud.qp_num = rep->qpn;
2153                 event.param.ud.qkey = rep->qkey;
2154                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2155                 event.status = 0;
2156                 break;
2157         default:
2158                 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d",
2159                        ib_event->event);
2160                 goto out;
2161         }
2162
2163         ret = id_priv->id.event_handler(&id_priv->id, &event);
2164         if (ret) {
2165                 /* Destroy the CM ID by returning a non-zero value. */
2166                 id_priv->cm_id.ib = NULL;
2167                 cma_exch(id_priv, CMA_DESTROYING);
2168                 cma_enable_remove(id_priv);
2169                 rdma_destroy_id(&id_priv->id);
2170                 return ret;
2171         }
2172 out:
2173         cma_enable_remove(id_priv);
2174         return ret;
2175 }
2176
2177 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2178                               struct rdma_conn_param *conn_param)
2179 {
2180         struct ib_cm_sidr_req_param req;
2181         struct rdma_route *route;
2182         int ret;
2183
2184         req.private_data_len = sizeof(struct cma_hdr) +
2185                                conn_param->private_data_len;
2186         req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2187         if (!req.private_data)
2188                 return -ENOMEM;
2189
2190         if (conn_param->private_data && conn_param->private_data_len)
2191                 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2192                        conn_param->private_data, conn_param->private_data_len);
2193
2194         route = &id_priv->id.route;
2195         ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2196         if (ret)
2197                 goto out;
2198
2199         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device,
2200                                             cma_sidr_rep_handler, id_priv);
2201         if (IS_ERR(id_priv->cm_id.ib)) {
2202                 ret = PTR_ERR(id_priv->cm_id.ib);
2203                 goto out;
2204         }
2205
2206         req.path = route->path_rec;
2207         req.service_id = cma_get_service_id(id_priv->id.ps,
2208                                             &route->addr.dst_addr);
2209         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2210         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2211
2212         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2213         if (ret) {
2214                 ib_destroy_cm_id(id_priv->cm_id.ib);
2215                 id_priv->cm_id.ib = NULL;
2216         }
2217 out:
2218         kfree(req.private_data);
2219         return ret;
2220 }
2221
2222 static int cma_connect_ib(struct rdma_id_private *id_priv,
2223                           struct rdma_conn_param *conn_param)
2224 {
2225         struct ib_cm_req_param req;
2226         struct rdma_route *route;
2227         void *private_data;
2228         int offset, ret;
2229
2230         memset(&req, 0, sizeof req);
2231         offset = cma_user_data_offset(id_priv->id.ps);
2232         req.private_data_len = offset + conn_param->private_data_len;
2233         private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2234         if (!private_data)
2235                 return -ENOMEM;
2236
2237         if (conn_param->private_data && conn_param->private_data_len)
2238                 memcpy(private_data + offset, conn_param->private_data,
2239                        conn_param->private_data_len);
2240
2241         id_priv->cm_id.ib = ib_create_cm_id(id_priv->id.device, cma_ib_handler,
2242                                             id_priv);
2243         if (IS_ERR(id_priv->cm_id.ib)) {
2244                 ret = PTR_ERR(id_priv->cm_id.ib);
2245                 goto out;
2246         }
2247
2248         route = &id_priv->id.route;
2249         ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2250         if (ret)
2251                 goto out;
2252         req.private_data = private_data;
2253
2254         req.primary_path = &route->path_rec[0];
2255         if (route->num_paths == 2)
2256                 req.alternate_path = &route->path_rec[1];
2257
2258         req.service_id = cma_get_service_id(id_priv->id.ps,
2259                                             &route->addr.dst_addr);
2260         req.qp_num = id_priv->qp_num;
2261         req.qp_type = IB_QPT_RC;
2262         req.starting_psn = id_priv->seq_num;
2263         req.responder_resources = conn_param->responder_resources;
2264         req.initiator_depth = conn_param->initiator_depth;
2265         req.flow_control = conn_param->flow_control;
2266         req.retry_count = conn_param->retry_count;
2267         req.rnr_retry_count = conn_param->rnr_retry_count;
2268         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2269         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2270         req.max_cm_retries = CMA_MAX_CM_RETRIES;
2271         req.srq = id_priv->srq ? 1 : 0;
2272
2273         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2274 out:
2275         if (ret && !IS_ERR(id_priv->cm_id.ib)) {
2276                 ib_destroy_cm_id(id_priv->cm_id.ib);
2277                 id_priv->cm_id.ib = NULL;
2278         }
2279
2280         kfree(private_data);
2281         return ret;
2282 }
2283
2284 static int cma_connect_iw(struct rdma_id_private *id_priv,
2285                           struct rdma_conn_param *conn_param)
2286 {
2287         struct iw_cm_id *cm_id;
2288         struct sockaddr_in* sin;
2289         int ret;
2290         struct iw_cm_conn_param iw_param;
2291
2292         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
2293         if (IS_ERR(cm_id)) {
2294                 ret = PTR_ERR(cm_id);
2295                 goto out;
2296         }
2297
2298         id_priv->cm_id.iw = cm_id;
2299
2300         sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2301         cm_id->local_addr = *sin;
2302
2303         sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2304         cm_id->remote_addr = *sin;
2305
2306         ret = cma_modify_qp_rtr(id_priv, conn_param);
2307         if (ret)
2308                 goto out;
2309
2310         iw_param.ord = conn_param->initiator_depth;
2311         iw_param.ird = conn_param->responder_resources;
2312         iw_param.private_data = conn_param->private_data;
2313         iw_param.private_data_len = conn_param->private_data_len;
2314         if (id_priv->id.qp)
2315                 iw_param.qpn = id_priv->qp_num;
2316         else
2317                 iw_param.qpn = conn_param->qp_num;
2318         ret = iw_cm_connect(cm_id, &iw_param);
2319 out:
2320         if (ret && !IS_ERR(cm_id)) {
2321                 iw_destroy_cm_id(cm_id);
2322                 id_priv->cm_id.iw = NULL;
2323         }
2324         return ret;
2325 }
2326
2327 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2328 {
2329         struct rdma_id_private *id_priv;
2330         int ret;
2331
2332         id_priv = container_of(id, struct rdma_id_private, id);
2333         if (!cma_comp_exch(id_priv, CMA_ROUTE_RESOLVED, CMA_CONNECT))
2334                 return -EINVAL;
2335
2336         if (!id->qp) {
2337                 id_priv->qp_num = conn_param->qp_num;
2338                 id_priv->srq = conn_param->srq;
2339         }
2340
2341         switch (rdma_node_get_transport(id->device->node_type)) {
2342         case RDMA_TRANSPORT_IB:
2343                 if (cma_is_ud_ps(id->ps))
2344                         ret = cma_resolve_ib_udp(id_priv, conn_param);
2345                 else
2346                         ret = cma_connect_ib(id_priv, conn_param);
2347                 break;
2348         case RDMA_TRANSPORT_IWARP:
2349                 ret = cma_connect_iw(id_priv, conn_param);
2350                 break;
2351         default:
2352                 ret = -ENOSYS;
2353                 break;
2354         }
2355         if (ret)
2356                 goto err;
2357
2358         return 0;
2359 err:
2360         cma_comp_exch(id_priv, CMA_CONNECT, CMA_ROUTE_RESOLVED);
2361         return ret;
2362 }
2363 EXPORT_SYMBOL(rdma_connect);
2364
2365 static int cma_accept_ib(struct rdma_id_private *id_priv,
2366                          struct rdma_conn_param *conn_param)
2367 {
2368         struct ib_cm_rep_param rep;
2369         int ret;
2370
2371         ret = cma_modify_qp_rtr(id_priv, conn_param);
2372         if (ret)
2373                 goto out;
2374
2375         ret = cma_modify_qp_rts(id_priv, conn_param);
2376         if (ret)
2377                 goto out;
2378
2379         memset(&rep, 0, sizeof rep);
2380         rep.qp_num = id_priv->qp_num;
2381         rep.starting_psn = id_priv->seq_num;
2382         rep.private_data = conn_param->private_data;
2383         rep.private_data_len = conn_param->private_data_len;
2384         rep.responder_resources = conn_param->responder_resources;
2385         rep.initiator_depth = conn_param->initiator_depth;
2386         rep.failover_accepted = 0;
2387         rep.flow_control = conn_param->flow_control;
2388         rep.rnr_retry_count = conn_param->rnr_retry_count;
2389         rep.srq = id_priv->srq ? 1 : 0;
2390
2391         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2392 out:
2393         return ret;
2394 }
2395
2396 static int cma_accept_iw(struct rdma_id_private *id_priv,
2397                   struct rdma_conn_param *conn_param)
2398 {
2399         struct iw_cm_conn_param iw_param;
2400         int ret;
2401
2402         ret = cma_modify_qp_rtr(id_priv, conn_param);
2403         if (ret)
2404                 return ret;
2405
2406         iw_param.ord = conn_param->initiator_depth;
2407         iw_param.ird = conn_param->responder_resources;
2408         iw_param.private_data = conn_param->private_data;
2409         iw_param.private_data_len = conn_param->private_data_len;
2410         if (id_priv->id.qp) {
2411                 iw_param.qpn = id_priv->qp_num;
2412         } else
2413                 iw_param.qpn = conn_param->qp_num;
2414
2415         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2416 }
2417
2418 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2419                              enum ib_cm_sidr_status status,
2420                              const void *private_data, int private_data_len)
2421 {
2422         struct ib_cm_sidr_rep_param rep;
2423
2424         memset(&rep, 0, sizeof rep);
2425         rep.status = status;
2426         if (status == IB_SIDR_SUCCESS) {
2427                 rep.qp_num = id_priv->qp_num;
2428                 rep.qkey = id_priv->qkey;
2429         }
2430         rep.private_data = private_data;
2431         rep.private_data_len = private_data_len;
2432
2433         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2434 }
2435
2436 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2437 {
2438         struct rdma_id_private *id_priv;
2439         int ret;
2440
2441         id_priv = container_of(id, struct rdma_id_private, id);
2442         if (!cma_comp(id_priv, CMA_CONNECT))
2443                 return -EINVAL;
2444
2445         if (!id->qp && conn_param) {
2446                 id_priv->qp_num = conn_param->qp_num;
2447                 id_priv->srq = conn_param->srq;
2448         }
2449
2450         switch (rdma_node_get_transport(id->device->node_type)) {
2451         case RDMA_TRANSPORT_IB:
2452                 if (cma_is_ud_ps(id->ps))
2453                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2454                                                 conn_param->private_data,
2455                                                 conn_param->private_data_len);
2456                 else if (conn_param)
2457                         ret = cma_accept_ib(id_priv, conn_param);
2458                 else
2459                         ret = cma_rep_recv(id_priv);
2460                 break;
2461         case RDMA_TRANSPORT_IWARP:
2462                 ret = cma_accept_iw(id_priv, conn_param);
2463                 break;
2464         default:
2465                 ret = -ENOSYS;
2466                 break;
2467         }
2468
2469         if (ret)
2470                 goto reject;
2471
2472         return 0;
2473 reject:
2474         cma_modify_qp_err(id_priv);
2475         rdma_reject(id, NULL, 0);
2476         return ret;
2477 }
2478 EXPORT_SYMBOL(rdma_accept);
2479
2480 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2481 {
2482         struct rdma_id_private *id_priv;
2483         int ret;
2484
2485         id_priv = container_of(id, struct rdma_id_private, id);
2486         if (!cma_has_cm_dev(id_priv))
2487                 return -EINVAL;
2488
2489         switch (id->device->node_type) {
2490         case RDMA_NODE_IB_CA:
2491                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2492                 break;
2493         default:
2494                 ret = 0;
2495                 break;
2496         }
2497         return ret;
2498 }
2499 EXPORT_SYMBOL(rdma_notify);
2500
2501 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2502                 u8 private_data_len)
2503 {
2504         struct rdma_id_private *id_priv;
2505         int ret;
2506
2507         id_priv = container_of(id, struct rdma_id_private, id);
2508         if (!cma_has_cm_dev(id_priv))
2509                 return -EINVAL;
2510
2511         switch (rdma_node_get_transport(id->device->node_type)) {
2512         case RDMA_TRANSPORT_IB:
2513                 if (cma_is_ud_ps(id->ps))
2514                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2515                                                 private_data, private_data_len);
2516                 else
2517                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
2518                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
2519                                              0, private_data, private_data_len);
2520                 break;
2521         case RDMA_TRANSPORT_IWARP:
2522                 ret = iw_cm_reject(id_priv->cm_id.iw,
2523                                    private_data, private_data_len);
2524                 break;
2525         default:
2526                 ret = -ENOSYS;
2527                 break;
2528         }
2529         return ret;
2530 }
2531 EXPORT_SYMBOL(rdma_reject);
2532
2533 int rdma_disconnect(struct rdma_cm_id *id)
2534 {
2535         struct rdma_id_private *id_priv;
2536         int ret;
2537
2538         id_priv = container_of(id, struct rdma_id_private, id);
2539         if (!cma_has_cm_dev(id_priv))
2540                 return -EINVAL;
2541
2542         switch (rdma_node_get_transport(id->device->node_type)) {
2543         case RDMA_TRANSPORT_IB:
2544                 ret = cma_modify_qp_err(id_priv);
2545                 if (ret)
2546                         goto out;
2547                 /* Initiate or respond to a disconnect. */
2548                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
2549                         ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
2550                 break;
2551         case RDMA_TRANSPORT_IWARP:
2552                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
2553                 break;
2554         default:
2555                 ret = -EINVAL;
2556                 break;
2557         }
2558 out:
2559         return ret;
2560 }
2561 EXPORT_SYMBOL(rdma_disconnect);
2562
2563 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
2564 {
2565         struct rdma_id_private *id_priv;
2566         struct cma_multicast *mc = multicast->context;
2567         struct rdma_cm_event event;
2568         int ret;
2569
2570         id_priv = mc->id_priv;
2571         if (cma_disable_remove(id_priv, CMA_ADDR_BOUND) &&
2572             cma_disable_remove(id_priv, CMA_ADDR_RESOLVED))
2573                 return 0;
2574
2575         mutex_lock(&id_priv->qp_mutex);
2576         if (!status && id_priv->id.qp)
2577                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
2578                                          multicast->rec.mlid);
2579         mutex_unlock(&id_priv->qp_mutex);
2580
2581         memset(&event, 0, sizeof event);
2582         event.status = status;
2583         event.param.ud.private_data = mc->context;
2584         if (!status) {
2585                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
2586                 ib_init_ah_from_mcmember(id_priv->id.device,
2587                                          id_priv->id.port_num, &multicast->rec,
2588                                          &event.param.ud.ah_attr);
2589                 event.param.ud.qp_num = 0xFFFFFF;
2590                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
2591         } else
2592                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
2593
2594         ret = id_priv->id.event_handler(&id_priv->id, &event);
2595         if (ret) {
2596                 cma_exch(id_priv, CMA_DESTROYING);
2597                 cma_enable_remove(id_priv);
2598                 rdma_destroy_id(&id_priv->id);
2599                 return 0;
2600         }
2601
2602         cma_enable_remove(id_priv);
2603         return 0;
2604 }
2605
2606 static void cma_set_mgid(struct rdma_id_private *id_priv,
2607                          struct sockaddr *addr, union ib_gid *mgid)
2608 {
2609         unsigned char mc_map[MAX_ADDR_LEN];
2610         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2611         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
2612         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
2613
2614         if (cma_any_addr(addr)) {
2615                 memset(mgid, 0, sizeof *mgid);
2616         } else if ((addr->sa_family == AF_INET6) &&
2617                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFF10A01B) ==
2618                                                                  0xFF10A01B)) {
2619                 /* IPv6 address is an SA assigned MGID. */
2620                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
2621         } else {
2622                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
2623                 if (id_priv->id.ps == RDMA_PS_UDP)
2624                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
2625                 *mgid = *(union ib_gid *) (mc_map + 4);
2626         }
2627 }
2628
2629 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
2630                                  struct cma_multicast *mc)
2631 {
2632         struct ib_sa_mcmember_rec rec;
2633         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2634         ib_sa_comp_mask comp_mask;
2635         int ret;
2636
2637         ib_addr_get_mgid(dev_addr, &rec.mgid);
2638         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
2639                                      &rec.mgid, &rec);
2640         if (ret)
2641                 return ret;
2642
2643         cma_set_mgid(id_priv, &mc->addr, &rec.mgid);
2644         if (id_priv->id.ps == RDMA_PS_UDP)
2645                 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
2646         ib_addr_get_sgid(dev_addr, &rec.port_gid);
2647         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2648         rec.join_state = 1;
2649
2650         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
2651                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
2652                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
2653                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
2654                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
2655
2656         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
2657                                                 id_priv->id.port_num, &rec,
2658                                                 comp_mask, GFP_KERNEL,
2659                                                 cma_ib_mc_handler, mc);
2660         if (IS_ERR(mc->multicast.ib))
2661                 return PTR_ERR(mc->multicast.ib);
2662
2663         return 0;
2664 }
2665
2666 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
2667                         void *context)
2668 {
2669         struct rdma_id_private *id_priv;
2670         struct cma_multicast *mc;
2671         int ret;
2672
2673         id_priv = container_of(id, struct rdma_id_private, id);
2674         if (!cma_comp(id_priv, CMA_ADDR_BOUND) &&
2675             !cma_comp(id_priv, CMA_ADDR_RESOLVED))
2676                 return -EINVAL;
2677
2678         mc = kmalloc(sizeof *mc, GFP_KERNEL);
2679         if (!mc)
2680                 return -ENOMEM;
2681
2682         memcpy(&mc->addr, addr, ip_addr_size(addr));
2683         mc->context = context;
2684         mc->id_priv = id_priv;
2685
2686         spin_lock(&id_priv->lock);
2687         list_add(&mc->list, &id_priv->mc_list);
2688         spin_unlock(&id_priv->lock);
2689
2690         switch (rdma_node_get_transport(id->device->node_type)) {
2691         case RDMA_TRANSPORT_IB:
2692                 ret = cma_join_ib_multicast(id_priv, mc);
2693                 break;
2694         default:
2695                 ret = -ENOSYS;
2696                 break;
2697         }
2698
2699         if (ret) {
2700                 spin_lock_irq(&id_priv->lock);
2701                 list_del(&mc->list);
2702                 spin_unlock_irq(&id_priv->lock);
2703                 kfree(mc);
2704         }
2705         return ret;
2706 }
2707 EXPORT_SYMBOL(rdma_join_multicast);
2708
2709 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
2710 {
2711         struct rdma_id_private *id_priv;
2712         struct cma_multicast *mc;
2713
2714         id_priv = container_of(id, struct rdma_id_private, id);
2715         spin_lock_irq(&id_priv->lock);
2716         list_for_each_entry(mc, &id_priv->mc_list, list) {
2717                 if (!memcmp(&mc->addr, addr, ip_addr_size(addr))) {
2718                         list_del(&mc->list);
2719                         spin_unlock_irq(&id_priv->lock);
2720
2721                         if (id->qp)
2722                                 ib_detach_mcast(id->qp,
2723                                                 &mc->multicast.ib->rec.mgid,
2724                                                 mc->multicast.ib->rec.mlid);
2725                         ib_sa_free_multicast(mc->multicast.ib);
2726                         kfree(mc);
2727                         return;
2728                 }
2729         }
2730         spin_unlock_irq(&id_priv->lock);
2731 }
2732 EXPORT_SYMBOL(rdma_leave_multicast);
2733
2734 static void cma_add_one(struct ib_device *device)
2735 {
2736         struct cma_device *cma_dev;
2737         struct rdma_id_private *id_priv;
2738
2739         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
2740         if (!cma_dev)
2741                 return;
2742
2743         cma_dev->device = device;
2744
2745         init_completion(&cma_dev->comp);
2746         atomic_set(&cma_dev->refcount, 1);
2747         INIT_LIST_HEAD(&cma_dev->id_list);
2748         ib_set_client_data(device, &cma_client, cma_dev);
2749
2750         mutex_lock(&lock);
2751         list_add_tail(&cma_dev->list, &dev_list);
2752         list_for_each_entry(id_priv, &listen_any_list, list)
2753                 cma_listen_on_dev(id_priv, cma_dev);
2754         mutex_unlock(&lock);
2755 }
2756
2757 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
2758 {
2759         struct rdma_cm_event event;
2760         enum cma_state state;
2761
2762         /* Record that we want to remove the device */
2763         state = cma_exch(id_priv, CMA_DEVICE_REMOVAL);
2764         if (state == CMA_DESTROYING)
2765                 return 0;
2766
2767         cma_cancel_operation(id_priv, state);
2768         wait_event(id_priv->wait_remove, !atomic_read(&id_priv->dev_remove));
2769
2770         /* Check for destruction from another callback. */
2771         if (!cma_comp(id_priv, CMA_DEVICE_REMOVAL))
2772                 return 0;
2773
2774         memset(&event, 0, sizeof event);
2775         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
2776         return id_priv->id.event_handler(&id_priv->id, &event);
2777 }
2778
2779 static void cma_process_remove(struct cma_device *cma_dev)
2780 {
2781         struct rdma_id_private *id_priv;
2782         int ret;
2783
2784         mutex_lock(&lock);
2785         while (!list_empty(&cma_dev->id_list)) {
2786                 id_priv = list_entry(cma_dev->id_list.next,
2787                                      struct rdma_id_private, list);
2788
2789                 list_del(&id_priv->listen_list);
2790                 list_del_init(&id_priv->list);
2791                 atomic_inc(&id_priv->refcount);
2792                 mutex_unlock(&lock);
2793
2794                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
2795                 cma_deref_id(id_priv);
2796                 if (ret)
2797                         rdma_destroy_id(&id_priv->id);
2798
2799                 mutex_lock(&lock);
2800         }
2801         mutex_unlock(&lock);
2802
2803         cma_deref_dev(cma_dev);
2804         wait_for_completion(&cma_dev->comp);
2805 }
2806
2807 static void cma_remove_one(struct ib_device *device)
2808 {
2809         struct cma_device *cma_dev;
2810
2811         cma_dev = ib_get_client_data(device, &cma_client);
2812         if (!cma_dev)
2813                 return;
2814
2815         mutex_lock(&lock);
2816         list_del(&cma_dev->list);
2817         mutex_unlock(&lock);
2818
2819         cma_process_remove(cma_dev);
2820         kfree(cma_dev);
2821 }
2822
2823 static int cma_init(void)
2824 {
2825         int ret, low, high, remaining;
2826
2827         get_random_bytes(&next_port, sizeof next_port);
2828         inet_get_local_port_range(&low, &high);
2829         remaining = (high - low) + 1;
2830         next_port = ((unsigned int) next_port % remaining) + low;
2831
2832         cma_wq = create_singlethread_workqueue("rdma_cm");
2833         if (!cma_wq)
2834                 return -ENOMEM;
2835
2836         ib_sa_register_client(&sa_client);
2837         rdma_addr_register_client(&addr_client);
2838
2839         ret = ib_register_client(&cma_client);
2840         if (ret)
2841                 goto err;
2842         return 0;
2843
2844 err:
2845         rdma_addr_unregister_client(&addr_client);
2846         ib_sa_unregister_client(&sa_client);
2847         destroy_workqueue(cma_wq);
2848         return ret;
2849 }
2850
2851 static void cma_cleanup(void)
2852 {
2853         ib_unregister_client(&cma_client);
2854         rdma_addr_unregister_client(&addr_client);
2855         ib_sa_unregister_client(&sa_client);
2856         destroy_workqueue(cma_wq);
2857         idr_destroy(&sdp_ps);
2858         idr_destroy(&tcp_ps);
2859         idr_destroy(&udp_ps);
2860         idr_destroy(&ipoib_ps);
2861 }
2862
2863 module_init(cma_init);
2864 module_exit(cma_cleanup);