drivers/infiniband/core/cm.c

   1 /*
   2  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   3  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   4  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   5  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   6  *
   7  * This software is available to you under a choice of one of two
   8  * licenses.  You may choose to be licensed under the terms of the GNU
   9  * General Public License (GPL) Version 2, available from the file
  10  * COPYING in the main directory of this source tree, or the
  11  * OpenIB.org BSD license below:
  12  *
  13  *     Redistribution and use in source and binary forms, with or
  14  *     without modification, are permitted provided that the following
  15  *     conditions are met:
  16  *
  17  *      - Redistributions of source code must retain the above
  18  *        copyright notice, this list of conditions and the following
  19  *        disclaimer.
  20  *
  21  *      - Redistributions in binary form must reproduce the above
  22  *        copyright notice, this list of conditions and the following
  23  *        disclaimer in the documentation and/or other materials
  24  *        provided with the distribution.
  25  *
  26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  33  * SOFTWARE.
  34  */
  35
  36 #include <linux/completion.h>
  37 #include <linux/dma-mapping.h>
  38 #include <linux/device.h>
  39 #include <linux/module.h>
  40 #include <linux/err.h>
  41 #include <linux/idr.h>
  42 #include <linux/interrupt.h>
  43 #include <linux/random.h>
  44 #include <linux/rbtree.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/slab.h>
  47 #include <linux/sysfs.h>
  48 #include <linux/workqueue.h>
  49 #include <linux/kdev_t.h>
  50
  51 #include <rdma/ib_cache.h>
  52 #include <rdma/ib_cm.h>
  53 #include "cm_msgs.h"
  54
  55 MODULE_AUTHOR("Sean Hefty");
  56 MODULE_DESCRIPTION("InfiniBand CM");
  57 MODULE_LICENSE("Dual BSD/GPL");
  58
  59 static void cm_add_one(struct ib_device *device);
  60 static void cm_remove_one(struct ib_device *device);
  61
  62 static struct ib_client cm_client = {
  63         .name   = "cm",
  64         .add    = cm_add_one,
  65         .remove = cm_remove_one
  66 };
  67
  68 static struct ib_cm {
  69         spinlock_t lock;
  70         struct list_head device_list;
  71         rwlock_t device_lock;
  72         struct rb_root listen_service_table;
  73         u64 listen_service_id;
  74         /* struct rb_root peer_service_table; todo: fix peer to peer */
  75         struct rb_root remote_qp_table;
  76         struct rb_root remote_id_table;
  77         struct rb_root remote_sidr_table;
  78         struct idr local_id_table;
  79         __be32 random_id_operand;
  80         struct list_head timewait_list;
  81         struct workqueue_struct *wq;
  82 } cm;
  83
  84 /* Counter indexes ordered by attribute ID */
  85 enum {
  86         CM_REQ_COUNTER,
  87         CM_MRA_COUNTER,
  88         CM_REJ_COUNTER,
  89         CM_REP_COUNTER,
  90         CM_RTU_COUNTER,
  91         CM_DREQ_COUNTER,
  92         CM_DREP_COUNTER,
  93         CM_SIDR_REQ_COUNTER,
  94         CM_SIDR_REP_COUNTER,
  95         CM_LAP_COUNTER,
  96         CM_APR_COUNTER,
  97         CM_ATTR_COUNT,
  98         CM_ATTR_ID_OFFSET = 0x0010,
  99 };
 100
 101 enum {
 102         CM_XMIT,
 103         CM_XMIT_RETRIES,
 104         CM_RECV,
 105         CM_RECV_DUPLICATES,
 106         CM_COUNTER_GROUPS
 107 };
 108
 109 static char const counter_group_names[CM_COUNTER_GROUPS]
 110                                      [sizeof("cm_rx_duplicates")] = {
 111         "cm_tx_msgs", "cm_tx_retries",
 112         "cm_rx_msgs", "cm_rx_duplicates"
 113 };
 114
 115 struct cm_counter_group {
 116         struct kobject obj;
 117         atomic_long_t counter[CM_ATTR_COUNT];
 118 };
 119
 120 struct cm_counter_attribute {
 121         struct attribute attr;
 122         int index;
 123 };
 124
 125 #define CM_COUNTER_ATTR(_name, _index) \
 126 struct cm_counter_attribute cm_##_name##_counter_attr = { \
 127         .attr = { .name = __stringify(_name), .mode = 0444 }, \
 128         .index = _index \
 129 }
 130
 131 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
 132 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
 133 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
 134 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
 135 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
 136 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
 137 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
 138 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
 139 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
 140 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
 141 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
 142
 143 static struct attribute *cm_counter_default_attrs[] = {
 144         &cm_req_counter_attr.attr,
 145         &cm_mra_counter_attr.attr,
 146         &cm_rej_counter_attr.attr,
 147         &cm_rep_counter_attr.attr,
 148         &cm_rtu_counter_attr.attr,
 149         &cm_dreq_counter_attr.attr,
 150         &cm_drep_counter_attr.attr,
 151         &cm_sidr_req_counter_attr.attr,
 152         &cm_sidr_rep_counter_attr.attr,
 153         &cm_lap_counter_attr.attr,
 154         &cm_apr_counter_attr.attr,
 155         NULL
 156 };
 157
 158 struct cm_port {
 159         struct cm_device *cm_dev;
 160         struct ib_mad_agent *mad_agent;
 161         struct kobject port_obj;
 162         u8 port_num;
 163         struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
 164 };
 165
 166 struct cm_device {
 167         struct list_head list;
 168         struct ib_device *ib_device;
 169         struct device *device;
 170         u8 ack_delay;
 171         struct cm_port *port[0];
 172 };
 173
 174 struct cm_av {
 175         struct cm_port *port;
 176         union ib_gid dgid;
 177         struct ib_ah_attr ah_attr;
 178         u16 pkey_index;
 179         u8 timeout;
 180 };
 181
 182 struct cm_work {
 183         struct delayed_work work;
 184         struct list_head list;
 185         struct cm_port *port;
 186         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 187         __be32 local_id;                        /* Established / timewait */
 188         __be32 remote_id;
 189         struct ib_cm_event cm_event;
 190         struct ib_sa_path_rec path[0];
 191 };
 192
 193 struct cm_timewait_info {
 194         struct cm_work work;                    /* Must be first. */
 195         struct list_head list;
 196         struct rb_node remote_qp_node;
 197         struct rb_node remote_id_node;
 198         __be64 remote_ca_guid;
 199         __be32 remote_qpn;
 200         u8 inserted_remote_qp;
 201         u8 inserted_remote_id;
 202 };
 203
 204 struct cm_id_private {
 205         struct ib_cm_id id;
 206
 207         struct rb_node service_node;
 208         struct rb_node sidr_id_node;
 209         spinlock_t lock;        /* Do not acquire inside cm.lock */
 210         struct completion comp;
 211         atomic_t refcount;
 212
 213         struct ib_mad_send_buf *msg;
 214         struct cm_timewait_info *timewait_info;
 215         /* todo: use alternate port on send failure */
 216         struct cm_av av;
 217         struct cm_av alt_av;
 218         struct ib_cm_compare_data *compare_data;
 219
 220         void *private_data;
 221         __be64 tid;
 222         __be32 local_qpn;
 223         __be32 remote_qpn;
 224         enum ib_qp_type qp_type;
 225         __be32 sq_psn;
 226         __be32 rq_psn;
 227         int timeout_ms;
 228         enum ib_mtu path_mtu;
 229         __be16 pkey;
 230         u8 private_data_len;
 231         u8 max_cm_retries;
 232         u8 peer_to_peer;
 233         u8 responder_resources;
 234         u8 initiator_depth;
 235         u8 retry_count;
 236         u8 rnr_retry_count;
 237         u8 service_timeout;
 238         u8 target_ack_delay;
 239
 240         struct list_head work_list;
 241         atomic_t work_count;
 242 };
 243
 244 static void cm_work_handler(struct work_struct *work);
 245
 246 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 247 {
 248         if (atomic_dec_and_test(&cm_id_priv->refcount))
 249                 complete(&cm_id_priv->comp);
 250 }
 251
 252 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 253                         struct ib_mad_send_buf **msg)
 254 {
 255         struct ib_mad_agent *mad_agent;
 256         struct ib_mad_send_buf *m;
 257         struct ib_ah *ah;
 258
 259         mad_agent = cm_id_priv->av.port->mad_agent;
 260         ah = ib_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr);
 261         if (IS_ERR(ah))
 262                 return PTR_ERR(ah);
 263
 264         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 265                                cm_id_priv->av.pkey_index,
 266                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 267                                GFP_ATOMIC);
 268         if (IS_ERR(m)) {
 269                 ib_destroy_ah(ah);
 270                 return PTR_ERR(m);
 271         }
 272
 273         /* Timeout set by caller if response is expected. */
 274         m->ah = ah;
 275         m->retries = cm_id_priv->max_cm_retries;
 276
 277         atomic_inc(&cm_id_priv->refcount);
 278         m->context[0] = cm_id_priv;
 279         *msg = m;
 280         return 0;
 281 }
 282
 283 static int cm_alloc_response_msg(struct cm_port *port,
 284                                  struct ib_mad_recv_wc *mad_recv_wc,
 285                                  struct ib_mad_send_buf **msg)
 286 {
 287         struct ib_mad_send_buf *m;
 288         struct ib_ah *ah;
 289
 290         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 291                                   mad_recv_wc->recv_buf.grh, port->port_num);
 292         if (IS_ERR(ah))
 293                 return PTR_ERR(ah);
 294
 295         m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 296                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 297                                GFP_ATOMIC);
 298         if (IS_ERR(m)) {
 299                 ib_destroy_ah(ah);
 300                 return PTR_ERR(m);
 301         }
 302         m->ah = ah;
 303         *msg = m;
 304         return 0;
 305 }
 306
 307 static void cm_free_msg(struct ib_mad_send_buf *msg)
 308 {
 309         ib_destroy_ah(msg->ah);
 310         if (msg->context[0])
 311                 cm_deref_id(msg->context[0]);
 312         ib_free_send_mad(msg);
 313 }
 314
 315 static void * cm_copy_private_data(const void *private_data,
 316                                    u8 private_data_len)
 317 {
 318         void *data;
 319
 320         if (!private_data || !private_data_len)
 321                 return NULL;
 322
 323         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 324         if (!data)
 325                 return ERR_PTR(-ENOMEM);
 326
 327         return data;
 328 }
 329
 330 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 331                                  void *private_data, u8 private_data_len)
 332 {
 333         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 334                 kfree(cm_id_priv->private_data);
 335
 336         cm_id_priv->private_data = private_data;
 337         cm_id_priv->private_data_len = private_data_len;
 338 }
 339
 340 static void cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 341                                     struct ib_grh *grh, struct cm_av *av)
 342 {
 343         av->port = port;
 344         av->pkey_index = wc->pkey_index;
 345         ib_init_ah_from_wc(port->cm_dev->ib_device, port->port_num, wc,
 346                            grh, &av->ah_attr);
 347 }
 348
 349 static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
 350 {
 351         struct cm_device *cm_dev;
 352         struct cm_port *port = NULL;
 353         unsigned long flags;
 354         int ret;
 355         u8 p;
 356
 357         read_lock_irqsave(&cm.device_lock, flags);
 358         list_for_each_entry(cm_dev, &cm.device_list, list) {
 359                 if (!ib_find_cached_gid(cm_dev->ib_device, &path->sgid,
 360                                         &p, NULL)) {
 361                         port = cm_dev->port[p-1];
 362                         break;
 363                 }
 364         }
 365         read_unlock_irqrestore(&cm.device_lock, flags);
 366
 367         if (!port)
 368                 return -EINVAL;
 369
 370         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 371                                   be16_to_cpu(path->pkey), &av->pkey_index);
 372         if (ret)
 373                 return ret;
 374
 375         av->port = port;
 376         ib_init_ah_from_path(cm_dev->ib_device, port->port_num, path,
 377                              &av->ah_attr);
 378         av->timeout = path->packet_life_time + 1;
 379         return 0;
 380 }
 381
 382 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 383 {
 384         unsigned long flags;
 385         int ret, id;
 386         static int next_id;
 387
 388         do {
 389                 spin_lock_irqsave(&cm.lock, flags);
 390                 ret = idr_get_new_above(&cm.local_id_table, cm_id_priv,
 391                                         next_id, &id);
 392                 if (!ret)
 393                         next_id = ((unsigned) id + 1) & MAX_ID_MASK;
 394                 spin_unlock_irqrestore(&cm.lock, flags);
 395         } while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) );
 396
 397         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 398         return ret;
 399 }
 400
 401 static void cm_free_id(__be32 local_id)
 402 {
 403         spin_lock_irq(&cm.lock);
 404         idr_remove(&cm.local_id_table,
 405                    (__force int) (local_id ^ cm.random_id_operand));
 406         spin_unlock_irq(&cm.lock);
 407 }
 408
 409 static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
 410 {
 411         struct cm_id_private *cm_id_priv;
 412
 413         cm_id_priv = idr_find(&cm.local_id_table,
 414                               (__force int) (local_id ^ cm.random_id_operand));
 415         if (cm_id_priv) {
 416                 if (cm_id_priv->id.remote_id == remote_id)
 417                         atomic_inc(&cm_id_priv->refcount);
 418                 else
 419                         cm_id_priv = NULL;
 420         }
 421
 422         return cm_id_priv;
 423 }
 424
 425 static struct cm_id_private * cm_acquire_id(__be32 local_id, __be32 remote_id)
 426 {
 427         struct cm_id_private *cm_id_priv;
 428
 429         spin_lock_irq(&cm.lock);
 430         cm_id_priv = cm_get_id(local_id, remote_id);
 431         spin_unlock_irq(&cm.lock);
 432
 433         return cm_id_priv;
 434 }
 435
 436 static void cm_mask_copy(u8 *dst, u8 *src, u8 *mask)
 437 {
 438         int i;
 439
 440         for (i = 0; i < IB_CM_COMPARE_SIZE / sizeof(unsigned long); i++)
 441                 ((unsigned long *) dst)[i] = ((unsigned long *) src)[i] &
 442                                              ((unsigned long *) mask)[i];
 443 }
 444
 445 static int cm_compare_data(struct ib_cm_compare_data *src_data,
 446                            struct ib_cm_compare_data *dst_data)
 447 {
 448         u8 src[IB_CM_COMPARE_SIZE];
 449         u8 dst[IB_CM_COMPARE_SIZE];
 450
 451         if (!src_data || !dst_data)
 452                 return 0;
 453
 454         cm_mask_copy(src, src_data->data, dst_data->mask);
 455         cm_mask_copy(dst, dst_data->data, src_data->mask);
 456         return memcmp(src, dst, IB_CM_COMPARE_SIZE);
 457 }
 458
 459 static int cm_compare_private_data(u8 *private_data,
 460                                    struct ib_cm_compare_data *dst_data)
 461 {
 462         u8 src[IB_CM_COMPARE_SIZE];
 463
 464         if (!dst_data)
 465                 return 0;
 466
 467         cm_mask_copy(src, private_data, dst_data->mask);
 468         return memcmp(src, dst_data->data, IB_CM_COMPARE_SIZE);
 469 }
 470
 471 /*
 472  * Trivial helpers to strip endian annotation and compare; the
 473  * endianness doesn't actually matter since we just need a stable
 474  * order for the RB tree.
 475  */
 476 static int be32_lt(__be32 a, __be32 b)
 477 {
 478         return (__force u32) a < (__force u32) b;
 479 }
 480
 481 static int be32_gt(__be32 a, __be32 b)
 482 {
 483         return (__force u32) a > (__force u32) b;
 484 }
 485
 486 static int be64_lt(__be64 a, __be64 b)
 487 {
 488         return (__force u64) a < (__force u64) b;
 489 }
 490
 491 static int be64_gt(__be64 a, __be64 b)
 492 {
 493         return (__force u64) a > (__force u64) b;
 494 }
 495
 496 static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
 497 {
 498         struct rb_node **link = &cm.listen_service_table.rb_node;
 499         struct rb_node *parent = NULL;
 500         struct cm_id_private *cur_cm_id_priv;
 501         __be64 service_id = cm_id_priv->id.service_id;
 502         __be64 service_mask = cm_id_priv->id.service_mask;
 503         int data_cmp;
 504
 505         while (*link) {
 506                 parent = *link;
 507                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 508                                           service_node);
 509                 data_cmp = cm_compare_data(cm_id_priv->compare_data,
 510                                            cur_cm_id_priv->compare_data);
 511                 if ((cur_cm_id_priv->id.service_mask & service_id) ==
 512                     (service_mask & cur_cm_id_priv->id.service_id) &&
 513                     (cm_id_priv->id.device == cur_cm_id_priv->id.device) &&
 514                     !data_cmp)
 515                         return cur_cm_id_priv;
 516
 517                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 518                         link = &(*link)->rb_left;
 519                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 520                         link = &(*link)->rb_right;
 521                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 522                         link = &(*link)->rb_left;
 523                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 524                         link = &(*link)->rb_right;
 525                 else if (data_cmp < 0)
 526                         link = &(*link)->rb_left;
 527                 else
 528                         link = &(*link)->rb_right;
 529         }
 530         rb_link_node(&cm_id_priv->service_node, parent, link);
 531         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 532         return NULL;
 533 }
 534
 535 static struct cm_id_private * cm_find_listen(struct ib_device *device,
 536                                              __be64 service_id,
 537                                              u8 *private_data)
 538 {
 539         struct rb_node *node = cm.listen_service_table.rb_node;
 540         struct cm_id_private *cm_id_priv;
 541         int data_cmp;
 542
 543         while (node) {
 544                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 545                 data_cmp = cm_compare_private_data(private_data,
 546                                                    cm_id_priv->compare_data);
 547                 if ((cm_id_priv->id.service_mask & service_id) ==
 548                      cm_id_priv->id.service_id &&
 549                     (cm_id_priv->id.device == device) && !data_cmp)
 550                         return cm_id_priv;
 551
 552                 if (device < cm_id_priv->id.device)
 553                         node = node->rb_left;
 554                 else if (device > cm_id_priv->id.device)
 555                         node = node->rb_right;
 556                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 557                         node = node->rb_left;
 558                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 559                         node = node->rb_right;
 560                 else if (data_cmp < 0)
 561                         node = node->rb_left;
 562                 else
 563                         node = node->rb_right;
 564         }
 565         return NULL;
 566 }
 567
 568 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
 569                                                      *timewait_info)
 570 {
 571         struct rb_node **link = &cm.remote_id_table.rb_node;
 572         struct rb_node *parent = NULL;
 573         struct cm_timewait_info *cur_timewait_info;
 574         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 575         __be32 remote_id = timewait_info->work.remote_id;
 576
 577         while (*link) {
 578                 parent = *link;
 579                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 580                                              remote_id_node);
 581                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 582                         link = &(*link)->rb_left;
 583                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 584                         link = &(*link)->rb_right;
 585                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 586                         link = &(*link)->rb_left;
 587                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 588                         link = &(*link)->rb_right;
 589                 else
 590                         return cur_timewait_info;
 591         }
 592         timewait_info->inserted_remote_id = 1;
 593         rb_link_node(&timewait_info->remote_id_node, parent, link);
 594         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 595         return NULL;
 596 }
 597
 598 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
 599                                                    __be32 remote_id)
 600 {
 601         struct rb_node *node = cm.remote_id_table.rb_node;
 602         struct cm_timewait_info *timewait_info;
 603
 604         while (node) {
 605                 timewait_info = rb_entry(node, struct cm_timewait_info,
 606                                          remote_id_node);
 607                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 608                         node = node->rb_left;
 609                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 610                         node = node->rb_right;
 611                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 612                         node = node->rb_left;
 613                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 614                         node = node->rb_right;
 615                 else
 616                         return timewait_info;
 617         }
 618         return NULL;
 619 }
 620
 621 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
 622                                                       *timewait_info)
 623 {
 624         struct rb_node **link = &cm.remote_qp_table.rb_node;
 625         struct rb_node *parent = NULL;
 626         struct cm_timewait_info *cur_timewait_info;
 627         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 628         __be32 remote_qpn = timewait_info->remote_qpn;
 629
 630         while (*link) {
 631                 parent = *link;
 632                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 633                                              remote_qp_node);
 634                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 635                         link = &(*link)->rb_left;
 636                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 637                         link = &(*link)->rb_right;
 638                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 639                         link = &(*link)->rb_left;
 640                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 641                         link = &(*link)->rb_right;
 642                 else
 643                         return cur_timewait_info;
 644         }
 645         timewait_info->inserted_remote_qp = 1;
 646         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 647         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 648         return NULL;
 649 }
 650
 651 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
 652                                                     *cm_id_priv)
 653 {
 654         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 655         struct rb_node *parent = NULL;
 656         struct cm_id_private *cur_cm_id_priv;
 657         union ib_gid *port_gid = &cm_id_priv->av.dgid;
 658         __be32 remote_id = cm_id_priv->id.remote_id;
 659
 660         while (*link) {
 661                 parent = *link;
 662                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 663                                           sidr_id_node);
 664                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 665                         link = &(*link)->rb_left;
 666                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 667                         link = &(*link)->rb_right;
 668                 else {
 669                         int cmp;
 670                         cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
 671                                      sizeof *port_gid);
 672                         if (cmp < 0)
 673                                 link = &(*link)->rb_left;
 674                         else if (cmp > 0)
 675                                 link = &(*link)->rb_right;
 676                         else
 677                                 return cur_cm_id_priv;
 678                 }
 679         }
 680         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 681         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 682         return NULL;
 683 }
 684
 685 static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
 686                                enum ib_cm_sidr_status status)
 687 {
 688         struct ib_cm_sidr_rep_param param;
 689
 690         memset(&param, 0, sizeof param);
 691         param.status = status;
 692         ib_send_cm_sidr_rep(&cm_id_priv->id, &param);
 693 }
 694
 695 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 696                                  ib_cm_handler cm_handler,
 697                                  void *context)
 698 {
 699         struct cm_id_private *cm_id_priv;
 700         int ret;
 701
 702         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 703         if (!cm_id_priv)
 704                 return ERR_PTR(-ENOMEM);
 705
 706         cm_id_priv->id.state = IB_CM_IDLE;
 707         cm_id_priv->id.device = device;
 708         cm_id_priv->id.cm_handler = cm_handler;
 709         cm_id_priv->id.context = context;
 710         cm_id_priv->id.remote_cm_qpn = 1;
 711         ret = cm_alloc_id(cm_id_priv);
 712         if (ret)
 713                 goto error;
 714
 715         spin_lock_init(&cm_id_priv->lock);
 716         init_completion(&cm_id_priv->comp);
 717         INIT_LIST_HEAD(&cm_id_priv->work_list);
 718         atomic_set(&cm_id_priv->work_count, -1);
 719         atomic_set(&cm_id_priv->refcount, 1);
 720         return &cm_id_priv->id;
 721
 722 error:
 723         kfree(cm_id_priv);
 724         return ERR_PTR(-ENOMEM);
 725 }
 726 EXPORT_SYMBOL(ib_create_cm_id);
 727
 728 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
 729 {
 730         struct cm_work *work;
 731
 732         if (list_empty(&cm_id_priv->work_list))
 733                 return NULL;
 734
 735         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 736         list_del(&work->list);
 737         return work;
 738 }
 739
 740 static void cm_free_work(struct cm_work *work)
 741 {
 742         if (work->mad_recv_wc)
 743                 ib_free_recv_mad(work->mad_recv_wc);
 744         kfree(work);
 745 }
 746
 747 static inline int cm_convert_to_ms(int iba_time)
 748 {
 749         /* approximate conversion to ms from 4.096us x 2^iba_time */
 750         return 1 << max(iba_time - 8, 0);
 751 }
 752
 753 /*
 754  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 755  * Because of how ack_timeout is stored, adding one doubles the timeout.
 756  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 757  * increment it (round up) only if the other is within 50%.
 758  */
 759 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 760 {
 761         int ack_timeout = packet_life_time + 1;
 762
 763         if (ack_timeout >= ca_ack_delay)
 764                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 765         else
 766                 ack_timeout = ca_ack_delay +
 767                               (ack_timeout >= (ca_ack_delay - 1));
 768
 769         return min(31, ack_timeout);
 770 }
 771
 772 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
 773 {
 774         if (timewait_info->inserted_remote_id) {
 775                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 776                 timewait_info->inserted_remote_id = 0;
 777         }
 778
 779         if (timewait_info->inserted_remote_qp) {
 780                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 781                 timewait_info->inserted_remote_qp = 0;
 782         }
 783 }
 784
 785 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
 786 {
 787         struct cm_timewait_info *timewait_info;
 788
 789         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 790         if (!timewait_info)
 791                 return ERR_PTR(-ENOMEM);
 792
 793         timewait_info->work.local_id = local_id;
 794         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 795         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 796         return timewait_info;
 797 }
 798
 799 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 800 {
 801         int wait_time;
 802         unsigned long flags;
 803
 804         spin_lock_irqsave(&cm.lock, flags);
 805         cm_cleanup_timewait(cm_id_priv->timewait_info);
 806         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 807         spin_unlock_irqrestore(&cm.lock, flags);
 808
 809         /*
 810          * The cm_id could be destroyed by the user before we exit timewait.
 811          * To protect against this, we search for the cm_id after exiting
 812          * timewait before notifying the user that we've exited timewait.
 813          */
 814         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 815         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 816         queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
 817                            msecs_to_jiffies(wait_time));
 818         cm_id_priv->timewait_info = NULL;
 819 }
 820
 821 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
 822 {
 823         unsigned long flags;
 824
 825         cm_id_priv->id.state = IB_CM_IDLE;
 826         if (cm_id_priv->timewait_info) {
 827                 spin_lock_irqsave(&cm.lock, flags);
 828                 cm_cleanup_timewait(cm_id_priv->timewait_info);
 829                 spin_unlock_irqrestore(&cm.lock, flags);
 830                 kfree(cm_id_priv->timewait_info);
 831                 cm_id_priv->timewait_info = NULL;
 832         }
 833 }
 834
 835 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
 836 {
 837         struct cm_id_private *cm_id_priv;
 838         struct cm_work *work;
 839
 840         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 841 retest:
 842         spin_lock_irq(&cm_id_priv->lock);
 843         switch (cm_id->state) {
 844         case IB_CM_LISTEN:
 845                 cm_id->state = IB_CM_IDLE;
 846                 spin_unlock_irq(&cm_id_priv->lock);
 847                 spin_lock_irq(&cm.lock);
 848                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
 849                 spin_unlock_irq(&cm.lock);
 850                 break;
 851         case IB_CM_SIDR_REQ_SENT:
 852                 cm_id->state = IB_CM_IDLE;
 853                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 854                 spin_unlock_irq(&cm_id_priv->lock);
 855                 break;
 856         case IB_CM_SIDR_REQ_RCVD:
 857                 spin_unlock_irq(&cm_id_priv->lock);
 858                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
 859                 break;
 860         case IB_CM_REQ_SENT:
 861                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 862                 spin_unlock_irq(&cm_id_priv->lock);
 863                 ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
 864                                &cm_id_priv->id.device->node_guid,
 865                                sizeof cm_id_priv->id.device->node_guid,
 866                                NULL, 0);
 867                 break;
 868         case IB_CM_REQ_RCVD:
 869                 if (err == -ENOMEM) {
 870                         /* Do not reject to allow future retries. */
 871                         cm_reset_to_idle(cm_id_priv);
 872                         spin_unlock_irq(&cm_id_priv->lock);
 873                 } else {
 874                         spin_unlock_irq(&cm_id_priv->lock);
 875                         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
 876                                        NULL, 0, NULL, 0);
 877                 }
 878                 break;
 879         case IB_CM_MRA_REQ_RCVD:
 880         case IB_CM_REP_SENT:
 881         case IB_CM_MRA_REP_RCVD:
 882                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 883                 /* Fall through */
 884         case IB_CM_MRA_REQ_SENT:
 885         case IB_CM_REP_RCVD:
 886         case IB_CM_MRA_REP_SENT:
 887                 spin_unlock_irq(&cm_id_priv->lock);
 888                 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
 889                                NULL, 0, NULL, 0);
 890                 break;
 891         case IB_CM_ESTABLISHED:
 892                 spin_unlock_irq(&cm_id_priv->lock);
 893                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT)
 894                         break;
 895                 ib_send_cm_dreq(cm_id, NULL, 0);
 896                 goto retest;
 897         case IB_CM_DREQ_SENT:
 898                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 899                 cm_enter_timewait(cm_id_priv);
 900                 spin_unlock_irq(&cm_id_priv->lock);
 901                 break;
 902         case IB_CM_DREQ_RCVD:
 903                 spin_unlock_irq(&cm_id_priv->lock);
 904                 ib_send_cm_drep(cm_id, NULL, 0);
 905                 break;
 906         default:
 907                 spin_unlock_irq(&cm_id_priv->lock);
 908                 break;
 909         }
 910
 911         cm_free_id(cm_id->local_id);
 912         cm_deref_id(cm_id_priv);
 913         wait_for_completion(&cm_id_priv->comp);
 914         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
 915                 cm_free_work(work);
 916         kfree(cm_id_priv->compare_data);
 917         kfree(cm_id_priv->private_data);
 918         kfree(cm_id_priv);
 919 }
 920
 921 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
 922 {
 923         cm_destroy_id(cm_id, 0);
 924 }
 925 EXPORT_SYMBOL(ib_destroy_cm_id);
 926
 927 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask,
 928                  struct ib_cm_compare_data *compare_data)
 929 {
 930         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
 931         unsigned long flags;
 932         int ret = 0;
 933
 934         service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
 935         service_id &= service_mask;
 936         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
 937             (service_id != IB_CM_ASSIGN_SERVICE_ID))
 938                 return -EINVAL;
 939
 940         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 941         if (cm_id->state != IB_CM_IDLE)
 942                 return -EINVAL;
 943
 944         if (compare_data) {
 945                 cm_id_priv->compare_data = kzalloc(sizeof *compare_data,
 946                                                    GFP_KERNEL);
 947                 if (!cm_id_priv->compare_data)
 948                         return -ENOMEM;
 949                 cm_mask_copy(cm_id_priv->compare_data->data,
 950                              compare_data->data, compare_data->mask);
 951                 memcpy(cm_id_priv->compare_data->mask, compare_data->mask,
 952                        IB_CM_COMPARE_SIZE);
 953         }
 954
 955         cm_id->state = IB_CM_LISTEN;
 956
 957         spin_lock_irqsave(&cm.lock, flags);
 958         if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
 959                 cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
 960                 cm_id->service_mask = ~cpu_to_be64(0);
 961         } else {
 962                 cm_id->service_id = service_id;
 963                 cm_id->service_mask = service_mask;
 964         }
 965         cur_cm_id_priv = cm_insert_listen(cm_id_priv);
 966         spin_unlock_irqrestore(&cm.lock, flags);
 967
 968         if (cur_cm_id_priv) {
 969                 cm_id->state = IB_CM_IDLE;
 970                 kfree(cm_id_priv->compare_data);
 971                 cm_id_priv->compare_data = NULL;
 972                 ret = -EBUSY;
 973         }
 974         return ret;
 975 }
 976 EXPORT_SYMBOL(ib_cm_listen);
 977
 978 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv,
 979                           enum cm_msg_sequence msg_seq)
 980 {
 981         u64 hi_tid, low_tid;
 982
 983         hi_tid   = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
 984         low_tid  = (u64) ((__force u32)cm_id_priv->id.local_id |
 985                           (msg_seq << 30));
 986         return cpu_to_be64(hi_tid | low_tid);
 987 }
 988
 989 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
 990                               __be16 attr_id, __be64 tid)
 991 {
 992         hdr->base_version  = IB_MGMT_BASE_VERSION;
 993         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
 994         hdr->class_version = IB_CM_CLASS_VERSION;
 995         hdr->method        = IB_MGMT_METHOD_SEND;
 996         hdr->attr_id       = attr_id;
 997         hdr->tid           = tid;
 998 }
 999
1000 static void cm_format_req(struct cm_req_msg *req_msg,
1001                           struct cm_id_private *cm_id_priv,
1002                           struct ib_cm_req_param *param)
1003 {
1004         struct ib_sa_path_rec *pri_path = param->primary_path;
1005         struct ib_sa_path_rec *alt_path = param->alternate_path;
1006
1007         cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1008                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_REQ));
1009
1010         req_msg->local_comm_id = cm_id_priv->id.local_id;
1011         req_msg->service_id = param->service_id;
1012         req_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1013         cm_req_set_local_qpn(req_msg, cpu_to_be32(param->qp_num));
1014         cm_req_set_init_depth(req_msg, param->initiator_depth);
1015         cm_req_set_remote_resp_timeout(req_msg,
1016                                        param->remote_cm_response_timeout);
1017         cm_req_set_qp_type(req_msg, param->qp_type);
1018         cm_req_set_flow_ctrl(req_msg, param->flow_control);
1019         cm_req_set_starting_psn(req_msg, cpu_to_be32(param->starting_psn));
1020         cm_req_set_local_resp_timeout(req_msg,
1021                                       param->local_cm_response_timeout);
1022         req_msg->pkey = param->primary_path->pkey;
1023         cm_req_set_path_mtu(req_msg, param->primary_path->mtu);
1024         cm_req_set_max_cm_retries(req_msg, param->max_cm_retries);
1025
1026         if (param->qp_type != IB_QPT_XRC_INI) {
1027                 cm_req_set_resp_res(req_msg, param->responder_resources);
1028                 cm_req_set_retry_count(req_msg, param->retry_count);
1029                 cm_req_set_rnr_retry_count(req_msg, param->rnr_retry_count);
1030                 cm_req_set_srq(req_msg, param->srq);
1031         }
1032
1033         if (pri_path->hop_limit <= 1) {
1034                 req_msg->primary_local_lid = pri_path->slid;
1035                 req_msg->primary_remote_lid = pri_path->dlid;
1036         } else {
1037                 /* Work-around until there's a way to obtain remote LID info */
1038                 req_msg->primary_local_lid = IB_LID_PERMISSIVE;
1039                 req_msg->primary_remote_lid = IB_LID_PERMISSIVE;
1040         }
1041         req_msg->primary_local_gid = pri_path->sgid;
1042         req_msg->primary_remote_gid = pri_path->dgid;
1043         cm_req_set_primary_flow_label(req_msg, pri_path->flow_label);
1044         cm_req_set_primary_packet_rate(req_msg, pri_path->rate);
1045         req_msg->primary_traffic_class = pri_path->traffic_class;
1046         req_msg->primary_hop_limit = pri_path->hop_limit;
1047         cm_req_set_primary_sl(req_msg, pri_path->sl);
1048         cm_req_set_primary_subnet_local(req_msg, (pri_path->hop_limit <= 1));
1049         cm_req_set_primary_local_ack_timeout(req_msg,
1050                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1051                                pri_path->packet_life_time));
1052
1053         if (alt_path) {
1054                 if (alt_path->hop_limit <= 1) {
1055                         req_msg->alt_local_lid = alt_path->slid;
1056                         req_msg->alt_remote_lid = alt_path->dlid;
1057                 } else {
1058                         req_msg->alt_local_lid = IB_LID_PERMISSIVE;
1059                         req_msg->alt_remote_lid = IB_LID_PERMISSIVE;
1060                 }
1061                 req_msg->alt_local_gid = alt_path->sgid;
1062                 req_msg->alt_remote_gid = alt_path->dgid;
1063                 cm_req_set_alt_flow_label(req_msg,
1064                                           alt_path->flow_label);
1065                 cm_req_set_alt_packet_rate(req_msg, alt_path->rate);
1066                 req_msg->alt_traffic_class = alt_path->traffic_class;
1067                 req_msg->alt_hop_limit = alt_path->hop_limit;
1068                 cm_req_set_alt_sl(req_msg, alt_path->sl);
1069                 cm_req_set_alt_subnet_local(req_msg, (alt_path->hop_limit <= 1));
1070                 cm_req_set_alt_local_ack_timeout(req_msg,
1071                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1072                                        alt_path->packet_life_time));
1073         }
1074
1075         if (param->private_data && param->private_data_len)
1076                 memcpy(req_msg->private_data, param->private_data,
1077                        param->private_data_len);
1078 }
1079
1080 static int cm_validate_req_param(struct ib_cm_req_param *param)
1081 {
1082         /* peer-to-peer not supported */
1083         if (param->peer_to_peer)
1084                 return -EINVAL;
1085
1086         if (!param->primary_path)
1087                 return -EINVAL;
1088
1089         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1090             param->qp_type != IB_QPT_XRC_INI)
1091                 return -EINVAL;
1092
1093         if (param->private_data &&
1094             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1095                 return -EINVAL;
1096
1097         if (param->alternate_path &&
1098             (param->alternate_path->pkey != param->primary_path->pkey ||
1099              param->alternate_path->mtu != param->primary_path->mtu))
1100                 return -EINVAL;
1101
1102         return 0;
1103 }
1104
1105 int ib_send_cm_req(struct ib_cm_id *cm_id,
1106                    struct ib_cm_req_param *param)
1107 {
1108         struct cm_id_private *cm_id_priv;
1109         struct cm_req_msg *req_msg;
1110         unsigned long flags;
1111         int ret;
1112
1113         ret = cm_validate_req_param(param);
1114         if (ret)
1115                 return ret;
1116
1117         /* Verify that we're not in timewait. */
1118         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1119         spin_lock_irqsave(&cm_id_priv->lock, flags);
1120         if (cm_id->state != IB_CM_IDLE) {
1121                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1122                 ret = -EINVAL;
1123                 goto out;
1124         }
1125         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1126
1127         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1128                                                             id.local_id);
1129         if (IS_ERR(cm_id_priv->timewait_info)) {
1130                 ret = PTR_ERR(cm_id_priv->timewait_info);
1131                 goto out;
1132         }
1133
1134         ret = cm_init_av_by_path(param->primary_path, &cm_id_priv->av);
1135         if (ret)
1136                 goto error1;
1137         if (param->alternate_path) {
1138                 ret = cm_init_av_by_path(param->alternate_path,
1139                                          &cm_id_priv->alt_av);
1140                 if (ret)
1141                         goto error1;
1142         }
1143         cm_id->service_id = param->service_id;
1144         cm_id->service_mask = ~cpu_to_be64(0);
1145         cm_id_priv->timeout_ms = cm_convert_to_ms(
1146                                     param->primary_path->packet_life_time) * 2 +
1147                                  cm_convert_to_ms(
1148                                     param->remote_cm_response_timeout);
1149         cm_id_priv->max_cm_retries = param->max_cm_retries;
1150         cm_id_priv->initiator_depth = param->initiator_depth;
1151         cm_id_priv->responder_resources = param->responder_resources;
1152         cm_id_priv->retry_count = param->retry_count;
1153         cm_id_priv->path_mtu = param->primary_path->mtu;
1154         cm_id_priv->pkey = param->primary_path->pkey;
1155         cm_id_priv->qp_type = param->qp_type;
1156
1157         ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1158         if (ret)
1159                 goto error1;
1160
1161         req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1162         cm_format_req(req_msg, cm_id_priv, param);
1163         cm_id_priv->tid = req_msg->hdr.tid;
1164         cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1165         cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1166
1167         cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
1168         cm_id_priv->rq_psn = cm_req_get_starting_psn(req_msg);
1169
1170         spin_lock_irqsave(&cm_id_priv->lock, flags);
1171         ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1172         if (ret) {
1173                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1174                 goto error2;
1175         }
1176         BUG_ON(cm_id->state != IB_CM_IDLE);
1177         cm_id->state = IB_CM_REQ_SENT;
1178         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1179         return 0;
1180
1181 error2: cm_free_msg(cm_id_priv->msg);
1182 error1: kfree(cm_id_priv->timewait_info);
1183 out:    return ret;
1184 }
1185 EXPORT_SYMBOL(ib_send_cm_req);
1186
1187 static int cm_issue_rej(struct cm_port *port,
1188                         struct ib_mad_recv_wc *mad_recv_wc,
1189                         enum ib_cm_rej_reason reason,
1190                         enum cm_msg_response msg_rejected,
1191                         void *ari, u8 ari_length)
1192 {
1193         struct ib_mad_send_buf *msg = NULL;
1194         struct cm_rej_msg *rej_msg, *rcv_msg;
1195         int ret;
1196
1197         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1198         if (ret)
1199                 return ret;
1200
1201         /* We just need common CM header information.  Cast to any message. */
1202         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1203         rej_msg = (struct cm_rej_msg *) msg->mad;
1204
1205         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1206         rej_msg->remote_comm_id = rcv_msg->local_comm_id;
1207         rej_msg->local_comm_id = rcv_msg->remote_comm_id;
1208         cm_rej_set_msg_rejected(rej_msg, msg_rejected);
1209         rej_msg->reason = cpu_to_be16(reason);
1210
1211         if (ari && ari_length) {
1212                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1213                 memcpy(rej_msg->ari, ari, ari_length);
1214         }
1215
1216         ret = ib_post_send_mad(msg, NULL);
1217         if (ret)
1218                 cm_free_msg(msg);
1219
1220         return ret;
1221 }
1222
1223 static inline int cm_is_active_peer(__be64 local_ca_guid, __be64 remote_ca_guid,
1224                                     __be32 local_qpn, __be32 remote_qpn)
1225 {
1226         return (be64_to_cpu(local_ca_guid) > be64_to_cpu(remote_ca_guid) ||
1227                 ((local_ca_guid == remote_ca_guid) &&
1228                  (be32_to_cpu(local_qpn) > be32_to_cpu(remote_qpn))));
1229 }
1230
1231 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1232                                             struct ib_sa_path_rec *primary_path,
1233                                             struct ib_sa_path_rec *alt_path)
1234 {
1235         memset(primary_path, 0, sizeof *primary_path);
1236         primary_path->dgid = req_msg->primary_local_gid;
1237         primary_path->sgid = req_msg->primary_remote_gid;
1238         primary_path->dlid = req_msg->primary_local_lid;
1239         primary_path->slid = req_msg->primary_remote_lid;
1240         primary_path->flow_label = cm_req_get_primary_flow_label(req_msg);
1241         primary_path->hop_limit = req_msg->primary_hop_limit;
1242         primary_path->traffic_class = req_msg->primary_traffic_class;
1243         primary_path->reversible = 1;
1244         primary_path->pkey = req_msg->pkey;
1245         primary_path->sl = cm_req_get_primary_sl(req_msg);
1246         primary_path->mtu_selector = IB_SA_EQ;
1247         primary_path->mtu = cm_req_get_path_mtu(req_msg);
1248         primary_path->rate_selector = IB_SA_EQ;
1249         primary_path->rate = cm_req_get_primary_packet_rate(req_msg);
1250         primary_path->packet_life_time_selector = IB_SA_EQ;
1251         primary_path->packet_life_time =
1252                 cm_req_get_primary_local_ack_timeout(req_msg);
1253         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1254
1255         if (req_msg->alt_local_lid) {
1256                 memset(alt_path, 0, sizeof *alt_path);
1257                 alt_path->dgid = req_msg->alt_local_gid;
1258                 alt_path->sgid = req_msg->alt_remote_gid;
1259                 alt_path->dlid = req_msg->alt_local_lid;
1260                 alt_path->slid = req_msg->alt_remote_lid;
1261                 alt_path->flow_label = cm_req_get_alt_flow_label(req_msg);
1262                 alt_path->hop_limit = req_msg->alt_hop_limit;
1263                 alt_path->traffic_class = req_msg->alt_traffic_class;
1264                 alt_path->reversible = 1;
1265                 alt_path->pkey = req_msg->pkey;
1266                 alt_path->sl = cm_req_get_alt_sl(req_msg);
1267                 alt_path->mtu_selector = IB_SA_EQ;
1268                 alt_path->mtu = cm_req_get_path_mtu(req_msg);
1269                 alt_path->rate_selector = IB_SA_EQ;
1270                 alt_path->rate = cm_req_get_alt_packet_rate(req_msg);
1271                 alt_path->packet_life_time_selector = IB_SA_EQ;
1272                 alt_path->packet_life_time =
1273                         cm_req_get_alt_local_ack_timeout(req_msg);
1274                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1275         }
1276 }
1277
1278 static void cm_format_req_event(struct cm_work *work,
1279                                 struct cm_id_private *cm_id_priv,
1280                                 struct ib_cm_id *listen_id)
1281 {
1282         struct cm_req_msg *req_msg;
1283         struct ib_cm_req_event_param *param;
1284
1285         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1286         param = &work->cm_event.param.req_rcvd;
1287         param->listen_id = listen_id;
1288         param->port = cm_id_priv->av.port->port_num;
1289         param->primary_path = &work->path[0];
1290         if (req_msg->alt_local_lid)
1291                 param->alternate_path = &work->path[1];
1292         else
1293                 param->alternate_path = NULL;
1294         param->remote_ca_guid = req_msg->local_ca_guid;
1295         param->remote_qkey = be32_to_cpu(req_msg->local_qkey);
1296         param->remote_qpn = be32_to_cpu(cm_req_get_local_qpn(req_msg));
1297         param->qp_type = cm_req_get_qp_type(req_msg);
1298         param->starting_psn = be32_to_cpu(cm_req_get_starting_psn(req_msg));
1299         param->responder_resources = cm_req_get_init_depth(req_msg);
1300         param->initiator_depth = cm_req_get_resp_res(req_msg);
1301         param->local_cm_response_timeout =
1302                                         cm_req_get_remote_resp_timeout(req_msg);
1303         param->flow_control = cm_req_get_flow_ctrl(req_msg);
1304         param->remote_cm_response_timeout =
1305                                         cm_req_get_local_resp_timeout(req_msg);
1306         param->retry_count = cm_req_get_retry_count(req_msg);
1307         param->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1308         param->srq = cm_req_get_srq(req_msg);
1309         work->cm_event.private_data = &req_msg->private_data;
1310 }
1311
1312 static void cm_process_work(struct cm_id_private *cm_id_priv,
1313                             struct cm_work *work)
1314 {
1315         int ret;
1316
1317         /* We will typically only have the current event to report. */
1318         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1319         cm_free_work(work);
1320
1321         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1322                 spin_lock_irq(&cm_id_priv->lock);
1323                 work = cm_dequeue_work(cm_id_priv);
1324                 spin_unlock_irq(&cm_id_priv->lock);
1325                 BUG_ON(!work);
1326                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1327                                                 &work->cm_event);
1328                 cm_free_work(work);
1329         }
1330         cm_deref_id(cm_id_priv);
1331         if (ret)
1332                 cm_destroy_id(&cm_id_priv->id, ret);
1333 }
1334
1335 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1336                           struct cm_id_private *cm_id_priv,
1337                           enum cm_msg_response msg_mraed, u8 service_timeout,
1338                           const void *private_data, u8 private_data_len)
1339 {
1340         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1341         cm_mra_set_msg_mraed(mra_msg, msg_mraed);
1342         mra_msg->local_comm_id = cm_id_priv->id.local_id;
1343         mra_msg->remote_comm_id = cm_id_priv->id.remote_id;
1344         cm_mra_set_service_timeout(mra_msg, service_timeout);
1345
1346         if (private_data && private_data_len)
1347                 memcpy(mra_msg->private_data, private_data, private_data_len);
1348 }
1349
1350 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1351                           struct cm_id_private *cm_id_priv,
1352                           enum ib_cm_rej_reason reason,
1353                           void *ari,
1354                           u8 ari_length,
1355                           const void *private_data,
1356                           u8 private_data_len)
1357 {
1358         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1359         rej_msg->remote_comm_id = cm_id_priv->id.remote_id;
1360
1361         switch(cm_id_priv->id.state) {
1362         case IB_CM_REQ_RCVD:
1363                 rej_msg->local_comm_id = 0;
1364                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1365                 break;
1366         case IB_CM_MRA_REQ_SENT:
1367                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1368                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REQ);
1369                 break;
1370         case IB_CM_REP_RCVD:
1371         case IB_CM_MRA_REP_SENT:
1372                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1373                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_REP);
1374                 break;
1375         default:
1376                 rej_msg->local_comm_id = cm_id_priv->id.local_id;
1377                 cm_rej_set_msg_rejected(rej_msg, CM_MSG_RESPONSE_OTHER);
1378                 break;
1379         }
1380
1381         rej_msg->reason = cpu_to_be16(reason);
1382         if (ari && ari_length) {
1383                 cm_rej_set_reject_info_len(rej_msg, ari_length);
1384                 memcpy(rej_msg->ari, ari, ari_length);
1385         }
1386
1387         if (private_data && private_data_len)
1388                 memcpy(rej_msg->private_data, private_data, private_data_len);
1389 }
1390
1391 static void cm_dup_req_handler(struct cm_work *work,
1392                                struct cm_id_private *cm_id_priv)
1393 {
1394         struct ib_mad_send_buf *msg = NULL;
1395         int ret;
1396
1397         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1398                         counter[CM_REQ_COUNTER]);
1399
1400         /* Quick state check to discard duplicate REQs. */
1401         if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
1402                 return;
1403
1404         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1405         if (ret)
1406                 return;
1407
1408         spin_lock_irq(&cm_id_priv->lock);
1409         switch (cm_id_priv->id.state) {
1410         case IB_CM_MRA_REQ_SENT:
1411                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1412                               CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1413                               cm_id_priv->private_data,
1414                               cm_id_priv->private_data_len);
1415                 break;
1416         case IB_CM_TIMEWAIT:
1417                 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
1418                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
1419                 break;
1420         default:
1421                 goto unlock;
1422         }
1423         spin_unlock_irq(&cm_id_priv->lock);
1424
1425         ret = ib_post_send_mad(msg, NULL);
1426         if (ret)
1427                 goto free;
1428         return;
1429
1430 unlock: spin_unlock_irq(&cm_id_priv->lock);
1431 free:   cm_free_msg(msg);
1432 }
1433
1434 static struct cm_id_private * cm_match_req(struct cm_work *work,
1435                                            struct cm_id_private *cm_id_priv)
1436 {
1437         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1438         struct cm_timewait_info *timewait_info;
1439         struct cm_req_msg *req_msg;
1440
1441         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1442
1443         /* Check for possible duplicate REQ. */
1444         spin_lock_irq(&cm.lock);
1445         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1446         if (timewait_info) {
1447                 cur_cm_id_priv = cm_get_id(timewait_info->work.local_id,
1448                                            timewait_info->work.remote_id);
1449                 spin_unlock_irq(&cm.lock);
1450                 if (cur_cm_id_priv) {
1451                         cm_dup_req_handler(work, cur_cm_id_priv);
1452                         cm_deref_id(cur_cm_id_priv);
1453                 }
1454                 return NULL;
1455         }
1456
1457         /* Check for stale connections. */
1458         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1459         if (timewait_info) {
1460                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1461                 spin_unlock_irq(&cm.lock);
1462                 cm_issue_rej(work->port, work->mad_recv_wc,
1463                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1464                              NULL, 0);
1465                 return NULL;
1466         }
1467
1468         /* Find matching listen request. */
1469         listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
1470                                            req_msg->service_id,
1471                                            req_msg->private_data);
1472         if (!listen_cm_id_priv) {
1473                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1474                 spin_unlock_irq(&cm.lock);
1475                 cm_issue_rej(work->port, work->mad_recv_wc,
1476                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1477                              NULL, 0);
1478                 goto out;
1479         }
1480         atomic_inc(&listen_cm_id_priv->refcount);
1481         atomic_inc(&cm_id_priv->refcount);
1482         cm_id_priv->id.state = IB_CM_REQ_RCVD;
1483         atomic_inc(&cm_id_priv->work_count);
1484         spin_unlock_irq(&cm.lock);
1485 out:
1486         return listen_cm_id_priv;
1487 }
1488
1489 /*
1490  * Work-around for inter-subnet connections.  If the LIDs are permissive,
1491  * we need to override the LID/SL data in the REQ with the LID information
1492  * in the work completion.
1493  */
1494 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1495 {
1496         if (!cm_req_get_primary_subnet_local(req_msg)) {
1497                 if (req_msg->primary_local_lid == IB_LID_PERMISSIVE) {
1498                         req_msg->primary_local_lid = cpu_to_be16(wc->slid);
1499                         cm_req_set_primary_sl(req_msg, wc->sl);
1500                 }
1501
1502                 if (req_msg->primary_remote_lid == IB_LID_PERMISSIVE)
1503                         req_msg->primary_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1504         }
1505
1506         if (!cm_req_get_alt_subnet_local(req_msg)) {
1507                 if (req_msg->alt_local_lid == IB_LID_PERMISSIVE) {
1508                         req_msg->alt_local_lid = cpu_to_be16(wc->slid);
1509                         cm_req_set_alt_sl(req_msg, wc->sl);
1510                 }
1511
1512                 if (req_msg->alt_remote_lid == IB_LID_PERMISSIVE)
1513                         req_msg->alt_remote_lid = cpu_to_be16(wc->dlid_path_bits);
1514         }
1515 }
1516
1517 static int cm_req_handler(struct cm_work *work)
1518 {
1519         struct ib_cm_id *cm_id;
1520         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
1521         struct cm_req_msg *req_msg;
1522         int ret;
1523
1524         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1525
1526         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
1527         if (IS_ERR(cm_id))
1528                 return PTR_ERR(cm_id);
1529
1530         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1531         cm_id_priv->id.remote_id = req_msg->local_comm_id;
1532         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
1533                                 work->mad_recv_wc->recv_buf.grh,
1534                                 &cm_id_priv->av);
1535         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1536                                                             id.local_id);
1537         if (IS_ERR(cm_id_priv->timewait_info)) {
1538                 ret = PTR_ERR(cm_id_priv->timewait_info);
1539                 goto destroy;
1540         }
1541         cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
1542         cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
1543         cm_id_priv->timewait_info->remote_qpn = cm_req_get_local_qpn(req_msg);
1544
1545         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
1546         if (!listen_cm_id_priv) {
1547                 ret = -EINVAL;
1548                 kfree(cm_id_priv->timewait_info);
1549                 goto destroy;
1550         }
1551
1552         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
1553         cm_id_priv->id.context = listen_cm_id_priv->id.context;
1554         cm_id_priv->id.service_id = req_msg->service_id;
1555         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
1556
1557         cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
1558         cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
1559         ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
1560         if (ret) {
1561                 ib_get_cached_gid(work->port->cm_dev->ib_device,
1562                                   work->port->port_num, 0, &work->path[0].sgid);
1563                 ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
1564                                &work->path[0].sgid, sizeof work->path[0].sgid,
1565                                NULL, 0);
1566                 goto rejected;
1567         }
1568         if (req_msg->alt_local_lid) {
1569                 ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av);
1570                 if (ret) {
1571                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
1572                                        &work->path[0].sgid,
1573                                        sizeof work->path[0].sgid, NULL, 0);
1574                         goto rejected;
1575                 }
1576         }
1577         cm_id_priv->tid = req_msg->hdr.tid;
1578         cm_id_priv->timeout_ms = cm_convert_to_ms(
1579                                         cm_req_get_local_resp_timeout(req_msg));
1580         cm_id_priv->max_cm_retries = cm_req_get_max_cm_retries(req_msg);
1581         cm_id_priv->remote_qpn = cm_req_get_local_qpn(req_msg);
1582         cm_id_priv->initiator_depth = cm_req_get_resp_res(req_msg);
1583         cm_id_priv->responder_resources = cm_req_get_init_depth(req_msg);
1584         cm_id_priv->path_mtu = cm_req_get_path_mtu(req_msg);
1585         cm_id_priv->pkey = req_msg->pkey;
1586         cm_id_priv->sq_psn = cm_req_get_starting_psn(req_msg);
1587         cm_id_priv->retry_count = cm_req_get_retry_count(req_msg);
1588         cm_id_priv->rnr_retry_count = cm_req_get_rnr_retry_count(req_msg);
1589         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
1590
1591         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
1592         cm_process_work(cm_id_priv, work);
1593         cm_deref_id(listen_cm_id_priv);
1594         return 0;
1595
1596 rejected:
1597         atomic_dec(&cm_id_priv->refcount);
1598         cm_deref_id(listen_cm_id_priv);
1599 destroy:
1600         ib_destroy_cm_id(cm_id);
1601         return ret;
1602 }
1603
1604 static void cm_format_rep(struct cm_rep_msg *rep_msg,
1605                           struct cm_id_private *cm_id_priv,
1606                           struct ib_cm_rep_param *param)
1607 {
1608         cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
1609         rep_msg->local_comm_id = cm_id_priv->id.local_id;
1610         rep_msg->remote_comm_id = cm_id_priv->id.remote_id;
1611         cm_rep_set_starting_psn(rep_msg, cpu_to_be32(param->starting_psn));
1612         rep_msg->resp_resources = param->responder_resources;
1613         cm_rep_set_target_ack_delay(rep_msg,
1614                                     cm_id_priv->av.port->cm_dev->ack_delay);
1615         cm_rep_set_failover(rep_msg, param->failover_accepted);
1616         cm_rep_set_rnr_retry_count(rep_msg, param->rnr_retry_count);
1617         rep_msg->local_ca_guid = cm_id_priv->id.device->node_guid;
1618
1619         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
1620                 rep_msg->initiator_depth = param->initiator_depth;
1621                 cm_rep_set_flow_ctrl(rep_msg, param->flow_control);
1622                 cm_rep_set_srq(rep_msg, param->srq);
1623                 cm_rep_set_local_qpn(rep_msg, cpu_to_be32(param->qp_num));
1624         } else {
1625                 cm_rep_set_srq(rep_msg, 1);
1626                 cm_rep_set_local_eecn(rep_msg, cpu_to_be32(param->qp_num));
1627         }
1628
1629         if (param->private_data && param->private_data_len)
1630                 memcpy(rep_msg->private_data, param->private_data,
1631                        param->private_data_len);
1632 }
1633
1634 int ib_send_cm_rep(struct ib_cm_id *cm_id,
1635                    struct ib_cm_rep_param *param)
1636 {
1637         struct cm_id_private *cm_id_priv;
1638         struct ib_mad_send_buf *msg;
1639         struct cm_rep_msg *rep_msg;
1640         unsigned long flags;
1641         int ret;
1642
1643         if (param->private_data &&
1644             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
1645                 return -EINVAL;
1646
1647         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1648         spin_lock_irqsave(&cm_id_priv->lock, flags);
1649         if (cm_id->state != IB_CM_REQ_RCVD &&
1650             cm_id->state != IB_CM_MRA_REQ_SENT) {
1651                 ret = -EINVAL;
1652                 goto out;
1653         }
1654
1655         ret = cm_alloc_msg(cm_id_priv, &msg);
1656         if (ret)
1657                 goto out;
1658
1659         rep_msg = (struct cm_rep_msg *) msg->mad;
1660         cm_format_rep(rep_msg, cm_id_priv, param);
1661         msg->timeout_ms = cm_id_priv->timeout_ms;
1662         msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
1663
1664         ret = ib_post_send_mad(msg, NULL);
1665         if (ret) {
1666                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1667                 cm_free_msg(msg);
1668                 return ret;
1669         }
1670
1671         cm_id->state = IB_CM_REP_SENT;
1672         cm_id_priv->msg = msg;
1673         cm_id_priv->initiator_depth = param->initiator_depth;
1674         cm_id_priv->responder_resources = param->responder_resources;
1675         cm_id_priv->rq_psn = cm_rep_get_starting_psn(rep_msg);
1676         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
1677
1678 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1679         return ret;
1680 }
1681 EXPORT_SYMBOL(ib_send_cm_rep);
1682
1683 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
1684                           struct cm_id_private *cm_id_priv,
1685                           const void *private_data,
1686                           u8 private_data_len)
1687 {
1688         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
1689         rtu_msg->local_comm_id = cm_id_priv->id.local_id;
1690         rtu_msg->remote_comm_id = cm_id_priv->id.remote_id;
1691
1692         if (private_data && private_data_len)
1693                 memcpy(rtu_msg->private_data, private_data, private_data_len);
1694 }
1695
1696 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
1697                    const void *private_data,
1698                    u8 private_data_len)
1699 {
1700         struct cm_id_private *cm_id_priv;
1701         struct ib_mad_send_buf *msg;
1702         unsigned long flags;
1703         void *data;
1704         int ret;
1705
1706         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
1707                 return -EINVAL;
1708
1709         data = cm_copy_private_data(private_data, private_data_len);
1710         if (IS_ERR(data))
1711                 return PTR_ERR(data);
1712
1713         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1714         spin_lock_irqsave(&cm_id_priv->lock, flags);
1715         if (cm_id->state != IB_CM_REP_RCVD &&
1716             cm_id->state != IB_CM_MRA_REP_SENT) {
1717                 ret = -EINVAL;
1718                 goto error;
1719         }
1720
1721         ret = cm_alloc_msg(cm_id_priv, &msg);
1722         if (ret)
1723                 goto error;
1724
1725         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
1726                       private_data, private_data_len);
1727
1728         ret = ib_post_send_mad(msg, NULL);
1729         if (ret) {
1730                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1731                 cm_free_msg(msg);
1732                 kfree(data);
1733                 return ret;
1734         }
1735
1736         cm_id->state = IB_CM_ESTABLISHED;
1737         cm_set_private_data(cm_id_priv, data, private_data_len);
1738         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1739         return 0;
1740
1741 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1742         kfree(data);
1743         return ret;
1744 }
1745 EXPORT_SYMBOL(ib_send_cm_rtu);
1746
1747 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
1748 {
1749         struct cm_rep_msg *rep_msg;
1750         struct ib_cm_rep_event_param *param;
1751
1752         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
1753         param = &work->cm_event.param.rep_rcvd;
1754         param->remote_ca_guid = rep_msg->local_ca_guid;
1755         param->remote_qkey = be32_to_cpu(rep_msg->local_qkey);
1756         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
1757         param->starting_psn = be32_to_cpu(cm_rep_get_starting_psn(rep_msg));
1758         param->responder_resources = rep_msg->initiator_depth;
1759         param->initiator_depth = rep_msg->resp_resources;
1760         param->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
1761         param->failover_accepted = cm_rep_get_failover(rep_msg);
1762         param->flow_control = cm_rep_get_flow_ctrl(rep_msg);
1763         param->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
1764         param->srq = cm_rep_get_srq(rep_msg);
1765         work->cm_event.private_data = &rep_msg->private_data;
1766 }
1767
1768 static void cm_dup_rep_handler(struct cm_work *work)
1769 {
1770         struct cm_id_private *cm_id_priv;
1771         struct cm_rep_msg *rep_msg;
1772         struct ib_mad_send_buf *msg = NULL;
1773         int ret;
1774
1775         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
1776         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id,
1777                                    rep_msg->local_comm_id);
1778         if (!cm_id_priv)
1779                 return;
1780
1781         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1782                         counter[CM_REP_COUNTER]);
1783         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1784         if (ret)
1785                 goto deref;
1786
1787         spin_lock_irq(&cm_id_priv->lock);
1788         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
1789                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
1790                               cm_id_priv->private_data,
1791                               cm_id_priv->private_data_len);
1792         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
1793                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1794                               CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
1795                               cm_id_priv->private_data,
1796                               cm_id_priv->private_data_len);
1797         else
1798                 goto unlock;
1799         spin_unlock_irq(&cm_id_priv->lock);
1800
1801         ret = ib_post_send_mad(msg, NULL);
1802         if (ret)
1803                 goto free;
1804         goto deref;
1805
1806 unlock: spin_unlock_irq(&cm_id_priv->lock);
1807 free:   cm_free_msg(msg);
1808 deref:  cm_deref_id(cm_id_priv);
1809 }
1810
1811 static int cm_rep_handler(struct cm_work *work)
1812 {
1813         struct cm_id_private *cm_id_priv;
1814         struct cm_rep_msg *rep_msg;
1815         int ret;
1816
1817         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
1818         cm_id_priv = cm_acquire_id(rep_msg->remote_comm_id, 0);
1819         if (!cm_id_priv) {
1820                 cm_dup_rep_handler(work);
1821                 return -EINVAL;
1822         }
1823
1824         cm_format_rep_event(work, cm_id_priv->qp_type);
1825
1826         spin_lock_irq(&cm_id_priv->lock);
1827         switch (cm_id_priv->id.state) {
1828         case IB_CM_REQ_SENT:
1829         case IB_CM_MRA_REQ_RCVD:
1830                 break;
1831         default:
1832                 spin_unlock_irq(&cm_id_priv->lock);
1833                 ret = -EINVAL;
1834                 goto error;
1835         }
1836
1837         cm_id_priv->timewait_info->work.remote_id = rep_msg->local_comm_id;
1838         cm_id_priv->timewait_info->remote_ca_guid = rep_msg->local_ca_guid;
1839         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
1840
1841         spin_lock(&cm.lock);
1842         /* Check for duplicate REP. */
1843         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
1844                 spin_unlock(&cm.lock);
1845                 spin_unlock_irq(&cm_id_priv->lock);
1846                 ret = -EINVAL;
1847                 goto error;
1848         }
1849         /* Check for a stale connection. */
1850         if (cm_insert_remote_qpn(cm_id_priv->timewait_info)) {
1851                 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
1852                          &cm.remote_id_table);
1853                 cm_id_priv->timewait_info->inserted_remote_id = 0;
1854                 spin_unlock(&cm.lock);
1855                 spin_unlock_irq(&cm_id_priv->lock);
1856                 cm_issue_rej(work->port, work->mad_recv_wc,
1857                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
1858                              NULL, 0);
1859                 ret = -EINVAL;
1860                 goto error;
1861         }
1862         spin_unlock(&cm.lock);
1863
1864         cm_id_priv->id.state = IB_CM_REP_RCVD;
1865         cm_id_priv->id.remote_id = rep_msg->local_comm_id;
1866         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
1867         cm_id_priv->initiator_depth = rep_msg->resp_resources;
1868         cm_id_priv->responder_resources = rep_msg->initiator_depth;
1869         cm_id_priv->sq_psn = cm_rep_get_starting_psn(rep_msg);
1870         cm_id_priv->rnr_retry_count = cm_rep_get_rnr_retry_count(rep_msg);
1871         cm_id_priv->target_ack_delay = cm_rep_get_target_ack_delay(rep_msg);
1872         cm_id_priv->av.timeout =
1873                         cm_ack_timeout(cm_id_priv->target_ack_delay,
1874                                        cm_id_priv->av.timeout - 1);
1875         cm_id_priv->alt_av.timeout =
1876                         cm_ack_timeout(cm_id_priv->target_ack_delay,
1877                                        cm_id_priv->alt_av.timeout - 1);
1878
1879         /* todo: handle peer_to_peer */
1880
1881         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1882         ret = atomic_inc_and_test(&cm_id_priv->work_count);
1883         if (!ret)
1884                 list_add_tail(&work->list, &cm_id_priv->work_list);
1885         spin_unlock_irq(&cm_id_priv->lock);
1886
1887         if (ret)
1888                 cm_process_work(cm_id_priv, work);
1889         else
1890                 cm_deref_id(cm_id_priv);
1891         return 0;
1892
1893 error:
1894         cm_deref_id(cm_id_priv);
1895         return ret;
1896 }
1897
1898 static int cm_establish_handler(struct cm_work *work)
1899 {
1900         struct cm_id_private *cm_id_priv;
1901         int ret;
1902
1903         /* See comment in cm_establish about lookup. */
1904         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
1905         if (!cm_id_priv)
1906                 return -EINVAL;
1907
1908         spin_lock_irq(&cm_id_priv->lock);
1909         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
1910                 spin_unlock_irq(&cm_id_priv->lock);
1911                 goto out;
1912         }
1913
1914         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1915         ret = atomic_inc_and_test(&cm_id_priv->work_count);
1916         if (!ret)
1917                 list_add_tail(&work->list, &cm_id_priv->work_list);
1918         spin_unlock_irq(&cm_id_priv->lock);
1919
1920         if (ret)
1921                 cm_process_work(cm_id_priv, work);
1922         else
1923                 cm_deref_id(cm_id_priv);
1924         return 0;
1925 out:
1926         cm_deref_id(cm_id_priv);
1927         return -EINVAL;
1928 }
1929
1930 static int cm_rtu_handler(struct cm_work *work)
1931 {
1932         struct cm_id_private *cm_id_priv;
1933         struct cm_rtu_msg *rtu_msg;
1934         int ret;
1935
1936         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
1937         cm_id_priv = cm_acquire_id(rtu_msg->remote_comm_id,
1938                                    rtu_msg->local_comm_id);
1939         if (!cm_id_priv)
1940                 return -EINVAL;
1941
1942         work->cm_event.private_data = &rtu_msg->private_data;
1943
1944         spin_lock_irq(&cm_id_priv->lock);
1945         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
1946             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
1947                 spin_unlock_irq(&cm_id_priv->lock);
1948                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1949                                 counter[CM_RTU_COUNTER]);
1950                 goto out;
1951         }
1952         cm_id_priv->id.state = IB_CM_ESTABLISHED;
1953
1954         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1955         ret = atomic_inc_and_test(&cm_id_priv->work_count);
1956         if (!ret)
1957                 list_add_tail(&work->list, &cm_id_priv->work_list);
1958         spin_unlock_irq(&cm_id_priv->lock);
1959
1960         if (ret)
1961                 cm_process_work(cm_id_priv, work);
1962         else
1963                 cm_deref_id(cm_id_priv);
1964         return 0;
1965 out:
1966         cm_deref_id(cm_id_priv);
1967         return -EINVAL;
1968 }
1969
1970 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
1971                           struct cm_id_private *cm_id_priv,
1972                           const void *private_data,
1973                           u8 private_data_len)
1974 {
1975         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
1976                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_DREQ));
1977         dreq_msg->local_comm_id = cm_id_priv->id.local_id;
1978         dreq_msg->remote_comm_id = cm_id_priv->id.remote_id;
1979         cm_dreq_set_remote_qpn(dreq_msg, cm_id_priv->remote_qpn);
1980
1981         if (private_data && private_data_len)
1982                 memcpy(dreq_msg->private_data, private_data, private_data_len);
1983 }
1984
1985 int ib_send_cm_dreq(struct ib_cm_id *cm_id,
1986                     const void *private_data,
1987                     u8 private_data_len)
1988 {
1989         struct cm_id_private *cm_id_priv;
1990         struct ib_mad_send_buf *msg;
1991         unsigned long flags;
1992         int ret;
1993
1994         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
1995                 return -EINVAL;
1996
1997         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1998         spin_lock_irqsave(&cm_id_priv->lock, flags);
1999         if (cm_id->state != IB_CM_ESTABLISHED) {
2000                 ret = -EINVAL;
2001                 goto out;
2002         }
2003
2004         if (cm_id->lap_state == IB_CM_LAP_SENT ||
2005             cm_id->lap_state == IB_CM_MRA_LAP_RCVD)
2006                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2007
2008         ret = cm_alloc_msg(cm_id_priv, &msg);
2009         if (ret) {
2010                 cm_enter_timewait(cm_id_priv);
2011                 goto out;
2012         }
2013
2014         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2015                        private_data, private_data_len);
2016         msg->timeout_ms = cm_id_priv->timeout_ms;
2017         msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2018
2019         ret = ib_post_send_mad(msg, NULL);
2020         if (ret) {
2021                 cm_enter_timewait(cm_id_priv);
2022                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2023                 cm_free_msg(msg);
2024                 return ret;
2025         }
2026
2027         cm_id->state = IB_CM_DREQ_SENT;
2028         cm_id_priv->msg = msg;
2029 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2030         return ret;
2031 }
2032 EXPORT_SYMBOL(ib_send_cm_dreq);
2033
2034 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2035                           struct cm_id_private *cm_id_priv,
2036                           const void *private_data,
2037                           u8 private_data_len)
2038 {
2039         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2040         drep_msg->local_comm_id = cm_id_priv->id.local_id;
2041         drep_msg->remote_comm_id = cm_id_priv->id.remote_id;
2042
2043         if (private_data && private_data_len)
2044                 memcpy(drep_msg->private_data, private_data, private_data_len);
2045 }
2046
2047 int ib_send_cm_drep(struct ib_cm_id *cm_id,
2048                     const void *private_data,
2049                     u8 private_data_len)
2050 {
2051         struct cm_id_private *cm_id_priv;
2052         struct ib_mad_send_buf *msg;
2053         unsigned long flags;
2054         void *data;
2055         int ret;
2056
2057         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2058                 return -EINVAL;
2059
2060         data = cm_copy_private_data(private_data, private_data_len);
2061         if (IS_ERR(data))
2062                 return PTR_ERR(data);
2063
2064         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2065         spin_lock_irqsave(&cm_id_priv->lock, flags);
2066         if (cm_id->state != IB_CM_DREQ_RCVD) {
2067                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2068                 kfree(data);
2069                 return -EINVAL;
2070         }
2071
2072         cm_set_private_data(cm_id_priv, data, private_data_len);
2073         cm_enter_timewait(cm_id_priv);
2074
2075         ret = cm_alloc_msg(cm_id_priv, &msg);
2076         if (ret)
2077                 goto out;
2078
2079         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2080                        private_data, private_data_len);
2081
2082         ret = ib_post_send_mad(msg, NULL);
2083         if (ret) {
2084                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2085                 cm_free_msg(msg);
2086                 return ret;
2087         }
2088
2089 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2090         return ret;
2091 }
2092 EXPORT_SYMBOL(ib_send_cm_drep);
2093
2094 static int cm_issue_drep(struct cm_port *port,
2095                          struct ib_mad_recv_wc *mad_recv_wc)
2096 {
2097         struct ib_mad_send_buf *msg = NULL;
2098         struct cm_dreq_msg *dreq_msg;
2099         struct cm_drep_msg *drep_msg;
2100         int ret;
2101
2102         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2103         if (ret)
2104                 return ret;
2105
2106         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2107         drep_msg = (struct cm_drep_msg *) msg->mad;
2108
2109         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2110         drep_msg->remote_comm_id = dreq_msg->local_comm_id;
2111         drep_msg->local_comm_id = dreq_msg->remote_comm_id;
2112
2113         ret = ib_post_send_mad(msg, NULL);
2114         if (ret)
2115                 cm_free_msg(msg);
2116
2117         return ret;
2118 }
2119
2120 static int cm_dreq_handler(struct cm_work *work)
2121 {
2122         struct cm_id_private *cm_id_priv;
2123         struct cm_dreq_msg *dreq_msg;
2124         struct ib_mad_send_buf *msg = NULL;
2125         int ret;
2126
2127         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2128         cm_id_priv = cm_acquire_id(dreq_msg->remote_comm_id,
2129                                    dreq_msg->local_comm_id);
2130         if (!cm_id_priv) {
2131                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2132                                 counter[CM_DREQ_COUNTER]);
2133                 cm_issue_drep(work->port, work->mad_recv_wc);
2134                 return -EINVAL;
2135         }
2136
2137         work->cm_event.private_data = &dreq_msg->private_data;
2138
2139         spin_lock_irq(&cm_id_priv->lock);
2140         if (cm_id_priv->local_qpn != cm_dreq_get_remote_qpn(dreq_msg))
2141                 goto unlock;
2142
2143         switch (cm_id_priv->id.state) {
2144         case IB_CM_REP_SENT:
2145         case IB_CM_DREQ_SENT:
2146                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2147                 break;
2148         case IB_CM_ESTABLISHED:
2149                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2150                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2151                         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2152                 break;
2153         case IB_CM_MRA_REP_RCVD:
2154                 break;
2155         case IB_CM_TIMEWAIT:
2156                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2157                                 counter[CM_DREQ_COUNTER]);
2158                 if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
2159                         goto unlock;
2160
2161                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2162                                cm_id_priv->private_data,
2163                                cm_id_priv->private_data_len);
2164                 spin_unlock_irq(&cm_id_priv->lock);
2165
2166                 if (ib_post_send_mad(msg, NULL))
2167                         cm_free_msg(msg);
2168                 goto deref;
2169         case IB_CM_DREQ_RCVD:
2170                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2171                                 counter[CM_DREQ_COUNTER]);
2172                 goto unlock;
2173         default:
2174                 goto unlock;
2175         }
2176         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2177         cm_id_priv->tid = dreq_msg->hdr.tid;
2178         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2179         if (!ret)
2180                 list_add_tail(&work->list, &cm_id_priv->work_list);
2181         spin_unlock_irq(&cm_id_priv->lock);
2182
2183         if (ret)
2184                 cm_process_work(cm_id_priv, work);
2185         else
2186                 cm_deref_id(cm_id_priv);
2187         return 0;
2188
2189 unlock: spin_unlock_irq(&cm_id_priv->lock);
2190 deref:  cm_deref_id(cm_id_priv);
2191         return -EINVAL;
2192 }
2193
2194 static int cm_drep_handler(struct cm_work *work)
2195 {
2196         struct cm_id_private *cm_id_priv;
2197         struct cm_drep_msg *drep_msg;
2198         int ret;
2199
2200         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2201         cm_id_priv = cm_acquire_id(drep_msg->remote_comm_id,
2202                                    drep_msg->local_comm_id);
2203         if (!cm_id_priv)
2204                 return -EINVAL;
2205
2206         work->cm_event.private_data = &drep_msg->private_data;
2207
2208         spin_lock_irq(&cm_id_priv->lock);
2209         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2210             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2211                 spin_unlock_irq(&cm_id_priv->lock);
2212                 goto out;
2213         }
2214         cm_enter_timewait(cm_id_priv);
2215
2216         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2217         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2218         if (!ret)
2219                 list_add_tail(&work->list, &cm_id_priv->work_list);
2220         spin_unlock_irq(&cm_id_priv->lock);
2221
2222         if (ret)
2223                 cm_process_work(cm_id_priv, work);
2224         else
2225                 cm_deref_id(cm_id_priv);
2226         return 0;
2227 out:
2228         cm_deref_id(cm_id_priv);
2229         return -EINVAL;
2230 }
2231
2232 int ib_send_cm_rej(struct ib_cm_id *cm_id,
2233                    enum ib_cm_rej_reason reason,
2234                    void *ari,
2235                    u8 ari_length,
2236                    const void *private_data,
2237                    u8 private_data_len)
2238 {
2239         struct cm_id_private *cm_id_priv;
2240         struct ib_mad_send_buf *msg;
2241         unsigned long flags;
2242         int ret;
2243
2244         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2245             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2246                 return -EINVAL;
2247
2248         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2249
2250         spin_lock_irqsave(&cm_id_priv->lock, flags);
2251         switch (cm_id->state) {
2252         case IB_CM_REQ_SENT:
2253         case IB_CM_MRA_REQ_RCVD:
2254         case IB_CM_REQ_RCVD:
2255         case IB_CM_MRA_REQ_SENT:
2256         case IB_CM_REP_RCVD:
2257         case IB_CM_MRA_REP_SENT:
2258                 ret = cm_alloc_msg(cm_id_priv, &msg);
2259                 if (!ret)
2260                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2261                                       cm_id_priv, reason, ari, ari_length,
2262                                       private_data, private_data_len);
2263
2264                 cm_reset_to_idle(cm_id_priv);
2265                 break;
2266         case IB_CM_REP_SENT:
2267         case IB_CM_MRA_REP_RCVD:
2268                 ret = cm_alloc_msg(cm_id_priv, &msg);
2269                 if (!ret)
2270                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2271                                       cm_id_priv, reason, ari, ari_length,
2272                                       private_data, private_data_len);
2273
2274                 cm_enter_timewait(cm_id_priv);
2275                 break;
2276         default:
2277                 ret = -EINVAL;
2278                 goto out;
2279         }
2280
2281         if (ret)
2282                 goto out;
2283
2284         ret = ib_post_send_mad(msg, NULL);
2285         if (ret)
2286                 cm_free_msg(msg);
2287
2288 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2289         return ret;
2290 }
2291 EXPORT_SYMBOL(ib_send_cm_rej);
2292
2293 static void cm_format_rej_event(struct cm_work *work)
2294 {
2295         struct cm_rej_msg *rej_msg;
2296         struct ib_cm_rej_event_param *param;
2297
2298         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2299         param = &work->cm_event.param.rej_rcvd;
2300         param->ari = rej_msg->ari;
2301         param->ari_length = cm_rej_get_reject_info_len(rej_msg);
2302         param->reason = __be16_to_cpu(rej_msg->reason);
2303         work->cm_event.private_data = &rej_msg->private_data;
2304 }
2305
2306 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2307 {
2308         struct cm_timewait_info *timewait_info;
2309         struct cm_id_private *cm_id_priv;
2310         __be32 remote_id;
2311
2312         remote_id = rej_msg->local_comm_id;
2313
2314         if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_TIMEOUT) {
2315                 spin_lock_irq(&cm.lock);
2316                 timewait_info = cm_find_remote_id( *((__be64 *) rej_msg->ari),
2317                                                   remote_id);
2318                 if (!timewait_info) {
2319                         spin_unlock_irq(&cm.lock);
2320                         return NULL;
2321                 }
2322                 cm_id_priv = idr_find(&cm.local_id_table, (__force int)
2323                                       (timewait_info->work.local_id ^
2324                                        cm.random_id_operand));
2325                 if (cm_id_priv) {
2326                         if (cm_id_priv->id.remote_id == remote_id)
2327                                 atomic_inc(&cm_id_priv->refcount);
2328                         else
2329                                 cm_id_priv = NULL;
2330                 }
2331                 spin_unlock_irq(&cm.lock);
2332         } else if (cm_rej_get_msg_rejected(rej_msg) == CM_MSG_RESPONSE_REQ)
2333                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, 0);
2334         else
2335                 cm_id_priv = cm_acquire_id(rej_msg->remote_comm_id, remote_id);
2336
2337         return cm_id_priv;
2338 }
2339
2340 static int cm_rej_handler(struct cm_work *work)
2341 {
2342         struct cm_id_private *cm_id_priv;
2343         struct cm_rej_msg *rej_msg;
2344         int ret;
2345
2346         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2347         cm_id_priv = cm_acquire_rejected_id(rej_msg);
2348         if (!cm_id_priv)
2349                 return -EINVAL;
2350
2351         cm_format_rej_event(work);
2352
2353         spin_lock_irq(&cm_id_priv->lock);
2354         switch (cm_id_priv->id.state) {
2355         case IB_CM_REQ_SENT:
2356         case IB_CM_MRA_REQ_RCVD:
2357         case IB_CM_REP_SENT:
2358         case IB_CM_MRA_REP_RCVD:
2359                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2360                 /* fall through */
2361         case IB_CM_REQ_RCVD:
2362         case IB_CM_MRA_REQ_SENT:
2363                 if (__be16_to_cpu(rej_msg->reason) == IB_CM_REJ_STALE_CONN)
2364                         cm_enter_timewait(cm_id_priv);
2365                 else
2366                         cm_reset_to_idle(cm_id_priv);
2367                 break;
2368         case IB_CM_DREQ_SENT:
2369                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2370                 /* fall through */
2371         case IB_CM_REP_RCVD:
2372         case IB_CM_MRA_REP_SENT:
2373                 cm_enter_timewait(cm_id_priv);
2374                 break;
2375         case IB_CM_ESTABLISHED:
2376                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
2377                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
2378                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
2379                                 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
2380                                               cm_id_priv->msg);
2381                         cm_enter_timewait(cm_id_priv);
2382                         break;
2383                 }
2384                 /* fall through */
2385         default:
2386                 spin_unlock_irq(&cm_id_priv->lock);
2387                 ret = -EINVAL;
2388                 goto out;
2389         }
2390
2391         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2392         if (!ret)
2393                 list_add_tail(&work->list, &cm_id_priv->work_list);
2394         spin_unlock_irq(&cm_id_priv->lock);
2395
2396         if (ret)
2397                 cm_process_work(cm_id_priv, work);
2398         else
2399                 cm_deref_id(cm_id_priv);
2400         return 0;
2401 out:
2402         cm_deref_id(cm_id_priv);
2403         return -EINVAL;
2404 }
2405
2406 int ib_send_cm_mra(struct ib_cm_id *cm_id,
2407                    u8 service_timeout,
2408                    const void *private_data,
2409                    u8 private_data_len)
2410 {
2411         struct cm_id_private *cm_id_priv;
2412         struct ib_mad_send_buf *msg;
2413         enum ib_cm_state cm_state;
2414         enum ib_cm_lap_state lap_state;
2415         enum cm_msg_response msg_response;
2416         void *data;
2417         unsigned long flags;
2418         int ret;
2419
2420         if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
2421                 return -EINVAL;
2422
2423         data = cm_copy_private_data(private_data, private_data_len);
2424         if (IS_ERR(data))
2425                 return PTR_ERR(data);
2426
2427         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2428
2429         spin_lock_irqsave(&cm_id_priv->lock, flags);
2430         switch(cm_id_priv->id.state) {
2431         case IB_CM_REQ_RCVD:
2432                 cm_state = IB_CM_MRA_REQ_SENT;
2433                 lap_state = cm_id->lap_state;
2434                 msg_response = CM_MSG_RESPONSE_REQ;
2435                 break;
2436         case IB_CM_REP_RCVD:
2437                 cm_state = IB_CM_MRA_REP_SENT;
2438                 lap_state = cm_id->lap_state;
2439                 msg_response = CM_MSG_RESPONSE_REP;
2440                 break;
2441         case IB_CM_ESTABLISHED:
2442                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
2443                         cm_state = cm_id->state;
2444                         lap_state = IB_CM_MRA_LAP_SENT;
2445                         msg_response = CM_MSG_RESPONSE_OTHER;
2446                         break;
2447                 }
2448         default:
2449                 ret = -EINVAL;
2450                 goto error1;
2451         }
2452
2453         if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
2454                 ret = cm_alloc_msg(cm_id_priv, &msg);
2455                 if (ret)
2456                         goto error1;
2457
2458                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2459                               msg_response, service_timeout,
2460                               private_data, private_data_len);
2461                 ret = ib_post_send_mad(msg, NULL);
2462                 if (ret)
2463                         goto error2;
2464         }
2465
2466         cm_id->state = cm_state;
2467         cm_id->lap_state = lap_state;
2468         cm_id_priv->service_timeout = service_timeout;
2469         cm_set_private_data(cm_id_priv, data, private_data_len);
2470         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2471         return 0;
2472
2473 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2474         kfree(data);
2475         return ret;
2476
2477 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2478         kfree(data);
2479         cm_free_msg(msg);
2480         return ret;
2481 }
2482 EXPORT_SYMBOL(ib_send_cm_mra);
2483
2484 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
2485 {
2486         switch (cm_mra_get_msg_mraed(mra_msg)) {
2487         case CM_MSG_RESPONSE_REQ:
2488                 return cm_acquire_id(mra_msg->remote_comm_id, 0);
2489         case CM_MSG_RESPONSE_REP:
2490         case CM_MSG_RESPONSE_OTHER:
2491                 return cm_acquire_id(mra_msg->remote_comm_id,
2492                                      mra_msg->local_comm_id);
2493         default:
2494                 return NULL;
2495         }
2496 }
2497
2498 static int cm_mra_handler(struct cm_work *work)
2499 {
2500         struct cm_id_private *cm_id_priv;
2501         struct cm_mra_msg *mra_msg;
2502         int timeout, ret;
2503
2504         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
2505         cm_id_priv = cm_acquire_mraed_id(mra_msg);
2506         if (!cm_id_priv)
2507                 return -EINVAL;
2508
2509         work->cm_event.private_data = &mra_msg->private_data;
2510         work->cm_event.param.mra_rcvd.service_timeout =
2511                                         cm_mra_get_service_timeout(mra_msg);
2512         timeout = cm_convert_to_ms(cm_mra_get_service_timeout(mra_msg)) +
2513                   cm_convert_to_ms(cm_id_priv->av.timeout);
2514
2515         spin_lock_irq(&cm_id_priv->lock);
2516         switch (cm_id_priv->id.state) {
2517         case IB_CM_REQ_SENT:
2518                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||
2519                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2520                                   cm_id_priv->msg, timeout))
2521                         goto out;
2522                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
2523                 break;
2524         case IB_CM_REP_SENT:
2525                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||
2526                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2527                                   cm_id_priv->msg, timeout))
2528                         goto out;
2529                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
2530                 break;
2531         case IB_CM_ESTABLISHED:
2532                 if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||
2533                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
2534                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
2535                                   cm_id_priv->msg, timeout)) {
2536                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2537                                 atomic_long_inc(&work->port->
2538                                                 counter_group[CM_RECV_DUPLICATES].
2539                                                 counter[CM_MRA_COUNTER]);
2540                         goto out;
2541                 }
2542                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
2543                 break;
2544         case IB_CM_MRA_REQ_RCVD:
2545         case IB_CM_MRA_REP_RCVD:
2546                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2547                                 counter[CM_MRA_COUNTER]);
2548                 /* fall through */
2549         default:
2550                 goto out;
2551         }
2552
2553         cm_id_priv->msg->context[1] = (void *) (unsigned long)
2554                                       cm_id_priv->id.state;
2555         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2556         if (!ret)
2557                 list_add_tail(&work->list, &cm_id_priv->work_list);
2558         spin_unlock_irq(&cm_id_priv->lock);
2559
2560         if (ret)
2561                 cm_process_work(cm_id_priv, work);
2562         else
2563                 cm_deref_id(cm_id_priv);
2564         return 0;
2565 out:
2566         spin_unlock_irq(&cm_id_priv->lock);
2567         cm_deref_id(cm_id_priv);
2568         return -EINVAL;
2569 }
2570
2571 static void cm_format_lap(struct cm_lap_msg *lap_msg,
2572                           struct cm_id_private *cm_id_priv,
2573                           struct ib_sa_path_rec *alternate_path,
2574                           const void *private_data,
2575                           u8 private_data_len)
2576 {
2577         cm_format_mad_hdr(&lap_msg->hdr, CM_LAP_ATTR_ID,
2578                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_LAP));
2579         lap_msg->local_comm_id = cm_id_priv->id.local_id;
2580         lap_msg->remote_comm_id = cm_id_priv->id.remote_id;
2581         cm_lap_set_remote_qpn(lap_msg, cm_id_priv->remote_qpn);
2582         /* todo: need remote CM response timeout */
2583         cm_lap_set_remote_resp_timeout(lap_msg, 0x1F);
2584         lap_msg->alt_local_lid = alternate_path->slid;
2585         lap_msg->alt_remote_lid = alternate_path->dlid;
2586         lap_msg->alt_local_gid = alternate_path->sgid;
2587         lap_msg->alt_remote_gid = alternate_path->dgid;
2588         cm_lap_set_flow_label(lap_msg, alternate_path->flow_label);
2589         cm_lap_set_traffic_class(lap_msg, alternate_path->traffic_class);
2590         lap_msg->alt_hop_limit = alternate_path->hop_limit;
2591         cm_lap_set_packet_rate(lap_msg, alternate_path->rate);
2592         cm_lap_set_sl(lap_msg, alternate_path->sl);
2593         cm_lap_set_subnet_local(lap_msg, 1); /* local only... */
2594         cm_lap_set_local_ack_timeout(lap_msg,
2595                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
2596                                alternate_path->packet_life_time));
2597
2598         if (private_data && private_data_len)
2599                 memcpy(lap_msg->private_data, private_data, private_data_len);
2600 }
2601
2602 int ib_send_cm_lap(struct ib_cm_id *cm_id,
2603                    struct ib_sa_path_rec *alternate_path,
2604                    const void *private_data,
2605                    u8 private_data_len)
2606 {
2607         struct cm_id_private *cm_id_priv;
2608         struct ib_mad_send_buf *msg;
2609         unsigned long flags;
2610         int ret;
2611
2612         if (private_data && private_data_len > IB_CM_LAP_PRIVATE_DATA_SIZE)
2613                 return -EINVAL;
2614
2615         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2616         spin_lock_irqsave(&cm_id_priv->lock, flags);
2617         if (cm_id->state != IB_CM_ESTABLISHED ||
2618             (cm_id->lap_state != IB_CM_LAP_UNINIT &&
2619              cm_id->lap_state != IB_CM_LAP_IDLE)) {
2620                 ret = -EINVAL;
2621                 goto out;
2622         }
2623
2624         ret = cm_init_av_by_path(alternate_path, &cm_id_priv->alt_av);
2625         if (ret)
2626                 goto out;
2627         cm_id_priv->alt_av.timeout =
2628                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2629                                        cm_id_priv->alt_av.timeout - 1);
2630
2631         ret = cm_alloc_msg(cm_id_priv, &msg);
2632         if (ret)
2633                 goto out;
2634
2635         cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv,
2636                       alternate_path, private_data, private_data_len);
2637         msg->timeout_ms = cm_id_priv->timeout_ms;
2638         msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;
2639
2640         ret = ib_post_send_mad(msg, NULL);
2641         if (ret) {
2642                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2643                 cm_free_msg(msg);
2644                 return ret;
2645         }
2646
2647         cm_id->lap_state = IB_CM_LAP_SENT;
2648         cm_id_priv->msg = msg;
2649
2650 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2651         return ret;
2652 }
2653 EXPORT_SYMBOL(ib_send_cm_lap);
2654
2655 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
2656                                     struct ib_sa_path_rec *path,
2657                                     struct cm_lap_msg *lap_msg)
2658 {
2659         memset(path, 0, sizeof *path);
2660         path->dgid = lap_msg->alt_local_gid;
2661         path->sgid = lap_msg->alt_remote_gid;
2662         path->dlid = lap_msg->alt_local_lid;
2663         path->slid = lap_msg->alt_remote_lid;
2664         path->flow_label = cm_lap_get_flow_label(lap_msg);
2665         path->hop_limit = lap_msg->alt_hop_limit;
2666         path->traffic_class = cm_lap_get_traffic_class(lap_msg);
2667         path->reversible = 1;
2668         path->pkey = cm_id_priv->pkey;
2669         path->sl = cm_lap_get_sl(lap_msg);
2670         path->mtu_selector = IB_SA_EQ;
2671         path->mtu = cm_id_priv->path_mtu;
2672         path->rate_selector = IB_SA_EQ;
2673         path->rate = cm_lap_get_packet_rate(lap_msg);
2674         path->packet_life_time_selector = IB_SA_EQ;
2675         path->packet_life_time = cm_lap_get_local_ack_timeout(lap_msg);
2676         path->packet_life_time -= (path->packet_life_time > 0);
2677 }
2678
2679 static int cm_lap_handler(struct cm_work *work)
2680 {
2681         struct cm_id_private *cm_id_priv;
2682         struct cm_lap_msg *lap_msg;
2683         struct ib_cm_lap_event_param *param;
2684         struct ib_mad_send_buf *msg = NULL;
2685         int ret;
2686
2687         /* todo: verify LAP request and send reject APR if invalid. */
2688         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
2689         cm_id_priv = cm_acquire_id(lap_msg->remote_comm_id,
2690                                    lap_msg->local_comm_id);
2691         if (!cm_id_priv)
2692                 return -EINVAL;
2693
2694         param = &work->cm_event.param.lap_rcvd;
2695         param->alternate_path = &work->path[0];
2696         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
2697         work->cm_event.private_data = &lap_msg->private_data;
2698
2699         spin_lock_irq(&cm_id_priv->lock);
2700         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
2701                 goto unlock;
2702
2703         switch (cm_id_priv->id.lap_state) {
2704         case IB_CM_LAP_UNINIT:
2705         case IB_CM_LAP_IDLE:
2706                 break;
2707         case IB_CM_MRA_LAP_SENT:
2708                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2709                                 counter[CM_LAP_COUNTER]);
2710                 if (cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg))
2711                         goto unlock;
2712
2713                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2714                               CM_MSG_RESPONSE_OTHER,
2715                               cm_id_priv->service_timeout,
2716                               cm_id_priv->private_data,
2717                               cm_id_priv->private_data_len);
2718                 spin_unlock_irq(&cm_id_priv->lock);
2719
2720                 if (ib_post_send_mad(msg, NULL))
2721                         cm_free_msg(msg);
2722                 goto deref;
2723         case IB_CM_LAP_RCVD:
2724                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2725                                 counter[CM_LAP_COUNTER]);
2726                 goto unlock;
2727         default:
2728                 goto unlock;
2729         }
2730
2731         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
2732         cm_id_priv->tid = lap_msg->hdr.tid;
2733         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2734                                 work->mad_recv_wc->recv_buf.grh,
2735                                 &cm_id_priv->av);
2736         cm_init_av_by_path(param->alternate_path, &cm_id_priv->alt_av);
2737         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2738         if (!ret)
2739                 list_add_tail(&work->list, &cm_id_priv->work_list);
2740         spin_unlock_irq(&cm_id_priv->lock);
2741
2742         if (ret)
2743                 cm_process_work(cm_id_priv, work);
2744         else
2745                 cm_deref_id(cm_id_priv);
2746         return 0;
2747
2748 unlock: spin_unlock_irq(&cm_id_priv->lock);
2749 deref:  cm_deref_id(cm_id_priv);
2750         return -EINVAL;
2751 }
2752
2753 static void cm_format_apr(struct cm_apr_msg *apr_msg,
2754                           struct cm_id_private *cm_id_priv,
2755                           enum ib_cm_apr_status status,
2756                           void *info,
2757                           u8 info_length,
2758                           const void *private_data,
2759                           u8 private_data_len)
2760 {
2761         cm_format_mad_hdr(&apr_msg->hdr, CM_APR_ATTR_ID, cm_id_priv->tid);
2762         apr_msg->local_comm_id = cm_id_priv->id.local_id;
2763         apr_msg->remote_comm_id = cm_id_priv->id.remote_id;
2764         apr_msg->ap_status = (u8) status;
2765
2766         if (info && info_length) {
2767                 apr_msg->info_length = info_length;
2768                 memcpy(apr_msg->info, info, info_length);
2769         }
2770
2771         if (private_data && private_data_len)
2772                 memcpy(apr_msg->private_data, private_data, private_data_len);
2773 }
2774
2775 int ib_send_cm_apr(struct ib_cm_id *cm_id,
2776                    enum ib_cm_apr_status status,
2777                    void *info,
2778                    u8 info_length,
2779                    const void *private_data,
2780                    u8 private_data_len)
2781 {
2782         struct cm_id_private *cm_id_priv;
2783         struct ib_mad_send_buf *msg;
2784         unsigned long flags;
2785         int ret;
2786
2787         if ((private_data && private_data_len > IB_CM_APR_PRIVATE_DATA_SIZE) ||
2788             (info && info_length > IB_CM_APR_INFO_LENGTH))
2789                 return -EINVAL;
2790
2791         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2792         spin_lock_irqsave(&cm_id_priv->lock, flags);
2793         if (cm_id->state != IB_CM_ESTABLISHED ||
2794             (cm_id->lap_state != IB_CM_LAP_RCVD &&
2795              cm_id->lap_state != IB_CM_MRA_LAP_SENT)) {
2796                 ret = -EINVAL;
2797                 goto out;
2798         }
2799
2800         ret = cm_alloc_msg(cm_id_priv, &msg);
2801         if (ret)
2802                 goto out;
2803
2804         cm_format_apr((struct cm_apr_msg *) msg->mad, cm_id_priv, status,
2805                       info, info_length, private_data, private_data_len);
2806         ret = ib_post_send_mad(msg, NULL);
2807         if (ret) {
2808                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2809                 cm_free_msg(msg);
2810                 return ret;
2811         }
2812
2813         cm_id->lap_state = IB_CM_LAP_IDLE;
2814 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2815         return ret;
2816 }
2817 EXPORT_SYMBOL(ib_send_cm_apr);
2818
2819 static int cm_apr_handler(struct cm_work *work)
2820 {
2821         struct cm_id_private *cm_id_priv;
2822         struct cm_apr_msg *apr_msg;
2823         int ret;
2824
2825         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
2826         cm_id_priv = cm_acquire_id(apr_msg->remote_comm_id,
2827                                    apr_msg->local_comm_id);
2828         if (!cm_id_priv)
2829                 return -EINVAL; /* Unmatched reply. */
2830
2831         work->cm_event.param.apr_rcvd.ap_status = apr_msg->ap_status;
2832         work->cm_event.param.apr_rcvd.apr_info = &apr_msg->info;
2833         work->cm_event.param.apr_rcvd.info_len = apr_msg->info_length;
2834         work->cm_event.private_data = &apr_msg->private_data;
2835
2836         spin_lock_irq(&cm_id_priv->lock);
2837         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
2838             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
2839              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
2840                 spin_unlock_irq(&cm_id_priv->lock);
2841                 goto out;
2842         }
2843         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
2844         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2845         cm_id_priv->msg = NULL;
2846
2847         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2848         if (!ret)
2849                 list_add_tail(&work->list, &cm_id_priv->work_list);
2850         spin_unlock_irq(&cm_id_priv->lock);
2851
2852         if (ret)
2853                 cm_process_work(cm_id_priv, work);
2854         else
2855                 cm_deref_id(cm_id_priv);
2856         return 0;
2857 out:
2858         cm_deref_id(cm_id_priv);
2859         return -EINVAL;
2860 }
2861
2862 static int cm_timewait_handler(struct cm_work *work)
2863 {
2864         struct cm_timewait_info *timewait_info;
2865         struct cm_id_private *cm_id_priv;
2866         int ret;
2867
2868         timewait_info = (struct cm_timewait_info *)work;
2869         spin_lock_irq(&cm.lock);
2870         list_del(&timewait_info->list);
2871         spin_unlock_irq(&cm.lock);
2872
2873         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2874                                    timewait_info->work.remote_id);
2875         if (!cm_id_priv)
2876                 return -EINVAL;
2877
2878         spin_lock_irq(&cm_id_priv->lock);
2879         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
2880             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
2881                 spin_unlock_irq(&cm_id_priv->lock);
2882                 goto out;
2883         }
2884         cm_id_priv->id.state = IB_CM_IDLE;
2885         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2886         if (!ret)
2887                 list_add_tail(&work->list, &cm_id_priv->work_list);
2888         spin_unlock_irq(&cm_id_priv->lock);
2889
2890         if (ret)
2891                 cm_process_work(cm_id_priv, work);
2892         else
2893                 cm_deref_id(cm_id_priv);
2894         return 0;
2895 out:
2896         cm_deref_id(cm_id_priv);
2897         return -EINVAL;
2898 }
2899
2900 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
2901                                struct cm_id_private *cm_id_priv,
2902                                struct ib_cm_sidr_req_param *param)
2903 {
2904         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
2905                           cm_form_tid(cm_id_priv, CM_MSG_SEQUENCE_SIDR));
2906         sidr_req_msg->request_id = cm_id_priv->id.local_id;
2907         sidr_req_msg->pkey = param->path->pkey;
2908         sidr_req_msg->service_id = param->service_id;
2909
2910         if (param->private_data && param->private_data_len)
2911                 memcpy(sidr_req_msg->private_data, param->private_data,
2912                        param->private_data_len);
2913 }
2914
2915 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
2916                         struct ib_cm_sidr_req_param *param)
2917 {
2918         struct cm_id_private *cm_id_priv;
2919         struct ib_mad_send_buf *msg;
2920         unsigned long flags;
2921         int ret;
2922
2923         if (!param->path || (param->private_data &&
2924              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
2925                 return -EINVAL;
2926
2927         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2928         ret = cm_init_av_by_path(param->path, &cm_id_priv->av);
2929         if (ret)
2930                 goto out;
2931
2932         cm_id->service_id = param->service_id;
2933         cm_id->service_mask = ~cpu_to_be64(0);
2934         cm_id_priv->timeout_ms = param->timeout_ms;
2935         cm_id_priv->max_cm_retries = param->max_cm_retries;
2936         ret = cm_alloc_msg(cm_id_priv, &msg);
2937         if (ret)
2938                 goto out;
2939
2940         cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
2941                            param);
2942         msg->timeout_ms = cm_id_priv->timeout_ms;
2943         msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
2944
2945         spin_lock_irqsave(&cm_id_priv->lock, flags);
2946         if (cm_id->state == IB_CM_IDLE)
2947                 ret = ib_post_send_mad(msg, NULL);
2948         else
2949                 ret = -EINVAL;
2950
2951         if (ret) {
2952                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2953                 cm_free_msg(msg);
2954                 goto out;
2955         }
2956         cm_id->state = IB_CM_SIDR_REQ_SENT;
2957         cm_id_priv->msg = msg;
2958         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2959 out:
2960         return ret;
2961 }
2962 EXPORT_SYMBOL(ib_send_cm_sidr_req);
2963
2964 static void cm_format_sidr_req_event(struct cm_work *work,
2965                                      struct ib_cm_id *listen_id)
2966 {
2967         struct cm_sidr_req_msg *sidr_req_msg;
2968         struct ib_cm_sidr_req_event_param *param;
2969
2970         sidr_req_msg = (struct cm_sidr_req_msg *)
2971                                 work->mad_recv_wc->recv_buf.mad;
2972         param = &work->cm_event.param.sidr_req_rcvd;
2973         param->pkey = __be16_to_cpu(sidr_req_msg->pkey);
2974         param->listen_id = listen_id;
2975         param->port = work->port->port_num;
2976         work->cm_event.private_data = &sidr_req_msg->private_data;
2977 }
2978
2979 static int cm_sidr_req_handler(struct cm_work *work)
2980 {
2981         struct ib_cm_id *cm_id;
2982         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
2983         struct cm_sidr_req_msg *sidr_req_msg;
2984         struct ib_wc *wc;
2985
2986         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
2987         if (IS_ERR(cm_id))
2988                 return PTR_ERR(cm_id);
2989         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2990
2991         /* Record SGID/SLID and request ID for lookup. */
2992         sidr_req_msg = (struct cm_sidr_req_msg *)
2993                                 work->mad_recv_wc->recv_buf.mad;
2994         wc = work->mad_recv_wc->wc;
2995         cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
2996         cm_id_priv->av.dgid.global.interface_id = 0;
2997         cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2998                                 work->mad_recv_wc->recv_buf.grh,
2999                                 &cm_id_priv->av);
3000         cm_id_priv->id.remote_id = sidr_req_msg->request_id;
3001         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3002         atomic_inc(&cm_id_priv->work_count);
3003
3004         spin_lock_irq(&cm.lock);
3005         cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3006         if (cur_cm_id_priv) {
3007                 spin_unlock_irq(&cm.lock);
3008                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3009                                 counter[CM_SIDR_REQ_COUNTER]);
3010                 goto out; /* Duplicate message. */
3011         }
3012         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3013         cur_cm_id_priv = cm_find_listen(cm_id->device,
3014                                         sidr_req_msg->service_id,
3015                                         sidr_req_msg->private_data);
3016         if (!cur_cm_id_priv) {
3017                 spin_unlock_irq(&cm.lock);
3018                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);
3019                 goto out; /* No match. */
3020         }
3021         atomic_inc(&cur_cm_id_priv->refcount);
3022         atomic_inc(&cm_id_priv->refcount);
3023         spin_unlock_irq(&cm.lock);
3024
3025         cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler;
3026         cm_id_priv->id.context = cur_cm_id_priv->id.context;
3027         cm_id_priv->id.service_id = sidr_req_msg->service_id;
3028         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3029
3030         cm_format_sidr_req_event(work, &cur_cm_id_priv->id);
3031         cm_process_work(cm_id_priv, work);
3032         cm_deref_id(cur_cm_id_priv);
3033         return 0;
3034 out:
3035         ib_destroy_cm_id(&cm_id_priv->id);
3036         return -EINVAL;
3037 }
3038
3039 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3040                                struct cm_id_private *cm_id_priv,
3041                                struct ib_cm_sidr_rep_param *param)
3042 {
3043         cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3044                           cm_id_priv->tid);
3045         sidr_rep_msg->request_id = cm_id_priv->id.remote_id;
3046         sidr_rep_msg->status = param->status;
3047         cm_sidr_rep_set_qpn(sidr_rep_msg, cpu_to_be32(param->qp_num));
3048         sidr_rep_msg->service_id = cm_id_priv->id.service_id;
3049         sidr_rep_msg->qkey = cpu_to_be32(param->qkey);
3050
3051         if (param->info && param->info_length)
3052                 memcpy(sidr_rep_msg->info, param->info, param->info_length);
3053
3054         if (param->private_data && param->private_data_len)
3055                 memcpy(sidr_rep_msg->private_data, param->private_data,
3056                        param->private_data_len);
3057 }
3058
3059 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3060                         struct ib_cm_sidr_rep_param *param)
3061 {
3062         struct cm_id_private *cm_id_priv;
3063         struct ib_mad_send_buf *msg;
3064         unsigned long flags;
3065         int ret;
3066
3067         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3068             (param->private_data &&
3069              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3070                 return -EINVAL;
3071
3072         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3073         spin_lock_irqsave(&cm_id_priv->lock, flags);
3074         if (cm_id->state != IB_CM_SIDR_REQ_RCVD) {
3075                 ret = -EINVAL;
3076                 goto error;
3077         }
3078
3079         ret = cm_alloc_msg(cm_id_priv, &msg);
3080         if (ret)
3081                 goto error;
3082
3083         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3084                            param);
3085         ret = ib_post_send_mad(msg, NULL);
3086         if (ret) {
3087                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3088                 cm_free_msg(msg);
3089                 return ret;
3090         }
3091         cm_id->state = IB_CM_IDLE;
3092         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3093
3094         spin_lock_irqsave(&cm.lock, flags);
3095         rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3096         spin_unlock_irqrestore(&cm.lock, flags);
3097         return 0;
3098
3099 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3100         return ret;
3101 }
3102 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3103
3104 static void cm_format_sidr_rep_event(struct cm_work *work)
3105 {
3106         struct cm_sidr_rep_msg *sidr_rep_msg;
3107         struct ib_cm_sidr_rep_event_param *param;
3108
3109         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3110                                 work->mad_recv_wc->recv_buf.mad;
3111         param = &work->cm_event.param.sidr_rep_rcvd;
3112         param->status = sidr_rep_msg->status;
3113         param->qkey = be32_to_cpu(sidr_rep_msg->qkey);
3114         param->qpn = be32_to_cpu(cm_sidr_rep_get_qpn(sidr_rep_msg));
3115         param->info = &sidr_rep_msg->info;
3116         param->info_len = sidr_rep_msg->info_length;
3117         work->cm_event.private_data = &sidr_rep_msg->private_data;
3118 }
3119
3120 static int cm_sidr_rep_handler(struct cm_work *work)
3121 {
3122         struct cm_sidr_rep_msg *sidr_rep_msg;
3123         struct cm_id_private *cm_id_priv;
3124
3125         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3126                                 work->mad_recv_wc->recv_buf.mad;
3127         cm_id_priv = cm_acquire_id(sidr_rep_msg->request_id, 0);
3128         if (!cm_id_priv)
3129                 return -EINVAL; /* Unmatched reply. */
3130
3131         spin_lock_irq(&cm_id_priv->lock);
3132         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3133                 spin_unlock_irq(&cm_id_priv->lock);
3134                 goto out;
3135         }
3136         cm_id_priv->id.state = IB_CM_IDLE;
3137         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3138         spin_unlock_irq(&cm_id_priv->lock);
3139
3140         cm_format_sidr_rep_event(work);
3141         cm_process_work(cm_id_priv, work);
3142         return 0;
3143 out:
3144         cm_deref_id(cm_id_priv);
3145         return -EINVAL;
3146 }
3147
3148 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3149                                   enum ib_wc_status wc_status)
3150 {
3151         struct cm_id_private *cm_id_priv;
3152         struct ib_cm_event cm_event;
3153         enum ib_cm_state state;
3154         int ret;
3155
3156         memset(&cm_event, 0, sizeof cm_event);
3157         cm_id_priv = msg->context[0];
3158
3159         /* Discard old sends or ones without a response. */
3160         spin_lock_irq(&cm_id_priv->lock);
3161         state = (enum ib_cm_state) (unsigned long) msg->context[1];
3162         if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3163                 goto discard;
3164
3165         switch (state) {
3166         case IB_CM_REQ_SENT:
3167         case IB_CM_MRA_REQ_RCVD:
3168                 cm_reset_to_idle(cm_id_priv);
3169                 cm_event.event = IB_CM_REQ_ERROR;
3170                 break;
3171         case IB_CM_REP_SENT:
3172         case IB_CM_MRA_REP_RCVD:
3173                 cm_reset_to_idle(cm_id_priv);
3174                 cm_event.event = IB_CM_REP_ERROR;
3175                 break;
3176         case IB_CM_DREQ_SENT:
3177                 cm_enter_timewait(cm_id_priv);
3178                 cm_event.event = IB_CM_DREQ_ERROR;
3179                 break;
3180         case IB_CM_SIDR_REQ_SENT:
3181                 cm_id_priv->id.state = IB_CM_IDLE;
3182                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3183                 break;
3184         default:
3185                 goto discard;
3186         }
3187         spin_unlock_irq(&cm_id_priv->lock);
3188         cm_event.param.send_status = wc_status;
3189
3190         /* No other events can occur on the cm_id at this point. */
3191         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3192         cm_free_msg(msg);
3193         if (ret)
3194                 ib_destroy_cm_id(&cm_id_priv->id);
3195         return;
3196 discard:
3197         spin_unlock_irq(&cm_id_priv->lock);
3198         cm_free_msg(msg);
3199 }
3200
3201 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3202                             struct ib_mad_send_wc *mad_send_wc)
3203 {
3204         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3205         struct cm_port *port;
3206         u16 attr_index;
3207
3208         port = mad_agent->context;
3209         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3210                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3211
3212         /*
3213          * If the send was in response to a received message (context[0] is not
3214          * set to a cm_id), and is not a REJ, then it is a send that was
3215          * manually retried.
3216          */
3217         if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3218                 msg->retries = 1;
3219
3220         atomic_long_add(1 + msg->retries,
3221                         &port->counter_group[CM_XMIT].counter[attr_index]);
3222         if (msg->retries)
3223                 atomic_long_add(msg->retries,
3224                                 &port->counter_group[CM_XMIT_RETRIES].
3225                                 counter[attr_index]);
3226
3227         switch (mad_send_wc->status) {
3228         case IB_WC_SUCCESS:
3229         case IB_WC_WR_FLUSH_ERR:
3230                 cm_free_msg(msg);
3231                 break;
3232         default:
3233                 if (msg->context[0] && msg->context[1])
3234                         cm_process_send_error(msg, mad_send_wc->status);
3235                 else
3236                         cm_free_msg(msg);
3237                 break;
3238         }
3239 }
3240
3241 static void cm_work_handler(struct work_struct *_work)
3242 {
3243         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3244         int ret;
3245
3246         switch (work->cm_event.event) {
3247         case IB_CM_REQ_RECEIVED:
3248                 ret = cm_req_handler(work);
3249                 break;
3250         case IB_CM_MRA_RECEIVED:
3251                 ret = cm_mra_handler(work);
3252                 break;
3253         case IB_CM_REJ_RECEIVED:
3254                 ret = cm_rej_handler(work);
3255                 break;
3256         case IB_CM_REP_RECEIVED:
3257                 ret = cm_rep_handler(work);
3258                 break;
3259         case IB_CM_RTU_RECEIVED:
3260                 ret = cm_rtu_handler(work);
3261                 break;
3262         case IB_CM_USER_ESTABLISHED:
3263                 ret = cm_establish_handler(work);
3264                 break;
3265         case IB_CM_DREQ_RECEIVED:
3266                 ret = cm_dreq_handler(work);
3267                 break;
3268         case IB_CM_DREP_RECEIVED:
3269                 ret = cm_drep_handler(work);
3270                 break;
3271         case IB_CM_SIDR_REQ_RECEIVED:
3272                 ret = cm_sidr_req_handler(work);
3273                 break;
3274         case IB_CM_SIDR_REP_RECEIVED:
3275                 ret = cm_sidr_rep_handler(work);
3276                 break;
3277         case IB_CM_LAP_RECEIVED:
3278                 ret = cm_lap_handler(work);
3279                 break;
3280         case IB_CM_APR_RECEIVED:
3281                 ret = cm_apr_handler(work);
3282                 break;
3283         case IB_CM_TIMEWAIT_EXIT:
3284                 ret = cm_timewait_handler(work);
3285                 break;
3286         default:
3287                 ret = -EINVAL;
3288                 break;
3289         }
3290         if (ret)
3291                 cm_free_work(work);
3292 }
3293
3294 static int cm_establish(struct ib_cm_id *cm_id)
3295 {
3296         struct cm_id_private *cm_id_priv;
3297         struct cm_work *work;
3298         unsigned long flags;
3299         int ret = 0;
3300
3301         work = kmalloc(sizeof *work, GFP_ATOMIC);
3302         if (!work)
3303                 return -ENOMEM;
3304
3305         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3306         spin_lock_irqsave(&cm_id_priv->lock, flags);
3307         switch (cm_id->state)
3308         {
3309         case IB_CM_REP_SENT:
3310         case IB_CM_MRA_REP_RCVD:
3311                 cm_id->state = IB_CM_ESTABLISHED;
3312                 break;
3313         case IB_CM_ESTABLISHED:
3314                 ret = -EISCONN;
3315                 break;
3316         default:
3317                 ret = -EINVAL;
3318                 break;
3319         }
3320         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3321
3322         if (ret) {
3323                 kfree(work);
3324                 goto out;
3325         }
3326
3327         /*
3328          * The CM worker thread may try to destroy the cm_id before it
3329          * can execute this work item.  To prevent potential deadlock,
3330          * we need to find the cm_id once we're in the context of the
3331          * worker thread, rather than holding a reference on it.
3332          */
3333         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3334         work->local_id = cm_id->local_id;
3335         work->remote_id = cm_id->remote_id;
3336         work->mad_recv_wc = NULL;
3337         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3338         queue_delayed_work(cm.wq, &work->work, 0);
3339 out:
3340         return ret;
3341 }
3342
3343 static int cm_migrate(struct ib_cm_id *cm_id)
3344 {
3345         struct cm_id_private *cm_id_priv;
3346         unsigned long flags;
3347         int ret = 0;
3348
3349         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3350         spin_lock_irqsave(&cm_id_priv->lock, flags);
3351         if (cm_id->state == IB_CM_ESTABLISHED &&
3352             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3353              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3354                 cm_id->lap_state = IB_CM_LAP_IDLE;
3355                 cm_id_priv->av = cm_id_priv->alt_av;
3356         } else
3357                 ret = -EINVAL;
3358         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3359
3360         return ret;
3361 }
3362
3363 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3364 {
3365         int ret;
3366
3367         switch (event) {
3368         case IB_EVENT_COMM_EST:
3369                 ret = cm_establish(cm_id);
3370                 break;
3371         case IB_EVENT_PATH_MIG:
3372                 ret = cm_migrate(cm_id);
3373                 break;
3374         default:
3375                 ret = -EINVAL;
3376         }
3377         return ret;
3378 }
3379 EXPORT_SYMBOL(ib_cm_notify);
3380
3381 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3382                             struct ib_mad_recv_wc *mad_recv_wc)
3383 {
3384         struct cm_port *port = mad_agent->context;
3385         struct cm_work *work;
3386         enum ib_cm_event_type event;
3387         u16 attr_id;
3388         int paths = 0;
3389
3390         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3391         case CM_REQ_ATTR_ID:
3392                 paths = 1 + (((struct cm_req_msg *) mad_recv_wc->recv_buf.mad)->
3393                                                     alt_local_lid != 0);
3394                 event = IB_CM_REQ_RECEIVED;
3395                 break;
3396         case CM_MRA_ATTR_ID:
3397                 event = IB_CM_MRA_RECEIVED;
3398                 break;
3399         case CM_REJ_ATTR_ID:
3400                 event = IB_CM_REJ_RECEIVED;
3401                 break;
3402         case CM_REP_ATTR_ID:
3403                 event = IB_CM_REP_RECEIVED;
3404                 break;
3405         case CM_RTU_ATTR_ID:
3406                 event = IB_CM_RTU_RECEIVED;
3407                 break;
3408         case CM_DREQ_ATTR_ID:
3409                 event = IB_CM_DREQ_RECEIVED;
3410                 break;
3411         case CM_DREP_ATTR_ID:
3412                 event = IB_CM_DREP_RECEIVED;
3413                 break;
3414         case CM_SIDR_REQ_ATTR_ID:
3415                 event = IB_CM_SIDR_REQ_RECEIVED;
3416                 break;
3417         case CM_SIDR_REP_ATTR_ID:
3418                 event = IB_CM_SIDR_REP_RECEIVED;
3419                 break;
3420         case CM_LAP_ATTR_ID:
3421                 paths = 1;
3422                 event = IB_CM_LAP_RECEIVED;
3423                 break;
3424         case CM_APR_ATTR_ID:
3425                 event = IB_CM_APR_RECEIVED;
3426                 break;
3427         default:
3428                 ib_free_recv_mad(mad_recv_wc);
3429                 return;
3430         }
3431
3432         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
3433         atomic_long_inc(&port->counter_group[CM_RECV].
3434                         counter[attr_id - CM_ATTR_ID_OFFSET]);
3435
3436         work = kmalloc(sizeof *work + sizeof(struct ib_sa_path_rec) * paths,
3437                        GFP_KERNEL);
3438         if (!work) {
3439                 ib_free_recv_mad(mad_recv_wc);
3440                 return;
3441         }
3442
3443         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3444         work->cm_event.event = event;
3445         work->mad_recv_wc = mad_recv_wc;
3446         work->port = port;
3447         queue_delayed_work(cm.wq, &work->work, 0);
3448 }
3449
3450 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
3451                                 struct ib_qp_attr *qp_attr,
3452                                 int *qp_attr_mask)
3453 {
3454         unsigned long flags;
3455         int ret;
3456
3457         spin_lock_irqsave(&cm_id_priv->lock, flags);
3458         switch (cm_id_priv->id.state) {
3459         case IB_CM_REQ_SENT:
3460         case IB_CM_MRA_REQ_RCVD:
3461         case IB_CM_REQ_RCVD:
3462         case IB_CM_MRA_REQ_SENT:
3463         case IB_CM_REP_RCVD:
3464         case IB_CM_MRA_REP_SENT:
3465         case IB_CM_REP_SENT:
3466         case IB_CM_MRA_REP_RCVD:
3467         case IB_CM_ESTABLISHED:
3468                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
3469                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
3470                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
3471                 if (cm_id_priv->responder_resources)
3472                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
3473                                                     IB_ACCESS_REMOTE_ATOMIC;
3474                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
3475                 qp_attr->port_num = cm_id_priv->av.port->port_num;
3476                 ret = 0;
3477                 break;
3478         default:
3479                 ret = -EINVAL;
3480                 break;
3481         }
3482         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3483         return ret;
3484 }
3485
3486 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
3487                                struct ib_qp_attr *qp_attr,
3488                                int *qp_attr_mask)
3489 {
3490         unsigned long flags;
3491         int ret;
3492
3493         spin_lock_irqsave(&cm_id_priv->lock, flags);
3494         switch (cm_id_priv->id.state) {
3495         case IB_CM_REQ_RCVD:
3496         case IB_CM_MRA_REQ_SENT:
3497         case IB_CM_REP_RCVD:
3498         case IB_CM_MRA_REP_SENT:
3499         case IB_CM_REP_SENT:
3500         case IB_CM_MRA_REP_RCVD:
3501         case IB_CM_ESTABLISHED:
3502                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
3503                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
3504                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
3505                 qp_attr->path_mtu = cm_id_priv->path_mtu;
3506                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
3507                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
3508                 if (cm_id_priv->qp_type == IB_QPT_RC ||
3509                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
3510                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
3511                                          IB_QP_MIN_RNR_TIMER;
3512                         qp_attr->max_dest_rd_atomic =
3513                                         cm_id_priv->responder_resources;
3514                         qp_attr->min_rnr_timer = 0;
3515                 }
3516                 if (cm_id_priv->alt_av.ah_attr.dlid) {
3517                         *qp_attr_mask |= IB_QP_ALT_PATH;
3518                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
3519                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
3520                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
3521                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
3522                 }
3523                 ret = 0;
3524                 break;
3525         default:
3526                 ret = -EINVAL;
3527                 break;
3528         }
3529         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3530         return ret;
3531 }
3532
3533 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
3534                                struct ib_qp_attr *qp_attr,
3535                                int *qp_attr_mask)
3536 {
3537         unsigned long flags;
3538         int ret;
3539
3540         spin_lock_irqsave(&cm_id_priv->lock, flags);
3541         switch (cm_id_priv->id.state) {
3542         /* Allow transition to RTS before sending REP */
3543         case IB_CM_REQ_RCVD:
3544         case IB_CM_MRA_REQ_SENT:
3545
3546         case IB_CM_REP_RCVD:
3547         case IB_CM_MRA_REP_SENT:
3548         case IB_CM_REP_SENT:
3549         case IB_CM_MRA_REP_RCVD:
3550         case IB_CM_ESTABLISHED:
3551                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
3552                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
3553                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
3554                         switch (cm_id_priv->qp_type) {
3555                         case IB_QPT_RC:
3556                         case IB_QPT_XRC_INI:
3557                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
3558                                                  IB_QP_MAX_QP_RD_ATOMIC;
3559                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
3560                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
3561                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
3562                                 /* fall through */
3563                         case IB_QPT_XRC_TGT:
3564                                 *qp_attr_mask |= IB_QP_TIMEOUT;
3565                                 qp_attr->timeout = cm_id_priv->av.timeout;
3566                                 break;
3567                         default:
3568                                 break;
3569                         }
3570                         if (cm_id_priv->alt_av.ah_attr.dlid) {
3571                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
3572                                 qp_attr->path_mig_state = IB_MIG_REARM;
3573                         }
3574                 } else {
3575                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
3576                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
3577                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
3578                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
3579                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
3580                         qp_attr->path_mig_state = IB_MIG_REARM;
3581                 }
3582                 ret = 0;
3583                 break;
3584         default:
3585                 ret = -EINVAL;
3586                 break;
3587         }
3588         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3589         return ret;
3590 }
3591
3592 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
3593                        struct ib_qp_attr *qp_attr,
3594                        int *qp_attr_mask)
3595 {
3596         struct cm_id_private *cm_id_priv;
3597         int ret;
3598
3599         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3600         switch (qp_attr->qp_state) {
3601         case IB_QPS_INIT:
3602                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
3603                 break;
3604         case IB_QPS_RTR:
3605                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
3606                 break;
3607         case IB_QPS_RTS:
3608                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
3609                 break;
3610         default:
3611                 ret = -EINVAL;
3612                 break;
3613         }
3614         return ret;
3615 }
3616 EXPORT_SYMBOL(ib_cm_init_qp_attr);
3617
3618 static void cm_get_ack_delay(struct cm_device *cm_dev)
3619 {
3620         struct ib_device_attr attr;
3621
3622         if (ib_query_device(cm_dev->ib_device, &attr))
3623                 cm_dev->ack_delay = 0; /* acks will rely on packet life time */
3624         else
3625                 cm_dev->ack_delay = attr.local_ca_ack_delay;
3626 }
3627
3628 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
3629                                char *buf)
3630 {
3631         struct cm_counter_group *group;
3632         struct cm_counter_attribute *cm_attr;
3633
3634         group = container_of(obj, struct cm_counter_group, obj);
3635         cm_attr = container_of(attr, struct cm_counter_attribute, attr);
3636
3637         return sprintf(buf, "%ld\n",
3638                        atomic_long_read(&group->counter[cm_attr->index]));
3639 }
3640
3641 static const struct sysfs_ops cm_counter_ops = {
3642         .show = cm_show_counter
3643 };
3644
3645 static struct kobj_type cm_counter_obj_type = {
3646         .sysfs_ops = &cm_counter_ops,
3647         .default_attrs = cm_counter_default_attrs
3648 };
3649
3650 static void cm_release_port_obj(struct kobject *obj)
3651 {
3652         struct cm_port *cm_port;
3653
3654         cm_port = container_of(obj, struct cm_port, port_obj);
3655         kfree(cm_port);
3656 }
3657
3658 static struct kobj_type cm_port_obj_type = {
3659         .release = cm_release_port_obj
3660 };
3661
3662 static char *cm_devnode(struct device *dev, mode_t *mode)
3663 {
3664         if (mode)
3665                 *mode = 0666;
3666         return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
3667 }
3668
3669 struct class cm_class = {
3670         .owner   = THIS_MODULE,
3671         .name    = "infiniband_cm",
3672         .devnode = cm_devnode,
3673 };
3674 EXPORT_SYMBOL(cm_class);
3675
3676 static int cm_create_port_fs(struct cm_port *port)
3677 {
3678         int i, ret;
3679
3680         ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,
3681                                    &port->cm_dev->device->kobj,
3682                                    "%d", port->port_num);
3683         if (ret) {
3684                 kfree(port);
3685                 return ret;
3686         }
3687
3688         for (i = 0; i < CM_COUNTER_GROUPS; i++) {
3689                 ret = kobject_init_and_add(&port->counter_group[i].obj,
3690                                            &cm_counter_obj_type,
3691                                            &port->port_obj,
3692                                            "%s", counter_group_names[i]);
3693                 if (ret)
3694                         goto error;
3695         }
3696
3697         return 0;
3698
3699 error:
3700         while (i--)
3701                 kobject_put(&port->counter_group[i].obj);
3702         kobject_put(&port->port_obj);
3703         return ret;
3704
3705 }
3706
3707 static void cm_remove_port_fs(struct cm_port *port)
3708 {
3709         int i;
3710
3711         for (i = 0; i < CM_COUNTER_GROUPS; i++)
3712                 kobject_put(&port->counter_group[i].obj);
3713
3714         kobject_put(&port->port_obj);
3715 }
3716
3717 static void cm_add_one(struct ib_device *ib_device)
3718 {
3719         struct cm_device *cm_dev;
3720         struct cm_port *port;
3721         struct ib_mad_reg_req reg_req = {
3722                 .mgmt_class = IB_MGMT_CLASS_CM,
3723                 .mgmt_class_version = IB_CM_CLASS_VERSION
3724         };
3725         struct ib_port_modify port_modify = {
3726                 .set_port_cap_mask = IB_PORT_CM_SUP
3727         };
3728         unsigned long flags;
3729         int ret;
3730         u8 i;
3731
3732         if (rdma_node_get_transport(ib_device->node_type) != RDMA_TRANSPORT_IB)
3733                 return;
3734
3735         cm_dev = kzalloc(sizeof(*cm_dev) + sizeof(*port) *
3736                          ib_device->phys_port_cnt, GFP_KERNEL);
3737         if (!cm_dev)
3738                 return;
3739
3740         cm_dev->ib_device = ib_device;
3741         cm_get_ack_delay(cm_dev);
3742
3743         cm_dev->device = device_create(&cm_class, &ib_device->dev,
3744                                        MKDEV(0, 0), NULL,
3745                                        "%s", ib_device->name);
3746         if (IS_ERR(cm_dev->device)) {
3747                 kfree(cm_dev);
3748                 return;
3749         }
3750
3751         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
3752         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
3753                 port = kzalloc(sizeof *port, GFP_KERNEL);
3754                 if (!port)
3755                         goto error1;
3756
3757                 cm_dev->port[i-1] = port;
3758                 port->cm_dev = cm_dev;
3759                 port->port_num = i;
3760
3761                 ret = cm_create_port_fs(port);
3762                 if (ret)
3763                         goto error1;
3764
3765                 port->mad_agent = ib_register_mad_agent(ib_device, i,
3766                                                         IB_QPT_GSI,
3767                                                         &reg_req,
3768                                                         0,
3769                                                         cm_send_handler,
3770                                                         cm_recv_handler,
3771                                                         port);
3772                 if (IS_ERR(port->mad_agent))
3773                         goto error2;
3774
3775                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
3776                 if (ret)
3777                         goto error3;
3778         }
3779         ib_set_client_data(ib_device, &cm_client, cm_dev);
3780
3781         write_lock_irqsave(&cm.device_lock, flags);
3782         list_add_tail(&cm_dev->list, &cm.device_list);
3783         write_unlock_irqrestore(&cm.device_lock, flags);
3784         return;
3785
3786 error3:
3787         ib_unregister_mad_agent(port->mad_agent);
3788 error2:
3789         cm_remove_port_fs(port);
3790 error1:
3791         port_modify.set_port_cap_mask = 0;
3792         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
3793         while (--i) {
3794                 port = cm_dev->port[i-1];
3795                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
3796                 ib_unregister_mad_agent(port->mad_agent);
3797                 cm_remove_port_fs(port);
3798         }
3799         device_unregister(cm_dev->device);
3800         kfree(cm_dev);
3801 }
3802
3803 static void cm_remove_one(struct ib_device *ib_device)
3804 {
3805         struct cm_device *cm_dev;
3806         struct cm_port *port;
3807         struct ib_port_modify port_modify = {
3808                 .clr_port_cap_mask = IB_PORT_CM_SUP
3809         };
3810         unsigned long flags;
3811         int i;
3812
3813         cm_dev = ib_get_client_data(ib_device, &cm_client);
3814         if (!cm_dev)
3815                 return;
3816
3817         write_lock_irqsave(&cm.device_lock, flags);
3818         list_del(&cm_dev->list);
3819         write_unlock_irqrestore(&cm.device_lock, flags);
3820
3821         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
3822                 port = cm_dev->port[i-1];
3823                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
3824                 ib_unregister_mad_agent(port->mad_agent);
3825                 flush_workqueue(cm.wq);
3826                 cm_remove_port_fs(port);
3827         }
3828         device_unregister(cm_dev->device);
3829         kfree(cm_dev);
3830 }
3831
3832 static int __init ib_cm_init(void)
3833 {
3834         int ret;
3835
3836         memset(&cm, 0, sizeof cm);
3837         INIT_LIST_HEAD(&cm.device_list);
3838         rwlock_init(&cm.device_lock);
3839         spin_lock_init(&cm.lock);
3840         cm.listen_service_table = RB_ROOT;
3841         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
3842         cm.remote_id_table = RB_ROOT;
3843         cm.remote_qp_table = RB_ROOT;
3844         cm.remote_sidr_table = RB_ROOT;
3845         idr_init(&cm.local_id_table);
3846         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
3847         idr_pre_get(&cm.local_id_table, GFP_KERNEL);
3848         INIT_LIST_HEAD(&cm.timewait_list);
3849
3850         ret = class_register(&cm_class);
3851         if (ret)
3852                 return -ENOMEM;
3853
3854         cm.wq = create_workqueue("ib_cm");
3855         if (!cm.wq) {
3856                 ret = -ENOMEM;
3857                 goto error1;
3858         }
3859
3860         ret = ib_register_client(&cm_client);
3861         if (ret)
3862                 goto error2;
3863
3864         return 0;
3865 error2:
3866         destroy_workqueue(cm.wq);
3867 error1:
3868         class_unregister(&cm_class);
3869         return ret;
3870 }
3871
3872 static void __exit ib_cm_cleanup(void)
3873 {
3874         struct cm_timewait_info *timewait_info, *tmp;
3875
3876         spin_lock_irq(&cm.lock);
3877         list_for_each_entry(timewait_info, &cm.timewait_list, list)
3878                 cancel_delayed_work(&timewait_info->work.work);
3879         spin_unlock_irq(&cm.lock);
3880
3881         ib_unregister_client(&cm_client);
3882         destroy_workqueue(cm.wq);
3883
3884         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
3885                 list_del(&timewait_info->list);
3886                 kfree(timewait_info);
3887         }
3888
3889         class_unregister(&cm_class);
3890         idr_destroy(&cm.local_id_table);
3891 }
3892
3893 module_init(ib_cm_init);
3894 module_exit(ib_cm_cleanup);
3895