drivers/infiniband/hw/cxgb4/device.c

   1 /*
   2  * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/debugfs.h>
  35 #include <linux/vmalloc.h>
  36
  37 #include <rdma/ib_verbs.h>
  38
  39 #include "iw_cxgb4.h"
  40
  41 #define DRV_VERSION "0.1"
  42
  43 MODULE_AUTHOR("Steve Wise");
  44 MODULE_DESCRIPTION("Chelsio T4/T5 RDMA Driver");
  45 MODULE_LICENSE("Dual BSD/GPL");
  46 MODULE_VERSION(DRV_VERSION);
  47
  48 static int allow_db_fc_on_t5;
  49 module_param(allow_db_fc_on_t5, int, 0644);
  50 MODULE_PARM_DESC(allow_db_fc_on_t5,
  51                  "Allow DB Flow Control on T5 (default = 0)");
  52
  53 static int allow_db_coalescing_on_t5;
  54 module_param(allow_db_coalescing_on_t5, int, 0644);
  55 MODULE_PARM_DESC(allow_db_coalescing_on_t5,
  56                  "Allow DB Coalescing on T5 (default = 0)");
  57
  58 struct uld_ctx {
  59         struct list_head entry;
  60         struct cxgb4_lld_info lldi;
  61         struct c4iw_dev *dev;
  62 };
  63
  64 static LIST_HEAD(uld_ctx_list);
  65 static DEFINE_MUTEX(dev_mutex);
  66
  67 #define DB_FC_RESUME_SIZE 64
  68 #define DB_FC_RESUME_DELAY 1
  69 #define DB_FC_DRAIN_THRESH 0
  70
  71 static struct dentry *c4iw_debugfs_root;
  72
  73 struct c4iw_debugfs_data {
  74         struct c4iw_dev *devp;
  75         char *buf;
  76         int bufsize;
  77         int pos;
  78 };
  79
  80 /* registered cxgb4 netlink callbacks */
  81 static struct ibnl_client_cbs c4iw_nl_cb_table[] = {
  82         [RDMA_NL_IWPM_REG_PID] = {.dump = iwpm_register_pid_cb},
  83         [RDMA_NL_IWPM_ADD_MAPPING] = {.dump = iwpm_add_mapping_cb},
  84         [RDMA_NL_IWPM_QUERY_MAPPING] = {.dump = iwpm_add_and_query_mapping_cb},
  85         [RDMA_NL_IWPM_HANDLE_ERR] = {.dump = iwpm_mapping_error_cb},
  86         [RDMA_NL_IWPM_MAPINFO] = {.dump = iwpm_mapping_info_cb},
  87         [RDMA_NL_IWPM_MAPINFO_NUM] = {.dump = iwpm_ack_mapping_info_cb}
  88 };
  89
  90 static int count_idrs(int id, void *p, void *data)
  91 {
  92         int *countp = data;
  93
  94         *countp = *countp + 1;
  95         return 0;
  96 }
  97
  98 static ssize_t debugfs_read(struct file *file, char __user *buf, size_t count,
  99                             loff_t *ppos)
 100 {
 101         struct c4iw_debugfs_data *d = file->private_data;
 102
 103         return simple_read_from_buffer(buf, count, ppos, d->buf, d->pos);
 104 }
 105
 106 static int dump_qp(int id, void *p, void *data)
 107 {
 108         struct c4iw_qp *qp = p;
 109         struct c4iw_debugfs_data *qpd = data;
 110         int space;
 111         int cc;
 112
 113         if (id != qp->wq.sq.qid)
 114                 return 0;
 115
 116         space = qpd->bufsize - qpd->pos - 1;
 117         if (space == 0)
 118                 return 1;
 119
 120         if (qp->ep) {
 121                 if (qp->ep->com.local_addr.ss_family == AF_INET) {
 122                         struct sockaddr_in *lsin = (struct sockaddr_in *)
 123                                 &qp->ep->com.local_addr;
 124                         struct sockaddr_in *rsin = (struct sockaddr_in *)
 125                                 &qp->ep->com.remote_addr;
 126                         struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
 127                                 &qp->ep->com.mapped_local_addr;
 128                         struct sockaddr_in *mapped_rsin = (struct sockaddr_in *)
 129                                 &qp->ep->com.mapped_remote_addr;
 130
 131                         cc = snprintf(qpd->buf + qpd->pos, space,
 132                                       "rc qp sq id %u rq id %u state %u "
 133                                       "onchip %u ep tid %u state %u "
 134                                       "%pI4:%u/%u->%pI4:%u/%u\n",
 135                                       qp->wq.sq.qid, qp->wq.rq.qid,
 136                                       (int)qp->attr.state,
 137                                       qp->wq.sq.flags & T4_SQ_ONCHIP,
 138                                       qp->ep->hwtid, (int)qp->ep->com.state,
 139                                       &lsin->sin_addr, ntohs(lsin->sin_port),
 140                                       ntohs(mapped_lsin->sin_port),
 141                                       &rsin->sin_addr, ntohs(rsin->sin_port),
 142                                       ntohs(mapped_rsin->sin_port));
 143                 } else {
 144                         struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
 145                                 &qp->ep->com.local_addr;
 146                         struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
 147                                 &qp->ep->com.remote_addr;
 148                         struct sockaddr_in6 *mapped_lsin6 =
 149                                 (struct sockaddr_in6 *)
 150                                 &qp->ep->com.mapped_local_addr;
 151                         struct sockaddr_in6 *mapped_rsin6 =
 152                                 (struct sockaddr_in6 *)
 153                                 &qp->ep->com.mapped_remote_addr;
 154
 155                         cc = snprintf(qpd->buf + qpd->pos, space,
 156                                       "rc qp sq id %u rq id %u state %u "
 157                                       "onchip %u ep tid %u state %u "
 158                                       "%pI6:%u/%u->%pI6:%u/%u\n",
 159                                       qp->wq.sq.qid, qp->wq.rq.qid,
 160                                       (int)qp->attr.state,
 161                                       qp->wq.sq.flags & T4_SQ_ONCHIP,
 162                                       qp->ep->hwtid, (int)qp->ep->com.state,
 163                                       &lsin6->sin6_addr,
 164                                       ntohs(lsin6->sin6_port),
 165                                       ntohs(mapped_lsin6->sin6_port),
 166                                       &rsin6->sin6_addr,
 167                                       ntohs(rsin6->sin6_port),
 168                                       ntohs(mapped_rsin6->sin6_port));
 169                 }
 170         } else
 171                 cc = snprintf(qpd->buf + qpd->pos, space,
 172                              "qp sq id %u rq id %u state %u onchip %u\n",
 173                               qp->wq.sq.qid, qp->wq.rq.qid,
 174                               (int)qp->attr.state,
 175                               qp->wq.sq.flags & T4_SQ_ONCHIP);
 176         if (cc < space)
 177                 qpd->pos += cc;
 178         return 0;
 179 }
 180
 181 static int qp_release(struct inode *inode, struct file *file)
 182 {
 183         struct c4iw_debugfs_data *qpd = file->private_data;
 184         if (!qpd) {
 185                 printk(KERN_INFO "%s null qpd?\n", __func__);
 186                 return 0;
 187         }
 188         vfree(qpd->buf);
 189         kfree(qpd);
 190         return 0;
 191 }
 192
 193 static int qp_open(struct inode *inode, struct file *file)
 194 {
 195         struct c4iw_debugfs_data *qpd;
 196         int ret = 0;
 197         int count = 1;
 198
 199         qpd = kmalloc(sizeof *qpd, GFP_KERNEL);
 200         if (!qpd) {
 201                 ret = -ENOMEM;
 202                 goto out;
 203         }
 204         qpd->devp = inode->i_private;
 205         qpd->pos = 0;
 206
 207         spin_lock_irq(&qpd->devp->lock);
 208         idr_for_each(&qpd->devp->qpidr, count_idrs, &count);
 209         spin_unlock_irq(&qpd->devp->lock);
 210
 211         qpd->bufsize = count * 128;
 212         qpd->buf = vmalloc(qpd->bufsize);
 213         if (!qpd->buf) {
 214                 ret = -ENOMEM;
 215                 goto err1;
 216         }
 217
 218         spin_lock_irq(&qpd->devp->lock);
 219         idr_for_each(&qpd->devp->qpidr, dump_qp, qpd);
 220         spin_unlock_irq(&qpd->devp->lock);
 221
 222         qpd->buf[qpd->pos++] = 0;
 223         file->private_data = qpd;
 224         goto out;
 225 err1:
 226         kfree(qpd);
 227 out:
 228         return ret;
 229 }
 230
 231 static const struct file_operations qp_debugfs_fops = {
 232         .owner   = THIS_MODULE,
 233         .open    = qp_open,
 234         .release = qp_release,
 235         .read    = debugfs_read,
 236         .llseek  = default_llseek,
 237 };
 238
 239 static int dump_stag(int id, void *p, void *data)
 240 {
 241         struct c4iw_debugfs_data *stagd = data;
 242         int space;
 243         int cc;
 244         struct fw_ri_tpte tpte;
 245         int ret;
 246
 247         space = stagd->bufsize - stagd->pos - 1;
 248         if (space == 0)
 249                 return 1;
 250
 251         ret = cxgb4_read_tpte(stagd->devp->rdev.lldi.ports[0], (u32)id<<8,
 252                               (__be32 *)&tpte);
 253         if (ret) {
 254                 dev_err(&stagd->devp->rdev.lldi.pdev->dev,
 255                         "%s cxgb4_read_tpte err %d\n", __func__, ret);
 256                 return ret;
 257         }
 258         cc = snprintf(stagd->buf + stagd->pos, space,
 259                       "stag: idx 0x%x valid %d key 0x%x state %d pdid %d "
 260                       "perm 0x%x ps %d len 0x%llx va 0x%llx\n",
 261                       (u32)id<<8,
 262                       G_FW_RI_TPTE_VALID(ntohl(tpte.valid_to_pdid)),
 263                       G_FW_RI_TPTE_STAGKEY(ntohl(tpte.valid_to_pdid)),
 264                       G_FW_RI_TPTE_STAGSTATE(ntohl(tpte.valid_to_pdid)),
 265                       G_FW_RI_TPTE_PDID(ntohl(tpte.valid_to_pdid)),
 266                       G_FW_RI_TPTE_PERM(ntohl(tpte.locread_to_qpid)),
 267                       G_FW_RI_TPTE_PS(ntohl(tpte.locread_to_qpid)),
 268                       ((u64)ntohl(tpte.len_hi) << 32) | ntohl(tpte.len_lo),
 269                       ((u64)ntohl(tpte.va_hi) << 32) | ntohl(tpte.va_lo_fbo));
 270         if (cc < space)
 271                 stagd->pos += cc;
 272         return 0;
 273 }
 274
 275 static int stag_release(struct inode *inode, struct file *file)
 276 {
 277         struct c4iw_debugfs_data *stagd = file->private_data;
 278         if (!stagd) {
 279                 printk(KERN_INFO "%s null stagd?\n", __func__);
 280                 return 0;
 281         }
 282         vfree(stagd->buf);
 283         kfree(stagd);
 284         return 0;
 285 }
 286
 287 static int stag_open(struct inode *inode, struct file *file)
 288 {
 289         struct c4iw_debugfs_data *stagd;
 290         int ret = 0;
 291         int count = 1;
 292
 293         stagd = kmalloc(sizeof *stagd, GFP_KERNEL);
 294         if (!stagd) {
 295                 ret = -ENOMEM;
 296                 goto out;
 297         }
 298         stagd->devp = inode->i_private;
 299         stagd->pos = 0;
 300
 301         spin_lock_irq(&stagd->devp->lock);
 302         idr_for_each(&stagd->devp->mmidr, count_idrs, &count);
 303         spin_unlock_irq(&stagd->devp->lock);
 304
 305         stagd->bufsize = count * 256;
 306         stagd->buf = vmalloc(stagd->bufsize);
 307         if (!stagd->buf) {
 308                 ret = -ENOMEM;
 309                 goto err1;
 310         }
 311
 312         spin_lock_irq(&stagd->devp->lock);
 313         idr_for_each(&stagd->devp->mmidr, dump_stag, stagd);
 314         spin_unlock_irq(&stagd->devp->lock);
 315
 316         stagd->buf[stagd->pos++] = 0;
 317         file->private_data = stagd;
 318         goto out;
 319 err1:
 320         kfree(stagd);
 321 out:
 322         return ret;
 323 }
 324
 325 static const struct file_operations stag_debugfs_fops = {
 326         .owner   = THIS_MODULE,
 327         .open    = stag_open,
 328         .release = stag_release,
 329         .read    = debugfs_read,
 330         .llseek  = default_llseek,
 331 };
 332
 333 static char *db_state_str[] = {"NORMAL", "FLOW_CONTROL", "RECOVERY", "STOPPED"};
 334
 335 static int stats_show(struct seq_file *seq, void *v)
 336 {
 337         struct c4iw_dev *dev = seq->private;
 338
 339         seq_printf(seq, "   Object: %10s %10s %10s %10s\n", "Total", "Current",
 340                    "Max", "Fail");
 341         seq_printf(seq, "     PDID: %10llu %10llu %10llu %10llu\n",
 342                         dev->rdev.stats.pd.total, dev->rdev.stats.pd.cur,
 343                         dev->rdev.stats.pd.max, dev->rdev.stats.pd.fail);
 344         seq_printf(seq, "      QID: %10llu %10llu %10llu %10llu\n",
 345                         dev->rdev.stats.qid.total, dev->rdev.stats.qid.cur,
 346                         dev->rdev.stats.qid.max, dev->rdev.stats.qid.fail);
 347         seq_printf(seq, "   TPTMEM: %10llu %10llu %10llu %10llu\n",
 348                         dev->rdev.stats.stag.total, dev->rdev.stats.stag.cur,
 349                         dev->rdev.stats.stag.max, dev->rdev.stats.stag.fail);
 350         seq_printf(seq, "   PBLMEM: %10llu %10llu %10llu %10llu\n",
 351                         dev->rdev.stats.pbl.total, dev->rdev.stats.pbl.cur,
 352                         dev->rdev.stats.pbl.max, dev->rdev.stats.pbl.fail);
 353         seq_printf(seq, "   RQTMEM: %10llu %10llu %10llu %10llu\n",
 354                         dev->rdev.stats.rqt.total, dev->rdev.stats.rqt.cur,
 355                         dev->rdev.stats.rqt.max, dev->rdev.stats.rqt.fail);
 356         seq_printf(seq, "  OCQPMEM: %10llu %10llu %10llu %10llu\n",
 357                         dev->rdev.stats.ocqp.total, dev->rdev.stats.ocqp.cur,
 358                         dev->rdev.stats.ocqp.max, dev->rdev.stats.ocqp.fail);
 359         seq_printf(seq, "  DB FULL: %10llu\n", dev->rdev.stats.db_full);
 360         seq_printf(seq, " DB EMPTY: %10llu\n", dev->rdev.stats.db_empty);
 361         seq_printf(seq, "  DB DROP: %10llu\n", dev->rdev.stats.db_drop);
 362         seq_printf(seq, " DB State: %s Transitions %llu FC Interruptions %llu\n",
 363                    db_state_str[dev->db_state],
 364                    dev->rdev.stats.db_state_transitions,
 365                    dev->rdev.stats.db_fc_interruptions);
 366         seq_printf(seq, "TCAM_FULL: %10llu\n", dev->rdev.stats.tcam_full);
 367         seq_printf(seq, "ACT_OFLD_CONN_FAILS: %10llu\n",
 368                    dev->rdev.stats.act_ofld_conn_fails);
 369         seq_printf(seq, "PAS_OFLD_CONN_FAILS: %10llu\n",
 370                    dev->rdev.stats.pas_ofld_conn_fails);
 371         seq_printf(seq, "AVAILABLE IRD: %10u\n", dev->avail_ird);
 372         return 0;
 373 }
 374
 375 static int stats_open(struct inode *inode, struct file *file)
 376 {
 377         return single_open(file, stats_show, inode->i_private);
 378 }
 379
 380 static ssize_t stats_clear(struct file *file, const char __user *buf,
 381                 size_t count, loff_t *pos)
 382 {
 383         struct c4iw_dev *dev = ((struct seq_file *)file->private_data)->private;
 384
 385         mutex_lock(&dev->rdev.stats.lock);
 386         dev->rdev.stats.pd.max = 0;
 387         dev->rdev.stats.pd.fail = 0;
 388         dev->rdev.stats.qid.max = 0;
 389         dev->rdev.stats.qid.fail = 0;
 390         dev->rdev.stats.stag.max = 0;
 391         dev->rdev.stats.stag.fail = 0;
 392         dev->rdev.stats.pbl.max = 0;
 393         dev->rdev.stats.pbl.fail = 0;
 394         dev->rdev.stats.rqt.max = 0;
 395         dev->rdev.stats.rqt.fail = 0;
 396         dev->rdev.stats.ocqp.max = 0;
 397         dev->rdev.stats.ocqp.fail = 0;
 398         dev->rdev.stats.db_full = 0;
 399         dev->rdev.stats.db_empty = 0;
 400         dev->rdev.stats.db_drop = 0;
 401         dev->rdev.stats.db_state_transitions = 0;
 402         dev->rdev.stats.tcam_full = 0;
 403         dev->rdev.stats.act_ofld_conn_fails = 0;
 404         dev->rdev.stats.pas_ofld_conn_fails = 0;
 405         mutex_unlock(&dev->rdev.stats.lock);
 406         return count;
 407 }
 408
 409 static const struct file_operations stats_debugfs_fops = {
 410         .owner   = THIS_MODULE,
 411         .open    = stats_open,
 412         .release = single_release,
 413         .read    = seq_read,
 414         .llseek  = seq_lseek,
 415         .write   = stats_clear,
 416 };
 417
 418 static int dump_ep(int id, void *p, void *data)
 419 {
 420         struct c4iw_ep *ep = p;
 421         struct c4iw_debugfs_data *epd = data;
 422         int space;
 423         int cc;
 424
 425         space = epd->bufsize - epd->pos - 1;
 426         if (space == 0)
 427                 return 1;
 428
 429         if (ep->com.local_addr.ss_family == AF_INET) {
 430                 struct sockaddr_in *lsin = (struct sockaddr_in *)
 431                         &ep->com.local_addr;
 432                 struct sockaddr_in *rsin = (struct sockaddr_in *)
 433                         &ep->com.remote_addr;
 434                 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
 435                         &ep->com.mapped_local_addr;
 436                 struct sockaddr_in *mapped_rsin = (struct sockaddr_in *)
 437                         &ep->com.mapped_remote_addr;
 438
 439                 cc = snprintf(epd->buf + epd->pos, space,
 440                               "ep %p cm_id %p qp %p state %d flags 0x%lx "
 441                               "history 0x%lx hwtid %d atid %d "
 442                               "%pI4:%d/%d <-> %pI4:%d/%d\n",
 443                               ep, ep->com.cm_id, ep->com.qp,
 444                               (int)ep->com.state, ep->com.flags,
 445                               ep->com.history, ep->hwtid, ep->atid,
 446                               &lsin->sin_addr, ntohs(lsin->sin_port),
 447                               ntohs(mapped_lsin->sin_port),
 448                               &rsin->sin_addr, ntohs(rsin->sin_port),
 449                               ntohs(mapped_rsin->sin_port));
 450         } else {
 451                 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
 452                         &ep->com.local_addr;
 453                 struct sockaddr_in6 *rsin6 = (struct sockaddr_in6 *)
 454                         &ep->com.remote_addr;
 455                 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *)
 456                         &ep->com.mapped_local_addr;
 457                 struct sockaddr_in6 *mapped_rsin6 = (struct sockaddr_in6 *)
 458                         &ep->com.mapped_remote_addr;
 459
 460                 cc = snprintf(epd->buf + epd->pos, space,
 461                               "ep %p cm_id %p qp %p state %d flags 0x%lx "
 462                               "history 0x%lx hwtid %d atid %d "
 463                               "%pI6:%d/%d <-> %pI6:%d/%d\n",
 464                               ep, ep->com.cm_id, ep->com.qp,
 465                               (int)ep->com.state, ep->com.flags,
 466                               ep->com.history, ep->hwtid, ep->atid,
 467                               &lsin6->sin6_addr, ntohs(lsin6->sin6_port),
 468                               ntohs(mapped_lsin6->sin6_port),
 469                               &rsin6->sin6_addr, ntohs(rsin6->sin6_port),
 470                               ntohs(mapped_rsin6->sin6_port));
 471         }
 472         if (cc < space)
 473                 epd->pos += cc;
 474         return 0;
 475 }
 476
 477 static int dump_listen_ep(int id, void *p, void *data)
 478 {
 479         struct c4iw_listen_ep *ep = p;
 480         struct c4iw_debugfs_data *epd = data;
 481         int space;
 482         int cc;
 483
 484         space = epd->bufsize - epd->pos - 1;
 485         if (space == 0)
 486                 return 1;
 487
 488         if (ep->com.local_addr.ss_family == AF_INET) {
 489                 struct sockaddr_in *lsin = (struct sockaddr_in *)
 490                         &ep->com.local_addr;
 491                 struct sockaddr_in *mapped_lsin = (struct sockaddr_in *)
 492                         &ep->com.mapped_local_addr;
 493
 494                 cc = snprintf(epd->buf + epd->pos, space,
 495                               "ep %p cm_id %p state %d flags 0x%lx stid %d "
 496                               "backlog %d %pI4:%d/%d\n",
 497                               ep, ep->com.cm_id, (int)ep->com.state,
 498                               ep->com.flags, ep->stid, ep->backlog,
 499                               &lsin->sin_addr, ntohs(lsin->sin_port),
 500                               ntohs(mapped_lsin->sin_port));
 501         } else {
 502                 struct sockaddr_in6 *lsin6 = (struct sockaddr_in6 *)
 503                         &ep->com.local_addr;
 504                 struct sockaddr_in6 *mapped_lsin6 = (struct sockaddr_in6 *)
 505                         &ep->com.mapped_local_addr;
 506
 507                 cc = snprintf(epd->buf + epd->pos, space,
 508                               "ep %p cm_id %p state %d flags 0x%lx stid %d "
 509                               "backlog %d %pI6:%d/%d\n",
 510                               ep, ep->com.cm_id, (int)ep->com.state,
 511                               ep->com.flags, ep->stid, ep->backlog,
 512                               &lsin6->sin6_addr, ntohs(lsin6->sin6_port),
 513                               ntohs(mapped_lsin6->sin6_port));
 514         }
 515         if (cc < space)
 516                 epd->pos += cc;
 517         return 0;
 518 }
 519
 520 static int ep_release(struct inode *inode, struct file *file)
 521 {
 522         struct c4iw_debugfs_data *epd = file->private_data;
 523         if (!epd) {
 524                 pr_info("%s null qpd?\n", __func__);
 525                 return 0;
 526         }
 527         vfree(epd->buf);
 528         kfree(epd);
 529         return 0;
 530 }
 531
 532 static int ep_open(struct inode *inode, struct file *file)
 533 {
 534         struct c4iw_debugfs_data *epd;
 535         int ret = 0;
 536         int count = 1;
 537
 538         epd = kmalloc(sizeof(*epd), GFP_KERNEL);
 539         if (!epd) {
 540                 ret = -ENOMEM;
 541                 goto out;
 542         }
 543         epd->devp = inode->i_private;
 544         epd->pos = 0;
 545
 546         spin_lock_irq(&epd->devp->lock);
 547         idr_for_each(&epd->devp->hwtid_idr, count_idrs, &count);
 548         idr_for_each(&epd->devp->atid_idr, count_idrs, &count);
 549         idr_for_each(&epd->devp->stid_idr, count_idrs, &count);
 550         spin_unlock_irq(&epd->devp->lock);
 551
 552         epd->bufsize = count * 160;
 553         epd->buf = vmalloc(epd->bufsize);
 554         if (!epd->buf) {
 555                 ret = -ENOMEM;
 556                 goto err1;
 557         }
 558
 559         spin_lock_irq(&epd->devp->lock);
 560         idr_for_each(&epd->devp->hwtid_idr, dump_ep, epd);
 561         idr_for_each(&epd->devp->atid_idr, dump_ep, epd);
 562         idr_for_each(&epd->devp->stid_idr, dump_listen_ep, epd);
 563         spin_unlock_irq(&epd->devp->lock);
 564
 565         file->private_data = epd;
 566         goto out;
 567 err1:
 568         kfree(epd);
 569 out:
 570         return ret;
 571 }
 572
 573 static const struct file_operations ep_debugfs_fops = {
 574         .owner   = THIS_MODULE,
 575         .open    = ep_open,
 576         .release = ep_release,
 577         .read    = debugfs_read,
 578 };
 579
 580 static int setup_debugfs(struct c4iw_dev *devp)
 581 {
 582         struct dentry *de;
 583
 584         if (!devp->debugfs_root)
 585                 return -1;
 586
 587         de = debugfs_create_file("qps", S_IWUSR, devp->debugfs_root,
 588                                  (void *)devp, &qp_debugfs_fops);
 589         if (de && de->d_inode)
 590                 de->d_inode->i_size = 4096;
 591
 592         de = debugfs_create_file("stags", S_IWUSR, devp->debugfs_root,
 593                                  (void *)devp, &stag_debugfs_fops);
 594         if (de && de->d_inode)
 595                 de->d_inode->i_size = 4096;
 596
 597         de = debugfs_create_file("stats", S_IWUSR, devp->debugfs_root,
 598                         (void *)devp, &stats_debugfs_fops);
 599         if (de && de->d_inode)
 600                 de->d_inode->i_size = 4096;
 601
 602         de = debugfs_create_file("eps", S_IWUSR, devp->debugfs_root,
 603                         (void *)devp, &ep_debugfs_fops);
 604         if (de && de->d_inode)
 605                 de->d_inode->i_size = 4096;
 606
 607         return 0;
 608 }
 609
 610 void c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
 611                                struct c4iw_dev_ucontext *uctx)
 612 {
 613         struct list_head *pos, *nxt;
 614         struct c4iw_qid_list *entry;
 615
 616         mutex_lock(&uctx->lock);
 617         list_for_each_safe(pos, nxt, &uctx->qpids) {
 618                 entry = list_entry(pos, struct c4iw_qid_list, entry);
 619                 list_del_init(&entry->entry);
 620                 if (!(entry->qid & rdev->qpmask)) {
 621                         c4iw_put_resource(&rdev->resource.qid_table,
 622                                           entry->qid);
 623                         mutex_lock(&rdev->stats.lock);
 624                         rdev->stats.qid.cur -= rdev->qpmask + 1;
 625                         mutex_unlock(&rdev->stats.lock);
 626                 }
 627                 kfree(entry);
 628         }
 629
 630         list_for_each_safe(pos, nxt, &uctx->qpids) {
 631                 entry = list_entry(pos, struct c4iw_qid_list, entry);
 632                 list_del_init(&entry->entry);
 633                 kfree(entry);
 634         }
 635         mutex_unlock(&uctx->lock);
 636 }
 637
 638 void c4iw_init_dev_ucontext(struct c4iw_rdev *rdev,
 639                             struct c4iw_dev_ucontext *uctx)
 640 {
 641         INIT_LIST_HEAD(&uctx->qpids);
 642         INIT_LIST_HEAD(&uctx->cqids);
 643         mutex_init(&uctx->lock);
 644 }
 645
 646 /* Caller takes care of locking if needed */
 647 static int c4iw_rdev_open(struct c4iw_rdev *rdev)
 648 {
 649         int err;
 650
 651         c4iw_init_dev_ucontext(rdev, &rdev->uctx);
 652
 653         /*
 654          * qpshift is the number of bits to shift the qpid left in order
 655          * to get the correct address of the doorbell for that qp.
 656          */
 657         rdev->qpshift = PAGE_SHIFT - ilog2(rdev->lldi.udb_density);
 658         rdev->qpmask = rdev->lldi.udb_density - 1;
 659         rdev->cqshift = PAGE_SHIFT - ilog2(rdev->lldi.ucq_density);
 660         rdev->cqmask = rdev->lldi.ucq_density - 1;
 661         PDBG("%s dev %s stag start 0x%0x size 0x%0x num stags %d "
 662              "pbl start 0x%0x size 0x%0x rq start 0x%0x size 0x%0x "
 663              "qp qid start %u size %u cq qid start %u size %u\n",
 664              __func__, pci_name(rdev->lldi.pdev), rdev->lldi.vr->stag.start,
 665              rdev->lldi.vr->stag.size, c4iw_num_stags(rdev),
 666              rdev->lldi.vr->pbl.start,
 667              rdev->lldi.vr->pbl.size, rdev->lldi.vr->rq.start,
 668              rdev->lldi.vr->rq.size,
 669              rdev->lldi.vr->qp.start,
 670              rdev->lldi.vr->qp.size,
 671              rdev->lldi.vr->cq.start,
 672              rdev->lldi.vr->cq.size);
 673         PDBG("udb len 0x%x udb base %llx db_reg %p gts_reg %p qpshift %lu "
 674              "qpmask 0x%x cqshift %lu cqmask 0x%x\n",
 675              (unsigned)pci_resource_len(rdev->lldi.pdev, 2),
 676              (u64)pci_resource_start(rdev->lldi.pdev, 2),
 677              rdev->lldi.db_reg,
 678              rdev->lldi.gts_reg,
 679              rdev->qpshift, rdev->qpmask,
 680              rdev->cqshift, rdev->cqmask);
 681
 682         if (c4iw_num_stags(rdev) == 0) {
 683                 err = -EINVAL;
 684                 goto err1;
 685         }
 686
 687         rdev->stats.pd.total = T4_MAX_NUM_PD;
 688         rdev->stats.stag.total = rdev->lldi.vr->stag.size;
 689         rdev->stats.pbl.total = rdev->lldi.vr->pbl.size;
 690         rdev->stats.rqt.total = rdev->lldi.vr->rq.size;
 691         rdev->stats.ocqp.total = rdev->lldi.vr->ocq.size;
 692         rdev->stats.qid.total = rdev->lldi.vr->qp.size;
 693
 694         err = c4iw_init_resource(rdev, c4iw_num_stags(rdev), T4_MAX_NUM_PD);
 695         if (err) {
 696                 printk(KERN_ERR MOD "error %d initializing resources\n", err);
 697                 goto err1;
 698         }
 699         err = c4iw_pblpool_create(rdev);
 700         if (err) {
 701                 printk(KERN_ERR MOD "error %d initializing pbl pool\n", err);
 702                 goto err2;
 703         }
 704         err = c4iw_rqtpool_create(rdev);
 705         if (err) {
 706                 printk(KERN_ERR MOD "error %d initializing rqt pool\n", err);
 707                 goto err3;
 708         }
 709         err = c4iw_ocqp_pool_create(rdev);
 710         if (err) {
 711                 printk(KERN_ERR MOD "error %d initializing ocqp pool\n", err);
 712                 goto err4;
 713         }
 714         rdev->status_page = (struct t4_dev_status_page *)
 715                             __get_free_page(GFP_KERNEL);
 716         if (!rdev->status_page) {
 717                 pr_err(MOD "error allocating status page\n");
 718                 goto err4;
 719         }
 720         return 0;
 721 err4:
 722         c4iw_rqtpool_destroy(rdev);
 723 err3:
 724         c4iw_pblpool_destroy(rdev);
 725 err2:
 726         c4iw_destroy_resource(&rdev->resource);
 727 err1:
 728         return err;
 729 }
 730
 731 static void c4iw_rdev_close(struct c4iw_rdev *rdev)
 732 {
 733         free_page((unsigned long)rdev->status_page);
 734         c4iw_pblpool_destroy(rdev);
 735         c4iw_rqtpool_destroy(rdev);
 736         c4iw_destroy_resource(&rdev->resource);
 737 }
 738
 739 static void c4iw_dealloc(struct uld_ctx *ctx)
 740 {
 741         c4iw_rdev_close(&ctx->dev->rdev);
 742         idr_destroy(&ctx->dev->cqidr);
 743         idr_destroy(&ctx->dev->qpidr);
 744         idr_destroy(&ctx->dev->mmidr);
 745         idr_destroy(&ctx->dev->hwtid_idr);
 746         idr_destroy(&ctx->dev->stid_idr);
 747         idr_destroy(&ctx->dev->atid_idr);
 748         if (ctx->dev->rdev.bar2_kva)
 749                 iounmap(ctx->dev->rdev.bar2_kva);
 750         if (ctx->dev->rdev.oc_mw_kva)
 751                 iounmap(ctx->dev->rdev.oc_mw_kva);
 752         ib_dealloc_device(&ctx->dev->ibdev);
 753         iwpm_exit(RDMA_NL_C4IW);
 754         ctx->dev = NULL;
 755 }
 756
 757 static void c4iw_remove(struct uld_ctx *ctx)
 758 {
 759         PDBG("%s c4iw_dev %p\n", __func__,  ctx->dev);
 760         c4iw_unregister_device(ctx->dev);
 761         c4iw_dealloc(ctx);
 762 }
 763
 764 static int rdma_supported(const struct cxgb4_lld_info *infop)
 765 {
 766         return infop->vr->stag.size > 0 && infop->vr->pbl.size > 0 &&
 767                infop->vr->rq.size > 0 && infop->vr->qp.size > 0 &&
 768                infop->vr->cq.size > 0;
 769 }
 770
 771 static struct c4iw_dev *c4iw_alloc(const struct cxgb4_lld_info *infop)
 772 {
 773         struct c4iw_dev *devp;
 774         int ret;
 775
 776         if (!rdma_supported(infop)) {
 777                 printk(KERN_INFO MOD "%s: RDMA not supported on this device.\n",
 778                        pci_name(infop->pdev));
 779                 return ERR_PTR(-ENOSYS);
 780         }
 781         if (!ocqp_supported(infop))
 782                 pr_info("%s: On-Chip Queues not supported on this device.\n",
 783                         pci_name(infop->pdev));
 784
 785         devp = (struct c4iw_dev *)ib_alloc_device(sizeof(*devp));
 786         if (!devp) {
 787                 printk(KERN_ERR MOD "Cannot allocate ib device\n");
 788                 return ERR_PTR(-ENOMEM);
 789         }
 790         devp->rdev.lldi = *infop;
 791
 792         /* init various hw-queue params based on lld info */
 793         PDBG("%s: Ing. padding boundary is %d, egrsstatuspagesize = %d\n",
 794              __func__, devp->rdev.lldi.sge_ingpadboundary,
 795              devp->rdev.lldi.sge_egrstatuspagesize);
 796
 797         devp->rdev.hw_queue.t4_eq_status_entries =
 798                 devp->rdev.lldi.sge_ingpadboundary > 64 ? 2 : 1;
 799         devp->rdev.hw_queue.t4_max_eq_size =
 800                 65520 - devp->rdev.hw_queue.t4_eq_status_entries;
 801         devp->rdev.hw_queue.t4_max_iq_size = 65520 - 1;
 802         devp->rdev.hw_queue.t4_max_rq_size =
 803                 8192 - devp->rdev.hw_queue.t4_eq_status_entries;
 804         devp->rdev.hw_queue.t4_max_sq_size =
 805                 devp->rdev.hw_queue.t4_max_eq_size - 1;
 806         devp->rdev.hw_queue.t4_max_qp_depth =
 807                 devp->rdev.hw_queue.t4_max_rq_size - 1;
 808         devp->rdev.hw_queue.t4_max_cq_depth =
 809                 devp->rdev.hw_queue.t4_max_iq_size - 1;
 810         devp->rdev.hw_queue.t4_stat_len =
 811                 devp->rdev.lldi.sge_egrstatuspagesize;
 812
 813         /*
 814          * For T5 devices, we map all of BAR2 with WC.
 815          * For T4 devices with onchip qp mem, we map only that part
 816          * of BAR2 with WC.
 817          */
 818         devp->rdev.bar2_pa = pci_resource_start(devp->rdev.lldi.pdev, 2);
 819         if (is_t5(devp->rdev.lldi.adapter_type)) {
 820                 devp->rdev.bar2_kva = ioremap_wc(devp->rdev.bar2_pa,
 821                         pci_resource_len(devp->rdev.lldi.pdev, 2));
 822                 if (!devp->rdev.bar2_kva) {
 823                         pr_err(MOD "Unable to ioremap BAR2\n");
 824                         ib_dealloc_device(&devp->ibdev);
 825                         return ERR_PTR(-EINVAL);
 826                 }
 827         } else if (ocqp_supported(infop)) {
 828                 devp->rdev.oc_mw_pa =
 829                         pci_resource_start(devp->rdev.lldi.pdev, 2) +
 830                         pci_resource_len(devp->rdev.lldi.pdev, 2) -
 831                         roundup_pow_of_two(devp->rdev.lldi.vr->ocq.size);
 832                 devp->rdev.oc_mw_kva = ioremap_wc(devp->rdev.oc_mw_pa,
 833                         devp->rdev.lldi.vr->ocq.size);
 834                 if (!devp->rdev.oc_mw_kva) {
 835                         pr_err(MOD "Unable to ioremap onchip mem\n");
 836                         ib_dealloc_device(&devp->ibdev);
 837                         return ERR_PTR(-EINVAL);
 838                 }
 839         }
 840
 841         PDBG(KERN_INFO MOD "ocq memory: "
 842                "hw_start 0x%x size %u mw_pa 0x%lx mw_kva %p\n",
 843                devp->rdev.lldi.vr->ocq.start, devp->rdev.lldi.vr->ocq.size,
 844                devp->rdev.oc_mw_pa, devp->rdev.oc_mw_kva);
 845
 846         ret = c4iw_rdev_open(&devp->rdev);
 847         if (ret) {
 848                 printk(KERN_ERR MOD "Unable to open CXIO rdev err %d\n", ret);
 849                 ib_dealloc_device(&devp->ibdev);
 850                 return ERR_PTR(ret);
 851         }
 852
 853         idr_init(&devp->cqidr);
 854         idr_init(&devp->qpidr);
 855         idr_init(&devp->mmidr);
 856         idr_init(&devp->hwtid_idr);
 857         idr_init(&devp->stid_idr);
 858         idr_init(&devp->atid_idr);
 859         spin_lock_init(&devp->lock);
 860         mutex_init(&devp->rdev.stats.lock);
 861         mutex_init(&devp->db_mutex);
 862         INIT_LIST_HEAD(&devp->db_fc_list);
 863         devp->avail_ird = devp->rdev.lldi.max_ird_adapter;
 864
 865         if (c4iw_debugfs_root) {
 866                 devp->debugfs_root = debugfs_create_dir(
 867                                         pci_name(devp->rdev.lldi.pdev),
 868                                         c4iw_debugfs_root);
 869                 setup_debugfs(devp);
 870         }
 871
 872         ret = iwpm_init(RDMA_NL_C4IW);
 873         if (ret) {
 874                 pr_err("port mapper initialization failed with %d\n", ret);
 875                 ib_dealloc_device(&devp->ibdev);
 876                 return ERR_PTR(ret);
 877         }
 878
 879         return devp;
 880 }
 881
 882 static void *c4iw_uld_add(const struct cxgb4_lld_info *infop)
 883 {
 884         struct uld_ctx *ctx;
 885         static int vers_printed;
 886         int i;
 887
 888         if (!vers_printed++)
 889                 pr_info("Chelsio T4/T5 RDMA Driver - version %s\n",
 890                         DRV_VERSION);
 891
 892         ctx = kzalloc(sizeof *ctx, GFP_KERNEL);
 893         if (!ctx) {
 894                 ctx = ERR_PTR(-ENOMEM);
 895                 goto out;
 896         }
 897         ctx->lldi = *infop;
 898
 899         PDBG("%s found device %s nchan %u nrxq %u ntxq %u nports %u\n",
 900              __func__, pci_name(ctx->lldi.pdev),
 901              ctx->lldi.nchan, ctx->lldi.nrxq,
 902              ctx->lldi.ntxq, ctx->lldi.nports);
 903
 904         mutex_lock(&dev_mutex);
 905         list_add_tail(&ctx->entry, &uld_ctx_list);
 906         mutex_unlock(&dev_mutex);
 907
 908         for (i = 0; i < ctx->lldi.nrxq; i++)
 909                 PDBG("rxqid[%u] %u\n", i, ctx->lldi.rxq_ids[i]);
 910 out:
 911         return ctx;
 912 }
 913
 914 static inline struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
 915                                                  const __be64 *rsp,
 916                                                  u32 pktshift)
 917 {
 918         struct sk_buff *skb;
 919
 920         /*
 921          * Allocate space for cpl_pass_accept_req which will be synthesized by
 922          * driver. Once the driver synthesizes the request the skb will go
 923          * through the regular cpl_pass_accept_req processing.
 924          * The math here assumes sizeof cpl_pass_accept_req >= sizeof
 925          * cpl_rx_pkt.
 926          */
 927         skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req) +
 928                         sizeof(struct rss_header) - pktshift, GFP_ATOMIC);
 929         if (unlikely(!skb))
 930                 return NULL;
 931
 932          __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) +
 933                    sizeof(struct rss_header) - pktshift);
 934
 935         /*
 936          * This skb will contain:
 937          *   rss_header from the rspq descriptor (1 flit)
 938          *   cpl_rx_pkt struct from the rspq descriptor (2 flits)
 939          *   space for the difference between the size of an
 940          *      rx_pkt and pass_accept_req cpl (1 flit)
 941          *   the packet data from the gl
 942          */
 943         skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_pass_accept_req) +
 944                                 sizeof(struct rss_header));
 945         skb_copy_to_linear_data_offset(skb, sizeof(struct rss_header) +
 946                                        sizeof(struct cpl_pass_accept_req),
 947                                        gl->va + pktshift,
 948                                        gl->tot_len - pktshift);
 949         return skb;
 950 }
 951
 952 static inline int recv_rx_pkt(struct c4iw_dev *dev, const struct pkt_gl *gl,
 953                            const __be64 *rsp)
 954 {
 955         unsigned int opcode = *(u8 *)rsp;
 956         struct sk_buff *skb;
 957
 958         if (opcode != CPL_RX_PKT)
 959                 goto out;
 960
 961         skb = copy_gl_to_skb_pkt(gl , rsp, dev->rdev.lldi.sge_pktshift);
 962         if (skb == NULL)
 963                 goto out;
 964
 965         if (c4iw_handlers[opcode] == NULL) {
 966                 pr_info("%s no handler opcode 0x%x...\n", __func__,
 967                        opcode);
 968                 kfree_skb(skb);
 969                 goto out;
 970         }
 971         c4iw_handlers[opcode](dev, skb);
 972         return 1;
 973 out:
 974         return 0;
 975 }
 976
 977 static int c4iw_uld_rx_handler(void *handle, const __be64 *rsp,
 978                         const struct pkt_gl *gl)
 979 {
 980         struct uld_ctx *ctx = handle;
 981         struct c4iw_dev *dev = ctx->dev;
 982         struct sk_buff *skb;
 983         u8 opcode;
 984
 985         if (gl == NULL) {
 986                 /* omit RSS and rsp_ctrl at end of descriptor */
 987                 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8;
 988
 989                 skb = alloc_skb(256, GFP_ATOMIC);
 990                 if (!skb)
 991                         goto nomem;
 992                 __skb_put(skb, len);
 993                 skb_copy_to_linear_data(skb, &rsp[1], len);
 994         } else if (gl == CXGB4_MSG_AN) {
 995                 const struct rsp_ctrl *rc = (void *)rsp;
 996
 997                 u32 qid = be32_to_cpu(rc->pldbuflen_qid);
 998                 c4iw_ev_handler(dev, qid);
 999                 return 0;
1000         } else if (unlikely(*(u8 *)rsp != *(u8 *)gl->va)) {
1001                 if (recv_rx_pkt(dev, gl, rsp))
1002                         return 0;
1003
1004                 pr_info("%s: unexpected FL contents at %p, " \
1005                        "RSS %#llx, FL %#llx, len %u\n",
1006                        pci_name(ctx->lldi.pdev), gl->va,
1007                        (unsigned long long)be64_to_cpu(*rsp),
1008                        (unsigned long long)be64_to_cpu(
1009                        *(__force __be64 *)gl->va),
1010                        gl->tot_len);
1011
1012                 return 0;
1013         } else {
1014                 skb = cxgb4_pktgl_to_skb(gl, 128, 128);
1015                 if (unlikely(!skb))
1016                         goto nomem;
1017         }
1018
1019         opcode = *(u8 *)rsp;
1020         if (c4iw_handlers[opcode]) {
1021                 c4iw_handlers[opcode](dev, skb);
1022         } else {
1023                 pr_info("%s no handler opcode 0x%x...\n", __func__,
1024                        opcode);
1025                 kfree_skb(skb);
1026         }
1027
1028         return 0;
1029 nomem:
1030         return -1;
1031 }
1032
1033 static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state)
1034 {
1035         struct uld_ctx *ctx = handle;
1036
1037         PDBG("%s new_state %u\n", __func__, new_state);
1038         switch (new_state) {
1039         case CXGB4_STATE_UP:
1040                 printk(KERN_INFO MOD "%s: Up\n", pci_name(ctx->lldi.pdev));
1041                 if (!ctx->dev) {
1042                         int ret;
1043
1044                         ctx->dev = c4iw_alloc(&ctx->lldi);
1045                         if (IS_ERR(ctx->dev)) {
1046                                 printk(KERN_ERR MOD
1047                                        "%s: initialization failed: %ld\n",
1048                                        pci_name(ctx->lldi.pdev),
1049                                        PTR_ERR(ctx->dev));
1050                                 ctx->dev = NULL;
1051                                 break;
1052                         }
1053                         ret = c4iw_register_device(ctx->dev);
1054                         if (ret) {
1055                                 printk(KERN_ERR MOD
1056                                        "%s: RDMA registration failed: %d\n",
1057                                        pci_name(ctx->lldi.pdev), ret);
1058                                 c4iw_dealloc(ctx);
1059                         }
1060                 }
1061                 break;
1062         case CXGB4_STATE_DOWN:
1063                 printk(KERN_INFO MOD "%s: Down\n",
1064                        pci_name(ctx->lldi.pdev));
1065                 if (ctx->dev)
1066                         c4iw_remove(ctx);
1067                 break;
1068         case CXGB4_STATE_START_RECOVERY:
1069                 printk(KERN_INFO MOD "%s: Fatal Error\n",
1070                        pci_name(ctx->lldi.pdev));
1071                 if (ctx->dev) {
1072                         struct ib_event event;
1073
1074                         ctx->dev->rdev.flags |= T4_FATAL_ERROR;
1075                         memset(&event, 0, sizeof event);
1076                         event.event  = IB_EVENT_DEVICE_FATAL;
1077                         event.device = &ctx->dev->ibdev;
1078                         ib_dispatch_event(&event);
1079                         c4iw_remove(ctx);
1080                 }
1081                 break;
1082         case CXGB4_STATE_DETACH:
1083                 printk(KERN_INFO MOD "%s: Detach\n",
1084                        pci_name(ctx->lldi.pdev));
1085                 if (ctx->dev)
1086                         c4iw_remove(ctx);
1087                 break;
1088         }
1089         return 0;
1090 }
1091
1092 static int disable_qp_db(int id, void *p, void *data)
1093 {
1094         struct c4iw_qp *qp = p;
1095
1096         t4_disable_wq_db(&qp->wq);
1097         return 0;
1098 }
1099
1100 static void stop_queues(struct uld_ctx *ctx)
1101 {
1102         unsigned long flags;
1103
1104         spin_lock_irqsave(&ctx->dev->lock, flags);
1105         ctx->dev->rdev.stats.db_state_transitions++;
1106         ctx->dev->db_state = STOPPED;
1107         if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED)
1108                 idr_for_each(&ctx->dev->qpidr, disable_qp_db, NULL);
1109         else
1110                 ctx->dev->rdev.status_page->db_off = 1;
1111         spin_unlock_irqrestore(&ctx->dev->lock, flags);
1112 }
1113
1114 static int enable_qp_db(int id, void *p, void *data)
1115 {
1116         struct c4iw_qp *qp = p;
1117
1118         t4_enable_wq_db(&qp->wq);
1119         return 0;
1120 }
1121
1122 static void resume_rc_qp(struct c4iw_qp *qp)
1123 {
1124         spin_lock(&qp->lock);
1125         t4_ring_sq_db(&qp->wq, qp->wq.sq.wq_pidx_inc,
1126                       is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
1127         qp->wq.sq.wq_pidx_inc = 0;
1128         t4_ring_rq_db(&qp->wq, qp->wq.rq.wq_pidx_inc,
1129                       is_t5(qp->rhp->rdev.lldi.adapter_type), NULL);
1130         qp->wq.rq.wq_pidx_inc = 0;
1131         spin_unlock(&qp->lock);
1132 }
1133
1134 static void resume_a_chunk(struct uld_ctx *ctx)
1135 {
1136         int i;
1137         struct c4iw_qp *qp;
1138
1139         for (i = 0; i < DB_FC_RESUME_SIZE; i++) {
1140                 qp = list_first_entry(&ctx->dev->db_fc_list, struct c4iw_qp,
1141                                       db_fc_entry);
1142                 list_del_init(&qp->db_fc_entry);
1143                 resume_rc_qp(qp);
1144                 if (list_empty(&ctx->dev->db_fc_list))
1145                         break;
1146         }
1147 }
1148
1149 static void resume_queues(struct uld_ctx *ctx)
1150 {
1151         spin_lock_irq(&ctx->dev->lock);
1152         if (ctx->dev->db_state != STOPPED)
1153                 goto out;
1154         ctx->dev->db_state = FLOW_CONTROL;
1155         while (1) {
1156                 if (list_empty(&ctx->dev->db_fc_list)) {
1157                         WARN_ON(ctx->dev->db_state != FLOW_CONTROL);
1158                         ctx->dev->db_state = NORMAL;
1159                         ctx->dev->rdev.stats.db_state_transitions++;
1160                         if (ctx->dev->rdev.flags & T4_STATUS_PAGE_DISABLED) {
1161                                 idr_for_each(&ctx->dev->qpidr, enable_qp_db,
1162                                              NULL);
1163                         } else {
1164                                 ctx->dev->rdev.status_page->db_off = 0;
1165                         }
1166                         break;
1167                 } else {
1168                         if (cxgb4_dbfifo_count(ctx->dev->rdev.lldi.ports[0], 1)
1169                             < (ctx->dev->rdev.lldi.dbfifo_int_thresh <<
1170                                DB_FC_DRAIN_THRESH)) {
1171                                 resume_a_chunk(ctx);
1172                         }
1173                         if (!list_empty(&ctx->dev->db_fc_list)) {
1174                                 spin_unlock_irq(&ctx->dev->lock);
1175                                 if (DB_FC_RESUME_DELAY) {
1176                                         set_current_state(TASK_UNINTERRUPTIBLE);
1177                                         schedule_timeout(DB_FC_RESUME_DELAY);
1178                                 }
1179                                 spin_lock_irq(&ctx->dev->lock);
1180                                 if (ctx->dev->db_state != FLOW_CONTROL)
1181                                         break;
1182                         }
1183                 }
1184         }
1185 out:
1186         if (ctx->dev->db_state != NORMAL)
1187                 ctx->dev->rdev.stats.db_fc_interruptions++;
1188         spin_unlock_irq(&ctx->dev->lock);
1189 }
1190
1191 struct qp_list {
1192         unsigned idx;
1193         struct c4iw_qp **qps;
1194 };
1195
1196 static int add_and_ref_qp(int id, void *p, void *data)
1197 {
1198         struct qp_list *qp_listp = data;
1199         struct c4iw_qp *qp = p;
1200
1201         c4iw_qp_add_ref(&qp->ibqp);
1202         qp_listp->qps[qp_listp->idx++] = qp;
1203         return 0;
1204 }
1205
1206 static int count_qps(int id, void *p, void *data)
1207 {
1208         unsigned *countp = data;
1209         (*countp)++;
1210         return 0;
1211 }
1212
1213 static void deref_qps(struct qp_list *qp_list)
1214 {
1215         int idx;
1216
1217         for (idx = 0; idx < qp_list->idx; idx++)
1218                 c4iw_qp_rem_ref(&qp_list->qps[idx]->ibqp);
1219 }
1220
1221 static void recover_lost_dbs(struct uld_ctx *ctx, struct qp_list *qp_list)
1222 {
1223         int idx;
1224         int ret;
1225
1226         for (idx = 0; idx < qp_list->idx; idx++) {
1227                 struct c4iw_qp *qp = qp_list->qps[idx];
1228
1229                 spin_lock_irq(&qp->rhp->lock);
1230                 spin_lock(&qp->lock);
1231                 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0],
1232                                           qp->wq.sq.qid,
1233                                           t4_sq_host_wq_pidx(&qp->wq),
1234                                           t4_sq_wq_size(&qp->wq));
1235                 if (ret) {
1236                         pr_err(KERN_ERR MOD "%s: Fatal error - "
1237                                "DB overflow recovery failed - "
1238                                "error syncing SQ qid %u\n",
1239                                pci_name(ctx->lldi.pdev), qp->wq.sq.qid);
1240                         spin_unlock(&qp->lock);
1241                         spin_unlock_irq(&qp->rhp->lock);
1242                         return;
1243                 }
1244                 qp->wq.sq.wq_pidx_inc = 0;
1245
1246                 ret = cxgb4_sync_txq_pidx(qp->rhp->rdev.lldi.ports[0],
1247                                           qp->wq.rq.qid,
1248                                           t4_rq_host_wq_pidx(&qp->wq),
1249                                           t4_rq_wq_size(&qp->wq));
1250
1251                 if (ret) {
1252                         pr_err(KERN_ERR MOD "%s: Fatal error - "
1253                                "DB overflow recovery failed - "
1254                                "error syncing RQ qid %u\n",
1255                                pci_name(ctx->lldi.pdev), qp->wq.rq.qid);
1256                         spin_unlock(&qp->lock);
1257                         spin_unlock_irq(&qp->rhp->lock);
1258                         return;
1259                 }
1260                 qp->wq.rq.wq_pidx_inc = 0;
1261                 spin_unlock(&qp->lock);
1262                 spin_unlock_irq(&qp->rhp->lock);
1263
1264                 /* Wait for the dbfifo to drain */
1265                 while (cxgb4_dbfifo_count(qp->rhp->rdev.lldi.ports[0], 1) > 0) {
1266                         set_current_state(TASK_UNINTERRUPTIBLE);
1267                         schedule_timeout(usecs_to_jiffies(10));
1268                 }
1269         }
1270 }
1271
1272 static void recover_queues(struct uld_ctx *ctx)
1273 {
1274         int count = 0;
1275         struct qp_list qp_list;
1276         int ret;
1277
1278         /* slow everybody down */
1279         set_current_state(TASK_UNINTERRUPTIBLE);
1280         schedule_timeout(usecs_to_jiffies(1000));
1281
1282         /* flush the SGE contexts */
1283         ret = cxgb4_flush_eq_cache(ctx->dev->rdev.lldi.ports[0]);
1284         if (ret) {
1285                 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n",
1286                        pci_name(ctx->lldi.pdev));
1287                 return;
1288         }
1289
1290         /* Count active queues so we can build a list of queues to recover */
1291         spin_lock_irq(&ctx->dev->lock);
1292         WARN_ON(ctx->dev->db_state != STOPPED);
1293         ctx->dev->db_state = RECOVERY;
1294         idr_for_each(&ctx->dev->qpidr, count_qps, &count);
1295
1296         qp_list.qps = kzalloc(count * sizeof *qp_list.qps, GFP_ATOMIC);
1297         if (!qp_list.qps) {
1298                 printk(KERN_ERR MOD "%s: Fatal error - DB overflow recovery failed\n",
1299                        pci_name(ctx->lldi.pdev));
1300                 spin_unlock_irq(&ctx->dev->lock);
1301                 return;
1302         }
1303         qp_list.idx = 0;
1304
1305         /* add and ref each qp so it doesn't get freed */
1306         idr_for_each(&ctx->dev->qpidr, add_and_ref_qp, &qp_list);
1307
1308         spin_unlock_irq(&ctx->dev->lock);
1309
1310         /* now traverse the list in a safe context to recover the db state*/
1311         recover_lost_dbs(ctx, &qp_list);
1312
1313         /* we're almost done!  deref the qps and clean up */
1314         deref_qps(&qp_list);
1315         kfree(qp_list.qps);
1316
1317         spin_lock_irq(&ctx->dev->lock);
1318         WARN_ON(ctx->dev->db_state != RECOVERY);
1319         ctx->dev->db_state = STOPPED;
1320         spin_unlock_irq(&ctx->dev->lock);
1321 }
1322
1323 static int c4iw_uld_control(void *handle, enum cxgb4_control control, ...)
1324 {
1325         struct uld_ctx *ctx = handle;
1326
1327         switch (control) {
1328         case CXGB4_CONTROL_DB_FULL:
1329                 stop_queues(ctx);
1330                 ctx->dev->rdev.stats.db_full++;
1331                 break;
1332         case CXGB4_CONTROL_DB_EMPTY:
1333                 resume_queues(ctx);
1334                 mutex_lock(&ctx->dev->rdev.stats.lock);
1335                 ctx->dev->rdev.stats.db_empty++;
1336                 mutex_unlock(&ctx->dev->rdev.stats.lock);
1337                 break;
1338         case CXGB4_CONTROL_DB_DROP:
1339                 recover_queues(ctx);
1340                 mutex_lock(&ctx->dev->rdev.stats.lock);
1341                 ctx->dev->rdev.stats.db_drop++;
1342                 mutex_unlock(&ctx->dev->rdev.stats.lock);
1343                 break;
1344         default:
1345                 printk(KERN_WARNING MOD "%s: unknown control cmd %u\n",
1346                        pci_name(ctx->lldi.pdev), control);
1347                 break;
1348         }
1349         return 0;
1350 }
1351
1352 static struct cxgb4_uld_info c4iw_uld_info = {
1353         .name = DRV_NAME,
1354         .add = c4iw_uld_add,
1355         .rx_handler = c4iw_uld_rx_handler,
1356         .state_change = c4iw_uld_state_change,
1357         .control = c4iw_uld_control,
1358 };
1359
1360 static int __init c4iw_init_module(void)
1361 {
1362         int err;
1363
1364         err = c4iw_cm_init();
1365         if (err)
1366                 return err;
1367
1368         c4iw_debugfs_root = debugfs_create_dir(DRV_NAME, NULL);
1369         if (!c4iw_debugfs_root)
1370                 printk(KERN_WARNING MOD
1371                        "could not create debugfs entry, continuing\n");
1372
1373         if (ibnl_add_client(RDMA_NL_C4IW, RDMA_NL_IWPM_NUM_OPS,
1374                             c4iw_nl_cb_table))
1375                 pr_err("%s[%u]: Failed to add netlink callback\n"
1376                        , __func__, __LINE__);
1377
1378         cxgb4_register_uld(CXGB4_ULD_RDMA, &c4iw_uld_info);
1379
1380         return 0;
1381 }
1382
1383 static void __exit c4iw_exit_module(void)
1384 {
1385         struct uld_ctx *ctx, *tmp;
1386
1387         mutex_lock(&dev_mutex);
1388         list_for_each_entry_safe(ctx, tmp, &uld_ctx_list, entry) {
1389                 if (ctx->dev)
1390                         c4iw_remove(ctx);
1391                 kfree(ctx);
1392         }
1393         mutex_unlock(&dev_mutex);
1394         cxgb4_unregister_uld(CXGB4_ULD_RDMA);
1395         ibnl_remove_client(RDMA_NL_C4IW);
1396         c4iw_cm_term();
1397         debugfs_remove_recursive(c4iw_debugfs_root);
1398 }
1399
1400 module_init(c4iw_init_module);
1401 module_exit(c4iw_exit_module);