drivers/net/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if_vlan.h>
  45 #include <linux/init.h>
  46 #include <linux/log2.h>
  47 #include <linux/mdio.h>
  48 #include <linux/module.h>
  49 #include <linux/moduleparam.h>
  50 #include <linux/mutex.h>
  51 #include <linux/netdevice.h>
  52 #include <linux/pci.h>
  53 #include <linux/aer.h>
  54 #include <linux/rtnetlink.h>
  55 #include <linux/sched.h>
  56 #include <linux/seq_file.h>
  57 #include <linux/sockios.h>
  58 #include <linux/vmalloc.h>
  59 #include <linux/workqueue.h>
  60 #include <net/neighbour.h>
  61 #include <net/netevent.h>
  62 #include <asm/uaccess.h>
  63
  64 #include "cxgb4.h"
  65 #include "t4_regs.h"
  66 #include "t4_msg.h"
  67 #include "t4fw_api.h"
  68 #include "l2t.h"
  69
  70 #define DRV_VERSION "1.3.0-ko"
  71 #define DRV_DESC "Chelsio T4 Network Driver"
  72
  73 /*
  74  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  75  * under extreme memory pressure so it's largish to give the system time to
  76  * recover.
  77  */
  78 #define MAX_SGE_TIMERVAL 200U
  79
  80 #ifdef CONFIG_PCI_IOV
  81 /*
  82  * Virtual Function provisioning constants.  We need two extra Ingress Queues
  83  * with Interrupt capability to serve as the VF's Firmware Event Queue and
  84  * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
  85  * Lists associated with them).  For each Ethernet/Control Egress Queue and
  86  * for each Free List, we need an Egress Context.
  87  */
  88 enum {
  89         VFRES_NPORTS = 1,               /* # of "ports" per VF */
  90         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
  91
  92         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
  93         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
  94         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
  95         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
  96         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
  97         VFRES_TC = 0,                   /* PCI-E traffic class */
  98         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
  99
 100         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 101         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 102 };
 103
 104 /*
 105  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 106  * static and likely not to be useful in the long run.  We really need to
 107  * implement some form of persistent configuration which the firmware
 108  * controls.
 109  */
 110 static unsigned int pfvfres_pmask(struct adapter *adapter,
 111                                   unsigned int pf, unsigned int vf)
 112 {
 113         unsigned int portn, portvec;
 114
 115         /*
 116          * Give PF's access to all of the ports.
 117          */
 118         if (vf == 0)
 119                 return FW_PFVF_CMD_PMASK_MASK;
 120
 121         /*
 122          * For VFs, we'll assign them access to the ports based purely on the
 123          * PF.  We assign active ports in order, wrapping around if there are
 124          * fewer active ports than PFs: e.g. active port[pf % nports].
 125          * Unfortunately the adapter's port_info structs haven't been
 126          * initialized yet so we have to compute this.
 127          */
 128         if (adapter->params.nports == 0)
 129                 return 0;
 130
 131         portn = pf % adapter->params.nports;
 132         portvec = adapter->params.portvec;
 133         for (;;) {
 134                 /*
 135                  * Isolate the lowest set bit in the port vector.  If we're at
 136                  * the port number that we want, return that as the pmask.
 137                  * otherwise mask that bit out of the port vector and
 138                  * decrement our port number ...
 139                  */
 140                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 141                 if (portn == 0)
 142                         return pmask;
 143                 portn--;
 144                 portvec &= ~pmask;
 145         }
 146         /*NOTREACHED*/
 147 }
 148 #endif
 149
 150 enum {
 151         MEMWIN0_APERTURE = 65536,
 152         MEMWIN0_BASE     = 0x30000,
 153         MEMWIN1_APERTURE = 32768,
 154         MEMWIN1_BASE     = 0x28000,
 155         MEMWIN2_APERTURE = 2048,
 156         MEMWIN2_BASE     = 0x1b800,
 157 };
 158
 159 enum {
 160         MAX_TXQ_ENTRIES      = 16384,
 161         MAX_CTRL_TXQ_ENTRIES = 1024,
 162         MAX_RSPQ_ENTRIES     = 16384,
 163         MAX_RX_BUFFERS       = 16384,
 164         MIN_TXQ_ENTRIES      = 32,
 165         MIN_CTRL_TXQ_ENTRIES = 32,
 166         MIN_RSPQ_ENTRIES     = 128,
 167         MIN_FL_ENTRIES       = 16
 168 };
 169
 170 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 171                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 172                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 173
 174 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 175
 176 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 177         CH_DEVICE(0xa000, 0),  /* PE10K */
 178         CH_DEVICE(0x4001, 0),
 179         CH_DEVICE(0x4002, 0),
 180         CH_DEVICE(0x4003, 0),
 181         CH_DEVICE(0x4004, 0),
 182         CH_DEVICE(0x4005, 0),
 183         CH_DEVICE(0x4006, 0),
 184         CH_DEVICE(0x4007, 0),
 185         CH_DEVICE(0x4008, 0),
 186         CH_DEVICE(0x4009, 0),
 187         CH_DEVICE(0x400a, 0),
 188         { 0, }
 189 };
 190
 191 #define FW_FNAME "cxgb4/t4fw.bin"
 192
 193 MODULE_DESCRIPTION(DRV_DESC);
 194 MODULE_AUTHOR("Chelsio Communications");
 195 MODULE_LICENSE("Dual BSD/GPL");
 196 MODULE_VERSION(DRV_VERSION);
 197 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 198 MODULE_FIRMWARE(FW_FNAME);
 199
 200 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 201
 202 module_param(dflt_msg_enable, int, 0644);
 203 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 204
 205 /*
 206  * The driver uses the best interrupt scheme available on a platform in the
 207  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 208  * of these schemes the driver may consider as follows:
 209  *
 210  * msi = 2: choose from among all three options
 211  * msi = 1: only consider MSI and INTx interrupts
 212  * msi = 0: force INTx interrupts
 213  */
 214 static int msi = 2;
 215
 216 module_param(msi, int, 0644);
 217 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 218
 219 /*
 220  * Queue interrupt hold-off timer values.  Queues default to the first of these
 221  * upon creation.
 222  */
 223 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 224
 225 module_param_array(intr_holdoff, uint, NULL, 0644);
 226 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 227                  "0..4 in microseconds");
 228
 229 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 230
 231 module_param_array(intr_cnt, uint, NULL, 0644);
 232 MODULE_PARM_DESC(intr_cnt,
 233                  "thresholds 1..3 for queue interrupt packet counters");
 234
 235 static int vf_acls;
 236
 237 #ifdef CONFIG_PCI_IOV
 238 module_param(vf_acls, bool, 0644);
 239 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 240
 241 static unsigned int num_vf[4];
 242
 243 module_param_array(num_vf, uint, NULL, 0644);
 244 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 245 #endif
 246
 247 static struct dentry *cxgb4_debugfs_root;
 248
 249 static LIST_HEAD(adapter_list);
 250 static DEFINE_MUTEX(uld_mutex);
 251 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 252 static const char *uld_str[] = { "RDMA", "iSCSI" };
 253
 254 static void link_report(struct net_device *dev)
 255 {
 256         if (!netif_carrier_ok(dev))
 257                 netdev_info(dev, "link down\n");
 258         else {
 259                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 260
 261                 const char *s = "10Mbps";
 262                 const struct port_info *p = netdev_priv(dev);
 263
 264                 switch (p->link_cfg.speed) {
 265                 case SPEED_10000:
 266                         s = "10Gbps";
 267                         break;
 268                 case SPEED_1000:
 269                         s = "1000Mbps";
 270                         break;
 271                 case SPEED_100:
 272                         s = "100Mbps";
 273                         break;
 274                 }
 275
 276                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 277                             fc[p->link_cfg.fc]);
 278         }
 279 }
 280
 281 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 282 {
 283         struct net_device *dev = adapter->port[port_id];
 284
 285         /* Skip changes from disabled ports. */
 286         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 287                 if (link_stat)
 288                         netif_carrier_on(dev);
 289                 else
 290                         netif_carrier_off(dev);
 291
 292                 link_report(dev);
 293         }
 294 }
 295
 296 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 297 {
 298         static const char *mod_str[] = {
 299                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 300         };
 301
 302         const struct net_device *dev = adap->port[port_id];
 303         const struct port_info *pi = netdev_priv(dev);
 304
 305         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 306                 netdev_info(dev, "port module unplugged\n");
 307         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 308                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 309 }
 310
 311 /*
 312  * Configure the exact and hash address filters to handle a port's multicast
 313  * and secondary unicast MAC addresses.
 314  */
 315 static int set_addr_filters(const struct net_device *dev, bool sleep)
 316 {
 317         u64 mhash = 0;
 318         u64 uhash = 0;
 319         bool free = true;
 320         u16 filt_idx[7];
 321         const u8 *addr[7];
 322         int ret, naddr = 0;
 323         const struct netdev_hw_addr *ha;
 324         int uc_cnt = netdev_uc_count(dev);
 325         int mc_cnt = netdev_mc_count(dev);
 326         const struct port_info *pi = netdev_priv(dev);
 327         unsigned int mb = pi->adapter->fn;
 328
 329         /* first do the secondary unicast addresses */
 330         netdev_for_each_uc_addr(ha, dev) {
 331                 addr[naddr++] = ha->addr;
 332                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 333                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 334                                         naddr, addr, filt_idx, &uhash, sleep);
 335                         if (ret < 0)
 336                                 return ret;
 337
 338                         free = false;
 339                         naddr = 0;
 340                 }
 341         }
 342
 343         /* next set up the multicast addresses */
 344         netdev_for_each_mc_addr(ha, dev) {
 345                 addr[naddr++] = ha->addr;
 346                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 347                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 348                                         naddr, addr, filt_idx, &mhash, sleep);
 349                         if (ret < 0)
 350                                 return ret;
 351
 352                         free = false;
 353                         naddr = 0;
 354                 }
 355         }
 356
 357         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 358                                 uhash | mhash, sleep);
 359 }
 360
 361 /*
 362  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 363  * If @mtu is -1 it is left unchanged.
 364  */
 365 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 366 {
 367         int ret;
 368         struct port_info *pi = netdev_priv(dev);
 369
 370         ret = set_addr_filters(dev, sleep_ok);
 371         if (ret == 0)
 372                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 373                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 374                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 375                                     sleep_ok);
 376         return ret;
 377 }
 378
 379 /**
 380  *      link_start - enable a port
 381  *      @dev: the port to enable
 382  *
 383  *      Performs the MAC and PHY actions needed to enable a port.
 384  */
 385 static int link_start(struct net_device *dev)
 386 {
 387         int ret;
 388         struct port_info *pi = netdev_priv(dev);
 389         unsigned int mb = pi->adapter->fn;
 390
 391         /*
 392          * We do not set address filters and promiscuity here, the stack does
 393          * that step explicitly.
 394          */
 395         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 396                             pi->vlan_grp != NULL, true);
 397         if (ret == 0) {
 398                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 399                                     pi->xact_addr_filt, dev->dev_addr, true,
 400                                     true);
 401                 if (ret >= 0) {
 402                         pi->xact_addr_filt = ret;
 403                         ret = 0;
 404                 }
 405         }
 406         if (ret == 0)
 407                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 408                                     &pi->link_cfg);
 409         if (ret == 0)
 410                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 411         return ret;
 412 }
 413
 414 /*
 415  * Response queue handler for the FW event queue.
 416  */
 417 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 418                           const struct pkt_gl *gl)
 419 {
 420         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 421
 422         rsp++;                                          /* skip RSS header */
 423         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 424                 const struct cpl_sge_egr_update *p = (void *)rsp;
 425                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 426                 struct sge_txq *txq = q->adap->sge.egr_map[qid];
 427
 428                 txq->restarts++;
 429                 if ((u8 *)txq < (u8 *)q->adap->sge.ethrxq) {
 430                         struct sge_eth_txq *eq;
 431
 432                         eq = container_of(txq, struct sge_eth_txq, q);
 433                         netif_tx_wake_queue(eq->txq);
 434                 } else {
 435                         struct sge_ofld_txq *oq;
 436
 437                         oq = container_of(txq, struct sge_ofld_txq, q);
 438                         tasklet_schedule(&oq->qresume_tsk);
 439                 }
 440         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 441                 const struct cpl_fw6_msg *p = (void *)rsp;
 442
 443                 if (p->type == 0)
 444                         t4_handle_fw_rpl(q->adap, p->data);
 445         } else if (opcode == CPL_L2T_WRITE_RPL) {
 446                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 447
 448                 do_l2t_write_rpl(q->adap, p);
 449         } else
 450                 dev_err(q->adap->pdev_dev,
 451                         "unexpected CPL %#x on FW event queue\n", opcode);
 452         return 0;
 453 }
 454
 455 /**
 456  *      uldrx_handler - response queue handler for ULD queues
 457  *      @q: the response queue that received the packet
 458  *      @rsp: the response queue descriptor holding the offload message
 459  *      @gl: the gather list of packet fragments
 460  *
 461  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 462  *      the ULD, we just maintain statistics.
 463  */
 464 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 465                          const struct pkt_gl *gl)
 466 {
 467         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 468
 469         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 470                 rxq->stats.nomem++;
 471                 return -1;
 472         }
 473         if (gl == NULL)
 474                 rxq->stats.imm++;
 475         else if (gl == CXGB4_MSG_AN)
 476                 rxq->stats.an++;
 477         else
 478                 rxq->stats.pkts++;
 479         return 0;
 480 }
 481
 482 static void disable_msi(struct adapter *adapter)
 483 {
 484         if (adapter->flags & USING_MSIX) {
 485                 pci_disable_msix(adapter->pdev);
 486                 adapter->flags &= ~USING_MSIX;
 487         } else if (adapter->flags & USING_MSI) {
 488                 pci_disable_msi(adapter->pdev);
 489                 adapter->flags &= ~USING_MSI;
 490         }
 491 }
 492
 493 /*
 494  * Interrupt handler for non-data events used with MSI-X.
 495  */
 496 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 497 {
 498         struct adapter *adap = cookie;
 499
 500         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 501         if (v & PFSW) {
 502                 adap->swintr = 1;
 503                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 504         }
 505         t4_slow_intr_handler(adap);
 506         return IRQ_HANDLED;
 507 }
 508
 509 /*
 510  * Name the MSI-X interrupts.
 511  */
 512 static void name_msix_vecs(struct adapter *adap)
 513 {
 514         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc) - 1;
 515
 516         /* non-data interrupts */
 517         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 518         adap->msix_info[0].desc[n] = 0;
 519
 520         /* FW events */
 521         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", adap->name);
 522         adap->msix_info[1].desc[n] = 0;
 523
 524         /* Ethernet queues */
 525         for_each_port(adap, j) {
 526                 struct net_device *d = adap->port[j];
 527                 const struct port_info *pi = netdev_priv(d);
 528
 529                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 530                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 531                                  d->name, i);
 532                         adap->msix_info[msi_idx].desc[n] = 0;
 533                 }
 534         }
 535
 536         /* offload queues */
 537         for_each_ofldrxq(&adap->sge, i) {
 538                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-ofld%d",
 539                          adap->name, i);
 540                 adap->msix_info[msi_idx++].desc[n] = 0;
 541         }
 542         for_each_rdmarxq(&adap->sge, i) {
 543                 snprintf(adap->msix_info[msi_idx].desc, n, "%s-rdma%d",
 544                          adap->name, i);
 545                 adap->msix_info[msi_idx++].desc[n] = 0;
 546         }
 547 }
 548
 549 static int request_msix_queue_irqs(struct adapter *adap)
 550 {
 551         struct sge *s = &adap->sge;
 552         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
 553
 554         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 555                           adap->msix_info[1].desc, &s->fw_evtq);
 556         if (err)
 557                 return err;
 558
 559         for_each_ethrxq(s, ethqidx) {
 560                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 561                                   adap->msix_info[msi].desc,
 562                                   &s->ethrxq[ethqidx].rspq);
 563                 if (err)
 564                         goto unwind;
 565                 msi++;
 566         }
 567         for_each_ofldrxq(s, ofldqidx) {
 568                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 569                                   adap->msix_info[msi].desc,
 570                                   &s->ofldrxq[ofldqidx].rspq);
 571                 if (err)
 572                         goto unwind;
 573                 msi++;
 574         }
 575         for_each_rdmarxq(s, rdmaqidx) {
 576                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 577                                   adap->msix_info[msi].desc,
 578                                   &s->rdmarxq[rdmaqidx].rspq);
 579                 if (err)
 580                         goto unwind;
 581                 msi++;
 582         }
 583         return 0;
 584
 585 unwind:
 586         while (--rdmaqidx >= 0)
 587                 free_irq(adap->msix_info[--msi].vec,
 588                          &s->rdmarxq[rdmaqidx].rspq);
 589         while (--ofldqidx >= 0)
 590                 free_irq(adap->msix_info[--msi].vec,
 591                          &s->ofldrxq[ofldqidx].rspq);
 592         while (--ethqidx >= 0)
 593                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
 594         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 595         return err;
 596 }
 597
 598 static void free_msix_queue_irqs(struct adapter *adap)
 599 {
 600         int i, msi = 2;
 601         struct sge *s = &adap->sge;
 602
 603         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 604         for_each_ethrxq(s, i)
 605                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
 606         for_each_ofldrxq(s, i)
 607                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
 608         for_each_rdmarxq(s, i)
 609                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
 610 }
 611
 612 /**
 613  *      write_rss - write the RSS table for a given port
 614  *      @pi: the port
 615  *      @queues: array of queue indices for RSS
 616  *
 617  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 618  *      packets to the Rx queues in @queues.
 619  */
 620 static int write_rss(const struct port_info *pi, const u16 *queues)
 621 {
 622         u16 *rss;
 623         int i, err;
 624         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 625
 626         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 627         if (!rss)
 628                 return -ENOMEM;
 629
 630         /* map the queue indices to queue ids */
 631         for (i = 0; i < pi->rss_size; i++, queues++)
 632                 rss[i] = q[*queues].rspq.abs_id;
 633
 634         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 635                                   pi->rss_size, rss, pi->rss_size);
 636         kfree(rss);
 637         return err;
 638 }
 639
 640 /**
 641  *      setup_rss - configure RSS
 642  *      @adap: the adapter
 643  *
 644  *      Sets up RSS for each port.
 645  */
 646 static int setup_rss(struct adapter *adap)
 647 {
 648         int i, err;
 649
 650         for_each_port(adap, i) {
 651                 const struct port_info *pi = adap2pinfo(adap, i);
 652
 653                 err = write_rss(pi, pi->rss);
 654                 if (err)
 655                         return err;
 656         }
 657         return 0;
 658 }
 659
 660 /*
 661  * Wait until all NAPI handlers are descheduled.
 662  */
 663 static void quiesce_rx(struct adapter *adap)
 664 {
 665         int i;
 666
 667         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 668                 struct sge_rspq *q = adap->sge.ingr_map[i];
 669
 670                 if (q && q->handler)
 671                         napi_disable(&q->napi);
 672         }
 673 }
 674
 675 /*
 676  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 677  */
 678 static void enable_rx(struct adapter *adap)
 679 {
 680         int i;
 681
 682         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 683                 struct sge_rspq *q = adap->sge.ingr_map[i];
 684
 685                 if (!q)
 686                         continue;
 687                 if (q->handler)
 688                         napi_enable(&q->napi);
 689                 /* 0-increment GTS to start the timer and enable interrupts */
 690                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 691                              SEINTARM(q->intr_params) |
 692                              INGRESSQID(q->cntxt_id));
 693         }
 694 }
 695
 696 /**
 697  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 698  *      @adap: the adapter
 699  *
 700  *      Determines how many sets of SGE queues to use and initializes them.
 701  *      We support multiple queue sets per port if we have MSI-X, otherwise
 702  *      just one queue set per port.
 703  */
 704 static int setup_sge_queues(struct adapter *adap)
 705 {
 706         int err, msi_idx, i, j;
 707         struct sge *s = &adap->sge;
 708
 709         bitmap_zero(s->starving_fl, MAX_EGRQ);
 710         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 711
 712         if (adap->flags & USING_MSIX)
 713                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 714         else {
 715                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 716                                        NULL, NULL);
 717                 if (err)
 718                         return err;
 719                 msi_idx = -((int)s->intrq.abs_id + 1);
 720         }
 721
 722         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 723                                msi_idx, NULL, fwevtq_handler);
 724         if (err) {
 725 freeout:        t4_free_sge_resources(adap);
 726                 return err;
 727         }
 728
 729         for_each_port(adap, i) {
 730                 struct net_device *dev = adap->port[i];
 731                 struct port_info *pi = netdev_priv(dev);
 732                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 733                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 734
 735                 for (j = 0; j < pi->nqsets; j++, q++) {
 736                         if (msi_idx > 0)
 737                                 msi_idx++;
 738                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 739                                                msi_idx, &q->fl,
 740                                                t4_ethrx_handler);
 741                         if (err)
 742                                 goto freeout;
 743                         q->rspq.idx = j;
 744                         memset(&q->stats, 0, sizeof(q->stats));
 745                 }
 746                 for (j = 0; j < pi->nqsets; j++, t++) {
 747                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 748                                         netdev_get_tx_queue(dev, j),
 749                                         s->fw_evtq.cntxt_id);
 750                         if (err)
 751                                 goto freeout;
 752                 }
 753         }
 754
 755         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 756         for_each_ofldrxq(s, i) {
 757                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 758                 struct net_device *dev = adap->port[i / j];
 759
 760                 if (msi_idx > 0)
 761                         msi_idx++;
 762                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 763                                        &q->fl, uldrx_handler);
 764                 if (err)
 765                         goto freeout;
 766                 memset(&q->stats, 0, sizeof(q->stats));
 767                 s->ofld_rxq[i] = q->rspq.abs_id;
 768                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 769                                             s->fw_evtq.cntxt_id);
 770                 if (err)
 771                         goto freeout;
 772         }
 773
 774         for_each_rdmarxq(s, i) {
 775                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 776
 777                 if (msi_idx > 0)
 778                         msi_idx++;
 779                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 780                                        msi_idx, &q->fl, uldrx_handler);
 781                 if (err)
 782                         goto freeout;
 783                 memset(&q->stats, 0, sizeof(q->stats));
 784                 s->rdma_rxq[i] = q->rspq.abs_id;
 785         }
 786
 787         for_each_port(adap, i) {
 788                 /*
 789                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 790                  * have RDMA queues, and that's the right value.
 791                  */
 792                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 793                                             s->fw_evtq.cntxt_id,
 794                                             s->rdmarxq[i].rspq.cntxt_id);
 795                 if (err)
 796                         goto freeout;
 797         }
 798
 799         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
 800                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
 801                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
 802         return 0;
 803 }
 804
 805 /*
 806  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
 807  * started but failed, and a negative errno if flash load couldn't start.
 808  */
 809 static int upgrade_fw(struct adapter *adap)
 810 {
 811         int ret;
 812         u32 vers;
 813         const struct fw_hdr *hdr;
 814         const struct firmware *fw;
 815         struct device *dev = adap->pdev_dev;
 816
 817         ret = request_firmware(&fw, FW_FNAME, dev);
 818         if (ret < 0) {
 819                 dev_err(dev, "unable to load firmware image " FW_FNAME
 820                         ", error %d\n", ret);
 821                 return ret;
 822         }
 823
 824         hdr = (const struct fw_hdr *)fw->data;
 825         vers = ntohl(hdr->fw_ver);
 826         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
 827                 ret = -EINVAL;              /* wrong major version, won't do */
 828                 goto out;
 829         }
 830
 831         /*
 832          * If the flash FW is unusable or we found something newer, load it.
 833          */
 834         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
 835             vers > adap->params.fw_vers) {
 836                 ret = -t4_load_fw(adap, fw->data, fw->size);
 837                 if (!ret)
 838                         dev_info(dev, "firmware upgraded to version %pI4 from "
 839                                  FW_FNAME "\n", &hdr->fw_ver);
 840         }
 841 out:    release_firmware(fw);
 842         return ret;
 843 }
 844
 845 /*
 846  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 847  * The allocated memory is cleared.
 848  */
 849 void *t4_alloc_mem(size_t size)
 850 {
 851         void *p = kmalloc(size, GFP_KERNEL);
 852
 853         if (!p)
 854                 p = vmalloc(size);
 855         if (p)
 856                 memset(p, 0, size);
 857         return p;
 858 }
 859
 860 /*
 861  * Free memory allocated through alloc_mem().
 862  */
 863 void t4_free_mem(void *addr)
 864 {
 865         if (is_vmalloc_addr(addr))
 866                 vfree(addr);
 867         else
 868                 kfree(addr);
 869 }
 870
 871 static inline int is_offload(const struct adapter *adap)
 872 {
 873         return adap->params.offload;
 874 }
 875
 876 /*
 877  * Implementation of ethtool operations.
 878  */
 879
 880 static u32 get_msglevel(struct net_device *dev)
 881 {
 882         return netdev2adap(dev)->msg_enable;
 883 }
 884
 885 static void set_msglevel(struct net_device *dev, u32 val)
 886 {
 887         netdev2adap(dev)->msg_enable = val;
 888 }
 889
 890 static char stats_strings[][ETH_GSTRING_LEN] = {
 891         "TxOctetsOK         ",
 892         "TxFramesOK         ",
 893         "TxBroadcastFrames  ",
 894         "TxMulticastFrames  ",
 895         "TxUnicastFrames    ",
 896         "TxErrorFrames      ",
 897
 898         "TxFrames64         ",
 899         "TxFrames65To127    ",
 900         "TxFrames128To255   ",
 901         "TxFrames256To511   ",
 902         "TxFrames512To1023  ",
 903         "TxFrames1024To1518 ",
 904         "TxFrames1519ToMax  ",
 905
 906         "TxFramesDropped    ",
 907         "TxPauseFrames      ",
 908         "TxPPP0Frames       ",
 909         "TxPPP1Frames       ",
 910         "TxPPP2Frames       ",
 911         "TxPPP3Frames       ",
 912         "TxPPP4Frames       ",
 913         "TxPPP5Frames       ",
 914         "TxPPP6Frames       ",
 915         "TxPPP7Frames       ",
 916
 917         "RxOctetsOK         ",
 918         "RxFramesOK         ",
 919         "RxBroadcastFrames  ",
 920         "RxMulticastFrames  ",
 921         "RxUnicastFrames    ",
 922
 923         "RxFramesTooLong    ",
 924         "RxJabberErrors     ",
 925         "RxFCSErrors        ",
 926         "RxLengthErrors     ",
 927         "RxSymbolErrors     ",
 928         "RxRuntFrames       ",
 929
 930         "RxFrames64         ",
 931         "RxFrames65To127    ",
 932         "RxFrames128To255   ",
 933         "RxFrames256To511   ",
 934         "RxFrames512To1023  ",
 935         "RxFrames1024To1518 ",
 936         "RxFrames1519ToMax  ",
 937
 938         "RxPauseFrames      ",
 939         "RxPPP0Frames       ",
 940         "RxPPP1Frames       ",
 941         "RxPPP2Frames       ",
 942         "RxPPP3Frames       ",
 943         "RxPPP4Frames       ",
 944         "RxPPP5Frames       ",
 945         "RxPPP6Frames       ",
 946         "RxPPP7Frames       ",
 947
 948         "RxBG0FramesDropped ",
 949         "RxBG1FramesDropped ",
 950         "RxBG2FramesDropped ",
 951         "RxBG3FramesDropped ",
 952         "RxBG0FramesTrunc   ",
 953         "RxBG1FramesTrunc   ",
 954         "RxBG2FramesTrunc   ",
 955         "RxBG3FramesTrunc   ",
 956
 957         "TSO                ",
 958         "TxCsumOffload      ",
 959         "RxCsumGood         ",
 960         "VLANextractions    ",
 961         "VLANinsertions     ",
 962         "GROpackets         ",
 963         "GROmerged          ",
 964 };
 965
 966 static int get_sset_count(struct net_device *dev, int sset)
 967 {
 968         switch (sset) {
 969         case ETH_SS_STATS:
 970                 return ARRAY_SIZE(stats_strings);
 971         default:
 972                 return -EOPNOTSUPP;
 973         }
 974 }
 975
 976 #define T4_REGMAP_SIZE (160 * 1024)
 977
 978 static int get_regs_len(struct net_device *dev)
 979 {
 980         return T4_REGMAP_SIZE;
 981 }
 982
 983 static int get_eeprom_len(struct net_device *dev)
 984 {
 985         return EEPROMSIZE;
 986 }
 987
 988 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 989 {
 990         struct adapter *adapter = netdev2adap(dev);
 991
 992         strcpy(info->driver, KBUILD_MODNAME);
 993         strcpy(info->version, DRV_VERSION);
 994         strcpy(info->bus_info, pci_name(adapter->pdev));
 995
 996         if (!adapter->params.fw_vers)
 997                 strcpy(info->fw_version, "N/A");
 998         else
 999                 snprintf(info->fw_version, sizeof(info->fw_version),
1000                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1001                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1002                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1003                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1004                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1005                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1006                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1007                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1008                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1009 }
1010
1011 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1012 {
1013         if (stringset == ETH_SS_STATS)
1014                 memcpy(data, stats_strings, sizeof(stats_strings));
1015 }
1016
1017 /*
1018  * port stats maintained per queue of the port.  They should be in the same
1019  * order as in stats_strings above.
1020  */
1021 struct queue_port_stats {
1022         u64 tso;
1023         u64 tx_csum;
1024         u64 rx_csum;
1025         u64 vlan_ex;
1026         u64 vlan_ins;
1027         u64 gro_pkts;
1028         u64 gro_merged;
1029 };
1030
1031 static void collect_sge_port_stats(const struct adapter *adap,
1032                 const struct port_info *p, struct queue_port_stats *s)
1033 {
1034         int i;
1035         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1036         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1037
1038         memset(s, 0, sizeof(*s));
1039         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1040                 s->tso += tx->tso;
1041                 s->tx_csum += tx->tx_cso;
1042                 s->rx_csum += rx->stats.rx_cso;
1043                 s->vlan_ex += rx->stats.vlan_ex;
1044                 s->vlan_ins += tx->vlan_ins;
1045                 s->gro_pkts += rx->stats.lro_pkts;
1046                 s->gro_merged += rx->stats.lro_merged;
1047         }
1048 }
1049
1050 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1051                       u64 *data)
1052 {
1053         struct port_info *pi = netdev_priv(dev);
1054         struct adapter *adapter = pi->adapter;
1055
1056         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1057
1058         data += sizeof(struct port_stats) / sizeof(u64);
1059         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1060 }
1061
1062 /*
1063  * Return a version number to identify the type of adapter.  The scheme is:
1064  * - bits 0..9: chip version
1065  * - bits 10..15: chip revision
1066  * - bits 16..23: register dump version
1067  */
1068 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1069 {
1070         return 4 | (ap->params.rev << 10) | (1 << 16);
1071 }
1072
1073 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1074                            unsigned int end)
1075 {
1076         u32 *p = buf + start;
1077
1078         for ( ; start <= end; start += sizeof(u32))
1079                 *p++ = t4_read_reg(ap, start);
1080 }
1081
1082 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1083                      void *buf)
1084 {
1085         static const unsigned int reg_ranges[] = {
1086                 0x1008, 0x1108,
1087                 0x1180, 0x11b4,
1088                 0x11fc, 0x123c,
1089                 0x1300, 0x173c,
1090                 0x1800, 0x18fc,
1091                 0x3000, 0x30d8,
1092                 0x30e0, 0x5924,
1093                 0x5960, 0x59d4,
1094                 0x5a00, 0x5af8,
1095                 0x6000, 0x6098,
1096                 0x6100, 0x6150,
1097                 0x6200, 0x6208,
1098                 0x6240, 0x6248,
1099                 0x6280, 0x6338,
1100                 0x6370, 0x638c,
1101                 0x6400, 0x643c,
1102                 0x6500, 0x6524,
1103                 0x6a00, 0x6a38,
1104                 0x6a60, 0x6a78,
1105                 0x6b00, 0x6b84,
1106                 0x6bf0, 0x6c84,
1107                 0x6cf0, 0x6d84,
1108                 0x6df0, 0x6e84,
1109                 0x6ef0, 0x6f84,
1110                 0x6ff0, 0x7084,
1111                 0x70f0, 0x7184,
1112                 0x71f0, 0x7284,
1113                 0x72f0, 0x7384,
1114                 0x73f0, 0x7450,
1115                 0x7500, 0x7530,
1116                 0x7600, 0x761c,
1117                 0x7680, 0x76cc,
1118                 0x7700, 0x7798,
1119                 0x77c0, 0x77fc,
1120                 0x7900, 0x79fc,
1121                 0x7b00, 0x7c38,
1122                 0x7d00, 0x7efc,
1123                 0x8dc0, 0x8e1c,
1124                 0x8e30, 0x8e78,
1125                 0x8ea0, 0x8f6c,
1126                 0x8fc0, 0x9074,
1127                 0x90fc, 0x90fc,
1128                 0x9400, 0x9458,
1129                 0x9600, 0x96bc,
1130                 0x9800, 0x9808,
1131                 0x9820, 0x983c,
1132                 0x9850, 0x9864,
1133                 0x9c00, 0x9c6c,
1134                 0x9c80, 0x9cec,
1135                 0x9d00, 0x9d6c,
1136                 0x9d80, 0x9dec,
1137                 0x9e00, 0x9e6c,
1138                 0x9e80, 0x9eec,
1139                 0x9f00, 0x9f6c,
1140                 0x9f80, 0x9fec,
1141                 0xd004, 0xd03c,
1142                 0xdfc0, 0xdfe0,
1143                 0xe000, 0xea7c,
1144                 0xf000, 0x11190,
1145                 0x19040, 0x1906c,
1146                 0x19078, 0x19080,
1147                 0x1908c, 0x19124,
1148                 0x19150, 0x191b0,
1149                 0x191d0, 0x191e8,
1150                 0x19238, 0x1924c,
1151                 0x193f8, 0x19474,
1152                 0x19490, 0x194f8,
1153                 0x19800, 0x19f30,
1154                 0x1a000, 0x1a06c,
1155                 0x1a0b0, 0x1a120,
1156                 0x1a128, 0x1a138,
1157                 0x1a190, 0x1a1c4,
1158                 0x1a1fc, 0x1a1fc,
1159                 0x1e040, 0x1e04c,
1160                 0x1e284, 0x1e28c,
1161                 0x1e2c0, 0x1e2c0,
1162                 0x1e2e0, 0x1e2e0,
1163                 0x1e300, 0x1e384,
1164                 0x1e3c0, 0x1e3c8,
1165                 0x1e440, 0x1e44c,
1166                 0x1e684, 0x1e68c,
1167                 0x1e6c0, 0x1e6c0,
1168                 0x1e6e0, 0x1e6e0,
1169                 0x1e700, 0x1e784,
1170                 0x1e7c0, 0x1e7c8,
1171                 0x1e840, 0x1e84c,
1172                 0x1ea84, 0x1ea8c,
1173                 0x1eac0, 0x1eac0,
1174                 0x1eae0, 0x1eae0,
1175                 0x1eb00, 0x1eb84,
1176                 0x1ebc0, 0x1ebc8,
1177                 0x1ec40, 0x1ec4c,
1178                 0x1ee84, 0x1ee8c,
1179                 0x1eec0, 0x1eec0,
1180                 0x1eee0, 0x1eee0,
1181                 0x1ef00, 0x1ef84,
1182                 0x1efc0, 0x1efc8,
1183                 0x1f040, 0x1f04c,
1184                 0x1f284, 0x1f28c,
1185                 0x1f2c0, 0x1f2c0,
1186                 0x1f2e0, 0x1f2e0,
1187                 0x1f300, 0x1f384,
1188                 0x1f3c0, 0x1f3c8,
1189                 0x1f440, 0x1f44c,
1190                 0x1f684, 0x1f68c,
1191                 0x1f6c0, 0x1f6c0,
1192                 0x1f6e0, 0x1f6e0,
1193                 0x1f700, 0x1f784,
1194                 0x1f7c0, 0x1f7c8,
1195                 0x1f840, 0x1f84c,
1196                 0x1fa84, 0x1fa8c,
1197                 0x1fac0, 0x1fac0,
1198                 0x1fae0, 0x1fae0,
1199                 0x1fb00, 0x1fb84,
1200                 0x1fbc0, 0x1fbc8,
1201                 0x1fc40, 0x1fc4c,
1202                 0x1fe84, 0x1fe8c,
1203                 0x1fec0, 0x1fec0,
1204                 0x1fee0, 0x1fee0,
1205                 0x1ff00, 0x1ff84,
1206                 0x1ffc0, 0x1ffc8,
1207                 0x20000, 0x2002c,
1208                 0x20100, 0x2013c,
1209                 0x20190, 0x201c8,
1210                 0x20200, 0x20318,
1211                 0x20400, 0x20528,
1212                 0x20540, 0x20614,
1213                 0x21000, 0x21040,
1214                 0x2104c, 0x21060,
1215                 0x210c0, 0x210ec,
1216                 0x21200, 0x21268,
1217                 0x21270, 0x21284,
1218                 0x212fc, 0x21388,
1219                 0x21400, 0x21404,
1220                 0x21500, 0x21518,
1221                 0x2152c, 0x2153c,
1222                 0x21550, 0x21554,
1223                 0x21600, 0x21600,
1224                 0x21608, 0x21628,
1225                 0x21630, 0x2163c,
1226                 0x21700, 0x2171c,
1227                 0x21780, 0x2178c,
1228                 0x21800, 0x21c38,
1229                 0x21c80, 0x21d7c,
1230                 0x21e00, 0x21e04,
1231                 0x22000, 0x2202c,
1232                 0x22100, 0x2213c,
1233                 0x22190, 0x221c8,
1234                 0x22200, 0x22318,
1235                 0x22400, 0x22528,
1236                 0x22540, 0x22614,
1237                 0x23000, 0x23040,
1238                 0x2304c, 0x23060,
1239                 0x230c0, 0x230ec,
1240                 0x23200, 0x23268,
1241                 0x23270, 0x23284,
1242                 0x232fc, 0x23388,
1243                 0x23400, 0x23404,
1244                 0x23500, 0x23518,
1245                 0x2352c, 0x2353c,
1246                 0x23550, 0x23554,
1247                 0x23600, 0x23600,
1248                 0x23608, 0x23628,
1249                 0x23630, 0x2363c,
1250                 0x23700, 0x2371c,
1251                 0x23780, 0x2378c,
1252                 0x23800, 0x23c38,
1253                 0x23c80, 0x23d7c,
1254                 0x23e00, 0x23e04,
1255                 0x24000, 0x2402c,
1256                 0x24100, 0x2413c,
1257                 0x24190, 0x241c8,
1258                 0x24200, 0x24318,
1259                 0x24400, 0x24528,
1260                 0x24540, 0x24614,
1261                 0x25000, 0x25040,
1262                 0x2504c, 0x25060,
1263                 0x250c0, 0x250ec,
1264                 0x25200, 0x25268,
1265                 0x25270, 0x25284,
1266                 0x252fc, 0x25388,
1267                 0x25400, 0x25404,
1268                 0x25500, 0x25518,
1269                 0x2552c, 0x2553c,
1270                 0x25550, 0x25554,
1271                 0x25600, 0x25600,
1272                 0x25608, 0x25628,
1273                 0x25630, 0x2563c,
1274                 0x25700, 0x2571c,
1275                 0x25780, 0x2578c,
1276                 0x25800, 0x25c38,
1277                 0x25c80, 0x25d7c,
1278                 0x25e00, 0x25e04,
1279                 0x26000, 0x2602c,
1280                 0x26100, 0x2613c,
1281                 0x26190, 0x261c8,
1282                 0x26200, 0x26318,
1283                 0x26400, 0x26528,
1284                 0x26540, 0x26614,
1285                 0x27000, 0x27040,
1286                 0x2704c, 0x27060,
1287                 0x270c0, 0x270ec,
1288                 0x27200, 0x27268,
1289                 0x27270, 0x27284,
1290                 0x272fc, 0x27388,
1291                 0x27400, 0x27404,
1292                 0x27500, 0x27518,
1293                 0x2752c, 0x2753c,
1294                 0x27550, 0x27554,
1295                 0x27600, 0x27600,
1296                 0x27608, 0x27628,
1297                 0x27630, 0x2763c,
1298                 0x27700, 0x2771c,
1299                 0x27780, 0x2778c,
1300                 0x27800, 0x27c38,
1301                 0x27c80, 0x27d7c,
1302                 0x27e00, 0x27e04
1303         };
1304
1305         int i;
1306         struct adapter *ap = netdev2adap(dev);
1307
1308         regs->version = mk_adap_vers(ap);
1309
1310         memset(buf, 0, T4_REGMAP_SIZE);
1311         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1312                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1313 }
1314
1315 static int restart_autoneg(struct net_device *dev)
1316 {
1317         struct port_info *p = netdev_priv(dev);
1318
1319         if (!netif_running(dev))
1320                 return -EAGAIN;
1321         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1322                 return -EINVAL;
1323         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1324         return 0;
1325 }
1326
1327 static int identify_port(struct net_device *dev, u32 data)
1328 {
1329         struct adapter *adap = netdev2adap(dev);
1330
1331         if (data == 0)
1332                 data = 2;     /* default to 2 seconds */
1333
1334         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid,
1335                                 data * 5);
1336 }
1337
1338 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1339 {
1340         unsigned int v = 0;
1341
1342         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1343             type == FW_PORT_TYPE_BT_XAUI) {
1344                 v |= SUPPORTED_TP;
1345                 if (caps & FW_PORT_CAP_SPEED_100M)
1346                         v |= SUPPORTED_100baseT_Full;
1347                 if (caps & FW_PORT_CAP_SPEED_1G)
1348                         v |= SUPPORTED_1000baseT_Full;
1349                 if (caps & FW_PORT_CAP_SPEED_10G)
1350                         v |= SUPPORTED_10000baseT_Full;
1351         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1352                 v |= SUPPORTED_Backplane;
1353                 if (caps & FW_PORT_CAP_SPEED_1G)
1354                         v |= SUPPORTED_1000baseKX_Full;
1355                 if (caps & FW_PORT_CAP_SPEED_10G)
1356                         v |= SUPPORTED_10000baseKX4_Full;
1357         } else if (type == FW_PORT_TYPE_KR)
1358                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1359         else if (type == FW_PORT_TYPE_BP_AP)
1360                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC;
1361         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1362                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1363                 v |= SUPPORTED_FIBRE;
1364
1365         if (caps & FW_PORT_CAP_ANEG)
1366                 v |= SUPPORTED_Autoneg;
1367         return v;
1368 }
1369
1370 static unsigned int to_fw_linkcaps(unsigned int caps)
1371 {
1372         unsigned int v = 0;
1373
1374         if (caps & ADVERTISED_100baseT_Full)
1375                 v |= FW_PORT_CAP_SPEED_100M;
1376         if (caps & ADVERTISED_1000baseT_Full)
1377                 v |= FW_PORT_CAP_SPEED_1G;
1378         if (caps & ADVERTISED_10000baseT_Full)
1379                 v |= FW_PORT_CAP_SPEED_10G;
1380         return v;
1381 }
1382
1383 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1384 {
1385         const struct port_info *p = netdev_priv(dev);
1386
1387         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1388             p->port_type == FW_PORT_TYPE_BT_XFI ||
1389             p->port_type == FW_PORT_TYPE_BT_XAUI)
1390                 cmd->port = PORT_TP;
1391         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1392                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1393                 cmd->port = PORT_FIBRE;
1394         else if (p->port_type == FW_PORT_TYPE_SFP) {
1395                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1396                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1397                         cmd->port = PORT_DA;
1398                 else
1399                         cmd->port = PORT_FIBRE;
1400         } else
1401                 cmd->port = PORT_OTHER;
1402
1403         if (p->mdio_addr >= 0) {
1404                 cmd->phy_address = p->mdio_addr;
1405                 cmd->transceiver = XCVR_EXTERNAL;
1406                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1407                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1408         } else {
1409                 cmd->phy_address = 0;  /* not really, but no better option */
1410                 cmd->transceiver = XCVR_INTERNAL;
1411                 cmd->mdio_support = 0;
1412         }
1413
1414         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1415         cmd->advertising = from_fw_linkcaps(p->port_type,
1416                                             p->link_cfg.advertising);
1417         cmd->speed = netif_carrier_ok(dev) ? p->link_cfg.speed : 0;
1418         cmd->duplex = DUPLEX_FULL;
1419         cmd->autoneg = p->link_cfg.autoneg;
1420         cmd->maxtxpkt = 0;
1421         cmd->maxrxpkt = 0;
1422         return 0;
1423 }
1424
1425 static unsigned int speed_to_caps(int speed)
1426 {
1427         if (speed == SPEED_100)
1428                 return FW_PORT_CAP_SPEED_100M;
1429         if (speed == SPEED_1000)
1430                 return FW_PORT_CAP_SPEED_1G;
1431         if (speed == SPEED_10000)
1432                 return FW_PORT_CAP_SPEED_10G;
1433         return 0;
1434 }
1435
1436 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1437 {
1438         unsigned int cap;
1439         struct port_info *p = netdev_priv(dev);
1440         struct link_config *lc = &p->link_cfg;
1441
1442         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1443                 return -EINVAL;
1444
1445         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1446                 /*
1447                  * PHY offers a single speed.  See if that's what's
1448                  * being requested.
1449                  */
1450                 if (cmd->autoneg == AUTONEG_DISABLE &&
1451                     (lc->supported & speed_to_caps(cmd->speed)))
1452                                 return 0;
1453                 return -EINVAL;
1454         }
1455
1456         if (cmd->autoneg == AUTONEG_DISABLE) {
1457                 cap = speed_to_caps(cmd->speed);
1458
1459                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000 ||
1460                     cmd->speed == SPEED_10000)
1461                         return -EINVAL;
1462                 lc->requested_speed = cap;
1463                 lc->advertising = 0;
1464         } else {
1465                 cap = to_fw_linkcaps(cmd->advertising);
1466                 if (!(lc->supported & cap))
1467                         return -EINVAL;
1468                 lc->requested_speed = 0;
1469                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1470         }
1471         lc->autoneg = cmd->autoneg;
1472
1473         if (netif_running(dev))
1474                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1475                                      lc);
1476         return 0;
1477 }
1478
1479 static void get_pauseparam(struct net_device *dev,
1480                            struct ethtool_pauseparam *epause)
1481 {
1482         struct port_info *p = netdev_priv(dev);
1483
1484         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1485         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1486         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1487 }
1488
1489 static int set_pauseparam(struct net_device *dev,
1490                           struct ethtool_pauseparam *epause)
1491 {
1492         struct port_info *p = netdev_priv(dev);
1493         struct link_config *lc = &p->link_cfg;
1494
1495         if (epause->autoneg == AUTONEG_DISABLE)
1496                 lc->requested_fc = 0;
1497         else if (lc->supported & FW_PORT_CAP_ANEG)
1498                 lc->requested_fc = PAUSE_AUTONEG;
1499         else
1500                 return -EINVAL;
1501
1502         if (epause->rx_pause)
1503                 lc->requested_fc |= PAUSE_RX;
1504         if (epause->tx_pause)
1505                 lc->requested_fc |= PAUSE_TX;
1506         if (netif_running(dev))
1507                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1508                                      lc);
1509         return 0;
1510 }
1511
1512 static u32 get_rx_csum(struct net_device *dev)
1513 {
1514         struct port_info *p = netdev_priv(dev);
1515
1516         return p->rx_offload & RX_CSO;
1517 }
1518
1519 static int set_rx_csum(struct net_device *dev, u32 data)
1520 {
1521         struct port_info *p = netdev_priv(dev);
1522
1523         if (data)
1524                 p->rx_offload |= RX_CSO;
1525         else
1526                 p->rx_offload &= ~RX_CSO;
1527         return 0;
1528 }
1529
1530 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1531 {
1532         const struct port_info *pi = netdev_priv(dev);
1533         const struct sge *s = &pi->adapter->sge;
1534
1535         e->rx_max_pending = MAX_RX_BUFFERS;
1536         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1537         e->rx_jumbo_max_pending = 0;
1538         e->tx_max_pending = MAX_TXQ_ENTRIES;
1539
1540         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1541         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1542         e->rx_jumbo_pending = 0;
1543         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1544 }
1545
1546 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1547 {
1548         int i;
1549         const struct port_info *pi = netdev_priv(dev);
1550         struct adapter *adapter = pi->adapter;
1551         struct sge *s = &adapter->sge;
1552
1553         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1554             e->tx_pending > MAX_TXQ_ENTRIES ||
1555             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1556             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1557             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1558                 return -EINVAL;
1559
1560         if (adapter->flags & FULL_INIT_DONE)
1561                 return -EBUSY;
1562
1563         for (i = 0; i < pi->nqsets; ++i) {
1564                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1565                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1566                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1567         }
1568         return 0;
1569 }
1570
1571 static int closest_timer(const struct sge *s, int time)
1572 {
1573         int i, delta, match = 0, min_delta = INT_MAX;
1574
1575         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1576                 delta = time - s->timer_val[i];
1577                 if (delta < 0)
1578                         delta = -delta;
1579                 if (delta < min_delta) {
1580                         min_delta = delta;
1581                         match = i;
1582                 }
1583         }
1584         return match;
1585 }
1586
1587 static int closest_thres(const struct sge *s, int thres)
1588 {
1589         int i, delta, match = 0, min_delta = INT_MAX;
1590
1591         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1592                 delta = thres - s->counter_val[i];
1593                 if (delta < 0)
1594                         delta = -delta;
1595                 if (delta < min_delta) {
1596                         min_delta = delta;
1597                         match = i;
1598                 }
1599         }
1600         return match;
1601 }
1602
1603 /*
1604  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1605  */
1606 static unsigned int qtimer_val(const struct adapter *adap,
1607                                const struct sge_rspq *q)
1608 {
1609         unsigned int idx = q->intr_params >> 1;
1610
1611         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1612 }
1613
1614 /**
1615  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1616  *      @adap: the adapter
1617  *      @q: the Rx queue
1618  *      @us: the hold-off time in us, or 0 to disable timer
1619  *      @cnt: the hold-off packet count, or 0 to disable counter
1620  *
1621  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1622  *      one of the two needs to be enabled for the queue to generate interrupts.
1623  */
1624 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1625                                unsigned int us, unsigned int cnt)
1626 {
1627         if ((us | cnt) == 0)
1628                 cnt = 1;
1629
1630         if (cnt) {
1631                 int err;
1632                 u32 v, new_idx;
1633
1634                 new_idx = closest_thres(&adap->sge, cnt);
1635                 if (q->desc && q->pktcnt_idx != new_idx) {
1636                         /* the queue has already been created, update it */
1637                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1638                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1639                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1640                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1641                                             &new_idx);
1642                         if (err)
1643                                 return err;
1644                 }
1645                 q->pktcnt_idx = new_idx;
1646         }
1647
1648         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1649         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1650         return 0;
1651 }
1652
1653 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1654 {
1655         const struct port_info *pi = netdev_priv(dev);
1656         struct adapter *adap = pi->adapter;
1657
1658         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1659                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1660 }
1661
1662 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1663 {
1664         const struct port_info *pi = netdev_priv(dev);
1665         const struct adapter *adap = pi->adapter;
1666         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1667
1668         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1669         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1670                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1671         return 0;
1672 }
1673
1674 /*
1675  * Translate a physical EEPROM address to virtual.  The first 1K is accessed
1676  * through virtual addresses starting at 31K, the rest is accessed through
1677  * virtual addresses starting at 0.  This mapping is correct only for PF0.
1678  */
1679 static int eeprom_ptov(unsigned int phys_addr)
1680 {
1681         if (phys_addr < 1024)
1682                 return phys_addr + (31 << 10);
1683         if (phys_addr < EEPROMSIZE)
1684                 return phys_addr - 1024;
1685         return -EINVAL;
1686 }
1687
1688 /*
1689  * The next two routines implement eeprom read/write from physical addresses.
1690  * The physical->virtual translation is correct only for PF0.
1691  */
1692 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1693 {
1694         int vaddr = eeprom_ptov(phys_addr);
1695
1696         if (vaddr >= 0)
1697                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1698         return vaddr < 0 ? vaddr : 0;
1699 }
1700
1701 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1702 {
1703         int vaddr = eeprom_ptov(phys_addr);
1704
1705         if (vaddr >= 0)
1706                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1707         return vaddr < 0 ? vaddr : 0;
1708 }
1709
1710 #define EEPROM_MAGIC 0x38E2F10C
1711
1712 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1713                       u8 *data)
1714 {
1715         int i, err = 0;
1716         struct adapter *adapter = netdev2adap(dev);
1717
1718         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1719         if (!buf)
1720                 return -ENOMEM;
1721
1722         e->magic = EEPROM_MAGIC;
1723         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1724                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1725
1726         if (!err)
1727                 memcpy(data, buf + e->offset, e->len);
1728         kfree(buf);
1729         return err;
1730 }
1731
1732 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1733                       u8 *data)
1734 {
1735         u8 *buf;
1736         int err = 0;
1737         u32 aligned_offset, aligned_len, *p;
1738         struct adapter *adapter = netdev2adap(dev);
1739
1740         if (eeprom->magic != EEPROM_MAGIC)
1741                 return -EINVAL;
1742
1743         aligned_offset = eeprom->offset & ~3;
1744         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1745
1746         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1747                 /*
1748                  * RMW possibly needed for first or last words.
1749                  */
1750                 buf = kmalloc(aligned_len, GFP_KERNEL);
1751                 if (!buf)
1752                         return -ENOMEM;
1753                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1754                 if (!err && aligned_len > 4)
1755                         err = eeprom_rd_phys(adapter,
1756                                              aligned_offset + aligned_len - 4,
1757                                              (u32 *)&buf[aligned_len - 4]);
1758                 if (err)
1759                         goto out;
1760                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1761         } else
1762                 buf = data;
1763
1764         err = t4_seeprom_wp(adapter, false);
1765         if (err)
1766                 goto out;
1767
1768         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1769                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1770                 aligned_offset += 4;
1771         }
1772
1773         if (!err)
1774                 err = t4_seeprom_wp(adapter, true);
1775 out:
1776         if (buf != data)
1777                 kfree(buf);
1778         return err;
1779 }
1780
1781 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1782 {
1783         int ret;
1784         const struct firmware *fw;
1785         struct adapter *adap = netdev2adap(netdev);
1786
1787         ef->data[sizeof(ef->data) - 1] = '\0';
1788         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1789         if (ret < 0)
1790                 return ret;
1791
1792         ret = t4_load_fw(adap, fw->data, fw->size);
1793         release_firmware(fw);
1794         if (!ret)
1795                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1796         return ret;
1797 }
1798
1799 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1800 #define BCAST_CRC 0xa0ccc1a6
1801
1802 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1803 {
1804         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1805         wol->wolopts = netdev2adap(dev)->wol;
1806         memset(&wol->sopass, 0, sizeof(wol->sopass));
1807 }
1808
1809 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1810 {
1811         int err = 0;
1812         struct port_info *pi = netdev_priv(dev);
1813
1814         if (wol->wolopts & ~WOL_SUPPORTED)
1815                 return -EINVAL;
1816         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1817                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1818         if (wol->wolopts & WAKE_BCAST) {
1819                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1820                                         ~0ULL, 0, false);
1821                 if (!err)
1822                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1823                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1824         } else
1825                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1826         return err;
1827 }
1828
1829 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1830
1831 static int set_tso(struct net_device *dev, u32 value)
1832 {
1833         if (value)
1834                 dev->features |= TSO_FLAGS;
1835         else
1836                 dev->features &= ~TSO_FLAGS;
1837         return 0;
1838 }
1839
1840 static int set_flags(struct net_device *dev, u32 flags)
1841 {
1842         return ethtool_op_set_flags(dev, flags, ETH_FLAG_RXHASH);
1843 }
1844
1845 static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1846 {
1847         const struct port_info *pi = netdev_priv(dev);
1848         unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1849
1850         p->size = pi->rss_size;
1851         while (n--)
1852                 p->ring_index[n] = pi->rss[n];
1853         return 0;
1854 }
1855
1856 static int set_rss_table(struct net_device *dev,
1857                          const struct ethtool_rxfh_indir *p)
1858 {
1859         unsigned int i;
1860         struct port_info *pi = netdev_priv(dev);
1861
1862         if (p->size != pi->rss_size)
1863                 return -EINVAL;
1864         for (i = 0; i < p->size; i++)
1865                 if (p->ring_index[i] >= pi->nqsets)
1866                         return -EINVAL;
1867         for (i = 0; i < p->size; i++)
1868                 pi->rss[i] = p->ring_index[i];
1869         if (pi->adapter->flags & FULL_INIT_DONE)
1870                 return write_rss(pi, pi->rss);
1871         return 0;
1872 }
1873
1874 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1875                      void *rules)
1876 {
1877         const struct port_info *pi = netdev_priv(dev);
1878
1879         switch (info->cmd) {
1880         case ETHTOOL_GRXFH: {
1881                 unsigned int v = pi->rss_mode;
1882
1883                 info->data = 0;
1884                 switch (info->flow_type) {
1885                 case TCP_V4_FLOW:
1886                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1887                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1888                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1889                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1890                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1891                         break;
1892                 case UDP_V4_FLOW:
1893                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1894                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1895                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1896                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1897                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1898                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1899                         break;
1900                 case SCTP_V4_FLOW:
1901                 case AH_ESP_V4_FLOW:
1902                 case IPV4_FLOW:
1903                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1904                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1905                         break;
1906                 case TCP_V6_FLOW:
1907                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1908                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1909                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1910                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1911                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1912                         break;
1913                 case UDP_V6_FLOW:
1914                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1915                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1916                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1917                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1918                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1919                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1920                         break;
1921                 case SCTP_V6_FLOW:
1922                 case AH_ESP_V6_FLOW:
1923                 case IPV6_FLOW:
1924                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1925                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1926                         break;
1927                 }
1928                 return 0;
1929         }
1930         case ETHTOOL_GRXRINGS:
1931                 info->data = pi->nqsets;
1932                 return 0;
1933         }
1934         return -EOPNOTSUPP;
1935 }
1936
1937 static struct ethtool_ops cxgb_ethtool_ops = {
1938         .get_settings      = get_settings,
1939         .set_settings      = set_settings,
1940         .get_drvinfo       = get_drvinfo,
1941         .get_msglevel      = get_msglevel,
1942         .set_msglevel      = set_msglevel,
1943         .get_ringparam     = get_sge_param,
1944         .set_ringparam     = set_sge_param,
1945         .get_coalesce      = get_coalesce,
1946         .set_coalesce      = set_coalesce,
1947         .get_eeprom_len    = get_eeprom_len,
1948         .get_eeprom        = get_eeprom,
1949         .set_eeprom        = set_eeprom,
1950         .get_pauseparam    = get_pauseparam,
1951         .set_pauseparam    = set_pauseparam,
1952         .get_rx_csum       = get_rx_csum,
1953         .set_rx_csum       = set_rx_csum,
1954         .set_tx_csum       = ethtool_op_set_tx_ipv6_csum,
1955         .set_sg            = ethtool_op_set_sg,
1956         .get_link          = ethtool_op_get_link,
1957         .get_strings       = get_strings,
1958         .phys_id           = identify_port,
1959         .nway_reset        = restart_autoneg,
1960         .get_sset_count    = get_sset_count,
1961         .get_ethtool_stats = get_stats,
1962         .get_regs_len      = get_regs_len,
1963         .get_regs          = get_regs,
1964         .get_wol           = get_wol,
1965         .set_wol           = set_wol,
1966         .set_tso           = set_tso,
1967         .set_flags         = set_flags,
1968         .get_rxnfc         = get_rxnfc,
1969         .get_rxfh_indir    = get_rss_table,
1970         .set_rxfh_indir    = set_rss_table,
1971         .flash_device      = set_flash,
1972 };
1973
1974 /*
1975  * debugfs support
1976  */
1977
1978 static int mem_open(struct inode *inode, struct file *file)
1979 {
1980         file->private_data = inode->i_private;
1981         return 0;
1982 }
1983
1984 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
1985                         loff_t *ppos)
1986 {
1987         loff_t pos = *ppos;
1988         loff_t avail = file->f_path.dentry->d_inode->i_size;
1989         unsigned int mem = (uintptr_t)file->private_data & 3;
1990         struct adapter *adap = file->private_data - mem;
1991
1992         if (pos < 0)
1993                 return -EINVAL;
1994         if (pos >= avail)
1995                 return 0;
1996         if (count > avail - pos)
1997                 count = avail - pos;
1998
1999         while (count) {
2000                 size_t len;
2001                 int ret, ofst;
2002                 __be32 data[16];
2003
2004                 if (mem == MEM_MC)
2005                         ret = t4_mc_read(adap, pos, data, NULL);
2006                 else
2007                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2008                 if (ret)
2009                         return ret;
2010
2011                 ofst = pos % sizeof(data);
2012                 len = min(count, sizeof(data) - ofst);
2013                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2014                         return -EFAULT;
2015
2016                 buf += len;
2017                 pos += len;
2018                 count -= len;
2019         }
2020         count = pos - *ppos;
2021         *ppos = pos;
2022         return count;
2023 }
2024
2025 static const struct file_operations mem_debugfs_fops = {
2026         .owner   = THIS_MODULE,
2027         .open    = mem_open,
2028         .read    = mem_read,
2029         .llseek  = default_llseek,
2030 };
2031
2032 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2033                                       unsigned int idx, unsigned int size_mb)
2034 {
2035         struct dentry *de;
2036
2037         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2038                                  (void *)adap + idx, &mem_debugfs_fops);
2039         if (de && de->d_inode)
2040                 de->d_inode->i_size = size_mb << 20;
2041 }
2042
2043 static int __devinit setup_debugfs(struct adapter *adap)
2044 {
2045         int i;
2046
2047         if (IS_ERR_OR_NULL(adap->debugfs_root))
2048                 return -1;
2049
2050         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2051         if (i & EDRAM0_ENABLE)
2052                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2053         if (i & EDRAM1_ENABLE)
2054                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2055         if (i & EXT_MEM_ENABLE)
2056                 add_debugfs_mem(adap, "mc", MEM_MC,
2057                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2058         if (adap->l2t)
2059                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2060                                     &t4_l2t_fops);
2061         return 0;
2062 }
2063
2064 /*
2065  * upper-layer driver support
2066  */
2067
2068 /*
2069  * Allocate an active-open TID and set it to the supplied value.
2070  */
2071 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2072 {
2073         int atid = -1;
2074
2075         spin_lock_bh(&t->atid_lock);
2076         if (t->afree) {
2077                 union aopen_entry *p = t->afree;
2078
2079                 atid = p - t->atid_tab;
2080                 t->afree = p->next;
2081                 p->data = data;
2082                 t->atids_in_use++;
2083         }
2084         spin_unlock_bh(&t->atid_lock);
2085         return atid;
2086 }
2087 EXPORT_SYMBOL(cxgb4_alloc_atid);
2088
2089 /*
2090  * Release an active-open TID.
2091  */
2092 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2093 {
2094         union aopen_entry *p = &t->atid_tab[atid];
2095
2096         spin_lock_bh(&t->atid_lock);
2097         p->next = t->afree;
2098         t->afree = p;
2099         t->atids_in_use--;
2100         spin_unlock_bh(&t->atid_lock);
2101 }
2102 EXPORT_SYMBOL(cxgb4_free_atid);
2103
2104 /*
2105  * Allocate a server TID and set it to the supplied value.
2106  */
2107 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2108 {
2109         int stid;
2110
2111         spin_lock_bh(&t->stid_lock);
2112         if (family == PF_INET) {
2113                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2114                 if (stid < t->nstids)
2115                         __set_bit(stid, t->stid_bmap);
2116                 else
2117                         stid = -1;
2118         } else {
2119                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2120                 if (stid < 0)
2121                         stid = -1;
2122         }
2123         if (stid >= 0) {
2124                 t->stid_tab[stid].data = data;
2125                 stid += t->stid_base;
2126                 t->stids_in_use++;
2127         }
2128         spin_unlock_bh(&t->stid_lock);
2129         return stid;
2130 }
2131 EXPORT_SYMBOL(cxgb4_alloc_stid);
2132
2133 /*
2134  * Release a server TID.
2135  */
2136 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2137 {
2138         stid -= t->stid_base;
2139         spin_lock_bh(&t->stid_lock);
2140         if (family == PF_INET)
2141                 __clear_bit(stid, t->stid_bmap);
2142         else
2143                 bitmap_release_region(t->stid_bmap, stid, 2);
2144         t->stid_tab[stid].data = NULL;
2145         t->stids_in_use--;
2146         spin_unlock_bh(&t->stid_lock);
2147 }
2148 EXPORT_SYMBOL(cxgb4_free_stid);
2149
2150 /*
2151  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2152  */
2153 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2154                            unsigned int tid)
2155 {
2156         struct cpl_tid_release *req;
2157
2158         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2159         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2160         INIT_TP_WR(req, tid);
2161         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2162 }
2163
2164 /*
2165  * Queue a TID release request and if necessary schedule a work queue to
2166  * process it.
2167  */
2168 void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2169                              unsigned int tid)
2170 {
2171         void **p = &t->tid_tab[tid];
2172         struct adapter *adap = container_of(t, struct adapter, tids);
2173
2174         spin_lock_bh(&adap->tid_release_lock);
2175         *p = adap->tid_release_head;
2176         /* Low 2 bits encode the Tx channel number */
2177         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2178         if (!adap->tid_release_task_busy) {
2179                 adap->tid_release_task_busy = true;
2180                 schedule_work(&adap->tid_release_task);
2181         }
2182         spin_unlock_bh(&adap->tid_release_lock);
2183 }
2184 EXPORT_SYMBOL(cxgb4_queue_tid_release);
2185
2186 /*
2187  * Process the list of pending TID release requests.
2188  */
2189 static void process_tid_release_list(struct work_struct *work)
2190 {
2191         struct sk_buff *skb;
2192         struct adapter *adap;
2193
2194         adap = container_of(work, struct adapter, tid_release_task);
2195
2196         spin_lock_bh(&adap->tid_release_lock);
2197         while (adap->tid_release_head) {
2198                 void **p = adap->tid_release_head;
2199                 unsigned int chan = (uintptr_t)p & 3;
2200                 p = (void *)p - chan;
2201
2202                 adap->tid_release_head = *p;
2203                 *p = NULL;
2204                 spin_unlock_bh(&adap->tid_release_lock);
2205
2206                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2207                                          GFP_KERNEL)))
2208                         schedule_timeout_uninterruptible(1);
2209
2210                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2211                 t4_ofld_send(adap, skb);
2212                 spin_lock_bh(&adap->tid_release_lock);
2213         }
2214         adap->tid_release_task_busy = false;
2215         spin_unlock_bh(&adap->tid_release_lock);
2216 }
2217
2218 /*
2219  * Release a TID and inform HW.  If we are unable to allocate the release
2220  * message we defer to a work queue.
2221  */
2222 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2223 {
2224         void *old;
2225         struct sk_buff *skb;
2226         struct adapter *adap = container_of(t, struct adapter, tids);
2227
2228         old = t->tid_tab[tid];
2229         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2230         if (likely(skb)) {
2231                 t->tid_tab[tid] = NULL;
2232                 mk_tid_release(skb, chan, tid);
2233                 t4_ofld_send(adap, skb);
2234         } else
2235                 cxgb4_queue_tid_release(t, chan, tid);
2236         if (old)
2237                 atomic_dec(&t->tids_in_use);
2238 }
2239 EXPORT_SYMBOL(cxgb4_remove_tid);
2240
2241 /*
2242  * Allocate and initialize the TID tables.  Returns 0 on success.
2243  */
2244 static int tid_init(struct tid_info *t)
2245 {
2246         size_t size;
2247         unsigned int natids = t->natids;
2248
2249         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2250                t->nstids * sizeof(*t->stid_tab) +
2251                BITS_TO_LONGS(t->nstids) * sizeof(long);
2252         t->tid_tab = t4_alloc_mem(size);
2253         if (!t->tid_tab)
2254                 return -ENOMEM;
2255
2256         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2257         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2258         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2259         spin_lock_init(&t->stid_lock);
2260         spin_lock_init(&t->atid_lock);
2261
2262         t->stids_in_use = 0;
2263         t->afree = NULL;
2264         t->atids_in_use = 0;
2265         atomic_set(&t->tids_in_use, 0);
2266
2267         /* Setup the free list for atid_tab and clear the stid bitmap. */
2268         if (natids) {
2269                 while (--natids)
2270                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2271                 t->afree = t->atid_tab;
2272         }
2273         bitmap_zero(t->stid_bmap, t->nstids);
2274         return 0;
2275 }
2276
2277 /**
2278  *      cxgb4_create_server - create an IP server
2279  *      @dev: the device
2280  *      @stid: the server TID
2281  *      @sip: local IP address to bind server to
2282  *      @sport: the server's TCP port
2283  *      @queue: queue to direct messages from this server to
2284  *
2285  *      Create an IP server for the given port and address.
2286  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2287  */
2288 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2289                         __be32 sip, __be16 sport, unsigned int queue)
2290 {
2291         unsigned int chan;
2292         struct sk_buff *skb;
2293         struct adapter *adap;
2294         struct cpl_pass_open_req *req;
2295
2296         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2297         if (!skb)
2298                 return -ENOMEM;
2299
2300         adap = netdev2adap(dev);
2301         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2302         INIT_TP_WR(req, 0);
2303         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2304         req->local_port = sport;
2305         req->peer_port = htons(0);
2306         req->local_ip = sip;
2307         req->peer_ip = htonl(0);
2308         chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2309         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2310         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2311                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2312         return t4_mgmt_tx(adap, skb);
2313 }
2314 EXPORT_SYMBOL(cxgb4_create_server);
2315
2316 /**
2317  *      cxgb4_create_server6 - create an IPv6 server
2318  *      @dev: the device
2319  *      @stid: the server TID
2320  *      @sip: local IPv6 address to bind server to
2321  *      @sport: the server's TCP port
2322  *      @queue: queue to direct messages from this server to
2323  *
2324  *      Create an IPv6 server for the given port and address.
2325  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2326  */
2327 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
2328                          const struct in6_addr *sip, __be16 sport,
2329                          unsigned int queue)
2330 {
2331         unsigned int chan;
2332         struct sk_buff *skb;
2333         struct adapter *adap;
2334         struct cpl_pass_open_req6 *req;
2335
2336         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2337         if (!skb)
2338                 return -ENOMEM;
2339
2340         adap = netdev2adap(dev);
2341         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
2342         INIT_TP_WR(req, 0);
2343         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
2344         req->local_port = sport;
2345         req->peer_port = htons(0);
2346         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
2347         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
2348         req->peer_ip_hi = cpu_to_be64(0);
2349         req->peer_ip_lo = cpu_to_be64(0);
2350         chan = netdev2pinfo(adap->sge.ingr_map[queue]->netdev)->tx_chan;
2351         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2352         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2353                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2354         return t4_mgmt_tx(adap, skb);
2355 }
2356 EXPORT_SYMBOL(cxgb4_create_server6);
2357
2358 /**
2359  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2360  *      @mtus: the HW MTU table
2361  *      @mtu: the target MTU
2362  *      @idx: index of selected entry in the MTU table
2363  *
2364  *      Returns the index and the value in the HW MTU table that is closest to
2365  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2366  *      table, in which case that smallest available value is selected.
2367  */
2368 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2369                             unsigned int *idx)
2370 {
2371         unsigned int i = 0;
2372
2373         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2374                 ++i;
2375         if (idx)
2376                 *idx = i;
2377         return mtus[i];
2378 }
2379 EXPORT_SYMBOL(cxgb4_best_mtu);
2380
2381 /**
2382  *      cxgb4_port_chan - get the HW channel of a port
2383  *      @dev: the net device for the port
2384  *
2385  *      Return the HW Tx channel of the given port.
2386  */
2387 unsigned int cxgb4_port_chan(const struct net_device *dev)
2388 {
2389         return netdev2pinfo(dev)->tx_chan;
2390 }
2391 EXPORT_SYMBOL(cxgb4_port_chan);
2392
2393 /**
2394  *      cxgb4_port_viid - get the VI id of a port
2395  *      @dev: the net device for the port
2396  *
2397  *      Return the VI id of the given port.
2398  */
2399 unsigned int cxgb4_port_viid(const struct net_device *dev)
2400 {
2401         return netdev2pinfo(dev)->viid;
2402 }
2403 EXPORT_SYMBOL(cxgb4_port_viid);
2404
2405 /**
2406  *      cxgb4_port_idx - get the index of a port
2407  *      @dev: the net device for the port
2408  *
2409  *      Return the index of the given port.
2410  */
2411 unsigned int cxgb4_port_idx(const struct net_device *dev)
2412 {
2413         return netdev2pinfo(dev)->port_id;
2414 }
2415 EXPORT_SYMBOL(cxgb4_port_idx);
2416
2417 /**
2418  *      cxgb4_netdev_by_hwid - return the net device of a HW port
2419  *      @pdev: identifies the adapter
2420  *      @id: the HW port id
2421  *
2422  *      Return the net device associated with the interface with the given HW
2423  *      id.
2424  */
2425 struct net_device *cxgb4_netdev_by_hwid(struct pci_dev *pdev, unsigned int id)
2426 {
2427         const struct adapter *adap = pci_get_drvdata(pdev);
2428
2429         if (!adap || id >= NCHAN)
2430                 return NULL;
2431         id = adap->chan_map[id];
2432         return id < MAX_NPORTS ? adap->port[id] : NULL;
2433 }
2434 EXPORT_SYMBOL(cxgb4_netdev_by_hwid);
2435
2436 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2437                          struct tp_tcp_stats *v6)
2438 {
2439         struct adapter *adap = pci_get_drvdata(pdev);
2440
2441         spin_lock(&adap->stats_lock);
2442         t4_tp_get_tcp_stats(adap, v4, v6);
2443         spin_unlock(&adap->stats_lock);
2444 }
2445 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2446
2447 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2448                       const unsigned int *pgsz_order)
2449 {
2450         struct adapter *adap = netdev2adap(dev);
2451
2452         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2453         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2454                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2455                      HPZ3(pgsz_order[3]));
2456 }
2457 EXPORT_SYMBOL(cxgb4_iscsi_init);
2458
2459 static struct pci_driver cxgb4_driver;
2460
2461 static void check_neigh_update(struct neighbour *neigh)
2462 {
2463         const struct device *parent;
2464         const struct net_device *netdev = neigh->dev;
2465
2466         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2467                 netdev = vlan_dev_real_dev(netdev);
2468         parent = netdev->dev.parent;
2469         if (parent && parent->driver == &cxgb4_driver.driver)
2470                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2471 }
2472
2473 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2474                        void *data)
2475 {
2476         switch (event) {
2477         case NETEVENT_NEIGH_UPDATE:
2478                 check_neigh_update(data);
2479                 break;
2480         case NETEVENT_PMTU_UPDATE:
2481         case NETEVENT_REDIRECT:
2482         default:
2483                 break;
2484         }
2485         return 0;
2486 }
2487
2488 static bool netevent_registered;
2489 static struct notifier_block cxgb4_netevent_nb = {
2490         .notifier_call = netevent_cb
2491 };
2492
2493 static void uld_attach(struct adapter *adap, unsigned int uld)
2494 {
2495         void *handle;
2496         struct cxgb4_lld_info lli;
2497
2498         lli.pdev = adap->pdev;
2499         lli.l2t = adap->l2t;
2500         lli.tids = &adap->tids;
2501         lli.ports = adap->port;
2502         lli.vr = &adap->vres;
2503         lli.mtus = adap->params.mtus;
2504         if (uld == CXGB4_ULD_RDMA) {
2505                 lli.rxq_ids = adap->sge.rdma_rxq;
2506                 lli.nrxq = adap->sge.rdmaqs;
2507         } else if (uld == CXGB4_ULD_ISCSI) {
2508                 lli.rxq_ids = adap->sge.ofld_rxq;
2509                 lli.nrxq = adap->sge.ofldqsets;
2510         }
2511         lli.ntxq = adap->sge.ofldqsets;
2512         lli.nchan = adap->params.nports;
2513         lli.nports = adap->params.nports;
2514         lli.wr_cred = adap->params.ofldq_wr_cred;
2515         lli.adapter_type = adap->params.rev;
2516         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2517         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2518                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2519                         (adap->fn * 4));
2520         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2521                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2522                         (adap->fn * 4));
2523         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2524         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2525         lli.fw_vers = adap->params.fw_vers;
2526
2527         handle = ulds[uld].add(&lli);
2528         if (IS_ERR(handle)) {
2529                 dev_warn(adap->pdev_dev,
2530                          "could not attach to the %s driver, error %ld\n",
2531                          uld_str[uld], PTR_ERR(handle));
2532                 return;
2533         }
2534
2535         adap->uld_handle[uld] = handle;
2536
2537         if (!netevent_registered) {
2538                 register_netevent_notifier(&cxgb4_netevent_nb);
2539                 netevent_registered = true;
2540         }
2541
2542         if (adap->flags & FULL_INIT_DONE)
2543                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2544 }
2545
2546 static void attach_ulds(struct adapter *adap)
2547 {
2548         unsigned int i;
2549
2550         mutex_lock(&uld_mutex);
2551         list_add_tail(&adap->list_node, &adapter_list);
2552         for (i = 0; i < CXGB4_ULD_MAX; i++)
2553                 if (ulds[i].add)
2554                         uld_attach(adap, i);
2555         mutex_unlock(&uld_mutex);
2556 }
2557
2558 static void detach_ulds(struct adapter *adap)
2559 {
2560         unsigned int i;
2561
2562         mutex_lock(&uld_mutex);
2563         list_del(&adap->list_node);
2564         for (i = 0; i < CXGB4_ULD_MAX; i++)
2565                 if (adap->uld_handle[i]) {
2566                         ulds[i].state_change(adap->uld_handle[i],
2567                                              CXGB4_STATE_DETACH);
2568                         adap->uld_handle[i] = NULL;
2569                 }
2570         if (netevent_registered && list_empty(&adapter_list)) {
2571                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2572                 netevent_registered = false;
2573         }
2574         mutex_unlock(&uld_mutex);
2575 }
2576
2577 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2578 {
2579         unsigned int i;
2580
2581         mutex_lock(&uld_mutex);
2582         for (i = 0; i < CXGB4_ULD_MAX; i++)
2583                 if (adap->uld_handle[i])
2584                         ulds[i].state_change(adap->uld_handle[i], new_state);
2585         mutex_unlock(&uld_mutex);
2586 }
2587
2588 /**
2589  *      cxgb4_register_uld - register an upper-layer driver
2590  *      @type: the ULD type
2591  *      @p: the ULD methods
2592  *
2593  *      Registers an upper-layer driver with this driver and notifies the ULD
2594  *      about any presently available devices that support its type.  Returns
2595  *      %-EBUSY if a ULD of the same type is already registered.
2596  */
2597 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2598 {
2599         int ret = 0;
2600         struct adapter *adap;
2601
2602         if (type >= CXGB4_ULD_MAX)
2603                 return -EINVAL;
2604         mutex_lock(&uld_mutex);
2605         if (ulds[type].add) {
2606                 ret = -EBUSY;
2607                 goto out;
2608         }
2609         ulds[type] = *p;
2610         list_for_each_entry(adap, &adapter_list, list_node)
2611                 uld_attach(adap, type);
2612 out:    mutex_unlock(&uld_mutex);
2613         return ret;
2614 }
2615 EXPORT_SYMBOL(cxgb4_register_uld);
2616
2617 /**
2618  *      cxgb4_unregister_uld - unregister an upper-layer driver
2619  *      @type: the ULD type
2620  *
2621  *      Unregisters an existing upper-layer driver.
2622  */
2623 int cxgb4_unregister_uld(enum cxgb4_uld type)
2624 {
2625         struct adapter *adap;
2626
2627         if (type >= CXGB4_ULD_MAX)
2628                 return -EINVAL;
2629         mutex_lock(&uld_mutex);
2630         list_for_each_entry(adap, &adapter_list, list_node)
2631                 adap->uld_handle[type] = NULL;
2632         ulds[type].add = NULL;
2633         mutex_unlock(&uld_mutex);
2634         return 0;
2635 }
2636 EXPORT_SYMBOL(cxgb4_unregister_uld);
2637
2638 /**
2639  *      cxgb_up - enable the adapter
2640  *      @adap: adapter being enabled
2641  *
2642  *      Called when the first port is enabled, this function performs the
2643  *      actions necessary to make an adapter operational, such as completing
2644  *      the initialization of HW modules, and enabling interrupts.
2645  *
2646  *      Must be called with the rtnl lock held.
2647  */
2648 static int cxgb_up(struct adapter *adap)
2649 {
2650         int err;
2651
2652         err = setup_sge_queues(adap);
2653         if (err)
2654                 goto out;
2655         err = setup_rss(adap);
2656         if (err)
2657                 goto freeq;
2658
2659         if (adap->flags & USING_MSIX) {
2660                 name_msix_vecs(adap);
2661                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2662                                   adap->msix_info[0].desc, adap);
2663                 if (err)
2664                         goto irq_err;
2665
2666                 err = request_msix_queue_irqs(adap);
2667                 if (err) {
2668                         free_irq(adap->msix_info[0].vec, adap);
2669                         goto irq_err;
2670                 }
2671         } else {
2672                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2673                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2674                                   adap->name, adap);
2675                 if (err)
2676                         goto irq_err;
2677         }
2678         enable_rx(adap);
2679         t4_sge_start(adap);
2680         t4_intr_enable(adap);
2681         adap->flags |= FULL_INIT_DONE;
2682         notify_ulds(adap, CXGB4_STATE_UP);
2683  out:
2684         return err;
2685  irq_err:
2686         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2687  freeq:
2688         t4_free_sge_resources(adap);
2689         goto out;
2690 }
2691
2692 static void cxgb_down(struct adapter *adapter)
2693 {
2694         t4_intr_disable(adapter);
2695         cancel_work_sync(&adapter->tid_release_task);
2696         adapter->tid_release_task_busy = false;
2697         adapter->tid_release_head = NULL;
2698
2699         if (adapter->flags & USING_MSIX) {
2700                 free_msix_queue_irqs(adapter);
2701                 free_irq(adapter->msix_info[0].vec, adapter);
2702         } else
2703                 free_irq(adapter->pdev->irq, adapter);
2704         quiesce_rx(adapter);
2705         t4_sge_stop(adapter);
2706         t4_free_sge_resources(adapter);
2707         adapter->flags &= ~FULL_INIT_DONE;
2708 }
2709
2710 /*
2711  * net_device operations
2712  */
2713 static int cxgb_open(struct net_device *dev)
2714 {
2715         int err;
2716         struct port_info *pi = netdev_priv(dev);
2717         struct adapter *adapter = pi->adapter;
2718
2719         if (!(adapter->flags & FULL_INIT_DONE)) {
2720                 err = cxgb_up(adapter);
2721                 if (err < 0)
2722                         return err;
2723         }
2724
2725         dev->real_num_tx_queues = pi->nqsets;
2726         err = link_start(dev);
2727         if (!err)
2728                 netif_tx_start_all_queues(dev);
2729         return err;
2730 }
2731
2732 static int cxgb_close(struct net_device *dev)
2733 {
2734         struct port_info *pi = netdev_priv(dev);
2735         struct adapter *adapter = pi->adapter;
2736
2737         netif_tx_stop_all_queues(dev);
2738         netif_carrier_off(dev);
2739         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2740 }
2741
2742 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2743                                                 struct rtnl_link_stats64 *ns)
2744 {
2745         struct port_stats stats;
2746         struct port_info *p = netdev_priv(dev);
2747         struct adapter *adapter = p->adapter;
2748
2749         spin_lock(&adapter->stats_lock);
2750         t4_get_port_stats(adapter, p->tx_chan, &stats);
2751         spin_unlock(&adapter->stats_lock);
2752
2753         ns->tx_bytes   = stats.tx_octets;
2754         ns->tx_packets = stats.tx_frames;
2755         ns->rx_bytes   = stats.rx_octets;
2756         ns->rx_packets = stats.rx_frames;
2757         ns->multicast  = stats.rx_mcast_frames;
2758
2759         /* detailed rx_errors */
2760         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2761                                stats.rx_runt;
2762         ns->rx_over_errors   = 0;
2763         ns->rx_crc_errors    = stats.rx_fcs_err;
2764         ns->rx_frame_errors  = stats.rx_symbol_err;
2765         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
2766                                stats.rx_ovflow2 + stats.rx_ovflow3 +
2767                                stats.rx_trunc0 + stats.rx_trunc1 +
2768                                stats.rx_trunc2 + stats.rx_trunc3;
2769         ns->rx_missed_errors = 0;
2770
2771         /* detailed tx_errors */
2772         ns->tx_aborted_errors   = 0;
2773         ns->tx_carrier_errors   = 0;
2774         ns->tx_fifo_errors      = 0;
2775         ns->tx_heartbeat_errors = 0;
2776         ns->tx_window_errors    = 0;
2777
2778         ns->tx_errors = stats.tx_error_frames;
2779         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2780                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2781         return ns;
2782 }
2783
2784 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2785 {
2786         unsigned int mbox;
2787         int ret = 0, prtad, devad;
2788         struct port_info *pi = netdev_priv(dev);
2789         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2790
2791         switch (cmd) {
2792         case SIOCGMIIPHY:
2793                 if (pi->mdio_addr < 0)
2794                         return -EOPNOTSUPP;
2795                 data->phy_id = pi->mdio_addr;
2796                 break;
2797         case SIOCGMIIREG:
2798         case SIOCSMIIREG:
2799                 if (mdio_phy_id_is_c45(data->phy_id)) {
2800                         prtad = mdio_phy_id_prtad(data->phy_id);
2801                         devad = mdio_phy_id_devad(data->phy_id);
2802                 } else if (data->phy_id < 32) {
2803                         prtad = data->phy_id;
2804                         devad = 0;
2805                         data->reg_num &= 0x1f;
2806                 } else
2807                         return -EINVAL;
2808
2809                 mbox = pi->adapter->fn;
2810                 if (cmd == SIOCGMIIREG)
2811                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2812                                          data->reg_num, &data->val_out);
2813                 else
2814                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2815                                          data->reg_num, data->val_in);
2816                 break;
2817         default:
2818                 return -EOPNOTSUPP;
2819         }
2820         return ret;
2821 }
2822
2823 static void cxgb_set_rxmode(struct net_device *dev)
2824 {
2825         /* unfortunately we can't return errors to the stack */
2826         set_rxmode(dev, -1, false);
2827 }
2828
2829 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2830 {
2831         int ret;
2832         struct port_info *pi = netdev_priv(dev);
2833
2834         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
2835                 return -EINVAL;
2836         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2837                             -1, -1, -1, true);
2838         if (!ret)
2839                 dev->mtu = new_mtu;
2840         return ret;
2841 }
2842
2843 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2844 {
2845         int ret;
2846         struct sockaddr *addr = p;
2847         struct port_info *pi = netdev_priv(dev);
2848
2849         if (!is_valid_ether_addr(addr->sa_data))
2850                 return -EINVAL;
2851
2852         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2853                             pi->xact_addr_filt, addr->sa_data, true, true);
2854         if (ret < 0)
2855                 return ret;
2856
2857         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2858         pi->xact_addr_filt = ret;
2859         return 0;
2860 }
2861
2862 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2863 {
2864         struct port_info *pi = netdev_priv(dev);
2865
2866         pi->vlan_grp = grp;
2867         t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1, -1, -1, -1,
2868                       grp != NULL, true);
2869 }
2870
2871 #ifdef CONFIG_NET_POLL_CONTROLLER
2872 static void cxgb_netpoll(struct net_device *dev)
2873 {
2874         struct port_info *pi = netdev_priv(dev);
2875         struct adapter *adap = pi->adapter;
2876
2877         if (adap->flags & USING_MSIX) {
2878                 int i;
2879                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2880
2881                 for (i = pi->nqsets; i; i--, rx++)
2882                         t4_sge_intr_msix(0, &rx->rspq);
2883         } else
2884                 t4_intr_handler(adap)(0, adap);
2885 }
2886 #endif
2887
2888 static const struct net_device_ops cxgb4_netdev_ops = {
2889         .ndo_open             = cxgb_open,
2890         .ndo_stop             = cxgb_close,
2891         .ndo_start_xmit       = t4_eth_xmit,
2892         .ndo_get_stats64      = cxgb_get_stats,
2893         .ndo_set_rx_mode      = cxgb_set_rxmode,
2894         .ndo_set_mac_address  = cxgb_set_mac_addr,
2895         .ndo_validate_addr    = eth_validate_addr,
2896         .ndo_do_ioctl         = cxgb_ioctl,
2897         .ndo_change_mtu       = cxgb_change_mtu,
2898         .ndo_vlan_rx_register = vlan_rx_register,
2899 #ifdef CONFIG_NET_POLL_CONTROLLER
2900         .ndo_poll_controller  = cxgb_netpoll,
2901 #endif
2902 };
2903
2904 void t4_fatal_err(struct adapter *adap)
2905 {
2906         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2907         t4_intr_disable(adap);
2908         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2909 }
2910
2911 static void setup_memwin(struct adapter *adap)
2912 {
2913         u32 bar0;
2914
2915         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
2916         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2917                      (bar0 + MEMWIN0_BASE) | BIR(0) |
2918                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2919         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2920                      (bar0 + MEMWIN1_BASE) | BIR(0) |
2921                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2922         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2923                      (bar0 + MEMWIN2_BASE) | BIR(0) |
2924                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2925         if (adap->vres.ocq.size) {
2926                 unsigned int start, sz_kb;
2927
2928                 start = pci_resource_start(adap->pdev, 2) +
2929                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2930                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2931                 t4_write_reg(adap,
2932                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2933                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
2934                 t4_write_reg(adap,
2935                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2936                              adap->vres.ocq.start);
2937                 t4_read_reg(adap,
2938                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2939         }
2940 }
2941
2942 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2943 {
2944         u32 v;
2945         int ret;
2946
2947         /* get device capabilities */
2948         memset(c, 0, sizeof(*c));
2949         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2950                                FW_CMD_REQUEST | FW_CMD_READ);
2951         c->retval_len16 = htonl(FW_LEN16(*c));
2952         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
2953         if (ret < 0)
2954                 return ret;
2955
2956         /* select capabilities we'll be using */
2957         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2958                 if (!vf_acls)
2959                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2960                 else
2961                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2962         } else if (vf_acls) {
2963                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2964                 return ret;
2965         }
2966         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2967                                FW_CMD_REQUEST | FW_CMD_WRITE);
2968         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
2969         if (ret < 0)
2970                 return ret;
2971
2972         ret = t4_config_glbl_rss(adap, adap->fn,
2973                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2974                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2975                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2976         if (ret < 0)
2977                 return ret;
2978
2979         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2980                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2981         if (ret < 0)
2982                 return ret;
2983
2984         t4_sge_init(adap);
2985
2986         /* tweak some settings */
2987         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2988         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2989         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2990         v = t4_read_reg(adap, TP_PIO_DATA);
2991         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2992
2993         /* get basic stuff going */
2994         return t4_early_init(adap, adap->fn);
2995 }
2996
2997 /*
2998  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
2999  */
3000 #define MAX_ATIDS 8192U
3001
3002 /*
3003  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
3004  */
3005 static int adap_init0(struct adapter *adap)
3006 {
3007         int ret;
3008         u32 v, port_vec;
3009         enum dev_state state;
3010         u32 params[7], val[7];
3011         struct fw_caps_config_cmd c;
3012
3013         ret = t4_check_fw_version(adap);
3014         if (ret == -EINVAL || ret > 0) {
3015                 if (upgrade_fw(adap) >= 0)             /* recache FW version */
3016                         ret = t4_check_fw_version(adap);
3017         }
3018         if (ret < 0)
3019                 return ret;
3020
3021         /* contact FW, request master */
3022         ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
3023         if (ret < 0) {
3024                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
3025                         ret);
3026                 return ret;
3027         }
3028
3029         /* reset device */
3030         ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
3031         if (ret < 0)
3032                 goto bye;
3033
3034         for (v = 0; v < SGE_NTIMERS - 1; v++)
3035                 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
3036         adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
3037         adap->sge.counter_val[0] = 1;
3038         for (v = 1; v < SGE_NCOUNTERS; v++)
3039                 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
3040                                                THRESHOLD_3_MASK);
3041 #define FW_PARAM_DEV(param) \
3042         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
3043          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
3044
3045         params[0] = FW_PARAM_DEV(CCLK);
3046         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
3047         if (ret < 0)
3048                 goto bye;
3049         adap->params.vpd.cclk = val[0];
3050
3051         ret = adap_init1(adap, &c);
3052         if (ret < 0)
3053                 goto bye;
3054
3055 #define FW_PARAM_PFVF(param) \
3056         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3057          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3058          FW_PARAMS_PARAM_Y(adap->fn))
3059
3060         params[0] = FW_PARAM_DEV(PORTVEC);
3061         params[1] = FW_PARAM_PFVF(L2T_START);
3062         params[2] = FW_PARAM_PFVF(L2T_END);
3063         params[3] = FW_PARAM_PFVF(FILTER_START);
3064         params[4] = FW_PARAM_PFVF(FILTER_END);
3065         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 5, params, val);
3066         if (ret < 0)
3067                 goto bye;
3068         port_vec = val[0];
3069         adap->tids.ftid_base = val[3];
3070         adap->tids.nftids = val[4] - val[3] + 1;
3071
3072         if (c.ofldcaps) {
3073                 /* query offload-related parameters */
3074                 params[0] = FW_PARAM_DEV(NTID);
3075                 params[1] = FW_PARAM_PFVF(SERVER_START);
3076                 params[2] = FW_PARAM_PFVF(SERVER_END);
3077                 params[3] = FW_PARAM_PFVF(TDDP_START);
3078                 params[4] = FW_PARAM_PFVF(TDDP_END);
3079                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3080                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3081                                       val);
3082                 if (ret < 0)
3083                         goto bye;
3084                 adap->tids.ntids = val[0];
3085                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3086                 adap->tids.stid_base = val[1];
3087                 adap->tids.nstids = val[2] - val[1] + 1;
3088                 adap->vres.ddp.start = val[3];
3089                 adap->vres.ddp.size = val[4] - val[3] + 1;
3090                 adap->params.ofldq_wr_cred = val[5];
3091                 adap->params.offload = 1;
3092         }
3093         if (c.rdmacaps) {
3094                 params[0] = FW_PARAM_PFVF(STAG_START);
3095                 params[1] = FW_PARAM_PFVF(STAG_END);
3096                 params[2] = FW_PARAM_PFVF(RQ_START);
3097                 params[3] = FW_PARAM_PFVF(RQ_END);
3098                 params[4] = FW_PARAM_PFVF(PBL_START);
3099                 params[5] = FW_PARAM_PFVF(PBL_END);
3100                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3101                                       val);
3102                 if (ret < 0)
3103                         goto bye;
3104                 adap->vres.stag.start = val[0];
3105                 adap->vres.stag.size = val[1] - val[0] + 1;
3106                 adap->vres.rq.start = val[2];
3107                 adap->vres.rq.size = val[3] - val[2] + 1;
3108                 adap->vres.pbl.start = val[4];
3109                 adap->vres.pbl.size = val[5] - val[4] + 1;
3110
3111                 params[0] = FW_PARAM_PFVF(SQRQ_START);
3112                 params[1] = FW_PARAM_PFVF(SQRQ_END);
3113                 params[2] = FW_PARAM_PFVF(CQ_START);
3114                 params[3] = FW_PARAM_PFVF(CQ_END);
3115                 params[4] = FW_PARAM_PFVF(OCQ_START);
3116                 params[5] = FW_PARAM_PFVF(OCQ_END);
3117                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3118                                       val);
3119                 if (ret < 0)
3120                         goto bye;
3121                 adap->vres.qp.start = val[0];
3122                 adap->vres.qp.size = val[1] - val[0] + 1;
3123                 adap->vres.cq.start = val[2];
3124                 adap->vres.cq.size = val[3] - val[2] + 1;
3125                 adap->vres.ocq.start = val[4];
3126                 adap->vres.ocq.size = val[5] - val[4] + 1;
3127         }
3128         if (c.iscsicaps) {
3129                 params[0] = FW_PARAM_PFVF(ISCSI_START);
3130                 params[1] = FW_PARAM_PFVF(ISCSI_END);
3131                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3132                                       val);
3133                 if (ret < 0)
3134                         goto bye;
3135                 adap->vres.iscsi.start = val[0];
3136                 adap->vres.iscsi.size = val[1] - val[0] + 1;
3137         }
3138 #undef FW_PARAM_PFVF
3139 #undef FW_PARAM_DEV
3140
3141         adap->params.nports = hweight32(port_vec);
3142         adap->params.portvec = port_vec;
3143         adap->flags |= FW_OK;
3144
3145         /* These are finalized by FW initialization, load their values now */
3146         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3147         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3148         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3149         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3150                      adap->params.b_wnd);
3151
3152 #ifdef CONFIG_PCI_IOV
3153         /*
3154          * Provision resource limits for Virtual Functions.  We currently
3155          * grant them all the same static resource limits except for the Port
3156          * Access Rights Mask which we're assigning based on the PF.  All of
3157          * the static provisioning stuff for both the PF and VF really needs
3158          * to be managed in a persistent manner for each device which the
3159          * firmware controls.
3160          */
3161         {
3162                 int pf, vf;
3163
3164                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3165                         if (num_vf[pf] <= 0)
3166                                 continue;
3167
3168                         /* VF numbering starts at 1! */
3169                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3170                                 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3171                                                   VFRES_NEQ, VFRES_NETHCTRL,
3172                                                   VFRES_NIQFLINT, VFRES_NIQ,
3173                                                   VFRES_TC, VFRES_NVI,
3174                                                   FW_PFVF_CMD_CMASK_MASK,
3175                                                   pfvfres_pmask(adap, pf, vf),
3176                                                   VFRES_NEXACTF,
3177                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3178                                 if (ret < 0)
3179                                         dev_warn(adap->pdev_dev, "failed to "
3180                                                  "provision pf/vf=%d/%d; "
3181                                                  "err=%d\n", pf, vf, ret);
3182                         }
3183                 }
3184         }
3185 #endif
3186
3187         setup_memwin(adap);
3188         return 0;
3189
3190         /*
3191          * If a command timed out or failed with EIO FW does not operate within
3192          * its spec or something catastrophic happened to HW/FW, stop issuing
3193          * commands.
3194          */
3195 bye:    if (ret != -ETIMEDOUT && ret != -EIO)
3196                 t4_fw_bye(adap, adap->fn);
3197         return ret;
3198 }
3199
3200 /* EEH callbacks */
3201
3202 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3203                                          pci_channel_state_t state)
3204 {
3205         int i;
3206         struct adapter *adap = pci_get_drvdata(pdev);
3207
3208         if (!adap)
3209                 goto out;
3210
3211         rtnl_lock();
3212         adap->flags &= ~FW_OK;
3213         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3214         for_each_port(adap, i) {
3215                 struct net_device *dev = adap->port[i];
3216
3217                 netif_device_detach(dev);
3218                 netif_carrier_off(dev);
3219         }
3220         if (adap->flags & FULL_INIT_DONE)
3221                 cxgb_down(adap);
3222         rtnl_unlock();
3223         pci_disable_device(pdev);
3224 out:    return state == pci_channel_io_perm_failure ?
3225                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3226 }
3227
3228 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3229 {
3230         int i, ret;
3231         struct fw_caps_config_cmd c;
3232         struct adapter *adap = pci_get_drvdata(pdev);
3233
3234         if (!adap) {
3235                 pci_restore_state(pdev);
3236                 pci_save_state(pdev);
3237                 return PCI_ERS_RESULT_RECOVERED;
3238         }
3239
3240         if (pci_enable_device(pdev)) {
3241                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3242                 return PCI_ERS_RESULT_DISCONNECT;
3243         }
3244
3245         pci_set_master(pdev);
3246         pci_restore_state(pdev);
3247         pci_save_state(pdev);
3248         pci_cleanup_aer_uncorrect_error_status(pdev);
3249
3250         if (t4_wait_dev_ready(adap) < 0)
3251                 return PCI_ERS_RESULT_DISCONNECT;
3252         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3253                 return PCI_ERS_RESULT_DISCONNECT;
3254         adap->flags |= FW_OK;
3255         if (adap_init1(adap, &c))
3256                 return PCI_ERS_RESULT_DISCONNECT;
3257
3258         for_each_port(adap, i) {
3259                 struct port_info *p = adap2pinfo(adap, i);
3260
3261                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3262                                   NULL, NULL);
3263                 if (ret < 0)
3264                         return PCI_ERS_RESULT_DISCONNECT;
3265                 p->viid = ret;
3266                 p->xact_addr_filt = -1;
3267         }
3268
3269         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3270                      adap->params.b_wnd);
3271         setup_memwin(adap);
3272         if (cxgb_up(adap))
3273                 return PCI_ERS_RESULT_DISCONNECT;
3274         return PCI_ERS_RESULT_RECOVERED;
3275 }
3276
3277 static void eeh_resume(struct pci_dev *pdev)
3278 {
3279         int i;
3280         struct adapter *adap = pci_get_drvdata(pdev);
3281
3282         if (!adap)
3283                 return;
3284
3285         rtnl_lock();
3286         for_each_port(adap, i) {
3287                 struct net_device *dev = adap->port[i];
3288
3289                 if (netif_running(dev)) {
3290                         link_start(dev);
3291                         cxgb_set_rxmode(dev);
3292                 }
3293                 netif_device_attach(dev);
3294         }
3295         rtnl_unlock();
3296 }
3297
3298 static struct pci_error_handlers cxgb4_eeh = {
3299         .error_detected = eeh_err_detected,
3300         .slot_reset     = eeh_slot_reset,
3301         .resume         = eeh_resume,
3302 };
3303
3304 static inline bool is_10g_port(const struct link_config *lc)
3305 {
3306         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3307 }
3308
3309 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3310                              unsigned int size, unsigned int iqe_size)
3311 {
3312         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3313                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3314         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3315         q->iqe_len = iqe_size;
3316         q->size = size;
3317 }
3318
3319 /*
3320  * Perform default configuration of DMA queues depending on the number and type
3321  * of ports we found and the number of available CPUs.  Most settings can be
3322  * modified by the admin prior to actual use.
3323  */
3324 static void __devinit cfg_queues(struct adapter *adap)
3325 {
3326         struct sge *s = &adap->sge;
3327         int i, q10g = 0, n10g = 0, qidx = 0;
3328
3329         for_each_port(adap, i)
3330                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3331
3332         /*
3333          * We default to 1 queue per non-10G port and up to # of cores queues
3334          * per 10G port.
3335          */
3336         if (n10g)
3337                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3338         if (q10g > num_online_cpus())
3339                 q10g = num_online_cpus();
3340
3341         for_each_port(adap, i) {
3342                 struct port_info *pi = adap2pinfo(adap, i);
3343
3344                 pi->first_qset = qidx;
3345                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3346                 qidx += pi->nqsets;
3347         }
3348
3349         s->ethqsets = qidx;
3350         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
3351
3352         if (is_offload(adap)) {
3353                 /*
3354                  * For offload we use 1 queue/channel if all ports are up to 1G,
3355                  * otherwise we divide all available queues amongst the channels
3356                  * capped by the number of available cores.
3357                  */
3358                 if (n10g) {
3359                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3360                                   num_online_cpus());
3361                         s->ofldqsets = roundup(i, adap->params.nports);
3362                 } else
3363                         s->ofldqsets = adap->params.nports;
3364                 /* For RDMA one Rx queue per channel suffices */
3365                 s->rdmaqs = adap->params.nports;
3366         }
3367
3368         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3369                 struct sge_eth_rxq *r = &s->ethrxq[i];
3370
3371                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3372                 r->fl.size = 72;
3373         }
3374
3375         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3376                 s->ethtxq[i].q.size = 1024;
3377
3378         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3379                 s->ctrlq[i].q.size = 512;
3380
3381         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3382                 s->ofldtxq[i].q.size = 1024;
3383
3384         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3385                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3386
3387                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3388                 r->rspq.uld = CXGB4_ULD_ISCSI;
3389                 r->fl.size = 72;
3390         }
3391
3392         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3393                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3394
3395                 init_rspq(&r->rspq, 0, 0, 511, 64);
3396                 r->rspq.uld = CXGB4_ULD_RDMA;
3397                 r->fl.size = 72;
3398         }
3399
3400         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3401         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3402 }
3403
3404 /*
3405  * Reduce the number of Ethernet queues across all ports to at most n.
3406  * n provides at least one queue per port.
3407  */
3408 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3409 {
3410         int i;
3411         struct port_info *pi;
3412
3413         while (n < adap->sge.ethqsets)
3414                 for_each_port(adap, i) {
3415                         pi = adap2pinfo(adap, i);
3416                         if (pi->nqsets > 1) {
3417                                 pi->nqsets--;
3418                                 adap->sge.ethqsets--;
3419                                 if (adap->sge.ethqsets <= n)
3420                                         break;
3421                         }
3422                 }
3423
3424         n = 0;
3425         for_each_port(adap, i) {
3426                 pi = adap2pinfo(adap, i);
3427                 pi->first_qset = n;
3428                 n += pi->nqsets;
3429         }
3430 }
3431
3432 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3433 #define EXTRA_VECS 2
3434
3435 static int __devinit enable_msix(struct adapter *adap)
3436 {
3437         int ofld_need = 0;
3438         int i, err, want, need;
3439         struct sge *s = &adap->sge;
3440         unsigned int nchan = adap->params.nports;
3441         struct msix_entry entries[MAX_INGQ + 1];
3442
3443         for (i = 0; i < ARRAY_SIZE(entries); ++i)
3444                 entries[i].entry = i;
3445
3446         want = s->max_ethqsets + EXTRA_VECS;
3447         if (is_offload(adap)) {
3448                 want += s->rdmaqs + s->ofldqsets;
3449                 /* need nchan for each possible ULD */
3450                 ofld_need = 2 * nchan;
3451         }
3452         need = adap->params.nports + EXTRA_VECS + ofld_need;
3453
3454         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3455                 want = err;
3456
3457         if (!err) {
3458                 /*
3459                  * Distribute available vectors to the various queue groups.
3460                  * Every group gets its minimum requirement and NIC gets top
3461                  * priority for leftovers.
3462                  */
3463                 i = want - EXTRA_VECS - ofld_need;
3464                 if (i < s->max_ethqsets) {
3465                         s->max_ethqsets = i;
3466                         if (i < s->ethqsets)
3467                                 reduce_ethqs(adap, i);
3468                 }
3469                 if (is_offload(adap)) {
3470                         i = want - EXTRA_VECS - s->max_ethqsets;
3471                         i -= ofld_need - nchan;
3472                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
3473                 }
3474                 for (i = 0; i < want; ++i)
3475                         adap->msix_info[i].vec = entries[i].vector;
3476         } else if (err > 0)
3477                 dev_info(adap->pdev_dev,
3478                          "only %d MSI-X vectors left, not using MSI-X\n", err);
3479         return err;
3480 }
3481
3482 #undef EXTRA_VECS
3483
3484 static int __devinit init_rss(struct adapter *adap)
3485 {
3486         unsigned int i, j;
3487
3488         for_each_port(adap, i) {
3489                 struct port_info *pi = adap2pinfo(adap, i);
3490
3491                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3492                 if (!pi->rss)
3493                         return -ENOMEM;
3494                 for (j = 0; j < pi->rss_size; j++)
3495                         pi->rss[j] = j % pi->nqsets;
3496         }
3497         return 0;
3498 }
3499
3500 static void __devinit print_port_info(struct adapter *adap)
3501 {
3502         static const char *base[] = {
3503                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3504                 "KX", "KR", "KR SFP+", "KR FEC"
3505         };
3506
3507         int i;
3508         char buf[80];
3509         const char *spd = "";
3510
3511         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3512                 spd = " 2.5 GT/s";
3513         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3514                 spd = " 5 GT/s";
3515
3516         for_each_port(adap, i) {
3517                 struct net_device *dev = adap->port[i];
3518                 const struct port_info *pi = netdev_priv(dev);
3519                 char *bufp = buf;
3520
3521                 if (!test_bit(i, &adap->registered_device_map))
3522                         continue;
3523
3524                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3525                         bufp += sprintf(bufp, "100/");
3526                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3527                         bufp += sprintf(bufp, "1000/");
3528                 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3529                         bufp += sprintf(bufp, "10G/");
3530                 if (bufp != buf)
3531                         --bufp;
3532                 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3533
3534                 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3535                             adap->params.vpd.id, adap->params.rev,
3536                             buf, is_offload(adap) ? "R" : "",
3537                             adap->params.pci.width, spd,
3538                             (adap->flags & USING_MSIX) ? " MSI-X" :
3539                             (adap->flags & USING_MSI) ? " MSI" : "");
3540                 if (adap->name == dev->name)
3541                         netdev_info(dev, "S/N: %s, E/C: %s\n",
3542                                     adap->params.vpd.sn, adap->params.vpd.ec);
3543         }
3544 }
3545
3546 /*
3547  * Free the following resources:
3548  * - memory used for tables
3549  * - MSI/MSI-X
3550  * - net devices
3551  * - resources FW is holding for us
3552  */
3553 static void free_some_resources(struct adapter *adapter)
3554 {
3555         unsigned int i;
3556
3557         t4_free_mem(adapter->l2t);
3558         t4_free_mem(adapter->tids.tid_tab);
3559         disable_msi(adapter);
3560
3561         for_each_port(adapter, i)
3562                 if (adapter->port[i]) {
3563                         kfree(adap2pinfo(adapter, i)->rss);
3564                         free_netdev(adapter->port[i]);
3565                 }
3566         if (adapter->flags & FW_OK)
3567                 t4_fw_bye(adapter, adapter->fn);
3568 }
3569
3570 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3571                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3572
3573 static int __devinit init_one(struct pci_dev *pdev,
3574                               const struct pci_device_id *ent)
3575 {
3576         int func, i, err;
3577         struct port_info *pi;
3578         unsigned int highdma = 0;
3579         struct adapter *adapter = NULL;
3580
3581         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3582
3583         err = pci_request_regions(pdev, KBUILD_MODNAME);
3584         if (err) {
3585                 /* Just info, some other driver may have claimed the device. */
3586                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3587                 return err;
3588         }
3589
3590         /* We control everything through one PF */
3591         func = PCI_FUNC(pdev->devfn);
3592         if (func != ent->driver_data) {
3593                 pci_save_state(pdev);        /* to restore SR-IOV later */
3594                 goto sriov;
3595         }
3596
3597         err = pci_enable_device(pdev);
3598         if (err) {
3599                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3600                 goto out_release_regions;
3601         }
3602
3603         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3604                 highdma = NETIF_F_HIGHDMA;
3605                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3606                 if (err) {
3607                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3608                                 "coherent allocations\n");
3609                         goto out_disable_device;
3610                 }
3611         } else {
3612                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3613                 if (err) {
3614                         dev_err(&pdev->dev, "no usable DMA configuration\n");
3615                         goto out_disable_device;
3616                 }
3617         }
3618
3619         pci_enable_pcie_error_reporting(pdev);
3620         pci_set_master(pdev);
3621         pci_save_state(pdev);
3622
3623         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3624         if (!adapter) {
3625                 err = -ENOMEM;
3626                 goto out_disable_device;
3627         }
3628
3629         adapter->regs = pci_ioremap_bar(pdev, 0);
3630         if (!adapter->regs) {
3631                 dev_err(&pdev->dev, "cannot map device registers\n");
3632                 err = -ENOMEM;
3633                 goto out_free_adapter;
3634         }
3635
3636         adapter->pdev = pdev;
3637         adapter->pdev_dev = &pdev->dev;
3638         adapter->fn = func;
3639         adapter->name = pci_name(pdev);
3640         adapter->msg_enable = dflt_msg_enable;
3641         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3642
3643         spin_lock_init(&adapter->stats_lock);
3644         spin_lock_init(&adapter->tid_release_lock);
3645
3646         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3647
3648         err = t4_prep_adapter(adapter);
3649         if (err)
3650                 goto out_unmap_bar;
3651         err = adap_init0(adapter);
3652         if (err)
3653                 goto out_unmap_bar;
3654
3655         for_each_port(adapter, i) {
3656                 struct net_device *netdev;
3657
3658                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3659                                            MAX_ETH_QSETS);
3660                 if (!netdev) {
3661                         err = -ENOMEM;
3662                         goto out_free_dev;
3663                 }
3664
3665                 SET_NETDEV_DEV(netdev, &pdev->dev);
3666
3667                 adapter->port[i] = netdev;
3668                 pi = netdev_priv(netdev);
3669                 pi->adapter = adapter;
3670                 pi->xact_addr_filt = -1;
3671                 pi->rx_offload = RX_CSO;
3672                 pi->port_id = i;
3673                 netif_carrier_off(netdev);
3674                 netif_tx_stop_all_queues(netdev);
3675                 netdev->irq = pdev->irq;
3676
3677                 netdev->features |= NETIF_F_SG | TSO_FLAGS;
3678                 netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
3679                 netdev->features |= NETIF_F_GRO | NETIF_F_RXHASH | highdma;
3680                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3681                 netdev->vlan_features = netdev->features & VLAN_FEAT;
3682
3683                 netdev->netdev_ops = &cxgb4_netdev_ops;
3684                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3685         }
3686
3687         pci_set_drvdata(pdev, adapter);
3688
3689         if (adapter->flags & FW_OK) {
3690                 err = t4_port_init(adapter, func, func, 0);
3691                 if (err)
3692                         goto out_free_dev;
3693         }
3694
3695         /*
3696          * Configure queues and allocate tables now, they can be needed as
3697          * soon as the first register_netdev completes.
3698          */
3699         cfg_queues(adapter);
3700
3701         adapter->l2t = t4_init_l2t();
3702         if (!adapter->l2t) {
3703                 /* We tolerate a lack of L2T, giving up some functionality */
3704                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3705                 adapter->params.offload = 0;
3706         }
3707
3708         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3709                 dev_warn(&pdev->dev, "could not allocate TID table, "
3710                          "continuing\n");
3711                 adapter->params.offload = 0;
3712         }
3713
3714         /* See what interrupts we'll be using */
3715         if (msi > 1 && enable_msix(adapter) == 0)
3716                 adapter->flags |= USING_MSIX;
3717         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3718                 adapter->flags |= USING_MSI;
3719
3720         err = init_rss(adapter);
3721         if (err)
3722                 goto out_free_dev;
3723
3724         /*
3725          * The card is now ready to go.  If any errors occur during device
3726          * registration we do not fail the whole card but rather proceed only
3727          * with the ports we manage to register successfully.  However we must
3728          * register at least one net device.
3729          */
3730         for_each_port(adapter, i) {
3731                 err = register_netdev(adapter->port[i]);
3732                 if (err)
3733                         dev_warn(&pdev->dev,
3734                                  "cannot register net device %s, skipping\n",
3735                                  adapter->port[i]->name);
3736                 else {
3737                         /*
3738                          * Change the name we use for messages to the name of
3739                          * the first successfully registered interface.
3740                          */
3741                         if (!adapter->registered_device_map)
3742                                 adapter->name = adapter->port[i]->name;
3743
3744                         __set_bit(i, &adapter->registered_device_map);
3745                         adapter->chan_map[adap2pinfo(adapter, i)->tx_chan] = i;
3746                 }
3747         }
3748         if (!adapter->registered_device_map) {
3749                 dev_err(&pdev->dev, "could not register any net devices\n");
3750                 goto out_free_dev;
3751         }
3752
3753         if (cxgb4_debugfs_root) {
3754                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3755                                                            cxgb4_debugfs_root);
3756                 setup_debugfs(adapter);
3757         }
3758
3759         if (is_offload(adapter))
3760                 attach_ulds(adapter);
3761
3762         print_port_info(adapter);
3763
3764 sriov:
3765 #ifdef CONFIG_PCI_IOV
3766         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3767                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3768                         dev_info(&pdev->dev,
3769                                  "instantiated %u virtual functions\n",
3770                                  num_vf[func]);
3771 #endif
3772         return 0;
3773
3774  out_free_dev:
3775         free_some_resources(adapter);
3776  out_unmap_bar:
3777         iounmap(adapter->regs);
3778  out_free_adapter:
3779         kfree(adapter);
3780  out_disable_device:
3781         pci_disable_pcie_error_reporting(pdev);
3782         pci_disable_device(pdev);
3783  out_release_regions:
3784         pci_release_regions(pdev);
3785         pci_set_drvdata(pdev, NULL);
3786         return err;
3787 }
3788
3789 static void __devexit remove_one(struct pci_dev *pdev)
3790 {
3791         struct adapter *adapter = pci_get_drvdata(pdev);
3792
3793         pci_disable_sriov(pdev);
3794
3795         if (adapter) {
3796                 int i;
3797
3798                 if (is_offload(adapter))
3799                         detach_ulds(adapter);
3800
3801                 for_each_port(adapter, i)
3802                         if (test_bit(i, &adapter->registered_device_map))
3803                                 unregister_netdev(adapter->port[i]);
3804
3805                 if (adapter->debugfs_root)
3806                         debugfs_remove_recursive(adapter->debugfs_root);
3807
3808                 if (adapter->flags & FULL_INIT_DONE)
3809                         cxgb_down(adapter);
3810
3811                 free_some_resources(adapter);
3812                 iounmap(adapter->regs);
3813                 kfree(adapter);
3814                 pci_disable_pcie_error_reporting(pdev);
3815                 pci_disable_device(pdev);
3816                 pci_release_regions(pdev);
3817                 pci_set_drvdata(pdev, NULL);
3818         } else if (PCI_FUNC(pdev->devfn) > 0)
3819                 pci_release_regions(pdev);
3820 }
3821
3822 static struct pci_driver cxgb4_driver = {
3823         .name     = KBUILD_MODNAME,
3824         .id_table = cxgb4_pci_tbl,
3825         .probe    = init_one,
3826         .remove   = __devexit_p(remove_one),
3827         .err_handler = &cxgb4_eeh,
3828 };
3829
3830 static int __init cxgb4_init_module(void)
3831 {
3832         int ret;
3833
3834         /* Debugfs support is optional, just warn if this fails */
3835         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3836         if (!cxgb4_debugfs_root)
3837                 pr_warning("could not create debugfs entry, continuing\n");
3838
3839         ret = pci_register_driver(&cxgb4_driver);
3840         if (ret < 0)
3841                 debugfs_remove(cxgb4_debugfs_root);
3842         return ret;
3843 }
3844
3845 static void __exit cxgb4_cleanup_module(void)
3846 {
3847         pci_unregister_driver(&cxgb4_driver);
3848         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
3849 }
3850
3851 module_init(cxgb4_init_module);
3852 module_exit(cxgb4_cleanup_module);