drivers/net/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if_vlan.h>
  45 #include <linux/init.h>
  46 #include <linux/log2.h>
  47 #include <linux/mdio.h>
  48 #include <linux/module.h>
  49 #include <linux/moduleparam.h>
  50 #include <linux/mutex.h>
  51 #include <linux/netdevice.h>
  52 #include <linux/pci.h>
  53 #include <linux/aer.h>
  54 #include <linux/rtnetlink.h>
  55 #include <linux/sched.h>
  56 #include <linux/seq_file.h>
  57 #include <linux/sockios.h>
  58 #include <linux/vmalloc.h>
  59 #include <linux/workqueue.h>
  60 #include <net/neighbour.h>
  61 #include <net/netevent.h>
  62 #include <asm/uaccess.h>
  63
  64 #include "cxgb4.h"
  65 #include "t4_regs.h"
  66 #include "t4_msg.h"
  67 #include "t4fw_api.h"
  68 #include "l2t.h"
  69
  70 #define DRV_VERSION "1.3.0-ko"
  71 #define DRV_DESC "Chelsio T4 Network Driver"
  72
  73 /*
  74  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  75  * under extreme memory pressure so it's largish to give the system time to
  76  * recover.
  77  */
  78 #define MAX_SGE_TIMERVAL 200U
  79
  80 #ifdef CONFIG_PCI_IOV
  81 /*
  82  * Virtual Function provisioning constants.  We need two extra Ingress Queues
  83  * with Interrupt capability to serve as the VF's Firmware Event Queue and
  84  * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
  85  * Lists associated with them).  For each Ethernet/Control Egress Queue and
  86  * for each Free List, we need an Egress Context.
  87  */
  88 enum {
  89         VFRES_NPORTS = 1,               /* # of "ports" per VF */
  90         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
  91
  92         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
  93         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
  94         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
  95         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
  96         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
  97         VFRES_TC = 0,                   /* PCI-E traffic class */
  98         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
  99
 100         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 101         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 102 };
 103
 104 /*
 105  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 106  * static and likely not to be useful in the long run.  We really need to
 107  * implement some form of persistent configuration which the firmware
 108  * controls.
 109  */
 110 static unsigned int pfvfres_pmask(struct adapter *adapter,
 111                                   unsigned int pf, unsigned int vf)
 112 {
 113         unsigned int portn, portvec;
 114
 115         /*
 116          * Give PF's access to all of the ports.
 117          */
 118         if (vf == 0)
 119                 return FW_PFVF_CMD_PMASK_MASK;
 120
 121         /*
 122          * For VFs, we'll assign them access to the ports based purely on the
 123          * PF.  We assign active ports in order, wrapping around if there are
 124          * fewer active ports than PFs: e.g. active port[pf % nports].
 125          * Unfortunately the adapter's port_info structs haven't been
 126          * initialized yet so we have to compute this.
 127          */
 128         if (adapter->params.nports == 0)
 129                 return 0;
 130
 131         portn = pf % adapter->params.nports;
 132         portvec = adapter->params.portvec;
 133         for (;;) {
 134                 /*
 135                  * Isolate the lowest set bit in the port vector.  If we're at
 136                  * the port number that we want, return that as the pmask.
 137                  * otherwise mask that bit out of the port vector and
 138                  * decrement our port number ...
 139                  */
 140                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 141                 if (portn == 0)
 142                         return pmask;
 143                 portn--;
 144                 portvec &= ~pmask;
 145         }
 146         /*NOTREACHED*/
 147 }
 148 #endif
 149
 150 enum {
 151         MEMWIN0_APERTURE = 65536,
 152         MEMWIN0_BASE     = 0x30000,
 153         MEMWIN1_APERTURE = 32768,
 154         MEMWIN1_BASE     = 0x28000,
 155         MEMWIN2_APERTURE = 2048,
 156         MEMWIN2_BASE     = 0x1b800,
 157 };
 158
 159 enum {
 160         MAX_TXQ_ENTRIES      = 16384,
 161         MAX_CTRL_TXQ_ENTRIES = 1024,
 162         MAX_RSPQ_ENTRIES     = 16384,
 163         MAX_RX_BUFFERS       = 16384,
 164         MIN_TXQ_ENTRIES      = 32,
 165         MIN_CTRL_TXQ_ENTRIES = 32,
 166         MIN_RSPQ_ENTRIES     = 128,
 167         MIN_FL_ENTRIES       = 16
 168 };
 169
 170 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 171                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 172                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 173
 174 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 175
 176 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 177         CH_DEVICE(0xa000, 0),  /* PE10K */
 178         CH_DEVICE(0x4001, -1),
 179         CH_DEVICE(0x4002, -1),
 180         CH_DEVICE(0x4003, -1),
 181         CH_DEVICE(0x4004, -1),
 182         CH_DEVICE(0x4005, -1),
 183         CH_DEVICE(0x4006, -1),
 184         CH_DEVICE(0x4007, -1),
 185         CH_DEVICE(0x4008, -1),
 186         CH_DEVICE(0x4009, -1),
 187         CH_DEVICE(0x400a, -1),
 188         CH_DEVICE(0x4401, 4),
 189         CH_DEVICE(0x4402, 4),
 190         CH_DEVICE(0x4403, 4),
 191         CH_DEVICE(0x4404, 4),
 192         CH_DEVICE(0x4405, 4),
 193         CH_DEVICE(0x4406, 4),
 194         CH_DEVICE(0x4407, 4),
 195         CH_DEVICE(0x4408, 4),
 196         CH_DEVICE(0x4409, 4),
 197         CH_DEVICE(0x440a, 4),
 198         { 0, }
 199 };
 200
 201 #define FW_FNAME "cxgb4/t4fw.bin"
 202
 203 MODULE_DESCRIPTION(DRV_DESC);
 204 MODULE_AUTHOR("Chelsio Communications");
 205 MODULE_LICENSE("Dual BSD/GPL");
 206 MODULE_VERSION(DRV_VERSION);
 207 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 208 MODULE_FIRMWARE(FW_FNAME);
 209
 210 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 211
 212 module_param(dflt_msg_enable, int, 0644);
 213 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 214
 215 /*
 216  * The driver uses the best interrupt scheme available on a platform in the
 217  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 218  * of these schemes the driver may consider as follows:
 219  *
 220  * msi = 2: choose from among all three options
 221  * msi = 1: only consider MSI and INTx interrupts
 222  * msi = 0: force INTx interrupts
 223  */
 224 static int msi = 2;
 225
 226 module_param(msi, int, 0644);
 227 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 228
 229 /*
 230  * Queue interrupt hold-off timer values.  Queues default to the first of these
 231  * upon creation.
 232  */
 233 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 234
 235 module_param_array(intr_holdoff, uint, NULL, 0644);
 236 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 237                  "0..4 in microseconds");
 238
 239 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 240
 241 module_param_array(intr_cnt, uint, NULL, 0644);
 242 MODULE_PARM_DESC(intr_cnt,
 243                  "thresholds 1..3 for queue interrupt packet counters");
 244
 245 static int vf_acls;
 246
 247 #ifdef CONFIG_PCI_IOV
 248 module_param(vf_acls, bool, 0644);
 249 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 250
 251 static unsigned int num_vf[4];
 252
 253 module_param_array(num_vf, uint, NULL, 0644);
 254 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 255 #endif
 256
 257 static struct dentry *cxgb4_debugfs_root;
 258
 259 static LIST_HEAD(adapter_list);
 260 static DEFINE_MUTEX(uld_mutex);
 261 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 262 static const char *uld_str[] = { "RDMA", "iSCSI" };
 263
 264 static void link_report(struct net_device *dev)
 265 {
 266         if (!netif_carrier_ok(dev))
 267                 netdev_info(dev, "link down\n");
 268         else {
 269                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 270
 271                 const char *s = "10Mbps";
 272                 const struct port_info *p = netdev_priv(dev);
 273
 274                 switch (p->link_cfg.speed) {
 275                 case SPEED_10000:
 276                         s = "10Gbps";
 277                         break;
 278                 case SPEED_1000:
 279                         s = "1000Mbps";
 280                         break;
 281                 case SPEED_100:
 282                         s = "100Mbps";
 283                         break;
 284                 }
 285
 286                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 287                             fc[p->link_cfg.fc]);
 288         }
 289 }
 290
 291 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 292 {
 293         struct net_device *dev = adapter->port[port_id];
 294
 295         /* Skip changes from disabled ports. */
 296         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 297                 if (link_stat)
 298                         netif_carrier_on(dev);
 299                 else
 300                         netif_carrier_off(dev);
 301
 302                 link_report(dev);
 303         }
 304 }
 305
 306 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 307 {
 308         static const char *mod_str[] = {
 309                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 310         };
 311
 312         const struct net_device *dev = adap->port[port_id];
 313         const struct port_info *pi = netdev_priv(dev);
 314
 315         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 316                 netdev_info(dev, "port module unplugged\n");
 317         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 318                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 319 }
 320
 321 /*
 322  * Configure the exact and hash address filters to handle a port's multicast
 323  * and secondary unicast MAC addresses.
 324  */
 325 static int set_addr_filters(const struct net_device *dev, bool sleep)
 326 {
 327         u64 mhash = 0;
 328         u64 uhash = 0;
 329         bool free = true;
 330         u16 filt_idx[7];
 331         const u8 *addr[7];
 332         int ret, naddr = 0;
 333         const struct netdev_hw_addr *ha;
 334         int uc_cnt = netdev_uc_count(dev);
 335         int mc_cnt = netdev_mc_count(dev);
 336         const struct port_info *pi = netdev_priv(dev);
 337         unsigned int mb = pi->adapter->fn;
 338
 339         /* first do the secondary unicast addresses */
 340         netdev_for_each_uc_addr(ha, dev) {
 341                 addr[naddr++] = ha->addr;
 342                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 343                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 344                                         naddr, addr, filt_idx, &uhash, sleep);
 345                         if (ret < 0)
 346                                 return ret;
 347
 348                         free = false;
 349                         naddr = 0;
 350                 }
 351         }
 352
 353         /* next set up the multicast addresses */
 354         netdev_for_each_mc_addr(ha, dev) {
 355                 addr[naddr++] = ha->addr;
 356                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 357                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 358                                         naddr, addr, filt_idx, &mhash, sleep);
 359                         if (ret < 0)
 360                                 return ret;
 361
 362                         free = false;
 363                         naddr = 0;
 364                 }
 365         }
 366
 367         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 368                                 uhash | mhash, sleep);
 369 }
 370
 371 /*
 372  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 373  * If @mtu is -1 it is left unchanged.
 374  */
 375 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 376 {
 377         int ret;
 378         struct port_info *pi = netdev_priv(dev);
 379
 380         ret = set_addr_filters(dev, sleep_ok);
 381         if (ret == 0)
 382                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 383                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 384                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 385                                     sleep_ok);
 386         return ret;
 387 }
 388
 389 /**
 390  *      link_start - enable a port
 391  *      @dev: the port to enable
 392  *
 393  *      Performs the MAC and PHY actions needed to enable a port.
 394  */
 395 static int link_start(struct net_device *dev)
 396 {
 397         int ret;
 398         struct port_info *pi = netdev_priv(dev);
 399         unsigned int mb = pi->adapter->fn;
 400
 401         /*
 402          * We do not set address filters and promiscuity here, the stack does
 403          * that step explicitly.
 404          */
 405         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 406                             !!(dev->features & NETIF_F_HW_VLAN_RX), true);
 407         if (ret == 0) {
 408                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 409                                     pi->xact_addr_filt, dev->dev_addr, true,
 410                                     true);
 411                 if (ret >= 0) {
 412                         pi->xact_addr_filt = ret;
 413                         ret = 0;
 414                 }
 415         }
 416         if (ret == 0)
 417                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 418                                     &pi->link_cfg);
 419         if (ret == 0)
 420                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 421         return ret;
 422 }
 423
 424 /*
 425  * Response queue handler for the FW event queue.
 426  */
 427 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 428                           const struct pkt_gl *gl)
 429 {
 430         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 431
 432         rsp++;                                          /* skip RSS header */
 433         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 434                 const struct cpl_sge_egr_update *p = (void *)rsp;
 435                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 436                 struct sge_txq *txq;
 437
 438                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 439                 txq->restarts++;
 440                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 441                         struct sge_eth_txq *eq;
 442
 443                         eq = container_of(txq, struct sge_eth_txq, q);
 444                         netif_tx_wake_queue(eq->txq);
 445                 } else {
 446                         struct sge_ofld_txq *oq;
 447
 448                         oq = container_of(txq, struct sge_ofld_txq, q);
 449                         tasklet_schedule(&oq->qresume_tsk);
 450                 }
 451         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 452                 const struct cpl_fw6_msg *p = (void *)rsp;
 453
 454                 if (p->type == 0)
 455                         t4_handle_fw_rpl(q->adap, p->data);
 456         } else if (opcode == CPL_L2T_WRITE_RPL) {
 457                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 458
 459                 do_l2t_write_rpl(q->adap, p);
 460         } else
 461                 dev_err(q->adap->pdev_dev,
 462                         "unexpected CPL %#x on FW event queue\n", opcode);
 463         return 0;
 464 }
 465
 466 /**
 467  *      uldrx_handler - response queue handler for ULD queues
 468  *      @q: the response queue that received the packet
 469  *      @rsp: the response queue descriptor holding the offload message
 470  *      @gl: the gather list of packet fragments
 471  *
 472  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 473  *      the ULD, we just maintain statistics.
 474  */
 475 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 476                          const struct pkt_gl *gl)
 477 {
 478         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 479
 480         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 481                 rxq->stats.nomem++;
 482                 return -1;
 483         }
 484         if (gl == NULL)
 485                 rxq->stats.imm++;
 486         else if (gl == CXGB4_MSG_AN)
 487                 rxq->stats.an++;
 488         else
 489                 rxq->stats.pkts++;
 490         return 0;
 491 }
 492
 493 static void disable_msi(struct adapter *adapter)
 494 {
 495         if (adapter->flags & USING_MSIX) {
 496                 pci_disable_msix(adapter->pdev);
 497                 adapter->flags &= ~USING_MSIX;
 498         } else if (adapter->flags & USING_MSI) {
 499                 pci_disable_msi(adapter->pdev);
 500                 adapter->flags &= ~USING_MSI;
 501         }
 502 }
 503
 504 /*
 505  * Interrupt handler for non-data events used with MSI-X.
 506  */
 507 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 508 {
 509         struct adapter *adap = cookie;
 510
 511         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 512         if (v & PFSW) {
 513                 adap->swintr = 1;
 514                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 515         }
 516         t4_slow_intr_handler(adap);
 517         return IRQ_HANDLED;
 518 }
 519
 520 /*
 521  * Name the MSI-X interrupts.
 522  */
 523 static void name_msix_vecs(struct adapter *adap)
 524 {
 525         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 526
 527         /* non-data interrupts */
 528         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 529
 530         /* FW events */
 531         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 532                  adap->port[0]->name);
 533
 534         /* Ethernet queues */
 535         for_each_port(adap, j) {
 536                 struct net_device *d = adap->port[j];
 537                 const struct port_info *pi = netdev_priv(d);
 538
 539                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 540                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 541                                  d->name, i);
 542         }
 543
 544         /* offload queues */
 545         for_each_ofldrxq(&adap->sge, i)
 546                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 547                          adap->port[0]->name, i);
 548
 549         for_each_rdmarxq(&adap->sge, i)
 550                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 551                          adap->port[0]->name, i);
 552 }
 553
 554 static int request_msix_queue_irqs(struct adapter *adap)
 555 {
 556         struct sge *s = &adap->sge;
 557         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
 558
 559         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 560                           adap->msix_info[1].desc, &s->fw_evtq);
 561         if (err)
 562                 return err;
 563
 564         for_each_ethrxq(s, ethqidx) {
 565                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 566                                   adap->msix_info[msi].desc,
 567                                   &s->ethrxq[ethqidx].rspq);
 568                 if (err)
 569                         goto unwind;
 570                 msi++;
 571         }
 572         for_each_ofldrxq(s, ofldqidx) {
 573                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 574                                   adap->msix_info[msi].desc,
 575                                   &s->ofldrxq[ofldqidx].rspq);
 576                 if (err)
 577                         goto unwind;
 578                 msi++;
 579         }
 580         for_each_rdmarxq(s, rdmaqidx) {
 581                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 582                                   adap->msix_info[msi].desc,
 583                                   &s->rdmarxq[rdmaqidx].rspq);
 584                 if (err)
 585                         goto unwind;
 586                 msi++;
 587         }
 588         return 0;
 589
 590 unwind:
 591         while (--rdmaqidx >= 0)
 592                 free_irq(adap->msix_info[--msi].vec,
 593                          &s->rdmarxq[rdmaqidx].rspq);
 594         while (--ofldqidx >= 0)
 595                 free_irq(adap->msix_info[--msi].vec,
 596                          &s->ofldrxq[ofldqidx].rspq);
 597         while (--ethqidx >= 0)
 598                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
 599         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 600         return err;
 601 }
 602
 603 static void free_msix_queue_irqs(struct adapter *adap)
 604 {
 605         int i, msi = 2;
 606         struct sge *s = &adap->sge;
 607
 608         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 609         for_each_ethrxq(s, i)
 610                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
 611         for_each_ofldrxq(s, i)
 612                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
 613         for_each_rdmarxq(s, i)
 614                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
 615 }
 616
 617 /**
 618  *      write_rss - write the RSS table for a given port
 619  *      @pi: the port
 620  *      @queues: array of queue indices for RSS
 621  *
 622  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 623  *      packets to the Rx queues in @queues.
 624  */
 625 static int write_rss(const struct port_info *pi, const u16 *queues)
 626 {
 627         u16 *rss;
 628         int i, err;
 629         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 630
 631         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 632         if (!rss)
 633                 return -ENOMEM;
 634
 635         /* map the queue indices to queue ids */
 636         for (i = 0; i < pi->rss_size; i++, queues++)
 637                 rss[i] = q[*queues].rspq.abs_id;
 638
 639         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 640                                   pi->rss_size, rss, pi->rss_size);
 641         kfree(rss);
 642         return err;
 643 }
 644
 645 /**
 646  *      setup_rss - configure RSS
 647  *      @adap: the adapter
 648  *
 649  *      Sets up RSS for each port.
 650  */
 651 static int setup_rss(struct adapter *adap)
 652 {
 653         int i, err;
 654
 655         for_each_port(adap, i) {
 656                 const struct port_info *pi = adap2pinfo(adap, i);
 657
 658                 err = write_rss(pi, pi->rss);
 659                 if (err)
 660                         return err;
 661         }
 662         return 0;
 663 }
 664
 665 /*
 666  * Return the channel of the ingress queue with the given qid.
 667  */
 668 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 669 {
 670         qid -= p->ingr_start;
 671         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 672 }
 673
 674 /*
 675  * Wait until all NAPI handlers are descheduled.
 676  */
 677 static void quiesce_rx(struct adapter *adap)
 678 {
 679         int i;
 680
 681         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 682                 struct sge_rspq *q = adap->sge.ingr_map[i];
 683
 684                 if (q && q->handler)
 685                         napi_disable(&q->napi);
 686         }
 687 }
 688
 689 /*
 690  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 691  */
 692 static void enable_rx(struct adapter *adap)
 693 {
 694         int i;
 695
 696         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 697                 struct sge_rspq *q = adap->sge.ingr_map[i];
 698
 699                 if (!q)
 700                         continue;
 701                 if (q->handler)
 702                         napi_enable(&q->napi);
 703                 /* 0-increment GTS to start the timer and enable interrupts */
 704                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 705                              SEINTARM(q->intr_params) |
 706                              INGRESSQID(q->cntxt_id));
 707         }
 708 }
 709
 710 /**
 711  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 712  *      @adap: the adapter
 713  *
 714  *      Determines how many sets of SGE queues to use and initializes them.
 715  *      We support multiple queue sets per port if we have MSI-X, otherwise
 716  *      just one queue set per port.
 717  */
 718 static int setup_sge_queues(struct adapter *adap)
 719 {
 720         int err, msi_idx, i, j;
 721         struct sge *s = &adap->sge;
 722
 723         bitmap_zero(s->starving_fl, MAX_EGRQ);
 724         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 725
 726         if (adap->flags & USING_MSIX)
 727                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 728         else {
 729                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 730                                        NULL, NULL);
 731                 if (err)
 732                         return err;
 733                 msi_idx = -((int)s->intrq.abs_id + 1);
 734         }
 735
 736         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 737                                msi_idx, NULL, fwevtq_handler);
 738         if (err) {
 739 freeout:        t4_free_sge_resources(adap);
 740                 return err;
 741         }
 742
 743         for_each_port(adap, i) {
 744                 struct net_device *dev = adap->port[i];
 745                 struct port_info *pi = netdev_priv(dev);
 746                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 747                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 748
 749                 for (j = 0; j < pi->nqsets; j++, q++) {
 750                         if (msi_idx > 0)
 751                                 msi_idx++;
 752                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 753                                                msi_idx, &q->fl,
 754                                                t4_ethrx_handler);
 755                         if (err)
 756                                 goto freeout;
 757                         q->rspq.idx = j;
 758                         memset(&q->stats, 0, sizeof(q->stats));
 759                 }
 760                 for (j = 0; j < pi->nqsets; j++, t++) {
 761                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 762                                         netdev_get_tx_queue(dev, j),
 763                                         s->fw_evtq.cntxt_id);
 764                         if (err)
 765                                 goto freeout;
 766                 }
 767         }
 768
 769         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 770         for_each_ofldrxq(s, i) {
 771                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 772                 struct net_device *dev = adap->port[i / j];
 773
 774                 if (msi_idx > 0)
 775                         msi_idx++;
 776                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 777                                        &q->fl, uldrx_handler);
 778                 if (err)
 779                         goto freeout;
 780                 memset(&q->stats, 0, sizeof(q->stats));
 781                 s->ofld_rxq[i] = q->rspq.abs_id;
 782                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 783                                             s->fw_evtq.cntxt_id);
 784                 if (err)
 785                         goto freeout;
 786         }
 787
 788         for_each_rdmarxq(s, i) {
 789                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 790
 791                 if (msi_idx > 0)
 792                         msi_idx++;
 793                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 794                                        msi_idx, &q->fl, uldrx_handler);
 795                 if (err)
 796                         goto freeout;
 797                 memset(&q->stats, 0, sizeof(q->stats));
 798                 s->rdma_rxq[i] = q->rspq.abs_id;
 799         }
 800
 801         for_each_port(adap, i) {
 802                 /*
 803                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 804                  * have RDMA queues, and that's the right value.
 805                  */
 806                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 807                                             s->fw_evtq.cntxt_id,
 808                                             s->rdmarxq[i].rspq.cntxt_id);
 809                 if (err)
 810                         goto freeout;
 811         }
 812
 813         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
 814                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
 815                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
 816         return 0;
 817 }
 818
 819 /*
 820  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
 821  * started but failed, and a negative errno if flash load couldn't start.
 822  */
 823 static int upgrade_fw(struct adapter *adap)
 824 {
 825         int ret;
 826         u32 vers;
 827         const struct fw_hdr *hdr;
 828         const struct firmware *fw;
 829         struct device *dev = adap->pdev_dev;
 830
 831         ret = request_firmware(&fw, FW_FNAME, dev);
 832         if (ret < 0) {
 833                 dev_err(dev, "unable to load firmware image " FW_FNAME
 834                         ", error %d\n", ret);
 835                 return ret;
 836         }
 837
 838         hdr = (const struct fw_hdr *)fw->data;
 839         vers = ntohl(hdr->fw_ver);
 840         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
 841                 ret = -EINVAL;              /* wrong major version, won't do */
 842                 goto out;
 843         }
 844
 845         /*
 846          * If the flash FW is unusable or we found something newer, load it.
 847          */
 848         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
 849             vers > adap->params.fw_vers) {
 850                 ret = -t4_load_fw(adap, fw->data, fw->size);
 851                 if (!ret)
 852                         dev_info(dev, "firmware upgraded to version %pI4 from "
 853                                  FW_FNAME "\n", &hdr->fw_ver);
 854         }
 855 out:    release_firmware(fw);
 856         return ret;
 857 }
 858
 859 /*
 860  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 861  * The allocated memory is cleared.
 862  */
 863 void *t4_alloc_mem(size_t size)
 864 {
 865         void *p = kzalloc(size, GFP_KERNEL);
 866
 867         if (!p)
 868                 p = vzalloc(size);
 869         return p;
 870 }
 871
 872 /*
 873  * Free memory allocated through alloc_mem().
 874  */
 875 static void t4_free_mem(void *addr)
 876 {
 877         if (is_vmalloc_addr(addr))
 878                 vfree(addr);
 879         else
 880                 kfree(addr);
 881 }
 882
 883 static inline int is_offload(const struct adapter *adap)
 884 {
 885         return adap->params.offload;
 886 }
 887
 888 /*
 889  * Implementation of ethtool operations.
 890  */
 891
 892 static u32 get_msglevel(struct net_device *dev)
 893 {
 894         return netdev2adap(dev)->msg_enable;
 895 }
 896
 897 static void set_msglevel(struct net_device *dev, u32 val)
 898 {
 899         netdev2adap(dev)->msg_enable = val;
 900 }
 901
 902 static char stats_strings[][ETH_GSTRING_LEN] = {
 903         "TxOctetsOK         ",
 904         "TxFramesOK         ",
 905         "TxBroadcastFrames  ",
 906         "TxMulticastFrames  ",
 907         "TxUnicastFrames    ",
 908         "TxErrorFrames      ",
 909
 910         "TxFrames64         ",
 911         "TxFrames65To127    ",
 912         "TxFrames128To255   ",
 913         "TxFrames256To511   ",
 914         "TxFrames512To1023  ",
 915         "TxFrames1024To1518 ",
 916         "TxFrames1519ToMax  ",
 917
 918         "TxFramesDropped    ",
 919         "TxPauseFrames      ",
 920         "TxPPP0Frames       ",
 921         "TxPPP1Frames       ",
 922         "TxPPP2Frames       ",
 923         "TxPPP3Frames       ",
 924         "TxPPP4Frames       ",
 925         "TxPPP5Frames       ",
 926         "TxPPP6Frames       ",
 927         "TxPPP7Frames       ",
 928
 929         "RxOctetsOK         ",
 930         "RxFramesOK         ",
 931         "RxBroadcastFrames  ",
 932         "RxMulticastFrames  ",
 933         "RxUnicastFrames    ",
 934
 935         "RxFramesTooLong    ",
 936         "RxJabberErrors     ",
 937         "RxFCSErrors        ",
 938         "RxLengthErrors     ",
 939         "RxSymbolErrors     ",
 940         "RxRuntFrames       ",
 941
 942         "RxFrames64         ",
 943         "RxFrames65To127    ",
 944         "RxFrames128To255   ",
 945         "RxFrames256To511   ",
 946         "RxFrames512To1023  ",
 947         "RxFrames1024To1518 ",
 948         "RxFrames1519ToMax  ",
 949
 950         "RxPauseFrames      ",
 951         "RxPPP0Frames       ",
 952         "RxPPP1Frames       ",
 953         "RxPPP2Frames       ",
 954         "RxPPP3Frames       ",
 955         "RxPPP4Frames       ",
 956         "RxPPP5Frames       ",
 957         "RxPPP6Frames       ",
 958         "RxPPP7Frames       ",
 959
 960         "RxBG0FramesDropped ",
 961         "RxBG1FramesDropped ",
 962         "RxBG2FramesDropped ",
 963         "RxBG3FramesDropped ",
 964         "RxBG0FramesTrunc   ",
 965         "RxBG1FramesTrunc   ",
 966         "RxBG2FramesTrunc   ",
 967         "RxBG3FramesTrunc   ",
 968
 969         "TSO                ",
 970         "TxCsumOffload      ",
 971         "RxCsumGood         ",
 972         "VLANextractions    ",
 973         "VLANinsertions     ",
 974         "GROpackets         ",
 975         "GROmerged          ",
 976 };
 977
 978 static int get_sset_count(struct net_device *dev, int sset)
 979 {
 980         switch (sset) {
 981         case ETH_SS_STATS:
 982                 return ARRAY_SIZE(stats_strings);
 983         default:
 984                 return -EOPNOTSUPP;
 985         }
 986 }
 987
 988 #define T4_REGMAP_SIZE (160 * 1024)
 989
 990 static int get_regs_len(struct net_device *dev)
 991 {
 992         return T4_REGMAP_SIZE;
 993 }
 994
 995 static int get_eeprom_len(struct net_device *dev)
 996 {
 997         return EEPROMSIZE;
 998 }
 999
1000 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1001 {
1002         struct adapter *adapter = netdev2adap(dev);
1003
1004         strcpy(info->driver, KBUILD_MODNAME);
1005         strcpy(info->version, DRV_VERSION);
1006         strcpy(info->bus_info, pci_name(adapter->pdev));
1007
1008         if (!adapter->params.fw_vers)
1009                 strcpy(info->fw_version, "N/A");
1010         else
1011                 snprintf(info->fw_version, sizeof(info->fw_version),
1012                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1013                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1014                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1015                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1016                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1017                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1018                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1019                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1020                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1021 }
1022
1023 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1024 {
1025         if (stringset == ETH_SS_STATS)
1026                 memcpy(data, stats_strings, sizeof(stats_strings));
1027 }
1028
1029 /*
1030  * port stats maintained per queue of the port.  They should be in the same
1031  * order as in stats_strings above.
1032  */
1033 struct queue_port_stats {
1034         u64 tso;
1035         u64 tx_csum;
1036         u64 rx_csum;
1037         u64 vlan_ex;
1038         u64 vlan_ins;
1039         u64 gro_pkts;
1040         u64 gro_merged;
1041 };
1042
1043 static void collect_sge_port_stats(const struct adapter *adap,
1044                 const struct port_info *p, struct queue_port_stats *s)
1045 {
1046         int i;
1047         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1048         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1049
1050         memset(s, 0, sizeof(*s));
1051         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1052                 s->tso += tx->tso;
1053                 s->tx_csum += tx->tx_cso;
1054                 s->rx_csum += rx->stats.rx_cso;
1055                 s->vlan_ex += rx->stats.vlan_ex;
1056                 s->vlan_ins += tx->vlan_ins;
1057                 s->gro_pkts += rx->stats.lro_pkts;
1058                 s->gro_merged += rx->stats.lro_merged;
1059         }
1060 }
1061
1062 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1063                       u64 *data)
1064 {
1065         struct port_info *pi = netdev_priv(dev);
1066         struct adapter *adapter = pi->adapter;
1067
1068         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1069
1070         data += sizeof(struct port_stats) / sizeof(u64);
1071         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1072 }
1073
1074 /*
1075  * Return a version number to identify the type of adapter.  The scheme is:
1076  * - bits 0..9: chip version
1077  * - bits 10..15: chip revision
1078  * - bits 16..23: register dump version
1079  */
1080 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1081 {
1082         return 4 | (ap->params.rev << 10) | (1 << 16);
1083 }
1084
1085 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1086                            unsigned int end)
1087 {
1088         u32 *p = buf + start;
1089
1090         for ( ; start <= end; start += sizeof(u32))
1091                 *p++ = t4_read_reg(ap, start);
1092 }
1093
1094 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1095                      void *buf)
1096 {
1097         static const unsigned int reg_ranges[] = {
1098                 0x1008, 0x1108,
1099                 0x1180, 0x11b4,
1100                 0x11fc, 0x123c,
1101                 0x1300, 0x173c,
1102                 0x1800, 0x18fc,
1103                 0x3000, 0x30d8,
1104                 0x30e0, 0x5924,
1105                 0x5960, 0x59d4,
1106                 0x5a00, 0x5af8,
1107                 0x6000, 0x6098,
1108                 0x6100, 0x6150,
1109                 0x6200, 0x6208,
1110                 0x6240, 0x6248,
1111                 0x6280, 0x6338,
1112                 0x6370, 0x638c,
1113                 0x6400, 0x643c,
1114                 0x6500, 0x6524,
1115                 0x6a00, 0x6a38,
1116                 0x6a60, 0x6a78,
1117                 0x6b00, 0x6b84,
1118                 0x6bf0, 0x6c84,
1119                 0x6cf0, 0x6d84,
1120                 0x6df0, 0x6e84,
1121                 0x6ef0, 0x6f84,
1122                 0x6ff0, 0x7084,
1123                 0x70f0, 0x7184,
1124                 0x71f0, 0x7284,
1125                 0x72f0, 0x7384,
1126                 0x73f0, 0x7450,
1127                 0x7500, 0x7530,
1128                 0x7600, 0x761c,
1129                 0x7680, 0x76cc,
1130                 0x7700, 0x7798,
1131                 0x77c0, 0x77fc,
1132                 0x7900, 0x79fc,
1133                 0x7b00, 0x7c38,
1134                 0x7d00, 0x7efc,
1135                 0x8dc0, 0x8e1c,
1136                 0x8e30, 0x8e78,
1137                 0x8ea0, 0x8f6c,
1138                 0x8fc0, 0x9074,
1139                 0x90fc, 0x90fc,
1140                 0x9400, 0x9458,
1141                 0x9600, 0x96bc,
1142                 0x9800, 0x9808,
1143                 0x9820, 0x983c,
1144                 0x9850, 0x9864,
1145                 0x9c00, 0x9c6c,
1146                 0x9c80, 0x9cec,
1147                 0x9d00, 0x9d6c,
1148                 0x9d80, 0x9dec,
1149                 0x9e00, 0x9e6c,
1150                 0x9e80, 0x9eec,
1151                 0x9f00, 0x9f6c,
1152                 0x9f80, 0x9fec,
1153                 0xd004, 0xd03c,
1154                 0xdfc0, 0xdfe0,
1155                 0xe000, 0xea7c,
1156                 0xf000, 0x11190,
1157                 0x19040, 0x1906c,
1158                 0x19078, 0x19080,
1159                 0x1908c, 0x19124,
1160                 0x19150, 0x191b0,
1161                 0x191d0, 0x191e8,
1162                 0x19238, 0x1924c,
1163                 0x193f8, 0x19474,
1164                 0x19490, 0x194f8,
1165                 0x19800, 0x19f30,
1166                 0x1a000, 0x1a06c,
1167                 0x1a0b0, 0x1a120,
1168                 0x1a128, 0x1a138,
1169                 0x1a190, 0x1a1c4,
1170                 0x1a1fc, 0x1a1fc,
1171                 0x1e040, 0x1e04c,
1172                 0x1e284, 0x1e28c,
1173                 0x1e2c0, 0x1e2c0,
1174                 0x1e2e0, 0x1e2e0,
1175                 0x1e300, 0x1e384,
1176                 0x1e3c0, 0x1e3c8,
1177                 0x1e440, 0x1e44c,
1178                 0x1e684, 0x1e68c,
1179                 0x1e6c0, 0x1e6c0,
1180                 0x1e6e0, 0x1e6e0,
1181                 0x1e700, 0x1e784,
1182                 0x1e7c0, 0x1e7c8,
1183                 0x1e840, 0x1e84c,
1184                 0x1ea84, 0x1ea8c,
1185                 0x1eac0, 0x1eac0,
1186                 0x1eae0, 0x1eae0,
1187                 0x1eb00, 0x1eb84,
1188                 0x1ebc0, 0x1ebc8,
1189                 0x1ec40, 0x1ec4c,
1190                 0x1ee84, 0x1ee8c,
1191                 0x1eec0, 0x1eec0,
1192                 0x1eee0, 0x1eee0,
1193                 0x1ef00, 0x1ef84,
1194                 0x1efc0, 0x1efc8,
1195                 0x1f040, 0x1f04c,
1196                 0x1f284, 0x1f28c,
1197                 0x1f2c0, 0x1f2c0,
1198                 0x1f2e0, 0x1f2e0,
1199                 0x1f300, 0x1f384,
1200                 0x1f3c0, 0x1f3c8,
1201                 0x1f440, 0x1f44c,
1202                 0x1f684, 0x1f68c,
1203                 0x1f6c0, 0x1f6c0,
1204                 0x1f6e0, 0x1f6e0,
1205                 0x1f700, 0x1f784,
1206                 0x1f7c0, 0x1f7c8,
1207                 0x1f840, 0x1f84c,
1208                 0x1fa84, 0x1fa8c,
1209                 0x1fac0, 0x1fac0,
1210                 0x1fae0, 0x1fae0,
1211                 0x1fb00, 0x1fb84,
1212                 0x1fbc0, 0x1fbc8,
1213                 0x1fc40, 0x1fc4c,
1214                 0x1fe84, 0x1fe8c,
1215                 0x1fec0, 0x1fec0,
1216                 0x1fee0, 0x1fee0,
1217                 0x1ff00, 0x1ff84,
1218                 0x1ffc0, 0x1ffc8,
1219                 0x20000, 0x2002c,
1220                 0x20100, 0x2013c,
1221                 0x20190, 0x201c8,
1222                 0x20200, 0x20318,
1223                 0x20400, 0x20528,
1224                 0x20540, 0x20614,
1225                 0x21000, 0x21040,
1226                 0x2104c, 0x21060,
1227                 0x210c0, 0x210ec,
1228                 0x21200, 0x21268,
1229                 0x21270, 0x21284,
1230                 0x212fc, 0x21388,
1231                 0x21400, 0x21404,
1232                 0x21500, 0x21518,
1233                 0x2152c, 0x2153c,
1234                 0x21550, 0x21554,
1235                 0x21600, 0x21600,
1236                 0x21608, 0x21628,
1237                 0x21630, 0x2163c,
1238                 0x21700, 0x2171c,
1239                 0x21780, 0x2178c,
1240                 0x21800, 0x21c38,
1241                 0x21c80, 0x21d7c,
1242                 0x21e00, 0x21e04,
1243                 0x22000, 0x2202c,
1244                 0x22100, 0x2213c,
1245                 0x22190, 0x221c8,
1246                 0x22200, 0x22318,
1247                 0x22400, 0x22528,
1248                 0x22540, 0x22614,
1249                 0x23000, 0x23040,
1250                 0x2304c, 0x23060,
1251                 0x230c0, 0x230ec,
1252                 0x23200, 0x23268,
1253                 0x23270, 0x23284,
1254                 0x232fc, 0x23388,
1255                 0x23400, 0x23404,
1256                 0x23500, 0x23518,
1257                 0x2352c, 0x2353c,
1258                 0x23550, 0x23554,
1259                 0x23600, 0x23600,
1260                 0x23608, 0x23628,
1261                 0x23630, 0x2363c,
1262                 0x23700, 0x2371c,
1263                 0x23780, 0x2378c,
1264                 0x23800, 0x23c38,
1265                 0x23c80, 0x23d7c,
1266                 0x23e00, 0x23e04,
1267                 0x24000, 0x2402c,
1268                 0x24100, 0x2413c,
1269                 0x24190, 0x241c8,
1270                 0x24200, 0x24318,
1271                 0x24400, 0x24528,
1272                 0x24540, 0x24614,
1273                 0x25000, 0x25040,
1274                 0x2504c, 0x25060,
1275                 0x250c0, 0x250ec,
1276                 0x25200, 0x25268,
1277                 0x25270, 0x25284,
1278                 0x252fc, 0x25388,
1279                 0x25400, 0x25404,
1280                 0x25500, 0x25518,
1281                 0x2552c, 0x2553c,
1282                 0x25550, 0x25554,
1283                 0x25600, 0x25600,
1284                 0x25608, 0x25628,
1285                 0x25630, 0x2563c,
1286                 0x25700, 0x2571c,
1287                 0x25780, 0x2578c,
1288                 0x25800, 0x25c38,
1289                 0x25c80, 0x25d7c,
1290                 0x25e00, 0x25e04,
1291                 0x26000, 0x2602c,
1292                 0x26100, 0x2613c,
1293                 0x26190, 0x261c8,
1294                 0x26200, 0x26318,
1295                 0x26400, 0x26528,
1296                 0x26540, 0x26614,
1297                 0x27000, 0x27040,
1298                 0x2704c, 0x27060,
1299                 0x270c0, 0x270ec,
1300                 0x27200, 0x27268,
1301                 0x27270, 0x27284,
1302                 0x272fc, 0x27388,
1303                 0x27400, 0x27404,
1304                 0x27500, 0x27518,
1305                 0x2752c, 0x2753c,
1306                 0x27550, 0x27554,
1307                 0x27600, 0x27600,
1308                 0x27608, 0x27628,
1309                 0x27630, 0x2763c,
1310                 0x27700, 0x2771c,
1311                 0x27780, 0x2778c,
1312                 0x27800, 0x27c38,
1313                 0x27c80, 0x27d7c,
1314                 0x27e00, 0x27e04
1315         };
1316
1317         int i;
1318         struct adapter *ap = netdev2adap(dev);
1319
1320         regs->version = mk_adap_vers(ap);
1321
1322         memset(buf, 0, T4_REGMAP_SIZE);
1323         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1324                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1325 }
1326
1327 static int restart_autoneg(struct net_device *dev)
1328 {
1329         struct port_info *p = netdev_priv(dev);
1330
1331         if (!netif_running(dev))
1332                 return -EAGAIN;
1333         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1334                 return -EINVAL;
1335         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1336         return 0;
1337 }
1338
1339 static int identify_port(struct net_device *dev,
1340                          enum ethtool_phys_id_state state)
1341 {
1342         unsigned int val;
1343         struct adapter *adap = netdev2adap(dev);
1344
1345         if (state == ETHTOOL_ID_ACTIVE)
1346                 val = 0xffff;
1347         else if (state == ETHTOOL_ID_INACTIVE)
1348                 val = 0;
1349         else
1350                 return -EINVAL;
1351
1352         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1353 }
1354
1355 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1356 {
1357         unsigned int v = 0;
1358
1359         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1360             type == FW_PORT_TYPE_BT_XAUI) {
1361                 v |= SUPPORTED_TP;
1362                 if (caps & FW_PORT_CAP_SPEED_100M)
1363                         v |= SUPPORTED_100baseT_Full;
1364                 if (caps & FW_PORT_CAP_SPEED_1G)
1365                         v |= SUPPORTED_1000baseT_Full;
1366                 if (caps & FW_PORT_CAP_SPEED_10G)
1367                         v |= SUPPORTED_10000baseT_Full;
1368         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1369                 v |= SUPPORTED_Backplane;
1370                 if (caps & FW_PORT_CAP_SPEED_1G)
1371                         v |= SUPPORTED_1000baseKX_Full;
1372                 if (caps & FW_PORT_CAP_SPEED_10G)
1373                         v |= SUPPORTED_10000baseKX4_Full;
1374         } else if (type == FW_PORT_TYPE_KR)
1375                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1376         else if (type == FW_PORT_TYPE_BP_AP)
1377                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1378                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1379         else if (type == FW_PORT_TYPE_BP4_AP)
1380                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1381                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1382                      SUPPORTED_10000baseKX4_Full;
1383         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1384                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1385                 v |= SUPPORTED_FIBRE;
1386
1387         if (caps & FW_PORT_CAP_ANEG)
1388                 v |= SUPPORTED_Autoneg;
1389         return v;
1390 }
1391
1392 static unsigned int to_fw_linkcaps(unsigned int caps)
1393 {
1394         unsigned int v = 0;
1395
1396         if (caps & ADVERTISED_100baseT_Full)
1397                 v |= FW_PORT_CAP_SPEED_100M;
1398         if (caps & ADVERTISED_1000baseT_Full)
1399                 v |= FW_PORT_CAP_SPEED_1G;
1400         if (caps & ADVERTISED_10000baseT_Full)
1401                 v |= FW_PORT_CAP_SPEED_10G;
1402         return v;
1403 }
1404
1405 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1406 {
1407         const struct port_info *p = netdev_priv(dev);
1408
1409         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1410             p->port_type == FW_PORT_TYPE_BT_XFI ||
1411             p->port_type == FW_PORT_TYPE_BT_XAUI)
1412                 cmd->port = PORT_TP;
1413         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1414                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1415                 cmd->port = PORT_FIBRE;
1416         else if (p->port_type == FW_PORT_TYPE_SFP) {
1417                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1418                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1419                         cmd->port = PORT_DA;
1420                 else
1421                         cmd->port = PORT_FIBRE;
1422         } else
1423                 cmd->port = PORT_OTHER;
1424
1425         if (p->mdio_addr >= 0) {
1426                 cmd->phy_address = p->mdio_addr;
1427                 cmd->transceiver = XCVR_EXTERNAL;
1428                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1429                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1430         } else {
1431                 cmd->phy_address = 0;  /* not really, but no better option */
1432                 cmd->transceiver = XCVR_INTERNAL;
1433                 cmd->mdio_support = 0;
1434         }
1435
1436         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1437         cmd->advertising = from_fw_linkcaps(p->port_type,
1438                                             p->link_cfg.advertising);
1439         ethtool_cmd_speed_set(cmd,
1440                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1441         cmd->duplex = DUPLEX_FULL;
1442         cmd->autoneg = p->link_cfg.autoneg;
1443         cmd->maxtxpkt = 0;
1444         cmd->maxrxpkt = 0;
1445         return 0;
1446 }
1447
1448 static unsigned int speed_to_caps(int speed)
1449 {
1450         if (speed == SPEED_100)
1451                 return FW_PORT_CAP_SPEED_100M;
1452         if (speed == SPEED_1000)
1453                 return FW_PORT_CAP_SPEED_1G;
1454         if (speed == SPEED_10000)
1455                 return FW_PORT_CAP_SPEED_10G;
1456         return 0;
1457 }
1458
1459 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1460 {
1461         unsigned int cap;
1462         struct port_info *p = netdev_priv(dev);
1463         struct link_config *lc = &p->link_cfg;
1464         u32 speed = ethtool_cmd_speed(cmd);
1465
1466         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1467                 return -EINVAL;
1468
1469         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1470                 /*
1471                  * PHY offers a single speed.  See if that's what's
1472                  * being requested.
1473                  */
1474                 if (cmd->autoneg == AUTONEG_DISABLE &&
1475                     (lc->supported & speed_to_caps(speed)))
1476                         return 0;
1477                 return -EINVAL;
1478         }
1479
1480         if (cmd->autoneg == AUTONEG_DISABLE) {
1481                 cap = speed_to_caps(speed);
1482
1483                 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1484                     (speed == SPEED_10000))
1485                         return -EINVAL;
1486                 lc->requested_speed = cap;
1487                 lc->advertising = 0;
1488         } else {
1489                 cap = to_fw_linkcaps(cmd->advertising);
1490                 if (!(lc->supported & cap))
1491                         return -EINVAL;
1492                 lc->requested_speed = 0;
1493                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1494         }
1495         lc->autoneg = cmd->autoneg;
1496
1497         if (netif_running(dev))
1498                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1499                                      lc);
1500         return 0;
1501 }
1502
1503 static void get_pauseparam(struct net_device *dev,
1504                            struct ethtool_pauseparam *epause)
1505 {
1506         struct port_info *p = netdev_priv(dev);
1507
1508         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1509         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1510         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1511 }
1512
1513 static int set_pauseparam(struct net_device *dev,
1514                           struct ethtool_pauseparam *epause)
1515 {
1516         struct port_info *p = netdev_priv(dev);
1517         struct link_config *lc = &p->link_cfg;
1518
1519         if (epause->autoneg == AUTONEG_DISABLE)
1520                 lc->requested_fc = 0;
1521         else if (lc->supported & FW_PORT_CAP_ANEG)
1522                 lc->requested_fc = PAUSE_AUTONEG;
1523         else
1524                 return -EINVAL;
1525
1526         if (epause->rx_pause)
1527                 lc->requested_fc |= PAUSE_RX;
1528         if (epause->tx_pause)
1529                 lc->requested_fc |= PAUSE_TX;
1530         if (netif_running(dev))
1531                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1532                                      lc);
1533         return 0;
1534 }
1535
1536 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1537 {
1538         const struct port_info *pi = netdev_priv(dev);
1539         const struct sge *s = &pi->adapter->sge;
1540
1541         e->rx_max_pending = MAX_RX_BUFFERS;
1542         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1543         e->rx_jumbo_max_pending = 0;
1544         e->tx_max_pending = MAX_TXQ_ENTRIES;
1545
1546         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1547         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1548         e->rx_jumbo_pending = 0;
1549         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1550 }
1551
1552 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1553 {
1554         int i;
1555         const struct port_info *pi = netdev_priv(dev);
1556         struct adapter *adapter = pi->adapter;
1557         struct sge *s = &adapter->sge;
1558
1559         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1560             e->tx_pending > MAX_TXQ_ENTRIES ||
1561             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1562             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1563             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1564                 return -EINVAL;
1565
1566         if (adapter->flags & FULL_INIT_DONE)
1567                 return -EBUSY;
1568
1569         for (i = 0; i < pi->nqsets; ++i) {
1570                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1571                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1572                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1573         }
1574         return 0;
1575 }
1576
1577 static int closest_timer(const struct sge *s, int time)
1578 {
1579         int i, delta, match = 0, min_delta = INT_MAX;
1580
1581         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1582                 delta = time - s->timer_val[i];
1583                 if (delta < 0)
1584                         delta = -delta;
1585                 if (delta < min_delta) {
1586                         min_delta = delta;
1587                         match = i;
1588                 }
1589         }
1590         return match;
1591 }
1592
1593 static int closest_thres(const struct sge *s, int thres)
1594 {
1595         int i, delta, match = 0, min_delta = INT_MAX;
1596
1597         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1598                 delta = thres - s->counter_val[i];
1599                 if (delta < 0)
1600                         delta = -delta;
1601                 if (delta < min_delta) {
1602                         min_delta = delta;
1603                         match = i;
1604                 }
1605         }
1606         return match;
1607 }
1608
1609 /*
1610  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1611  */
1612 static unsigned int qtimer_val(const struct adapter *adap,
1613                                const struct sge_rspq *q)
1614 {
1615         unsigned int idx = q->intr_params >> 1;
1616
1617         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1618 }
1619
1620 /**
1621  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1622  *      @adap: the adapter
1623  *      @q: the Rx queue
1624  *      @us: the hold-off time in us, or 0 to disable timer
1625  *      @cnt: the hold-off packet count, or 0 to disable counter
1626  *
1627  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1628  *      one of the two needs to be enabled for the queue to generate interrupts.
1629  */
1630 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1631                                unsigned int us, unsigned int cnt)
1632 {
1633         if ((us | cnt) == 0)
1634                 cnt = 1;
1635
1636         if (cnt) {
1637                 int err;
1638                 u32 v, new_idx;
1639
1640                 new_idx = closest_thres(&adap->sge, cnt);
1641                 if (q->desc && q->pktcnt_idx != new_idx) {
1642                         /* the queue has already been created, update it */
1643                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1644                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1645                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1646                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1647                                             &new_idx);
1648                         if (err)
1649                                 return err;
1650                 }
1651                 q->pktcnt_idx = new_idx;
1652         }
1653
1654         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1655         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1656         return 0;
1657 }
1658
1659 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1660 {
1661         const struct port_info *pi = netdev_priv(dev);
1662         struct adapter *adap = pi->adapter;
1663
1664         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1665                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1666 }
1667
1668 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1669 {
1670         const struct port_info *pi = netdev_priv(dev);
1671         const struct adapter *adap = pi->adapter;
1672         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1673
1674         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1675         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1676                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1677         return 0;
1678 }
1679
1680 /**
1681  *      eeprom_ptov - translate a physical EEPROM address to virtual
1682  *      @phys_addr: the physical EEPROM address
1683  *      @fn: the PCI function number
1684  *      @sz: size of function-specific area
1685  *
1686  *      Translate a physical EEPROM address to virtual.  The first 1K is
1687  *      accessed through virtual addresses starting at 31K, the rest is
1688  *      accessed through virtual addresses starting at 0.
1689  *
1690  *      The mapping is as follows:
1691  *      [0..1K) -> [31K..32K)
1692  *      [1K..1K+A) -> [31K-A..31K)
1693  *      [1K+A..ES) -> [0..ES-A-1K)
1694  *
1695  *      where A = @fn * @sz, and ES = EEPROM size.
1696  */
1697 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1698 {
1699         fn *= sz;
1700         if (phys_addr < 1024)
1701                 return phys_addr + (31 << 10);
1702         if (phys_addr < 1024 + fn)
1703                 return 31744 - fn + phys_addr - 1024;
1704         if (phys_addr < EEPROMSIZE)
1705                 return phys_addr - 1024 - fn;
1706         return -EINVAL;
1707 }
1708
1709 /*
1710  * The next two routines implement eeprom read/write from physical addresses.
1711  */
1712 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1713 {
1714         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1715
1716         if (vaddr >= 0)
1717                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1718         return vaddr < 0 ? vaddr : 0;
1719 }
1720
1721 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1722 {
1723         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1724
1725         if (vaddr >= 0)
1726                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1727         return vaddr < 0 ? vaddr : 0;
1728 }
1729
1730 #define EEPROM_MAGIC 0x38E2F10C
1731
1732 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1733                       u8 *data)
1734 {
1735         int i, err = 0;
1736         struct adapter *adapter = netdev2adap(dev);
1737
1738         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1739         if (!buf)
1740                 return -ENOMEM;
1741
1742         e->magic = EEPROM_MAGIC;
1743         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1744                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1745
1746         if (!err)
1747                 memcpy(data, buf + e->offset, e->len);
1748         kfree(buf);
1749         return err;
1750 }
1751
1752 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1753                       u8 *data)
1754 {
1755         u8 *buf;
1756         int err = 0;
1757         u32 aligned_offset, aligned_len, *p;
1758         struct adapter *adapter = netdev2adap(dev);
1759
1760         if (eeprom->magic != EEPROM_MAGIC)
1761                 return -EINVAL;
1762
1763         aligned_offset = eeprom->offset & ~3;
1764         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1765
1766         if (adapter->fn > 0) {
1767                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1768
1769                 if (aligned_offset < start ||
1770                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
1771                         return -EPERM;
1772         }
1773
1774         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1775                 /*
1776                  * RMW possibly needed for first or last words.
1777                  */
1778                 buf = kmalloc(aligned_len, GFP_KERNEL);
1779                 if (!buf)
1780                         return -ENOMEM;
1781                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1782                 if (!err && aligned_len > 4)
1783                         err = eeprom_rd_phys(adapter,
1784                                              aligned_offset + aligned_len - 4,
1785                                              (u32 *)&buf[aligned_len - 4]);
1786                 if (err)
1787                         goto out;
1788                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1789         } else
1790                 buf = data;
1791
1792         err = t4_seeprom_wp(adapter, false);
1793         if (err)
1794                 goto out;
1795
1796         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1797                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1798                 aligned_offset += 4;
1799         }
1800
1801         if (!err)
1802                 err = t4_seeprom_wp(adapter, true);
1803 out:
1804         if (buf != data)
1805                 kfree(buf);
1806         return err;
1807 }
1808
1809 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1810 {
1811         int ret;
1812         const struct firmware *fw;
1813         struct adapter *adap = netdev2adap(netdev);
1814
1815         ef->data[sizeof(ef->data) - 1] = '\0';
1816         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1817         if (ret < 0)
1818                 return ret;
1819
1820         ret = t4_load_fw(adap, fw->data, fw->size);
1821         release_firmware(fw);
1822         if (!ret)
1823                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1824         return ret;
1825 }
1826
1827 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1828 #define BCAST_CRC 0xa0ccc1a6
1829
1830 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1831 {
1832         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1833         wol->wolopts = netdev2adap(dev)->wol;
1834         memset(&wol->sopass, 0, sizeof(wol->sopass));
1835 }
1836
1837 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1838 {
1839         int err = 0;
1840         struct port_info *pi = netdev_priv(dev);
1841
1842         if (wol->wolopts & ~WOL_SUPPORTED)
1843                 return -EINVAL;
1844         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1845                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1846         if (wol->wolopts & WAKE_BCAST) {
1847                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1848                                         ~0ULL, 0, false);
1849                 if (!err)
1850                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1851                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1852         } else
1853                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1854         return err;
1855 }
1856
1857 static int cxgb_set_features(struct net_device *dev, u32 features)
1858 {
1859         const struct port_info *pi = netdev_priv(dev);
1860         u32 changed = dev->features ^ features;
1861         int err;
1862
1863         if (!(changed & NETIF_F_HW_VLAN_RX))
1864                 return 0;
1865
1866         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1867                             -1, -1, -1,
1868                             !!(features & NETIF_F_HW_VLAN_RX), true);
1869         if (unlikely(err))
1870                 dev->features = features ^ NETIF_F_HW_VLAN_RX;
1871         return err;
1872 }
1873
1874 static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1875 {
1876         const struct port_info *pi = netdev_priv(dev);
1877         unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1878
1879         p->size = pi->rss_size;
1880         while (n--)
1881                 p->ring_index[n] = pi->rss[n];
1882         return 0;
1883 }
1884
1885 static int set_rss_table(struct net_device *dev,
1886                          const struct ethtool_rxfh_indir *p)
1887 {
1888         unsigned int i;
1889         struct port_info *pi = netdev_priv(dev);
1890
1891         if (p->size != pi->rss_size)
1892                 return -EINVAL;
1893         for (i = 0; i < p->size; i++)
1894                 if (p->ring_index[i] >= pi->nqsets)
1895                         return -EINVAL;
1896         for (i = 0; i < p->size; i++)
1897                 pi->rss[i] = p->ring_index[i];
1898         if (pi->adapter->flags & FULL_INIT_DONE)
1899                 return write_rss(pi, pi->rss);
1900         return 0;
1901 }
1902
1903 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1904                      void *rules)
1905 {
1906         const struct port_info *pi = netdev_priv(dev);
1907
1908         switch (info->cmd) {
1909         case ETHTOOL_GRXFH: {
1910                 unsigned int v = pi->rss_mode;
1911
1912                 info->data = 0;
1913                 switch (info->flow_type) {
1914                 case TCP_V4_FLOW:
1915                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1916                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1917                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1918                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1919                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1920                         break;
1921                 case UDP_V4_FLOW:
1922                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1923                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1924                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1925                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1926                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1927                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1928                         break;
1929                 case SCTP_V4_FLOW:
1930                 case AH_ESP_V4_FLOW:
1931                 case IPV4_FLOW:
1932                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1933                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1934                         break;
1935                 case TCP_V6_FLOW:
1936                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1937                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1938                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1939                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1940                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1941                         break;
1942                 case UDP_V6_FLOW:
1943                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1944                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1945                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1946                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1947                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1948                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1949                         break;
1950                 case SCTP_V6_FLOW:
1951                 case AH_ESP_V6_FLOW:
1952                 case IPV6_FLOW:
1953                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1954                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1955                         break;
1956                 }
1957                 return 0;
1958         }
1959         case ETHTOOL_GRXRINGS:
1960                 info->data = pi->nqsets;
1961                 return 0;
1962         }
1963         return -EOPNOTSUPP;
1964 }
1965
1966 static struct ethtool_ops cxgb_ethtool_ops = {
1967         .get_settings      = get_settings,
1968         .set_settings      = set_settings,
1969         .get_drvinfo       = get_drvinfo,
1970         .get_msglevel      = get_msglevel,
1971         .set_msglevel      = set_msglevel,
1972         .get_ringparam     = get_sge_param,
1973         .set_ringparam     = set_sge_param,
1974         .get_coalesce      = get_coalesce,
1975         .set_coalesce      = set_coalesce,
1976         .get_eeprom_len    = get_eeprom_len,
1977         .get_eeprom        = get_eeprom,
1978         .set_eeprom        = set_eeprom,
1979         .get_pauseparam    = get_pauseparam,
1980         .set_pauseparam    = set_pauseparam,
1981         .get_link          = ethtool_op_get_link,
1982         .get_strings       = get_strings,
1983         .set_phys_id       = identify_port,
1984         .nway_reset        = restart_autoneg,
1985         .get_sset_count    = get_sset_count,
1986         .get_ethtool_stats = get_stats,
1987         .get_regs_len      = get_regs_len,
1988         .get_regs          = get_regs,
1989         .get_wol           = get_wol,
1990         .set_wol           = set_wol,
1991         .get_rxnfc         = get_rxnfc,
1992         .get_rxfh_indir    = get_rss_table,
1993         .set_rxfh_indir    = set_rss_table,
1994         .flash_device      = set_flash,
1995 };
1996
1997 /*
1998  * debugfs support
1999  */
2000
2001 static int mem_open(struct inode *inode, struct file *file)
2002 {
2003         file->private_data = inode->i_private;
2004         return 0;
2005 }
2006
2007 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2008                         loff_t *ppos)
2009 {
2010         loff_t pos = *ppos;
2011         loff_t avail = file->f_path.dentry->d_inode->i_size;
2012         unsigned int mem = (uintptr_t)file->private_data & 3;
2013         struct adapter *adap = file->private_data - mem;
2014
2015         if (pos < 0)
2016                 return -EINVAL;
2017         if (pos >= avail)
2018                 return 0;
2019         if (count > avail - pos)
2020                 count = avail - pos;
2021
2022         while (count) {
2023                 size_t len;
2024                 int ret, ofst;
2025                 __be32 data[16];
2026
2027                 if (mem == MEM_MC)
2028                         ret = t4_mc_read(adap, pos, data, NULL);
2029                 else
2030                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2031                 if (ret)
2032                         return ret;
2033
2034                 ofst = pos % sizeof(data);
2035                 len = min(count, sizeof(data) - ofst);
2036                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2037                         return -EFAULT;
2038
2039                 buf += len;
2040                 pos += len;
2041                 count -= len;
2042         }
2043         count = pos - *ppos;
2044         *ppos = pos;
2045         return count;
2046 }
2047
2048 static const struct file_operations mem_debugfs_fops = {
2049         .owner   = THIS_MODULE,
2050         .open    = mem_open,
2051         .read    = mem_read,
2052         .llseek  = default_llseek,
2053 };
2054
2055 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2056                                       unsigned int idx, unsigned int size_mb)
2057 {
2058         struct dentry *de;
2059
2060         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2061                                  (void *)adap + idx, &mem_debugfs_fops);
2062         if (de && de->d_inode)
2063                 de->d_inode->i_size = size_mb << 20;
2064 }
2065
2066 static int __devinit setup_debugfs(struct adapter *adap)
2067 {
2068         int i;
2069
2070         if (IS_ERR_OR_NULL(adap->debugfs_root))
2071                 return -1;
2072
2073         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2074         if (i & EDRAM0_ENABLE)
2075                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2076         if (i & EDRAM1_ENABLE)
2077                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2078         if (i & EXT_MEM_ENABLE)
2079                 add_debugfs_mem(adap, "mc", MEM_MC,
2080                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2081         if (adap->l2t)
2082                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2083                                     &t4_l2t_fops);
2084         return 0;
2085 }
2086
2087 /*
2088  * upper-layer driver support
2089  */
2090
2091 /*
2092  * Allocate an active-open TID and set it to the supplied value.
2093  */
2094 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2095 {
2096         int atid = -1;
2097
2098         spin_lock_bh(&t->atid_lock);
2099         if (t->afree) {
2100                 union aopen_entry *p = t->afree;
2101
2102                 atid = p - t->atid_tab;
2103                 t->afree = p->next;
2104                 p->data = data;
2105                 t->atids_in_use++;
2106         }
2107         spin_unlock_bh(&t->atid_lock);
2108         return atid;
2109 }
2110 EXPORT_SYMBOL(cxgb4_alloc_atid);
2111
2112 /*
2113  * Release an active-open TID.
2114  */
2115 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2116 {
2117         union aopen_entry *p = &t->atid_tab[atid];
2118
2119         spin_lock_bh(&t->atid_lock);
2120         p->next = t->afree;
2121         t->afree = p;
2122         t->atids_in_use--;
2123         spin_unlock_bh(&t->atid_lock);
2124 }
2125 EXPORT_SYMBOL(cxgb4_free_atid);
2126
2127 /*
2128  * Allocate a server TID and set it to the supplied value.
2129  */
2130 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2131 {
2132         int stid;
2133
2134         spin_lock_bh(&t->stid_lock);
2135         if (family == PF_INET) {
2136                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2137                 if (stid < t->nstids)
2138                         __set_bit(stid, t->stid_bmap);
2139                 else
2140                         stid = -1;
2141         } else {
2142                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2143                 if (stid < 0)
2144                         stid = -1;
2145         }
2146         if (stid >= 0) {
2147                 t->stid_tab[stid].data = data;
2148                 stid += t->stid_base;
2149                 t->stids_in_use++;
2150         }
2151         spin_unlock_bh(&t->stid_lock);
2152         return stid;
2153 }
2154 EXPORT_SYMBOL(cxgb4_alloc_stid);
2155
2156 /*
2157  * Release a server TID.
2158  */
2159 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2160 {
2161         stid -= t->stid_base;
2162         spin_lock_bh(&t->stid_lock);
2163         if (family == PF_INET)
2164                 __clear_bit(stid, t->stid_bmap);
2165         else
2166                 bitmap_release_region(t->stid_bmap, stid, 2);
2167         t->stid_tab[stid].data = NULL;
2168         t->stids_in_use--;
2169         spin_unlock_bh(&t->stid_lock);
2170 }
2171 EXPORT_SYMBOL(cxgb4_free_stid);
2172
2173 /*
2174  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2175  */
2176 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2177                            unsigned int tid)
2178 {
2179         struct cpl_tid_release *req;
2180
2181         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2182         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2183         INIT_TP_WR(req, tid);
2184         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2185 }
2186
2187 /*
2188  * Queue a TID release request and if necessary schedule a work queue to
2189  * process it.
2190  */
2191 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2192                                     unsigned int tid)
2193 {
2194         void **p = &t->tid_tab[tid];
2195         struct adapter *adap = container_of(t, struct adapter, tids);
2196
2197         spin_lock_bh(&adap->tid_release_lock);
2198         *p = adap->tid_release_head;
2199         /* Low 2 bits encode the Tx channel number */
2200         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2201         if (!adap->tid_release_task_busy) {
2202                 adap->tid_release_task_busy = true;
2203                 schedule_work(&adap->tid_release_task);
2204         }
2205         spin_unlock_bh(&adap->tid_release_lock);
2206 }
2207
2208 /*
2209  * Process the list of pending TID release requests.
2210  */
2211 static void process_tid_release_list(struct work_struct *work)
2212 {
2213         struct sk_buff *skb;
2214         struct adapter *adap;
2215
2216         adap = container_of(work, struct adapter, tid_release_task);
2217
2218         spin_lock_bh(&adap->tid_release_lock);
2219         while (adap->tid_release_head) {
2220                 void **p = adap->tid_release_head;
2221                 unsigned int chan = (uintptr_t)p & 3;
2222                 p = (void *)p - chan;
2223
2224                 adap->tid_release_head = *p;
2225                 *p = NULL;
2226                 spin_unlock_bh(&adap->tid_release_lock);
2227
2228                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2229                                          GFP_KERNEL)))
2230                         schedule_timeout_uninterruptible(1);
2231
2232                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2233                 t4_ofld_send(adap, skb);
2234                 spin_lock_bh(&adap->tid_release_lock);
2235         }
2236         adap->tid_release_task_busy = false;
2237         spin_unlock_bh(&adap->tid_release_lock);
2238 }
2239
2240 /*
2241  * Release a TID and inform HW.  If we are unable to allocate the release
2242  * message we defer to a work queue.
2243  */
2244 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2245 {
2246         void *old;
2247         struct sk_buff *skb;
2248         struct adapter *adap = container_of(t, struct adapter, tids);
2249
2250         old = t->tid_tab[tid];
2251         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2252         if (likely(skb)) {
2253                 t->tid_tab[tid] = NULL;
2254                 mk_tid_release(skb, chan, tid);
2255                 t4_ofld_send(adap, skb);
2256         } else
2257                 cxgb4_queue_tid_release(t, chan, tid);
2258         if (old)
2259                 atomic_dec(&t->tids_in_use);
2260 }
2261 EXPORT_SYMBOL(cxgb4_remove_tid);
2262
2263 /*
2264  * Allocate and initialize the TID tables.  Returns 0 on success.
2265  */
2266 static int tid_init(struct tid_info *t)
2267 {
2268         size_t size;
2269         unsigned int natids = t->natids;
2270
2271         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2272                t->nstids * sizeof(*t->stid_tab) +
2273                BITS_TO_LONGS(t->nstids) * sizeof(long);
2274         t->tid_tab = t4_alloc_mem(size);
2275         if (!t->tid_tab)
2276                 return -ENOMEM;
2277
2278         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2279         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2280         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2281         spin_lock_init(&t->stid_lock);
2282         spin_lock_init(&t->atid_lock);
2283
2284         t->stids_in_use = 0;
2285         t->afree = NULL;
2286         t->atids_in_use = 0;
2287         atomic_set(&t->tids_in_use, 0);
2288
2289         /* Setup the free list for atid_tab and clear the stid bitmap. */
2290         if (natids) {
2291                 while (--natids)
2292                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2293                 t->afree = t->atid_tab;
2294         }
2295         bitmap_zero(t->stid_bmap, t->nstids);
2296         return 0;
2297 }
2298
2299 /**
2300  *      cxgb4_create_server - create an IP server
2301  *      @dev: the device
2302  *      @stid: the server TID
2303  *      @sip: local IP address to bind server to
2304  *      @sport: the server's TCP port
2305  *      @queue: queue to direct messages from this server to
2306  *
2307  *      Create an IP server for the given port and address.
2308  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2309  */
2310 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2311                         __be32 sip, __be16 sport, unsigned int queue)
2312 {
2313         unsigned int chan;
2314         struct sk_buff *skb;
2315         struct adapter *adap;
2316         struct cpl_pass_open_req *req;
2317
2318         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2319         if (!skb)
2320                 return -ENOMEM;
2321
2322         adap = netdev2adap(dev);
2323         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2324         INIT_TP_WR(req, 0);
2325         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2326         req->local_port = sport;
2327         req->peer_port = htons(0);
2328         req->local_ip = sip;
2329         req->peer_ip = htonl(0);
2330         chan = rxq_to_chan(&adap->sge, queue);
2331         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2332         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2333                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2334         return t4_mgmt_tx(adap, skb);
2335 }
2336 EXPORT_SYMBOL(cxgb4_create_server);
2337
2338 /**
2339  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2340  *      @mtus: the HW MTU table
2341  *      @mtu: the target MTU
2342  *      @idx: index of selected entry in the MTU table
2343  *
2344  *      Returns the index and the value in the HW MTU table that is closest to
2345  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2346  *      table, in which case that smallest available value is selected.
2347  */
2348 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2349                             unsigned int *idx)
2350 {
2351         unsigned int i = 0;
2352
2353         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2354                 ++i;
2355         if (idx)
2356                 *idx = i;
2357         return mtus[i];
2358 }
2359 EXPORT_SYMBOL(cxgb4_best_mtu);
2360
2361 /**
2362  *      cxgb4_port_chan - get the HW channel of a port
2363  *      @dev: the net device for the port
2364  *
2365  *      Return the HW Tx channel of the given port.
2366  */
2367 unsigned int cxgb4_port_chan(const struct net_device *dev)
2368 {
2369         return netdev2pinfo(dev)->tx_chan;
2370 }
2371 EXPORT_SYMBOL(cxgb4_port_chan);
2372
2373 /**
2374  *      cxgb4_port_viid - get the VI id of a port
2375  *      @dev: the net device for the port
2376  *
2377  *      Return the VI id of the given port.
2378  */
2379 unsigned int cxgb4_port_viid(const struct net_device *dev)
2380 {
2381         return netdev2pinfo(dev)->viid;
2382 }
2383 EXPORT_SYMBOL(cxgb4_port_viid);
2384
2385 /**
2386  *      cxgb4_port_idx - get the index of a port
2387  *      @dev: the net device for the port
2388  *
2389  *      Return the index of the given port.
2390  */
2391 unsigned int cxgb4_port_idx(const struct net_device *dev)
2392 {
2393         return netdev2pinfo(dev)->port_id;
2394 }
2395 EXPORT_SYMBOL(cxgb4_port_idx);
2396
2397 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2398                          struct tp_tcp_stats *v6)
2399 {
2400         struct adapter *adap = pci_get_drvdata(pdev);
2401
2402         spin_lock(&adap->stats_lock);
2403         t4_tp_get_tcp_stats(adap, v4, v6);
2404         spin_unlock(&adap->stats_lock);
2405 }
2406 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2407
2408 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2409                       const unsigned int *pgsz_order)
2410 {
2411         struct adapter *adap = netdev2adap(dev);
2412
2413         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2414         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2415                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2416                      HPZ3(pgsz_order[3]));
2417 }
2418 EXPORT_SYMBOL(cxgb4_iscsi_init);
2419
2420 static struct pci_driver cxgb4_driver;
2421
2422 static void check_neigh_update(struct neighbour *neigh)
2423 {
2424         const struct device *parent;
2425         const struct net_device *netdev = neigh->dev;
2426
2427         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2428                 netdev = vlan_dev_real_dev(netdev);
2429         parent = netdev->dev.parent;
2430         if (parent && parent->driver == &cxgb4_driver.driver)
2431                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2432 }
2433
2434 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2435                        void *data)
2436 {
2437         switch (event) {
2438         case NETEVENT_NEIGH_UPDATE:
2439                 check_neigh_update(data);
2440                 break;
2441         case NETEVENT_REDIRECT:
2442         default:
2443                 break;
2444         }
2445         return 0;
2446 }
2447
2448 static bool netevent_registered;
2449 static struct notifier_block cxgb4_netevent_nb = {
2450         .notifier_call = netevent_cb
2451 };
2452
2453 static void uld_attach(struct adapter *adap, unsigned int uld)
2454 {
2455         void *handle;
2456         struct cxgb4_lld_info lli;
2457
2458         lli.pdev = adap->pdev;
2459         lli.l2t = adap->l2t;
2460         lli.tids = &adap->tids;
2461         lli.ports = adap->port;
2462         lli.vr = &adap->vres;
2463         lli.mtus = adap->params.mtus;
2464         if (uld == CXGB4_ULD_RDMA) {
2465                 lli.rxq_ids = adap->sge.rdma_rxq;
2466                 lli.nrxq = adap->sge.rdmaqs;
2467         } else if (uld == CXGB4_ULD_ISCSI) {
2468                 lli.rxq_ids = adap->sge.ofld_rxq;
2469                 lli.nrxq = adap->sge.ofldqsets;
2470         }
2471         lli.ntxq = adap->sge.ofldqsets;
2472         lli.nchan = adap->params.nports;
2473         lli.nports = adap->params.nports;
2474         lli.wr_cred = adap->params.ofldq_wr_cred;
2475         lli.adapter_type = adap->params.rev;
2476         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2477         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2478                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2479                         (adap->fn * 4));
2480         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2481                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2482                         (adap->fn * 4));
2483         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2484         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2485         lli.fw_vers = adap->params.fw_vers;
2486
2487         handle = ulds[uld].add(&lli);
2488         if (IS_ERR(handle)) {
2489                 dev_warn(adap->pdev_dev,
2490                          "could not attach to the %s driver, error %ld\n",
2491                          uld_str[uld], PTR_ERR(handle));
2492                 return;
2493         }
2494
2495         adap->uld_handle[uld] = handle;
2496
2497         if (!netevent_registered) {
2498                 register_netevent_notifier(&cxgb4_netevent_nb);
2499                 netevent_registered = true;
2500         }
2501
2502         if (adap->flags & FULL_INIT_DONE)
2503                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2504 }
2505
2506 static void attach_ulds(struct adapter *adap)
2507 {
2508         unsigned int i;
2509
2510         mutex_lock(&uld_mutex);
2511         list_add_tail(&adap->list_node, &adapter_list);
2512         for (i = 0; i < CXGB4_ULD_MAX; i++)
2513                 if (ulds[i].add)
2514                         uld_attach(adap, i);
2515         mutex_unlock(&uld_mutex);
2516 }
2517
2518 static void detach_ulds(struct adapter *adap)
2519 {
2520         unsigned int i;
2521
2522         mutex_lock(&uld_mutex);
2523         list_del(&adap->list_node);
2524         for (i = 0; i < CXGB4_ULD_MAX; i++)
2525                 if (adap->uld_handle[i]) {
2526                         ulds[i].state_change(adap->uld_handle[i],
2527                                              CXGB4_STATE_DETACH);
2528                         adap->uld_handle[i] = NULL;
2529                 }
2530         if (netevent_registered && list_empty(&adapter_list)) {
2531                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2532                 netevent_registered = false;
2533         }
2534         mutex_unlock(&uld_mutex);
2535 }
2536
2537 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2538 {
2539         unsigned int i;
2540
2541         mutex_lock(&uld_mutex);
2542         for (i = 0; i < CXGB4_ULD_MAX; i++)
2543                 if (adap->uld_handle[i])
2544                         ulds[i].state_change(adap->uld_handle[i], new_state);
2545         mutex_unlock(&uld_mutex);
2546 }
2547
2548 /**
2549  *      cxgb4_register_uld - register an upper-layer driver
2550  *      @type: the ULD type
2551  *      @p: the ULD methods
2552  *
2553  *      Registers an upper-layer driver with this driver and notifies the ULD
2554  *      about any presently available devices that support its type.  Returns
2555  *      %-EBUSY if a ULD of the same type is already registered.
2556  */
2557 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2558 {
2559         int ret = 0;
2560         struct adapter *adap;
2561
2562         if (type >= CXGB4_ULD_MAX)
2563                 return -EINVAL;
2564         mutex_lock(&uld_mutex);
2565         if (ulds[type].add) {
2566                 ret = -EBUSY;
2567                 goto out;
2568         }
2569         ulds[type] = *p;
2570         list_for_each_entry(adap, &adapter_list, list_node)
2571                 uld_attach(adap, type);
2572 out:    mutex_unlock(&uld_mutex);
2573         return ret;
2574 }
2575 EXPORT_SYMBOL(cxgb4_register_uld);
2576
2577 /**
2578  *      cxgb4_unregister_uld - unregister an upper-layer driver
2579  *      @type: the ULD type
2580  *
2581  *      Unregisters an existing upper-layer driver.
2582  */
2583 int cxgb4_unregister_uld(enum cxgb4_uld type)
2584 {
2585         struct adapter *adap;
2586
2587         if (type >= CXGB4_ULD_MAX)
2588                 return -EINVAL;
2589         mutex_lock(&uld_mutex);
2590         list_for_each_entry(adap, &adapter_list, list_node)
2591                 adap->uld_handle[type] = NULL;
2592         ulds[type].add = NULL;
2593         mutex_unlock(&uld_mutex);
2594         return 0;
2595 }
2596 EXPORT_SYMBOL(cxgb4_unregister_uld);
2597
2598 /**
2599  *      cxgb_up - enable the adapter
2600  *      @adap: adapter being enabled
2601  *
2602  *      Called when the first port is enabled, this function performs the
2603  *      actions necessary to make an adapter operational, such as completing
2604  *      the initialization of HW modules, and enabling interrupts.
2605  *
2606  *      Must be called with the rtnl lock held.
2607  */
2608 static int cxgb_up(struct adapter *adap)
2609 {
2610         int err;
2611
2612         err = setup_sge_queues(adap);
2613         if (err)
2614                 goto out;
2615         err = setup_rss(adap);
2616         if (err)
2617                 goto freeq;
2618
2619         if (adap->flags & USING_MSIX) {
2620                 name_msix_vecs(adap);
2621                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2622                                   adap->msix_info[0].desc, adap);
2623                 if (err)
2624                         goto irq_err;
2625
2626                 err = request_msix_queue_irqs(adap);
2627                 if (err) {
2628                         free_irq(adap->msix_info[0].vec, adap);
2629                         goto irq_err;
2630                 }
2631         } else {
2632                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2633                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2634                                   adap->port[0]->name, adap);
2635                 if (err)
2636                         goto irq_err;
2637         }
2638         enable_rx(adap);
2639         t4_sge_start(adap);
2640         t4_intr_enable(adap);
2641         adap->flags |= FULL_INIT_DONE;
2642         notify_ulds(adap, CXGB4_STATE_UP);
2643  out:
2644         return err;
2645  irq_err:
2646         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2647  freeq:
2648         t4_free_sge_resources(adap);
2649         goto out;
2650 }
2651
2652 static void cxgb_down(struct adapter *adapter)
2653 {
2654         t4_intr_disable(adapter);
2655         cancel_work_sync(&adapter->tid_release_task);
2656         adapter->tid_release_task_busy = false;
2657         adapter->tid_release_head = NULL;
2658
2659         if (adapter->flags & USING_MSIX) {
2660                 free_msix_queue_irqs(adapter);
2661                 free_irq(adapter->msix_info[0].vec, adapter);
2662         } else
2663                 free_irq(adapter->pdev->irq, adapter);
2664         quiesce_rx(adapter);
2665         t4_sge_stop(adapter);
2666         t4_free_sge_resources(adapter);
2667         adapter->flags &= ~FULL_INIT_DONE;
2668 }
2669
2670 /*
2671  * net_device operations
2672  */
2673 static int cxgb_open(struct net_device *dev)
2674 {
2675         int err;
2676         struct port_info *pi = netdev_priv(dev);
2677         struct adapter *adapter = pi->adapter;
2678
2679         netif_carrier_off(dev);
2680
2681         if (!(adapter->flags & FULL_INIT_DONE)) {
2682                 err = cxgb_up(adapter);
2683                 if (err < 0)
2684                         return err;
2685         }
2686
2687         err = link_start(dev);
2688         if (!err)
2689                 netif_tx_start_all_queues(dev);
2690         return err;
2691 }
2692
2693 static int cxgb_close(struct net_device *dev)
2694 {
2695         struct port_info *pi = netdev_priv(dev);
2696         struct adapter *adapter = pi->adapter;
2697
2698         netif_tx_stop_all_queues(dev);
2699         netif_carrier_off(dev);
2700         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2701 }
2702
2703 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2704                                                 struct rtnl_link_stats64 *ns)
2705 {
2706         struct port_stats stats;
2707         struct port_info *p = netdev_priv(dev);
2708         struct adapter *adapter = p->adapter;
2709
2710         spin_lock(&adapter->stats_lock);
2711         t4_get_port_stats(adapter, p->tx_chan, &stats);
2712         spin_unlock(&adapter->stats_lock);
2713
2714         ns->tx_bytes   = stats.tx_octets;
2715         ns->tx_packets = stats.tx_frames;
2716         ns->rx_bytes   = stats.rx_octets;
2717         ns->rx_packets = stats.rx_frames;
2718         ns->multicast  = stats.rx_mcast_frames;
2719
2720         /* detailed rx_errors */
2721         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2722                                stats.rx_runt;
2723         ns->rx_over_errors   = 0;
2724         ns->rx_crc_errors    = stats.rx_fcs_err;
2725         ns->rx_frame_errors  = stats.rx_symbol_err;
2726         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
2727                                stats.rx_ovflow2 + stats.rx_ovflow3 +
2728                                stats.rx_trunc0 + stats.rx_trunc1 +
2729                                stats.rx_trunc2 + stats.rx_trunc3;
2730         ns->rx_missed_errors = 0;
2731
2732         /* detailed tx_errors */
2733         ns->tx_aborted_errors   = 0;
2734         ns->tx_carrier_errors   = 0;
2735         ns->tx_fifo_errors      = 0;
2736         ns->tx_heartbeat_errors = 0;
2737         ns->tx_window_errors    = 0;
2738
2739         ns->tx_errors = stats.tx_error_frames;
2740         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2741                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2742         return ns;
2743 }
2744
2745 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2746 {
2747         unsigned int mbox;
2748         int ret = 0, prtad, devad;
2749         struct port_info *pi = netdev_priv(dev);
2750         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2751
2752         switch (cmd) {
2753         case SIOCGMIIPHY:
2754                 if (pi->mdio_addr < 0)
2755                         return -EOPNOTSUPP;
2756                 data->phy_id = pi->mdio_addr;
2757                 break;
2758         case SIOCGMIIREG:
2759         case SIOCSMIIREG:
2760                 if (mdio_phy_id_is_c45(data->phy_id)) {
2761                         prtad = mdio_phy_id_prtad(data->phy_id);
2762                         devad = mdio_phy_id_devad(data->phy_id);
2763                 } else if (data->phy_id < 32) {
2764                         prtad = data->phy_id;
2765                         devad = 0;
2766                         data->reg_num &= 0x1f;
2767                 } else
2768                         return -EINVAL;
2769
2770                 mbox = pi->adapter->fn;
2771                 if (cmd == SIOCGMIIREG)
2772                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2773                                          data->reg_num, &data->val_out);
2774                 else
2775                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2776                                          data->reg_num, data->val_in);
2777                 break;
2778         default:
2779                 return -EOPNOTSUPP;
2780         }
2781         return ret;
2782 }
2783
2784 static void cxgb_set_rxmode(struct net_device *dev)
2785 {
2786         /* unfortunately we can't return errors to the stack */
2787         set_rxmode(dev, -1, false);
2788 }
2789
2790 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2791 {
2792         int ret;
2793         struct port_info *pi = netdev_priv(dev);
2794
2795         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
2796                 return -EINVAL;
2797         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2798                             -1, -1, -1, true);
2799         if (!ret)
2800                 dev->mtu = new_mtu;
2801         return ret;
2802 }
2803
2804 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2805 {
2806         int ret;
2807         struct sockaddr *addr = p;
2808         struct port_info *pi = netdev_priv(dev);
2809
2810         if (!is_valid_ether_addr(addr->sa_data))
2811                 return -EINVAL;
2812
2813         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2814                             pi->xact_addr_filt, addr->sa_data, true, true);
2815         if (ret < 0)
2816                 return ret;
2817
2818         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2819         pi->xact_addr_filt = ret;
2820         return 0;
2821 }
2822
2823 #ifdef CONFIG_NET_POLL_CONTROLLER
2824 static void cxgb_netpoll(struct net_device *dev)
2825 {
2826         struct port_info *pi = netdev_priv(dev);
2827         struct adapter *adap = pi->adapter;
2828
2829         if (adap->flags & USING_MSIX) {
2830                 int i;
2831                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2832
2833                 for (i = pi->nqsets; i; i--, rx++)
2834                         t4_sge_intr_msix(0, &rx->rspq);
2835         } else
2836                 t4_intr_handler(adap)(0, adap);
2837 }
2838 #endif
2839
2840 static const struct net_device_ops cxgb4_netdev_ops = {
2841         .ndo_open             = cxgb_open,
2842         .ndo_stop             = cxgb_close,
2843         .ndo_start_xmit       = t4_eth_xmit,
2844         .ndo_get_stats64      = cxgb_get_stats,
2845         .ndo_set_rx_mode      = cxgb_set_rxmode,
2846         .ndo_set_mac_address  = cxgb_set_mac_addr,
2847         .ndo_set_features     = cxgb_set_features,
2848         .ndo_validate_addr    = eth_validate_addr,
2849         .ndo_do_ioctl         = cxgb_ioctl,
2850         .ndo_change_mtu       = cxgb_change_mtu,
2851 #ifdef CONFIG_NET_POLL_CONTROLLER
2852         .ndo_poll_controller  = cxgb_netpoll,
2853 #endif
2854 };
2855
2856 void t4_fatal_err(struct adapter *adap)
2857 {
2858         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2859         t4_intr_disable(adap);
2860         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2861 }
2862
2863 static void setup_memwin(struct adapter *adap)
2864 {
2865         u32 bar0;
2866
2867         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
2868         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2869                      (bar0 + MEMWIN0_BASE) | BIR(0) |
2870                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2871         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2872                      (bar0 + MEMWIN1_BASE) | BIR(0) |
2873                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2874         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2875                      (bar0 + MEMWIN2_BASE) | BIR(0) |
2876                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2877         if (adap->vres.ocq.size) {
2878                 unsigned int start, sz_kb;
2879
2880                 start = pci_resource_start(adap->pdev, 2) +
2881                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2882                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2883                 t4_write_reg(adap,
2884                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2885                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
2886                 t4_write_reg(adap,
2887                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2888                              adap->vres.ocq.start);
2889                 t4_read_reg(adap,
2890                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2891         }
2892 }
2893
2894 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2895 {
2896         u32 v;
2897         int ret;
2898
2899         /* get device capabilities */
2900         memset(c, 0, sizeof(*c));
2901         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2902                                FW_CMD_REQUEST | FW_CMD_READ);
2903         c->retval_len16 = htonl(FW_LEN16(*c));
2904         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
2905         if (ret < 0)
2906                 return ret;
2907
2908         /* select capabilities we'll be using */
2909         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2910                 if (!vf_acls)
2911                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2912                 else
2913                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2914         } else if (vf_acls) {
2915                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2916                 return ret;
2917         }
2918         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2919                                FW_CMD_REQUEST | FW_CMD_WRITE);
2920         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
2921         if (ret < 0)
2922                 return ret;
2923
2924         ret = t4_config_glbl_rss(adap, adap->fn,
2925                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2926                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2927                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2928         if (ret < 0)
2929                 return ret;
2930
2931         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2932                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2933         if (ret < 0)
2934                 return ret;
2935
2936         t4_sge_init(adap);
2937
2938         /* tweak some settings */
2939         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2940         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2941         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2942         v = t4_read_reg(adap, TP_PIO_DATA);
2943         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2944
2945         /* get basic stuff going */
2946         return t4_early_init(adap, adap->fn);
2947 }
2948
2949 /*
2950  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
2951  */
2952 #define MAX_ATIDS 8192U
2953
2954 /*
2955  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2956  */
2957 static int adap_init0(struct adapter *adap)
2958 {
2959         int ret;
2960         u32 v, port_vec;
2961         enum dev_state state;
2962         u32 params[7], val[7];
2963         struct fw_caps_config_cmd c;
2964
2965         ret = t4_check_fw_version(adap);
2966         if (ret == -EINVAL || ret > 0) {
2967                 if (upgrade_fw(adap) >= 0)             /* recache FW version */
2968                         ret = t4_check_fw_version(adap);
2969         }
2970         if (ret < 0)
2971                 return ret;
2972
2973         /* contact FW, request master */
2974         ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
2975         if (ret < 0) {
2976                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2977                         ret);
2978                 return ret;
2979         }
2980
2981         /* reset device */
2982         ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
2983         if (ret < 0)
2984                 goto bye;
2985
2986         for (v = 0; v < SGE_NTIMERS - 1; v++)
2987                 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2988         adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2989         adap->sge.counter_val[0] = 1;
2990         for (v = 1; v < SGE_NCOUNTERS; v++)
2991                 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2992                                                THRESHOLD_3_MASK);
2993 #define FW_PARAM_DEV(param) \
2994         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2995          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2996
2997         params[0] = FW_PARAM_DEV(CCLK);
2998         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
2999         if (ret < 0)
3000                 goto bye;
3001         adap->params.vpd.cclk = val[0];
3002
3003         ret = adap_init1(adap, &c);
3004         if (ret < 0)
3005                 goto bye;
3006
3007 #define FW_PARAM_PFVF(param) \
3008         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3009          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3010          FW_PARAMS_PARAM_Y(adap->fn))
3011
3012         params[0] = FW_PARAM_DEV(PORTVEC);
3013         params[1] = FW_PARAM_PFVF(L2T_START);
3014         params[2] = FW_PARAM_PFVF(L2T_END);
3015         params[3] = FW_PARAM_PFVF(FILTER_START);
3016         params[4] = FW_PARAM_PFVF(FILTER_END);
3017         params[5] = FW_PARAM_PFVF(IQFLINT_START);
3018         params[6] = FW_PARAM_PFVF(EQ_START);
3019         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
3020         if (ret < 0)
3021                 goto bye;
3022         port_vec = val[0];
3023         adap->tids.ftid_base = val[3];
3024         adap->tids.nftids = val[4] - val[3] + 1;
3025         adap->sge.ingr_start = val[5];
3026         adap->sge.egr_start = val[6];
3027
3028         if (c.ofldcaps) {
3029                 /* query offload-related parameters */
3030                 params[0] = FW_PARAM_DEV(NTID);
3031                 params[1] = FW_PARAM_PFVF(SERVER_START);
3032                 params[2] = FW_PARAM_PFVF(SERVER_END);
3033                 params[3] = FW_PARAM_PFVF(TDDP_START);
3034                 params[4] = FW_PARAM_PFVF(TDDP_END);
3035                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3036                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3037                                       val);
3038                 if (ret < 0)
3039                         goto bye;
3040                 adap->tids.ntids = val[0];
3041                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3042                 adap->tids.stid_base = val[1];
3043                 adap->tids.nstids = val[2] - val[1] + 1;
3044                 adap->vres.ddp.start = val[3];
3045                 adap->vres.ddp.size = val[4] - val[3] + 1;
3046                 adap->params.ofldq_wr_cred = val[5];
3047                 adap->params.offload = 1;
3048         }
3049         if (c.rdmacaps) {
3050                 params[0] = FW_PARAM_PFVF(STAG_START);
3051                 params[1] = FW_PARAM_PFVF(STAG_END);
3052                 params[2] = FW_PARAM_PFVF(RQ_START);
3053                 params[3] = FW_PARAM_PFVF(RQ_END);
3054                 params[4] = FW_PARAM_PFVF(PBL_START);
3055                 params[5] = FW_PARAM_PFVF(PBL_END);
3056                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3057                                       val);
3058                 if (ret < 0)
3059                         goto bye;
3060                 adap->vres.stag.start = val[0];
3061                 adap->vres.stag.size = val[1] - val[0] + 1;
3062                 adap->vres.rq.start = val[2];
3063                 adap->vres.rq.size = val[3] - val[2] + 1;
3064                 adap->vres.pbl.start = val[4];
3065                 adap->vres.pbl.size = val[5] - val[4] + 1;
3066
3067                 params[0] = FW_PARAM_PFVF(SQRQ_START);
3068                 params[1] = FW_PARAM_PFVF(SQRQ_END);
3069                 params[2] = FW_PARAM_PFVF(CQ_START);
3070                 params[3] = FW_PARAM_PFVF(CQ_END);
3071                 params[4] = FW_PARAM_PFVF(OCQ_START);
3072                 params[5] = FW_PARAM_PFVF(OCQ_END);
3073                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3074                                       val);
3075                 if (ret < 0)
3076                         goto bye;
3077                 adap->vres.qp.start = val[0];
3078                 adap->vres.qp.size = val[1] - val[0] + 1;
3079                 adap->vres.cq.start = val[2];
3080                 adap->vres.cq.size = val[3] - val[2] + 1;
3081                 adap->vres.ocq.start = val[4];
3082                 adap->vres.ocq.size = val[5] - val[4] + 1;
3083         }
3084         if (c.iscsicaps) {
3085                 params[0] = FW_PARAM_PFVF(ISCSI_START);
3086                 params[1] = FW_PARAM_PFVF(ISCSI_END);
3087                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3088                                       val);
3089                 if (ret < 0)
3090                         goto bye;
3091                 adap->vres.iscsi.start = val[0];
3092                 adap->vres.iscsi.size = val[1] - val[0] + 1;
3093         }
3094 #undef FW_PARAM_PFVF
3095 #undef FW_PARAM_DEV
3096
3097         adap->params.nports = hweight32(port_vec);
3098         adap->params.portvec = port_vec;
3099         adap->flags |= FW_OK;
3100
3101         /* These are finalized by FW initialization, load their values now */
3102         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3103         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3104         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3105         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3106                      adap->params.b_wnd);
3107
3108 #ifdef CONFIG_PCI_IOV
3109         /*
3110          * Provision resource limits for Virtual Functions.  We currently
3111          * grant them all the same static resource limits except for the Port
3112          * Access Rights Mask which we're assigning based on the PF.  All of
3113          * the static provisioning stuff for both the PF and VF really needs
3114          * to be managed in a persistent manner for each device which the
3115          * firmware controls.
3116          */
3117         {
3118                 int pf, vf;
3119
3120                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3121                         if (num_vf[pf] <= 0)
3122                                 continue;
3123
3124                         /* VF numbering starts at 1! */
3125                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3126                                 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3127                                                   VFRES_NEQ, VFRES_NETHCTRL,
3128                                                   VFRES_NIQFLINT, VFRES_NIQ,
3129                                                   VFRES_TC, VFRES_NVI,
3130                                                   FW_PFVF_CMD_CMASK_MASK,
3131                                                   pfvfres_pmask(adap, pf, vf),
3132                                                   VFRES_NEXACTF,
3133                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3134                                 if (ret < 0)
3135                                         dev_warn(adap->pdev_dev, "failed to "
3136                                                  "provision pf/vf=%d/%d; "
3137                                                  "err=%d\n", pf, vf, ret);
3138                         }
3139                 }
3140         }
3141 #endif
3142
3143         setup_memwin(adap);
3144         return 0;
3145
3146         /*
3147          * If a command timed out or failed with EIO FW does not operate within
3148          * its spec or something catastrophic happened to HW/FW, stop issuing
3149          * commands.
3150          */
3151 bye:    if (ret != -ETIMEDOUT && ret != -EIO)
3152                 t4_fw_bye(adap, adap->fn);
3153         return ret;
3154 }
3155
3156 /* EEH callbacks */
3157
3158 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3159                                          pci_channel_state_t state)
3160 {
3161         int i;
3162         struct adapter *adap = pci_get_drvdata(pdev);
3163
3164         if (!adap)
3165                 goto out;
3166
3167         rtnl_lock();
3168         adap->flags &= ~FW_OK;
3169         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3170         for_each_port(adap, i) {
3171                 struct net_device *dev = adap->port[i];
3172
3173                 netif_device_detach(dev);
3174                 netif_carrier_off(dev);
3175         }
3176         if (adap->flags & FULL_INIT_DONE)
3177                 cxgb_down(adap);
3178         rtnl_unlock();
3179         pci_disable_device(pdev);
3180 out:    return state == pci_channel_io_perm_failure ?
3181                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3182 }
3183
3184 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3185 {
3186         int i, ret;
3187         struct fw_caps_config_cmd c;
3188         struct adapter *adap = pci_get_drvdata(pdev);
3189
3190         if (!adap) {
3191                 pci_restore_state(pdev);
3192                 pci_save_state(pdev);
3193                 return PCI_ERS_RESULT_RECOVERED;
3194         }
3195
3196         if (pci_enable_device(pdev)) {
3197                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3198                 return PCI_ERS_RESULT_DISCONNECT;
3199         }
3200
3201         pci_set_master(pdev);
3202         pci_restore_state(pdev);
3203         pci_save_state(pdev);
3204         pci_cleanup_aer_uncorrect_error_status(pdev);
3205
3206         if (t4_wait_dev_ready(adap) < 0)
3207                 return PCI_ERS_RESULT_DISCONNECT;
3208         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3209                 return PCI_ERS_RESULT_DISCONNECT;
3210         adap->flags |= FW_OK;
3211         if (adap_init1(adap, &c))
3212                 return PCI_ERS_RESULT_DISCONNECT;
3213
3214         for_each_port(adap, i) {
3215                 struct port_info *p = adap2pinfo(adap, i);
3216
3217                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3218                                   NULL, NULL);
3219                 if (ret < 0)
3220                         return PCI_ERS_RESULT_DISCONNECT;
3221                 p->viid = ret;
3222                 p->xact_addr_filt = -1;
3223         }
3224
3225         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3226                      adap->params.b_wnd);
3227         setup_memwin(adap);
3228         if (cxgb_up(adap))
3229                 return PCI_ERS_RESULT_DISCONNECT;
3230         return PCI_ERS_RESULT_RECOVERED;
3231 }
3232
3233 static void eeh_resume(struct pci_dev *pdev)
3234 {
3235         int i;
3236         struct adapter *adap = pci_get_drvdata(pdev);
3237
3238         if (!adap)
3239                 return;
3240
3241         rtnl_lock();
3242         for_each_port(adap, i) {
3243                 struct net_device *dev = adap->port[i];
3244
3245                 if (netif_running(dev)) {
3246                         link_start(dev);
3247                         cxgb_set_rxmode(dev);
3248                 }
3249                 netif_device_attach(dev);
3250         }
3251         rtnl_unlock();
3252 }
3253
3254 static struct pci_error_handlers cxgb4_eeh = {
3255         .error_detected = eeh_err_detected,
3256         .slot_reset     = eeh_slot_reset,
3257         .resume         = eeh_resume,
3258 };
3259
3260 static inline bool is_10g_port(const struct link_config *lc)
3261 {
3262         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3263 }
3264
3265 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3266                              unsigned int size, unsigned int iqe_size)
3267 {
3268         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3269                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3270         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3271         q->iqe_len = iqe_size;
3272         q->size = size;
3273 }
3274
3275 /*
3276  * Perform default configuration of DMA queues depending on the number and type
3277  * of ports we found and the number of available CPUs.  Most settings can be
3278  * modified by the admin prior to actual use.
3279  */
3280 static void __devinit cfg_queues(struct adapter *adap)
3281 {
3282         struct sge *s = &adap->sge;
3283         int i, q10g = 0, n10g = 0, qidx = 0;
3284
3285         for_each_port(adap, i)
3286                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3287
3288         /*
3289          * We default to 1 queue per non-10G port and up to # of cores queues
3290          * per 10G port.
3291          */
3292         if (n10g)
3293                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3294         if (q10g > num_online_cpus())
3295                 q10g = num_online_cpus();
3296
3297         for_each_port(adap, i) {
3298                 struct port_info *pi = adap2pinfo(adap, i);
3299
3300                 pi->first_qset = qidx;
3301                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3302                 qidx += pi->nqsets;
3303         }
3304
3305         s->ethqsets = qidx;
3306         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
3307
3308         if (is_offload(adap)) {
3309                 /*
3310                  * For offload we use 1 queue/channel if all ports are up to 1G,
3311                  * otherwise we divide all available queues amongst the channels
3312                  * capped by the number of available cores.
3313                  */
3314                 if (n10g) {
3315                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3316                                   num_online_cpus());
3317                         s->ofldqsets = roundup(i, adap->params.nports);
3318                 } else
3319                         s->ofldqsets = adap->params.nports;
3320                 /* For RDMA one Rx queue per channel suffices */
3321                 s->rdmaqs = adap->params.nports;
3322         }
3323
3324         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3325                 struct sge_eth_rxq *r = &s->ethrxq[i];
3326
3327                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3328                 r->fl.size = 72;
3329         }
3330
3331         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3332                 s->ethtxq[i].q.size = 1024;
3333
3334         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3335                 s->ctrlq[i].q.size = 512;
3336
3337         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3338                 s->ofldtxq[i].q.size = 1024;
3339
3340         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3341                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3342
3343                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3344                 r->rspq.uld = CXGB4_ULD_ISCSI;
3345                 r->fl.size = 72;
3346         }
3347
3348         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3349                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3350
3351                 init_rspq(&r->rspq, 0, 0, 511, 64);
3352                 r->rspq.uld = CXGB4_ULD_RDMA;
3353                 r->fl.size = 72;
3354         }
3355
3356         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3357         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3358 }
3359
3360 /*
3361  * Reduce the number of Ethernet queues across all ports to at most n.
3362  * n provides at least one queue per port.
3363  */
3364 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3365 {
3366         int i;
3367         struct port_info *pi;
3368
3369         while (n < adap->sge.ethqsets)
3370                 for_each_port(adap, i) {
3371                         pi = adap2pinfo(adap, i);
3372                         if (pi->nqsets > 1) {
3373                                 pi->nqsets--;
3374                                 adap->sge.ethqsets--;
3375                                 if (adap->sge.ethqsets <= n)
3376                                         break;
3377                         }
3378                 }
3379
3380         n = 0;
3381         for_each_port(adap, i) {
3382                 pi = adap2pinfo(adap, i);
3383                 pi->first_qset = n;
3384                 n += pi->nqsets;
3385         }
3386 }
3387
3388 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3389 #define EXTRA_VECS 2
3390
3391 static int __devinit enable_msix(struct adapter *adap)
3392 {
3393         int ofld_need = 0;
3394         int i, err, want, need;
3395         struct sge *s = &adap->sge;
3396         unsigned int nchan = adap->params.nports;
3397         struct msix_entry entries[MAX_INGQ + 1];
3398
3399         for (i = 0; i < ARRAY_SIZE(entries); ++i)
3400                 entries[i].entry = i;
3401
3402         want = s->max_ethqsets + EXTRA_VECS;
3403         if (is_offload(adap)) {
3404                 want += s->rdmaqs + s->ofldqsets;
3405                 /* need nchan for each possible ULD */
3406                 ofld_need = 2 * nchan;
3407         }
3408         need = adap->params.nports + EXTRA_VECS + ofld_need;
3409
3410         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3411                 want = err;
3412
3413         if (!err) {
3414                 /*
3415                  * Distribute available vectors to the various queue groups.
3416                  * Every group gets its minimum requirement and NIC gets top
3417                  * priority for leftovers.
3418                  */
3419                 i = want - EXTRA_VECS - ofld_need;
3420                 if (i < s->max_ethqsets) {
3421                         s->max_ethqsets = i;
3422                         if (i < s->ethqsets)
3423                                 reduce_ethqs(adap, i);
3424                 }
3425                 if (is_offload(adap)) {
3426                         i = want - EXTRA_VECS - s->max_ethqsets;
3427                         i -= ofld_need - nchan;
3428                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
3429                 }
3430                 for (i = 0; i < want; ++i)
3431                         adap->msix_info[i].vec = entries[i].vector;
3432         } else if (err > 0)
3433                 dev_info(adap->pdev_dev,
3434                          "only %d MSI-X vectors left, not using MSI-X\n", err);
3435         return err;
3436 }
3437
3438 #undef EXTRA_VECS
3439
3440 static int __devinit init_rss(struct adapter *adap)
3441 {
3442         unsigned int i, j;
3443
3444         for_each_port(adap, i) {
3445                 struct port_info *pi = adap2pinfo(adap, i);
3446
3447                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3448                 if (!pi->rss)
3449                         return -ENOMEM;
3450                 for (j = 0; j < pi->rss_size; j++)
3451                         pi->rss[j] = j % pi->nqsets;
3452         }
3453         return 0;
3454 }
3455
3456 static void __devinit print_port_info(const struct net_device *dev)
3457 {
3458         static const char *base[] = {
3459                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3460                 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
3461         };
3462
3463         char buf[80];
3464         char *bufp = buf;
3465         const char *spd = "";
3466         const struct port_info *pi = netdev_priv(dev);
3467         const struct adapter *adap = pi->adapter;
3468
3469         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3470                 spd = " 2.5 GT/s";
3471         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3472                 spd = " 5 GT/s";
3473
3474         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3475                 bufp += sprintf(bufp, "100/");
3476         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3477                 bufp += sprintf(bufp, "1000/");
3478         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3479                 bufp += sprintf(bufp, "10G/");
3480         if (bufp != buf)
3481                 --bufp;
3482         sprintf(bufp, "BASE-%s", base[pi->port_type]);
3483
3484         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3485                     adap->params.vpd.id, adap->params.rev, buf,
3486                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
3487                     (adap->flags & USING_MSIX) ? " MSI-X" :
3488                     (adap->flags & USING_MSI) ? " MSI" : "");
3489         netdev_info(dev, "S/N: %s, E/C: %s\n",
3490                     adap->params.vpd.sn, adap->params.vpd.ec);
3491 }
3492
3493 static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
3494 {
3495         u16 v;
3496         int pos;
3497
3498         pos = pci_pcie_cap(dev);
3499         if (pos > 0) {
3500                 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
3501                 v |= PCI_EXP_DEVCTL_RELAX_EN;
3502                 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
3503         }
3504 }
3505
3506 /*
3507  * Free the following resources:
3508  * - memory used for tables
3509  * - MSI/MSI-X
3510  * - net devices
3511  * - resources FW is holding for us
3512  */
3513 static void free_some_resources(struct adapter *adapter)
3514 {
3515         unsigned int i;
3516
3517         t4_free_mem(adapter->l2t);
3518         t4_free_mem(adapter->tids.tid_tab);
3519         disable_msi(adapter);
3520
3521         for_each_port(adapter, i)
3522                 if (adapter->port[i]) {
3523                         kfree(adap2pinfo(adapter, i)->rss);
3524                         free_netdev(adapter->port[i]);
3525                 }
3526         if (adapter->flags & FW_OK)
3527                 t4_fw_bye(adapter, adapter->fn);
3528 }
3529
3530 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3531 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3532                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3533
3534 static int __devinit init_one(struct pci_dev *pdev,
3535                               const struct pci_device_id *ent)
3536 {
3537         int func, i, err;
3538         struct port_info *pi;
3539         unsigned int highdma = 0;
3540         struct adapter *adapter = NULL;
3541
3542         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3543
3544         err = pci_request_regions(pdev, KBUILD_MODNAME);
3545         if (err) {
3546                 /* Just info, some other driver may have claimed the device. */
3547                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3548                 return err;
3549         }
3550
3551         /* We control everything through one PF */
3552         func = PCI_FUNC(pdev->devfn);
3553         if (func != ent->driver_data) {
3554                 pci_save_state(pdev);        /* to restore SR-IOV later */
3555                 goto sriov;
3556         }
3557
3558         err = pci_enable_device(pdev);
3559         if (err) {
3560                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3561                 goto out_release_regions;
3562         }
3563
3564         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3565                 highdma = NETIF_F_HIGHDMA;
3566                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3567                 if (err) {
3568                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3569                                 "coherent allocations\n");
3570                         goto out_disable_device;
3571                 }
3572         } else {
3573                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3574                 if (err) {
3575                         dev_err(&pdev->dev, "no usable DMA configuration\n");
3576                         goto out_disable_device;
3577                 }
3578         }
3579
3580         pci_enable_pcie_error_reporting(pdev);
3581         enable_pcie_relaxed_ordering(pdev);
3582         pci_set_master(pdev);
3583         pci_save_state(pdev);
3584
3585         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3586         if (!adapter) {
3587                 err = -ENOMEM;
3588                 goto out_disable_device;
3589         }
3590
3591         adapter->regs = pci_ioremap_bar(pdev, 0);
3592         if (!adapter->regs) {
3593                 dev_err(&pdev->dev, "cannot map device registers\n");
3594                 err = -ENOMEM;
3595                 goto out_free_adapter;
3596         }
3597
3598         adapter->pdev = pdev;
3599         adapter->pdev_dev = &pdev->dev;
3600         adapter->fn = func;
3601         adapter->msg_enable = dflt_msg_enable;
3602         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3603
3604         spin_lock_init(&adapter->stats_lock);
3605         spin_lock_init(&adapter->tid_release_lock);
3606
3607         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3608
3609         err = t4_prep_adapter(adapter);
3610         if (err)
3611                 goto out_unmap_bar;
3612         err = adap_init0(adapter);
3613         if (err)
3614                 goto out_unmap_bar;
3615
3616         for_each_port(adapter, i) {
3617                 struct net_device *netdev;
3618
3619                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3620                                            MAX_ETH_QSETS);
3621                 if (!netdev) {
3622                         err = -ENOMEM;
3623                         goto out_free_dev;
3624                 }
3625
3626                 SET_NETDEV_DEV(netdev, &pdev->dev);
3627
3628                 adapter->port[i] = netdev;
3629                 pi = netdev_priv(netdev);
3630                 pi->adapter = adapter;
3631                 pi->xact_addr_filt = -1;
3632                 pi->port_id = i;
3633                 netdev->irq = pdev->irq;
3634
3635                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3636                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3637                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
3638                         NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3639                 netdev->features |= netdev->hw_features | highdma;
3640                 netdev->vlan_features = netdev->features & VLAN_FEAT;
3641
3642                 netdev->netdev_ops = &cxgb4_netdev_ops;
3643                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3644         }
3645
3646         pci_set_drvdata(pdev, adapter);
3647
3648         if (adapter->flags & FW_OK) {
3649                 err = t4_port_init(adapter, func, func, 0);
3650                 if (err)
3651                         goto out_free_dev;
3652         }
3653
3654         /*
3655          * Configure queues and allocate tables now, they can be needed as
3656          * soon as the first register_netdev completes.
3657          */
3658         cfg_queues(adapter);
3659
3660         adapter->l2t = t4_init_l2t();
3661         if (!adapter->l2t) {
3662                 /* We tolerate a lack of L2T, giving up some functionality */
3663                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3664                 adapter->params.offload = 0;
3665         }
3666
3667         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3668                 dev_warn(&pdev->dev, "could not allocate TID table, "
3669                          "continuing\n");
3670                 adapter->params.offload = 0;
3671         }
3672
3673         /* See what interrupts we'll be using */
3674         if (msi > 1 && enable_msix(adapter) == 0)
3675                 adapter->flags |= USING_MSIX;
3676         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3677                 adapter->flags |= USING_MSI;
3678
3679         err = init_rss(adapter);
3680         if (err)
3681                 goto out_free_dev;
3682
3683         /*
3684          * The card is now ready to go.  If any errors occur during device
3685          * registration we do not fail the whole card but rather proceed only
3686          * with the ports we manage to register successfully.  However we must
3687          * register at least one net device.
3688          */
3689         for_each_port(adapter, i) {
3690                 pi = adap2pinfo(adapter, i);
3691                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
3692                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
3693
3694                 err = register_netdev(adapter->port[i]);
3695                 if (err)
3696                         break;
3697                 adapter->chan_map[pi->tx_chan] = i;
3698                 print_port_info(adapter->port[i]);
3699         }
3700         if (i == 0) {
3701                 dev_err(&pdev->dev, "could not register any net devices\n");
3702                 goto out_free_dev;
3703         }
3704         if (err) {
3705                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
3706                 err = 0;
3707         }
3708
3709         if (cxgb4_debugfs_root) {
3710                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3711                                                            cxgb4_debugfs_root);
3712                 setup_debugfs(adapter);
3713         }
3714
3715         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
3716         pdev->needs_freset = 1;
3717
3718         if (is_offload(adapter))
3719                 attach_ulds(adapter);
3720
3721 sriov:
3722 #ifdef CONFIG_PCI_IOV
3723         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3724                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3725                         dev_info(&pdev->dev,
3726                                  "instantiated %u virtual functions\n",
3727                                  num_vf[func]);
3728 #endif
3729         return 0;
3730
3731  out_free_dev:
3732         free_some_resources(adapter);
3733  out_unmap_bar:
3734         iounmap(adapter->regs);
3735  out_free_adapter:
3736         kfree(adapter);
3737  out_disable_device:
3738         pci_disable_pcie_error_reporting(pdev);
3739         pci_disable_device(pdev);
3740  out_release_regions:
3741         pci_release_regions(pdev);
3742         pci_set_drvdata(pdev, NULL);
3743         return err;
3744 }
3745
3746 static void __devexit remove_one(struct pci_dev *pdev)
3747 {
3748         struct adapter *adapter = pci_get_drvdata(pdev);
3749
3750         pci_disable_sriov(pdev);
3751
3752         if (adapter) {
3753                 int i;
3754
3755                 if (is_offload(adapter))
3756                         detach_ulds(adapter);
3757
3758                 for_each_port(adapter, i)
3759                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
3760                                 unregister_netdev(adapter->port[i]);
3761
3762                 if (adapter->debugfs_root)
3763                         debugfs_remove_recursive(adapter->debugfs_root);
3764
3765                 if (adapter->flags & FULL_INIT_DONE)
3766                         cxgb_down(adapter);
3767
3768                 free_some_resources(adapter);
3769                 iounmap(adapter->regs);
3770                 kfree(adapter);
3771                 pci_disable_pcie_error_reporting(pdev);
3772                 pci_disable_device(pdev);
3773                 pci_release_regions(pdev);
3774                 pci_set_drvdata(pdev, NULL);
3775         } else
3776                 pci_release_regions(pdev);
3777 }
3778
3779 static struct pci_driver cxgb4_driver = {
3780         .name     = KBUILD_MODNAME,
3781         .id_table = cxgb4_pci_tbl,
3782         .probe    = init_one,
3783         .remove   = __devexit_p(remove_one),
3784         .err_handler = &cxgb4_eeh,
3785 };
3786
3787 static int __init cxgb4_init_module(void)
3788 {
3789         int ret;
3790
3791         /* Debugfs support is optional, just warn if this fails */
3792         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3793         if (!cxgb4_debugfs_root)
3794                 pr_warning("could not create debugfs entry, continuing\n");
3795
3796         ret = pci_register_driver(&cxgb4_driver);
3797         if (ret < 0)
3798                 debugfs_remove(cxgb4_debugfs_root);
3799         return ret;
3800 }
3801
3802 static void __exit cxgb4_cleanup_module(void)
3803 {
3804         pci_unregister_driver(&cxgb4_driver);
3805         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
3806 }
3807
3808 module_init(cxgb4_init_module);
3809 module_exit(cxgb4_cleanup_module);