drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <asm/uaccess.h>
  64
  65 #include "cxgb4.h"
  66 #include "t4_regs.h"
  67 #include "t4_msg.h"
  68 #include "t4fw_api.h"
  69 #include "l2t.h"
  70
  71 #define DRV_VERSION "1.3.0-ko"
  72 #define DRV_DESC "Chelsio T4 Network Driver"
  73
  74 /*
  75  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  76  * under extreme memory pressure so it's largish to give the system time to
  77  * recover.
  78  */
  79 #define MAX_SGE_TIMERVAL 200U
  80
  81 #ifdef CONFIG_PCI_IOV
  82 /*
  83  * Virtual Function provisioning constants.  We need two extra Ingress Queues
  84  * with Interrupt capability to serve as the VF's Firmware Event Queue and
  85  * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
  86  * Lists associated with them).  For each Ethernet/Control Egress Queue and
  87  * for each Free List, we need an Egress Context.
  88  */
  89 enum {
  90         VFRES_NPORTS = 1,               /* # of "ports" per VF */
  91         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
  92
  93         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
  94         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
  95         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
  96         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
  97         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
  98         VFRES_TC = 0,                   /* PCI-E traffic class */
  99         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
 100
 101         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 102         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 103 };
 104
 105 /*
 106  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 107  * static and likely not to be useful in the long run.  We really need to
 108  * implement some form of persistent configuration which the firmware
 109  * controls.
 110  */
 111 static unsigned int pfvfres_pmask(struct adapter *adapter,
 112                                   unsigned int pf, unsigned int vf)
 113 {
 114         unsigned int portn, portvec;
 115
 116         /*
 117          * Give PF's access to all of the ports.
 118          */
 119         if (vf == 0)
 120                 return FW_PFVF_CMD_PMASK_MASK;
 121
 122         /*
 123          * For VFs, we'll assign them access to the ports based purely on the
 124          * PF.  We assign active ports in order, wrapping around if there are
 125          * fewer active ports than PFs: e.g. active port[pf % nports].
 126          * Unfortunately the adapter's port_info structs haven't been
 127          * initialized yet so we have to compute this.
 128          */
 129         if (adapter->params.nports == 0)
 130                 return 0;
 131
 132         portn = pf % adapter->params.nports;
 133         portvec = adapter->params.portvec;
 134         for (;;) {
 135                 /*
 136                  * Isolate the lowest set bit in the port vector.  If we're at
 137                  * the port number that we want, return that as the pmask.
 138                  * otherwise mask that bit out of the port vector and
 139                  * decrement our port number ...
 140                  */
 141                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 142                 if (portn == 0)
 143                         return pmask;
 144                 portn--;
 145                 portvec &= ~pmask;
 146         }
 147         /*NOTREACHED*/
 148 }
 149 #endif
 150
 151 enum {
 152         MEMWIN0_APERTURE = 65536,
 153         MEMWIN0_BASE     = 0x30000,
 154         MEMWIN1_APERTURE = 32768,
 155         MEMWIN1_BASE     = 0x28000,
 156         MEMWIN2_APERTURE = 2048,
 157         MEMWIN2_BASE     = 0x1b800,
 158 };
 159
 160 enum {
 161         MAX_TXQ_ENTRIES      = 16384,
 162         MAX_CTRL_TXQ_ENTRIES = 1024,
 163         MAX_RSPQ_ENTRIES     = 16384,
 164         MAX_RX_BUFFERS       = 16384,
 165         MIN_TXQ_ENTRIES      = 32,
 166         MIN_CTRL_TXQ_ENTRIES = 32,
 167         MIN_RSPQ_ENTRIES     = 128,
 168         MIN_FL_ENTRIES       = 16
 169 };
 170
 171 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 172                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 173                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 174
 175 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 176
 177 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 178         CH_DEVICE(0xa000, 0),  /* PE10K */
 179         CH_DEVICE(0x4001, -1),
 180         CH_DEVICE(0x4002, -1),
 181         CH_DEVICE(0x4003, -1),
 182         CH_DEVICE(0x4004, -1),
 183         CH_DEVICE(0x4005, -1),
 184         CH_DEVICE(0x4006, -1),
 185         CH_DEVICE(0x4007, -1),
 186         CH_DEVICE(0x4008, -1),
 187         CH_DEVICE(0x4009, -1),
 188         CH_DEVICE(0x400a, -1),
 189         CH_DEVICE(0x4401, 4),
 190         CH_DEVICE(0x4402, 4),
 191         CH_DEVICE(0x4403, 4),
 192         CH_DEVICE(0x4404, 4),
 193         CH_DEVICE(0x4405, 4),
 194         CH_DEVICE(0x4406, 4),
 195         CH_DEVICE(0x4407, 4),
 196         CH_DEVICE(0x4408, 4),
 197         CH_DEVICE(0x4409, 4),
 198         CH_DEVICE(0x440a, 4),
 199         { 0, }
 200 };
 201
 202 #define FW_FNAME "cxgb4/t4fw.bin"
 203
 204 MODULE_DESCRIPTION(DRV_DESC);
 205 MODULE_AUTHOR("Chelsio Communications");
 206 MODULE_LICENSE("Dual BSD/GPL");
 207 MODULE_VERSION(DRV_VERSION);
 208 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 209 MODULE_FIRMWARE(FW_FNAME);
 210
 211 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 212
 213 module_param(dflt_msg_enable, int, 0644);
 214 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 215
 216 /*
 217  * The driver uses the best interrupt scheme available on a platform in the
 218  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 219  * of these schemes the driver may consider as follows:
 220  *
 221  * msi = 2: choose from among all three options
 222  * msi = 1: only consider MSI and INTx interrupts
 223  * msi = 0: force INTx interrupts
 224  */
 225 static int msi = 2;
 226
 227 module_param(msi, int, 0644);
 228 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 229
 230 /*
 231  * Queue interrupt hold-off timer values.  Queues default to the first of these
 232  * upon creation.
 233  */
 234 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 235
 236 module_param_array(intr_holdoff, uint, NULL, 0644);
 237 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 238                  "0..4 in microseconds");
 239
 240 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 241
 242 module_param_array(intr_cnt, uint, NULL, 0644);
 243 MODULE_PARM_DESC(intr_cnt,
 244                  "thresholds 1..3 for queue interrupt packet counters");
 245
 246 static int vf_acls;
 247
 248 #ifdef CONFIG_PCI_IOV
 249 module_param(vf_acls, bool, 0644);
 250 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 251
 252 static unsigned int num_vf[4];
 253
 254 module_param_array(num_vf, uint, NULL, 0644);
 255 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 256 #endif
 257
 258 static struct dentry *cxgb4_debugfs_root;
 259
 260 static LIST_HEAD(adapter_list);
 261 static DEFINE_MUTEX(uld_mutex);
 262 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 263 static const char *uld_str[] = { "RDMA", "iSCSI" };
 264
 265 static void link_report(struct net_device *dev)
 266 {
 267         if (!netif_carrier_ok(dev))
 268                 netdev_info(dev, "link down\n");
 269         else {
 270                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 271
 272                 const char *s = "10Mbps";
 273                 const struct port_info *p = netdev_priv(dev);
 274
 275                 switch (p->link_cfg.speed) {
 276                 case SPEED_10000:
 277                         s = "10Gbps";
 278                         break;
 279                 case SPEED_1000:
 280                         s = "1000Mbps";
 281                         break;
 282                 case SPEED_100:
 283                         s = "100Mbps";
 284                         break;
 285                 }
 286
 287                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 288                             fc[p->link_cfg.fc]);
 289         }
 290 }
 291
 292 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 293 {
 294         struct net_device *dev = adapter->port[port_id];
 295
 296         /* Skip changes from disabled ports. */
 297         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 298                 if (link_stat)
 299                         netif_carrier_on(dev);
 300                 else
 301                         netif_carrier_off(dev);
 302
 303                 link_report(dev);
 304         }
 305 }
 306
 307 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 308 {
 309         static const char *mod_str[] = {
 310                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 311         };
 312
 313         const struct net_device *dev = adap->port[port_id];
 314         const struct port_info *pi = netdev_priv(dev);
 315
 316         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 317                 netdev_info(dev, "port module unplugged\n");
 318         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 319                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 320 }
 321
 322 /*
 323  * Configure the exact and hash address filters to handle a port's multicast
 324  * and secondary unicast MAC addresses.
 325  */
 326 static int set_addr_filters(const struct net_device *dev, bool sleep)
 327 {
 328         u64 mhash = 0;
 329         u64 uhash = 0;
 330         bool free = true;
 331         u16 filt_idx[7];
 332         const u8 *addr[7];
 333         int ret, naddr = 0;
 334         const struct netdev_hw_addr *ha;
 335         int uc_cnt = netdev_uc_count(dev);
 336         int mc_cnt = netdev_mc_count(dev);
 337         const struct port_info *pi = netdev_priv(dev);
 338         unsigned int mb = pi->adapter->fn;
 339
 340         /* first do the secondary unicast addresses */
 341         netdev_for_each_uc_addr(ha, dev) {
 342                 addr[naddr++] = ha->addr;
 343                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 344                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 345                                         naddr, addr, filt_idx, &uhash, sleep);
 346                         if (ret < 0)
 347                                 return ret;
 348
 349                         free = false;
 350                         naddr = 0;
 351                 }
 352         }
 353
 354         /* next set up the multicast addresses */
 355         netdev_for_each_mc_addr(ha, dev) {
 356                 addr[naddr++] = ha->addr;
 357                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 358                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 359                                         naddr, addr, filt_idx, &mhash, sleep);
 360                         if (ret < 0)
 361                                 return ret;
 362
 363                         free = false;
 364                         naddr = 0;
 365                 }
 366         }
 367
 368         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 369                                 uhash | mhash, sleep);
 370 }
 371
 372 /*
 373  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 374  * If @mtu is -1 it is left unchanged.
 375  */
 376 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 377 {
 378         int ret;
 379         struct port_info *pi = netdev_priv(dev);
 380
 381         ret = set_addr_filters(dev, sleep_ok);
 382         if (ret == 0)
 383                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 384                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 385                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 386                                     sleep_ok);
 387         return ret;
 388 }
 389
 390 /**
 391  *      link_start - enable a port
 392  *      @dev: the port to enable
 393  *
 394  *      Performs the MAC and PHY actions needed to enable a port.
 395  */
 396 static int link_start(struct net_device *dev)
 397 {
 398         int ret;
 399         struct port_info *pi = netdev_priv(dev);
 400         unsigned int mb = pi->adapter->fn;
 401
 402         /*
 403          * We do not set address filters and promiscuity here, the stack does
 404          * that step explicitly.
 405          */
 406         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 407                             !!(dev->features & NETIF_F_HW_VLAN_RX), true);
 408         if (ret == 0) {
 409                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 410                                     pi->xact_addr_filt, dev->dev_addr, true,
 411                                     true);
 412                 if (ret >= 0) {
 413                         pi->xact_addr_filt = ret;
 414                         ret = 0;
 415                 }
 416         }
 417         if (ret == 0)
 418                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 419                                     &pi->link_cfg);
 420         if (ret == 0)
 421                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 422         return ret;
 423 }
 424
 425 /*
 426  * Response queue handler for the FW event queue.
 427  */
 428 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 429                           const struct pkt_gl *gl)
 430 {
 431         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 432
 433         rsp++;                                          /* skip RSS header */
 434         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 435                 const struct cpl_sge_egr_update *p = (void *)rsp;
 436                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 437                 struct sge_txq *txq;
 438
 439                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 440                 txq->restarts++;
 441                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 442                         struct sge_eth_txq *eq;
 443
 444                         eq = container_of(txq, struct sge_eth_txq, q);
 445                         netif_tx_wake_queue(eq->txq);
 446                 } else {
 447                         struct sge_ofld_txq *oq;
 448
 449                         oq = container_of(txq, struct sge_ofld_txq, q);
 450                         tasklet_schedule(&oq->qresume_tsk);
 451                 }
 452         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 453                 const struct cpl_fw6_msg *p = (void *)rsp;
 454
 455                 if (p->type == 0)
 456                         t4_handle_fw_rpl(q->adap, p->data);
 457         } else if (opcode == CPL_L2T_WRITE_RPL) {
 458                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 459
 460                 do_l2t_write_rpl(q->adap, p);
 461         } else
 462                 dev_err(q->adap->pdev_dev,
 463                         "unexpected CPL %#x on FW event queue\n", opcode);
 464         return 0;
 465 }
 466
 467 /**
 468  *      uldrx_handler - response queue handler for ULD queues
 469  *      @q: the response queue that received the packet
 470  *      @rsp: the response queue descriptor holding the offload message
 471  *      @gl: the gather list of packet fragments
 472  *
 473  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 474  *      the ULD, we just maintain statistics.
 475  */
 476 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 477                          const struct pkt_gl *gl)
 478 {
 479         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 480
 481         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 482                 rxq->stats.nomem++;
 483                 return -1;
 484         }
 485         if (gl == NULL)
 486                 rxq->stats.imm++;
 487         else if (gl == CXGB4_MSG_AN)
 488                 rxq->stats.an++;
 489         else
 490                 rxq->stats.pkts++;
 491         return 0;
 492 }
 493
 494 static void disable_msi(struct adapter *adapter)
 495 {
 496         if (adapter->flags & USING_MSIX) {
 497                 pci_disable_msix(adapter->pdev);
 498                 adapter->flags &= ~USING_MSIX;
 499         } else if (adapter->flags & USING_MSI) {
 500                 pci_disable_msi(adapter->pdev);
 501                 adapter->flags &= ~USING_MSI;
 502         }
 503 }
 504
 505 /*
 506  * Interrupt handler for non-data events used with MSI-X.
 507  */
 508 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 509 {
 510         struct adapter *adap = cookie;
 511
 512         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 513         if (v & PFSW) {
 514                 adap->swintr = 1;
 515                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 516         }
 517         t4_slow_intr_handler(adap);
 518         return IRQ_HANDLED;
 519 }
 520
 521 /*
 522  * Name the MSI-X interrupts.
 523  */
 524 static void name_msix_vecs(struct adapter *adap)
 525 {
 526         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 527
 528         /* non-data interrupts */
 529         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 530
 531         /* FW events */
 532         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 533                  adap->port[0]->name);
 534
 535         /* Ethernet queues */
 536         for_each_port(adap, j) {
 537                 struct net_device *d = adap->port[j];
 538                 const struct port_info *pi = netdev_priv(d);
 539
 540                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 541                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 542                                  d->name, i);
 543         }
 544
 545         /* offload queues */
 546         for_each_ofldrxq(&adap->sge, i)
 547                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 548                          adap->port[0]->name, i);
 549
 550         for_each_rdmarxq(&adap->sge, i)
 551                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 552                          adap->port[0]->name, i);
 553 }
 554
 555 static int request_msix_queue_irqs(struct adapter *adap)
 556 {
 557         struct sge *s = &adap->sge;
 558         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
 559
 560         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 561                           adap->msix_info[1].desc, &s->fw_evtq);
 562         if (err)
 563                 return err;
 564
 565         for_each_ethrxq(s, ethqidx) {
 566                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 567                                   adap->msix_info[msi].desc,
 568                                   &s->ethrxq[ethqidx].rspq);
 569                 if (err)
 570                         goto unwind;
 571                 msi++;
 572         }
 573         for_each_ofldrxq(s, ofldqidx) {
 574                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 575                                   adap->msix_info[msi].desc,
 576                                   &s->ofldrxq[ofldqidx].rspq);
 577                 if (err)
 578                         goto unwind;
 579                 msi++;
 580         }
 581         for_each_rdmarxq(s, rdmaqidx) {
 582                 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
 583                                   adap->msix_info[msi].desc,
 584                                   &s->rdmarxq[rdmaqidx].rspq);
 585                 if (err)
 586                         goto unwind;
 587                 msi++;
 588         }
 589         return 0;
 590
 591 unwind:
 592         while (--rdmaqidx >= 0)
 593                 free_irq(adap->msix_info[--msi].vec,
 594                          &s->rdmarxq[rdmaqidx].rspq);
 595         while (--ofldqidx >= 0)
 596                 free_irq(adap->msix_info[--msi].vec,
 597                          &s->ofldrxq[ofldqidx].rspq);
 598         while (--ethqidx >= 0)
 599                 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
 600         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 601         return err;
 602 }
 603
 604 static void free_msix_queue_irqs(struct adapter *adap)
 605 {
 606         int i, msi = 2;
 607         struct sge *s = &adap->sge;
 608
 609         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 610         for_each_ethrxq(s, i)
 611                 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
 612         for_each_ofldrxq(s, i)
 613                 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
 614         for_each_rdmarxq(s, i)
 615                 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
 616 }
 617
 618 /**
 619  *      write_rss - write the RSS table for a given port
 620  *      @pi: the port
 621  *      @queues: array of queue indices for RSS
 622  *
 623  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 624  *      packets to the Rx queues in @queues.
 625  */
 626 static int write_rss(const struct port_info *pi, const u16 *queues)
 627 {
 628         u16 *rss;
 629         int i, err;
 630         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 631
 632         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 633         if (!rss)
 634                 return -ENOMEM;
 635
 636         /* map the queue indices to queue ids */
 637         for (i = 0; i < pi->rss_size; i++, queues++)
 638                 rss[i] = q[*queues].rspq.abs_id;
 639
 640         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 641                                   pi->rss_size, rss, pi->rss_size);
 642         kfree(rss);
 643         return err;
 644 }
 645
 646 /**
 647  *      setup_rss - configure RSS
 648  *      @adap: the adapter
 649  *
 650  *      Sets up RSS for each port.
 651  */
 652 static int setup_rss(struct adapter *adap)
 653 {
 654         int i, err;
 655
 656         for_each_port(adap, i) {
 657                 const struct port_info *pi = adap2pinfo(adap, i);
 658
 659                 err = write_rss(pi, pi->rss);
 660                 if (err)
 661                         return err;
 662         }
 663         return 0;
 664 }
 665
 666 /*
 667  * Return the channel of the ingress queue with the given qid.
 668  */
 669 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 670 {
 671         qid -= p->ingr_start;
 672         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 673 }
 674
 675 /*
 676  * Wait until all NAPI handlers are descheduled.
 677  */
 678 static void quiesce_rx(struct adapter *adap)
 679 {
 680         int i;
 681
 682         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 683                 struct sge_rspq *q = adap->sge.ingr_map[i];
 684
 685                 if (q && q->handler)
 686                         napi_disable(&q->napi);
 687         }
 688 }
 689
 690 /*
 691  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 692  */
 693 static void enable_rx(struct adapter *adap)
 694 {
 695         int i;
 696
 697         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 698                 struct sge_rspq *q = adap->sge.ingr_map[i];
 699
 700                 if (!q)
 701                         continue;
 702                 if (q->handler)
 703                         napi_enable(&q->napi);
 704                 /* 0-increment GTS to start the timer and enable interrupts */
 705                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 706                              SEINTARM(q->intr_params) |
 707                              INGRESSQID(q->cntxt_id));
 708         }
 709 }
 710
 711 /**
 712  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 713  *      @adap: the adapter
 714  *
 715  *      Determines how many sets of SGE queues to use and initializes them.
 716  *      We support multiple queue sets per port if we have MSI-X, otherwise
 717  *      just one queue set per port.
 718  */
 719 static int setup_sge_queues(struct adapter *adap)
 720 {
 721         int err, msi_idx, i, j;
 722         struct sge *s = &adap->sge;
 723
 724         bitmap_zero(s->starving_fl, MAX_EGRQ);
 725         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 726
 727         if (adap->flags & USING_MSIX)
 728                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 729         else {
 730                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 731                                        NULL, NULL);
 732                 if (err)
 733                         return err;
 734                 msi_idx = -((int)s->intrq.abs_id + 1);
 735         }
 736
 737         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 738                                msi_idx, NULL, fwevtq_handler);
 739         if (err) {
 740 freeout:        t4_free_sge_resources(adap);
 741                 return err;
 742         }
 743
 744         for_each_port(adap, i) {
 745                 struct net_device *dev = adap->port[i];
 746                 struct port_info *pi = netdev_priv(dev);
 747                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 748                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 749
 750                 for (j = 0; j < pi->nqsets; j++, q++) {
 751                         if (msi_idx > 0)
 752                                 msi_idx++;
 753                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 754                                                msi_idx, &q->fl,
 755                                                t4_ethrx_handler);
 756                         if (err)
 757                                 goto freeout;
 758                         q->rspq.idx = j;
 759                         memset(&q->stats, 0, sizeof(q->stats));
 760                 }
 761                 for (j = 0; j < pi->nqsets; j++, t++) {
 762                         err = t4_sge_alloc_eth_txq(adap, t, dev,
 763                                         netdev_get_tx_queue(dev, j),
 764                                         s->fw_evtq.cntxt_id);
 765                         if (err)
 766                                 goto freeout;
 767                 }
 768         }
 769
 770         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
 771         for_each_ofldrxq(s, i) {
 772                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
 773                 struct net_device *dev = adap->port[i / j];
 774
 775                 if (msi_idx > 0)
 776                         msi_idx++;
 777                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
 778                                        &q->fl, uldrx_handler);
 779                 if (err)
 780                         goto freeout;
 781                 memset(&q->stats, 0, sizeof(q->stats));
 782                 s->ofld_rxq[i] = q->rspq.abs_id;
 783                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
 784                                             s->fw_evtq.cntxt_id);
 785                 if (err)
 786                         goto freeout;
 787         }
 788
 789         for_each_rdmarxq(s, i) {
 790                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
 791
 792                 if (msi_idx > 0)
 793                         msi_idx++;
 794                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
 795                                        msi_idx, &q->fl, uldrx_handler);
 796                 if (err)
 797                         goto freeout;
 798                 memset(&q->stats, 0, sizeof(q->stats));
 799                 s->rdma_rxq[i] = q->rspq.abs_id;
 800         }
 801
 802         for_each_port(adap, i) {
 803                 /*
 804                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
 805                  * have RDMA queues, and that's the right value.
 806                  */
 807                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
 808                                             s->fw_evtq.cntxt_id,
 809                                             s->rdmarxq[i].rspq.cntxt_id);
 810                 if (err)
 811                         goto freeout;
 812         }
 813
 814         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
 815                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
 816                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
 817         return 0;
 818 }
 819
 820 /*
 821  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
 822  * started but failed, and a negative errno if flash load couldn't start.
 823  */
 824 static int upgrade_fw(struct adapter *adap)
 825 {
 826         int ret;
 827         u32 vers;
 828         const struct fw_hdr *hdr;
 829         const struct firmware *fw;
 830         struct device *dev = adap->pdev_dev;
 831
 832         ret = request_firmware(&fw, FW_FNAME, dev);
 833         if (ret < 0) {
 834                 dev_err(dev, "unable to load firmware image " FW_FNAME
 835                         ", error %d\n", ret);
 836                 return ret;
 837         }
 838
 839         hdr = (const struct fw_hdr *)fw->data;
 840         vers = ntohl(hdr->fw_ver);
 841         if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
 842                 ret = -EINVAL;              /* wrong major version, won't do */
 843                 goto out;
 844         }
 845
 846         /*
 847          * If the flash FW is unusable or we found something newer, load it.
 848          */
 849         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
 850             vers > adap->params.fw_vers) {
 851                 ret = -t4_load_fw(adap, fw->data, fw->size);
 852                 if (!ret)
 853                         dev_info(dev, "firmware upgraded to version %pI4 from "
 854                                  FW_FNAME "\n", &hdr->fw_ver);
 855         }
 856 out:    release_firmware(fw);
 857         return ret;
 858 }
 859
 860 /*
 861  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
 862  * The allocated memory is cleared.
 863  */
 864 void *t4_alloc_mem(size_t size)
 865 {
 866         void *p = kzalloc(size, GFP_KERNEL);
 867
 868         if (!p)
 869                 p = vzalloc(size);
 870         return p;
 871 }
 872
 873 /*
 874  * Free memory allocated through alloc_mem().
 875  */
 876 static void t4_free_mem(void *addr)
 877 {
 878         if (is_vmalloc_addr(addr))
 879                 vfree(addr);
 880         else
 881                 kfree(addr);
 882 }
 883
 884 static inline int is_offload(const struct adapter *adap)
 885 {
 886         return adap->params.offload;
 887 }
 888
 889 /*
 890  * Implementation of ethtool operations.
 891  */
 892
 893 static u32 get_msglevel(struct net_device *dev)
 894 {
 895         return netdev2adap(dev)->msg_enable;
 896 }
 897
 898 static void set_msglevel(struct net_device *dev, u32 val)
 899 {
 900         netdev2adap(dev)->msg_enable = val;
 901 }
 902
 903 static char stats_strings[][ETH_GSTRING_LEN] = {
 904         "TxOctetsOK         ",
 905         "TxFramesOK         ",
 906         "TxBroadcastFrames  ",
 907         "TxMulticastFrames  ",
 908         "TxUnicastFrames    ",
 909         "TxErrorFrames      ",
 910
 911         "TxFrames64         ",
 912         "TxFrames65To127    ",
 913         "TxFrames128To255   ",
 914         "TxFrames256To511   ",
 915         "TxFrames512To1023  ",
 916         "TxFrames1024To1518 ",
 917         "TxFrames1519ToMax  ",
 918
 919         "TxFramesDropped    ",
 920         "TxPauseFrames      ",
 921         "TxPPP0Frames       ",
 922         "TxPPP1Frames       ",
 923         "TxPPP2Frames       ",
 924         "TxPPP3Frames       ",
 925         "TxPPP4Frames       ",
 926         "TxPPP5Frames       ",
 927         "TxPPP6Frames       ",
 928         "TxPPP7Frames       ",
 929
 930         "RxOctetsOK         ",
 931         "RxFramesOK         ",
 932         "RxBroadcastFrames  ",
 933         "RxMulticastFrames  ",
 934         "RxUnicastFrames    ",
 935
 936         "RxFramesTooLong    ",
 937         "RxJabberErrors     ",
 938         "RxFCSErrors        ",
 939         "RxLengthErrors     ",
 940         "RxSymbolErrors     ",
 941         "RxRuntFrames       ",
 942
 943         "RxFrames64         ",
 944         "RxFrames65To127    ",
 945         "RxFrames128To255   ",
 946         "RxFrames256To511   ",
 947         "RxFrames512To1023  ",
 948         "RxFrames1024To1518 ",
 949         "RxFrames1519ToMax  ",
 950
 951         "RxPauseFrames      ",
 952         "RxPPP0Frames       ",
 953         "RxPPP1Frames       ",
 954         "RxPPP2Frames       ",
 955         "RxPPP3Frames       ",
 956         "RxPPP4Frames       ",
 957         "RxPPP5Frames       ",
 958         "RxPPP6Frames       ",
 959         "RxPPP7Frames       ",
 960
 961         "RxBG0FramesDropped ",
 962         "RxBG1FramesDropped ",
 963         "RxBG2FramesDropped ",
 964         "RxBG3FramesDropped ",
 965         "RxBG0FramesTrunc   ",
 966         "RxBG1FramesTrunc   ",
 967         "RxBG2FramesTrunc   ",
 968         "RxBG3FramesTrunc   ",
 969
 970         "TSO                ",
 971         "TxCsumOffload      ",
 972         "RxCsumGood         ",
 973         "VLANextractions    ",
 974         "VLANinsertions     ",
 975         "GROpackets         ",
 976         "GROmerged          ",
 977 };
 978
 979 static int get_sset_count(struct net_device *dev, int sset)
 980 {
 981         switch (sset) {
 982         case ETH_SS_STATS:
 983                 return ARRAY_SIZE(stats_strings);
 984         default:
 985                 return -EOPNOTSUPP;
 986         }
 987 }
 988
 989 #define T4_REGMAP_SIZE (160 * 1024)
 990
 991 static int get_regs_len(struct net_device *dev)
 992 {
 993         return T4_REGMAP_SIZE;
 994 }
 995
 996 static int get_eeprom_len(struct net_device *dev)
 997 {
 998         return EEPROMSIZE;
 999 }
1000
1001 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1002 {
1003         struct adapter *adapter = netdev2adap(dev);
1004
1005         strcpy(info->driver, KBUILD_MODNAME);
1006         strcpy(info->version, DRV_VERSION);
1007         strcpy(info->bus_info, pci_name(adapter->pdev));
1008
1009         if (!adapter->params.fw_vers)
1010                 strcpy(info->fw_version, "N/A");
1011         else
1012                 snprintf(info->fw_version, sizeof(info->fw_version),
1013                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1014                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1015                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1016                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1017                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1018                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1019                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1020                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1021                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1022 }
1023
1024 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1025 {
1026         if (stringset == ETH_SS_STATS)
1027                 memcpy(data, stats_strings, sizeof(stats_strings));
1028 }
1029
1030 /*
1031  * port stats maintained per queue of the port.  They should be in the same
1032  * order as in stats_strings above.
1033  */
1034 struct queue_port_stats {
1035         u64 tso;
1036         u64 tx_csum;
1037         u64 rx_csum;
1038         u64 vlan_ex;
1039         u64 vlan_ins;
1040         u64 gro_pkts;
1041         u64 gro_merged;
1042 };
1043
1044 static void collect_sge_port_stats(const struct adapter *adap,
1045                 const struct port_info *p, struct queue_port_stats *s)
1046 {
1047         int i;
1048         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1049         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1050
1051         memset(s, 0, sizeof(*s));
1052         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1053                 s->tso += tx->tso;
1054                 s->tx_csum += tx->tx_cso;
1055                 s->rx_csum += rx->stats.rx_cso;
1056                 s->vlan_ex += rx->stats.vlan_ex;
1057                 s->vlan_ins += tx->vlan_ins;
1058                 s->gro_pkts += rx->stats.lro_pkts;
1059                 s->gro_merged += rx->stats.lro_merged;
1060         }
1061 }
1062
1063 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1064                       u64 *data)
1065 {
1066         struct port_info *pi = netdev_priv(dev);
1067         struct adapter *adapter = pi->adapter;
1068
1069         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1070
1071         data += sizeof(struct port_stats) / sizeof(u64);
1072         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1073 }
1074
1075 /*
1076  * Return a version number to identify the type of adapter.  The scheme is:
1077  * - bits 0..9: chip version
1078  * - bits 10..15: chip revision
1079  * - bits 16..23: register dump version
1080  */
1081 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1082 {
1083         return 4 | (ap->params.rev << 10) | (1 << 16);
1084 }
1085
1086 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1087                            unsigned int end)
1088 {
1089         u32 *p = buf + start;
1090
1091         for ( ; start <= end; start += sizeof(u32))
1092                 *p++ = t4_read_reg(ap, start);
1093 }
1094
1095 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1096                      void *buf)
1097 {
1098         static const unsigned int reg_ranges[] = {
1099                 0x1008, 0x1108,
1100                 0x1180, 0x11b4,
1101                 0x11fc, 0x123c,
1102                 0x1300, 0x173c,
1103                 0x1800, 0x18fc,
1104                 0x3000, 0x30d8,
1105                 0x30e0, 0x5924,
1106                 0x5960, 0x59d4,
1107                 0x5a00, 0x5af8,
1108                 0x6000, 0x6098,
1109                 0x6100, 0x6150,
1110                 0x6200, 0x6208,
1111                 0x6240, 0x6248,
1112                 0x6280, 0x6338,
1113                 0x6370, 0x638c,
1114                 0x6400, 0x643c,
1115                 0x6500, 0x6524,
1116                 0x6a00, 0x6a38,
1117                 0x6a60, 0x6a78,
1118                 0x6b00, 0x6b84,
1119                 0x6bf0, 0x6c84,
1120                 0x6cf0, 0x6d84,
1121                 0x6df0, 0x6e84,
1122                 0x6ef0, 0x6f84,
1123                 0x6ff0, 0x7084,
1124                 0x70f0, 0x7184,
1125                 0x71f0, 0x7284,
1126                 0x72f0, 0x7384,
1127                 0x73f0, 0x7450,
1128                 0x7500, 0x7530,
1129                 0x7600, 0x761c,
1130                 0x7680, 0x76cc,
1131                 0x7700, 0x7798,
1132                 0x77c0, 0x77fc,
1133                 0x7900, 0x79fc,
1134                 0x7b00, 0x7c38,
1135                 0x7d00, 0x7efc,
1136                 0x8dc0, 0x8e1c,
1137                 0x8e30, 0x8e78,
1138                 0x8ea0, 0x8f6c,
1139                 0x8fc0, 0x9074,
1140                 0x90fc, 0x90fc,
1141                 0x9400, 0x9458,
1142                 0x9600, 0x96bc,
1143                 0x9800, 0x9808,
1144                 0x9820, 0x983c,
1145                 0x9850, 0x9864,
1146                 0x9c00, 0x9c6c,
1147                 0x9c80, 0x9cec,
1148                 0x9d00, 0x9d6c,
1149                 0x9d80, 0x9dec,
1150                 0x9e00, 0x9e6c,
1151                 0x9e80, 0x9eec,
1152                 0x9f00, 0x9f6c,
1153                 0x9f80, 0x9fec,
1154                 0xd004, 0xd03c,
1155                 0xdfc0, 0xdfe0,
1156                 0xe000, 0xea7c,
1157                 0xf000, 0x11190,
1158                 0x19040, 0x1906c,
1159                 0x19078, 0x19080,
1160                 0x1908c, 0x19124,
1161                 0x19150, 0x191b0,
1162                 0x191d0, 0x191e8,
1163                 0x19238, 0x1924c,
1164                 0x193f8, 0x19474,
1165                 0x19490, 0x194f8,
1166                 0x19800, 0x19f30,
1167                 0x1a000, 0x1a06c,
1168                 0x1a0b0, 0x1a120,
1169                 0x1a128, 0x1a138,
1170                 0x1a190, 0x1a1c4,
1171                 0x1a1fc, 0x1a1fc,
1172                 0x1e040, 0x1e04c,
1173                 0x1e284, 0x1e28c,
1174                 0x1e2c0, 0x1e2c0,
1175                 0x1e2e0, 0x1e2e0,
1176                 0x1e300, 0x1e384,
1177                 0x1e3c0, 0x1e3c8,
1178                 0x1e440, 0x1e44c,
1179                 0x1e684, 0x1e68c,
1180                 0x1e6c0, 0x1e6c0,
1181                 0x1e6e0, 0x1e6e0,
1182                 0x1e700, 0x1e784,
1183                 0x1e7c0, 0x1e7c8,
1184                 0x1e840, 0x1e84c,
1185                 0x1ea84, 0x1ea8c,
1186                 0x1eac0, 0x1eac0,
1187                 0x1eae0, 0x1eae0,
1188                 0x1eb00, 0x1eb84,
1189                 0x1ebc0, 0x1ebc8,
1190                 0x1ec40, 0x1ec4c,
1191                 0x1ee84, 0x1ee8c,
1192                 0x1eec0, 0x1eec0,
1193                 0x1eee0, 0x1eee0,
1194                 0x1ef00, 0x1ef84,
1195                 0x1efc0, 0x1efc8,
1196                 0x1f040, 0x1f04c,
1197                 0x1f284, 0x1f28c,
1198                 0x1f2c0, 0x1f2c0,
1199                 0x1f2e0, 0x1f2e0,
1200                 0x1f300, 0x1f384,
1201                 0x1f3c0, 0x1f3c8,
1202                 0x1f440, 0x1f44c,
1203                 0x1f684, 0x1f68c,
1204                 0x1f6c0, 0x1f6c0,
1205                 0x1f6e0, 0x1f6e0,
1206                 0x1f700, 0x1f784,
1207                 0x1f7c0, 0x1f7c8,
1208                 0x1f840, 0x1f84c,
1209                 0x1fa84, 0x1fa8c,
1210                 0x1fac0, 0x1fac0,
1211                 0x1fae0, 0x1fae0,
1212                 0x1fb00, 0x1fb84,
1213                 0x1fbc0, 0x1fbc8,
1214                 0x1fc40, 0x1fc4c,
1215                 0x1fe84, 0x1fe8c,
1216                 0x1fec0, 0x1fec0,
1217                 0x1fee0, 0x1fee0,
1218                 0x1ff00, 0x1ff84,
1219                 0x1ffc0, 0x1ffc8,
1220                 0x20000, 0x2002c,
1221                 0x20100, 0x2013c,
1222                 0x20190, 0x201c8,
1223                 0x20200, 0x20318,
1224                 0x20400, 0x20528,
1225                 0x20540, 0x20614,
1226                 0x21000, 0x21040,
1227                 0x2104c, 0x21060,
1228                 0x210c0, 0x210ec,
1229                 0x21200, 0x21268,
1230                 0x21270, 0x21284,
1231                 0x212fc, 0x21388,
1232                 0x21400, 0x21404,
1233                 0x21500, 0x21518,
1234                 0x2152c, 0x2153c,
1235                 0x21550, 0x21554,
1236                 0x21600, 0x21600,
1237                 0x21608, 0x21628,
1238                 0x21630, 0x2163c,
1239                 0x21700, 0x2171c,
1240                 0x21780, 0x2178c,
1241                 0x21800, 0x21c38,
1242                 0x21c80, 0x21d7c,
1243                 0x21e00, 0x21e04,
1244                 0x22000, 0x2202c,
1245                 0x22100, 0x2213c,
1246                 0x22190, 0x221c8,
1247                 0x22200, 0x22318,
1248                 0x22400, 0x22528,
1249                 0x22540, 0x22614,
1250                 0x23000, 0x23040,
1251                 0x2304c, 0x23060,
1252                 0x230c0, 0x230ec,
1253                 0x23200, 0x23268,
1254                 0x23270, 0x23284,
1255                 0x232fc, 0x23388,
1256                 0x23400, 0x23404,
1257                 0x23500, 0x23518,
1258                 0x2352c, 0x2353c,
1259                 0x23550, 0x23554,
1260                 0x23600, 0x23600,
1261                 0x23608, 0x23628,
1262                 0x23630, 0x2363c,
1263                 0x23700, 0x2371c,
1264                 0x23780, 0x2378c,
1265                 0x23800, 0x23c38,
1266                 0x23c80, 0x23d7c,
1267                 0x23e00, 0x23e04,
1268                 0x24000, 0x2402c,
1269                 0x24100, 0x2413c,
1270                 0x24190, 0x241c8,
1271                 0x24200, 0x24318,
1272                 0x24400, 0x24528,
1273                 0x24540, 0x24614,
1274                 0x25000, 0x25040,
1275                 0x2504c, 0x25060,
1276                 0x250c0, 0x250ec,
1277                 0x25200, 0x25268,
1278                 0x25270, 0x25284,
1279                 0x252fc, 0x25388,
1280                 0x25400, 0x25404,
1281                 0x25500, 0x25518,
1282                 0x2552c, 0x2553c,
1283                 0x25550, 0x25554,
1284                 0x25600, 0x25600,
1285                 0x25608, 0x25628,
1286                 0x25630, 0x2563c,
1287                 0x25700, 0x2571c,
1288                 0x25780, 0x2578c,
1289                 0x25800, 0x25c38,
1290                 0x25c80, 0x25d7c,
1291                 0x25e00, 0x25e04,
1292                 0x26000, 0x2602c,
1293                 0x26100, 0x2613c,
1294                 0x26190, 0x261c8,
1295                 0x26200, 0x26318,
1296                 0x26400, 0x26528,
1297                 0x26540, 0x26614,
1298                 0x27000, 0x27040,
1299                 0x2704c, 0x27060,
1300                 0x270c0, 0x270ec,
1301                 0x27200, 0x27268,
1302                 0x27270, 0x27284,
1303                 0x272fc, 0x27388,
1304                 0x27400, 0x27404,
1305                 0x27500, 0x27518,
1306                 0x2752c, 0x2753c,
1307                 0x27550, 0x27554,
1308                 0x27600, 0x27600,
1309                 0x27608, 0x27628,
1310                 0x27630, 0x2763c,
1311                 0x27700, 0x2771c,
1312                 0x27780, 0x2778c,
1313                 0x27800, 0x27c38,
1314                 0x27c80, 0x27d7c,
1315                 0x27e00, 0x27e04
1316         };
1317
1318         int i;
1319         struct adapter *ap = netdev2adap(dev);
1320
1321         regs->version = mk_adap_vers(ap);
1322
1323         memset(buf, 0, T4_REGMAP_SIZE);
1324         for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1325                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1326 }
1327
1328 static int restart_autoneg(struct net_device *dev)
1329 {
1330         struct port_info *p = netdev_priv(dev);
1331
1332         if (!netif_running(dev))
1333                 return -EAGAIN;
1334         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1335                 return -EINVAL;
1336         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1337         return 0;
1338 }
1339
1340 static int identify_port(struct net_device *dev,
1341                          enum ethtool_phys_id_state state)
1342 {
1343         unsigned int val;
1344         struct adapter *adap = netdev2adap(dev);
1345
1346         if (state == ETHTOOL_ID_ACTIVE)
1347                 val = 0xffff;
1348         else if (state == ETHTOOL_ID_INACTIVE)
1349                 val = 0;
1350         else
1351                 return -EINVAL;
1352
1353         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1354 }
1355
1356 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1357 {
1358         unsigned int v = 0;
1359
1360         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1361             type == FW_PORT_TYPE_BT_XAUI) {
1362                 v |= SUPPORTED_TP;
1363                 if (caps & FW_PORT_CAP_SPEED_100M)
1364                         v |= SUPPORTED_100baseT_Full;
1365                 if (caps & FW_PORT_CAP_SPEED_1G)
1366                         v |= SUPPORTED_1000baseT_Full;
1367                 if (caps & FW_PORT_CAP_SPEED_10G)
1368                         v |= SUPPORTED_10000baseT_Full;
1369         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1370                 v |= SUPPORTED_Backplane;
1371                 if (caps & FW_PORT_CAP_SPEED_1G)
1372                         v |= SUPPORTED_1000baseKX_Full;
1373                 if (caps & FW_PORT_CAP_SPEED_10G)
1374                         v |= SUPPORTED_10000baseKX4_Full;
1375         } else if (type == FW_PORT_TYPE_KR)
1376                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1377         else if (type == FW_PORT_TYPE_BP_AP)
1378                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1379                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1380         else if (type == FW_PORT_TYPE_BP4_AP)
1381                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1382                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1383                      SUPPORTED_10000baseKX4_Full;
1384         else if (type == FW_PORT_TYPE_FIBER_XFI ||
1385                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1386                 v |= SUPPORTED_FIBRE;
1387
1388         if (caps & FW_PORT_CAP_ANEG)
1389                 v |= SUPPORTED_Autoneg;
1390         return v;
1391 }
1392
1393 static unsigned int to_fw_linkcaps(unsigned int caps)
1394 {
1395         unsigned int v = 0;
1396
1397         if (caps & ADVERTISED_100baseT_Full)
1398                 v |= FW_PORT_CAP_SPEED_100M;
1399         if (caps & ADVERTISED_1000baseT_Full)
1400                 v |= FW_PORT_CAP_SPEED_1G;
1401         if (caps & ADVERTISED_10000baseT_Full)
1402                 v |= FW_PORT_CAP_SPEED_10G;
1403         return v;
1404 }
1405
1406 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1407 {
1408         const struct port_info *p = netdev_priv(dev);
1409
1410         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1411             p->port_type == FW_PORT_TYPE_BT_XFI ||
1412             p->port_type == FW_PORT_TYPE_BT_XAUI)
1413                 cmd->port = PORT_TP;
1414         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1415                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1416                 cmd->port = PORT_FIBRE;
1417         else if (p->port_type == FW_PORT_TYPE_SFP) {
1418                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1419                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1420                         cmd->port = PORT_DA;
1421                 else
1422                         cmd->port = PORT_FIBRE;
1423         } else
1424                 cmd->port = PORT_OTHER;
1425
1426         if (p->mdio_addr >= 0) {
1427                 cmd->phy_address = p->mdio_addr;
1428                 cmd->transceiver = XCVR_EXTERNAL;
1429                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1430                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1431         } else {
1432                 cmd->phy_address = 0;  /* not really, but no better option */
1433                 cmd->transceiver = XCVR_INTERNAL;
1434                 cmd->mdio_support = 0;
1435         }
1436
1437         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1438         cmd->advertising = from_fw_linkcaps(p->port_type,
1439                                             p->link_cfg.advertising);
1440         ethtool_cmd_speed_set(cmd,
1441                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1442         cmd->duplex = DUPLEX_FULL;
1443         cmd->autoneg = p->link_cfg.autoneg;
1444         cmd->maxtxpkt = 0;
1445         cmd->maxrxpkt = 0;
1446         return 0;
1447 }
1448
1449 static unsigned int speed_to_caps(int speed)
1450 {
1451         if (speed == SPEED_100)
1452                 return FW_PORT_CAP_SPEED_100M;
1453         if (speed == SPEED_1000)
1454                 return FW_PORT_CAP_SPEED_1G;
1455         if (speed == SPEED_10000)
1456                 return FW_PORT_CAP_SPEED_10G;
1457         return 0;
1458 }
1459
1460 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1461 {
1462         unsigned int cap;
1463         struct port_info *p = netdev_priv(dev);
1464         struct link_config *lc = &p->link_cfg;
1465         u32 speed = ethtool_cmd_speed(cmd);
1466
1467         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
1468                 return -EINVAL;
1469
1470         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1471                 /*
1472                  * PHY offers a single speed.  See if that's what's
1473                  * being requested.
1474                  */
1475                 if (cmd->autoneg == AUTONEG_DISABLE &&
1476                     (lc->supported & speed_to_caps(speed)))
1477                         return 0;
1478                 return -EINVAL;
1479         }
1480
1481         if (cmd->autoneg == AUTONEG_DISABLE) {
1482                 cap = speed_to_caps(speed);
1483
1484                 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1485                     (speed == SPEED_10000))
1486                         return -EINVAL;
1487                 lc->requested_speed = cap;
1488                 lc->advertising = 0;
1489         } else {
1490                 cap = to_fw_linkcaps(cmd->advertising);
1491                 if (!(lc->supported & cap))
1492                         return -EINVAL;
1493                 lc->requested_speed = 0;
1494                 lc->advertising = cap | FW_PORT_CAP_ANEG;
1495         }
1496         lc->autoneg = cmd->autoneg;
1497
1498         if (netif_running(dev))
1499                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1500                                      lc);
1501         return 0;
1502 }
1503
1504 static void get_pauseparam(struct net_device *dev,
1505                            struct ethtool_pauseparam *epause)
1506 {
1507         struct port_info *p = netdev_priv(dev);
1508
1509         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1510         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1511         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1512 }
1513
1514 static int set_pauseparam(struct net_device *dev,
1515                           struct ethtool_pauseparam *epause)
1516 {
1517         struct port_info *p = netdev_priv(dev);
1518         struct link_config *lc = &p->link_cfg;
1519
1520         if (epause->autoneg == AUTONEG_DISABLE)
1521                 lc->requested_fc = 0;
1522         else if (lc->supported & FW_PORT_CAP_ANEG)
1523                 lc->requested_fc = PAUSE_AUTONEG;
1524         else
1525                 return -EINVAL;
1526
1527         if (epause->rx_pause)
1528                 lc->requested_fc |= PAUSE_RX;
1529         if (epause->tx_pause)
1530                 lc->requested_fc |= PAUSE_TX;
1531         if (netif_running(dev))
1532                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1533                                      lc);
1534         return 0;
1535 }
1536
1537 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1538 {
1539         const struct port_info *pi = netdev_priv(dev);
1540         const struct sge *s = &pi->adapter->sge;
1541
1542         e->rx_max_pending = MAX_RX_BUFFERS;
1543         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1544         e->rx_jumbo_max_pending = 0;
1545         e->tx_max_pending = MAX_TXQ_ENTRIES;
1546
1547         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1548         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1549         e->rx_jumbo_pending = 0;
1550         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1551 }
1552
1553 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1554 {
1555         int i;
1556         const struct port_info *pi = netdev_priv(dev);
1557         struct adapter *adapter = pi->adapter;
1558         struct sge *s = &adapter->sge;
1559
1560         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1561             e->tx_pending > MAX_TXQ_ENTRIES ||
1562             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1563             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1564             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1565                 return -EINVAL;
1566
1567         if (adapter->flags & FULL_INIT_DONE)
1568                 return -EBUSY;
1569
1570         for (i = 0; i < pi->nqsets; ++i) {
1571                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1572                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1573                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1574         }
1575         return 0;
1576 }
1577
1578 static int closest_timer(const struct sge *s, int time)
1579 {
1580         int i, delta, match = 0, min_delta = INT_MAX;
1581
1582         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1583                 delta = time - s->timer_val[i];
1584                 if (delta < 0)
1585                         delta = -delta;
1586                 if (delta < min_delta) {
1587                         min_delta = delta;
1588                         match = i;
1589                 }
1590         }
1591         return match;
1592 }
1593
1594 static int closest_thres(const struct sge *s, int thres)
1595 {
1596         int i, delta, match = 0, min_delta = INT_MAX;
1597
1598         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1599                 delta = thres - s->counter_val[i];
1600                 if (delta < 0)
1601                         delta = -delta;
1602                 if (delta < min_delta) {
1603                         min_delta = delta;
1604                         match = i;
1605                 }
1606         }
1607         return match;
1608 }
1609
1610 /*
1611  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1612  */
1613 static unsigned int qtimer_val(const struct adapter *adap,
1614                                const struct sge_rspq *q)
1615 {
1616         unsigned int idx = q->intr_params >> 1;
1617
1618         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1619 }
1620
1621 /**
1622  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
1623  *      @adap: the adapter
1624  *      @q: the Rx queue
1625  *      @us: the hold-off time in us, or 0 to disable timer
1626  *      @cnt: the hold-off packet count, or 0 to disable counter
1627  *
1628  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
1629  *      one of the two needs to be enabled for the queue to generate interrupts.
1630  */
1631 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1632                                unsigned int us, unsigned int cnt)
1633 {
1634         if ((us | cnt) == 0)
1635                 cnt = 1;
1636
1637         if (cnt) {
1638                 int err;
1639                 u32 v, new_idx;
1640
1641                 new_idx = closest_thres(&adap->sge, cnt);
1642                 if (q->desc && q->pktcnt_idx != new_idx) {
1643                         /* the queue has already been created, update it */
1644                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1645                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1646                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
1647                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1648                                             &new_idx);
1649                         if (err)
1650                                 return err;
1651                 }
1652                 q->pktcnt_idx = new_idx;
1653         }
1654
1655         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1656         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1657         return 0;
1658 }
1659
1660 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1661 {
1662         const struct port_info *pi = netdev_priv(dev);
1663         struct adapter *adap = pi->adapter;
1664
1665         return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1666                         c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1667 }
1668
1669 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1670 {
1671         const struct port_info *pi = netdev_priv(dev);
1672         const struct adapter *adap = pi->adapter;
1673         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1674
1675         c->rx_coalesce_usecs = qtimer_val(adap, rq);
1676         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1677                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1678         return 0;
1679 }
1680
1681 /**
1682  *      eeprom_ptov - translate a physical EEPROM address to virtual
1683  *      @phys_addr: the physical EEPROM address
1684  *      @fn: the PCI function number
1685  *      @sz: size of function-specific area
1686  *
1687  *      Translate a physical EEPROM address to virtual.  The first 1K is
1688  *      accessed through virtual addresses starting at 31K, the rest is
1689  *      accessed through virtual addresses starting at 0.
1690  *
1691  *      The mapping is as follows:
1692  *      [0..1K) -> [31K..32K)
1693  *      [1K..1K+A) -> [31K-A..31K)
1694  *      [1K+A..ES) -> [0..ES-A-1K)
1695  *
1696  *      where A = @fn * @sz, and ES = EEPROM size.
1697  */
1698 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1699 {
1700         fn *= sz;
1701         if (phys_addr < 1024)
1702                 return phys_addr + (31 << 10);
1703         if (phys_addr < 1024 + fn)
1704                 return 31744 - fn + phys_addr - 1024;
1705         if (phys_addr < EEPROMSIZE)
1706                 return phys_addr - 1024 - fn;
1707         return -EINVAL;
1708 }
1709
1710 /*
1711  * The next two routines implement eeprom read/write from physical addresses.
1712  */
1713 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1714 {
1715         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1716
1717         if (vaddr >= 0)
1718                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1719         return vaddr < 0 ? vaddr : 0;
1720 }
1721
1722 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1723 {
1724         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1725
1726         if (vaddr >= 0)
1727                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1728         return vaddr < 0 ? vaddr : 0;
1729 }
1730
1731 #define EEPROM_MAGIC 0x38E2F10C
1732
1733 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1734                       u8 *data)
1735 {
1736         int i, err = 0;
1737         struct adapter *adapter = netdev2adap(dev);
1738
1739         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1740         if (!buf)
1741                 return -ENOMEM;
1742
1743         e->magic = EEPROM_MAGIC;
1744         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1745                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1746
1747         if (!err)
1748                 memcpy(data, buf + e->offset, e->len);
1749         kfree(buf);
1750         return err;
1751 }
1752
1753 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1754                       u8 *data)
1755 {
1756         u8 *buf;
1757         int err = 0;
1758         u32 aligned_offset, aligned_len, *p;
1759         struct adapter *adapter = netdev2adap(dev);
1760
1761         if (eeprom->magic != EEPROM_MAGIC)
1762                 return -EINVAL;
1763
1764         aligned_offset = eeprom->offset & ~3;
1765         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1766
1767         if (adapter->fn > 0) {
1768                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1769
1770                 if (aligned_offset < start ||
1771                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
1772                         return -EPERM;
1773         }
1774
1775         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1776                 /*
1777                  * RMW possibly needed for first or last words.
1778                  */
1779                 buf = kmalloc(aligned_len, GFP_KERNEL);
1780                 if (!buf)
1781                         return -ENOMEM;
1782                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1783                 if (!err && aligned_len > 4)
1784                         err = eeprom_rd_phys(adapter,
1785                                              aligned_offset + aligned_len - 4,
1786                                              (u32 *)&buf[aligned_len - 4]);
1787                 if (err)
1788                         goto out;
1789                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1790         } else
1791                 buf = data;
1792
1793         err = t4_seeprom_wp(adapter, false);
1794         if (err)
1795                 goto out;
1796
1797         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1798                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1799                 aligned_offset += 4;
1800         }
1801
1802         if (!err)
1803                 err = t4_seeprom_wp(adapter, true);
1804 out:
1805         if (buf != data)
1806                 kfree(buf);
1807         return err;
1808 }
1809
1810 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1811 {
1812         int ret;
1813         const struct firmware *fw;
1814         struct adapter *adap = netdev2adap(netdev);
1815
1816         ef->data[sizeof(ef->data) - 1] = '\0';
1817         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1818         if (ret < 0)
1819                 return ret;
1820
1821         ret = t4_load_fw(adap, fw->data, fw->size);
1822         release_firmware(fw);
1823         if (!ret)
1824                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1825         return ret;
1826 }
1827
1828 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1829 #define BCAST_CRC 0xa0ccc1a6
1830
1831 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1832 {
1833         wol->supported = WAKE_BCAST | WAKE_MAGIC;
1834         wol->wolopts = netdev2adap(dev)->wol;
1835         memset(&wol->sopass, 0, sizeof(wol->sopass));
1836 }
1837
1838 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1839 {
1840         int err = 0;
1841         struct port_info *pi = netdev_priv(dev);
1842
1843         if (wol->wolopts & ~WOL_SUPPORTED)
1844                 return -EINVAL;
1845         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1846                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1847         if (wol->wolopts & WAKE_BCAST) {
1848                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1849                                         ~0ULL, 0, false);
1850                 if (!err)
1851                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1852                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
1853         } else
1854                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1855         return err;
1856 }
1857
1858 static int cxgb_set_features(struct net_device *dev, u32 features)
1859 {
1860         const struct port_info *pi = netdev_priv(dev);
1861         u32 changed = dev->features ^ features;
1862         int err;
1863
1864         if (!(changed & NETIF_F_HW_VLAN_RX))
1865                 return 0;
1866
1867         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1868                             -1, -1, -1,
1869                             !!(features & NETIF_F_HW_VLAN_RX), true);
1870         if (unlikely(err))
1871                 dev->features = features ^ NETIF_F_HW_VLAN_RX;
1872         return err;
1873 }
1874
1875 static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1876 {
1877         const struct port_info *pi = netdev_priv(dev);
1878         unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1879
1880         p->size = pi->rss_size;
1881         while (n--)
1882                 p->ring_index[n] = pi->rss[n];
1883         return 0;
1884 }
1885
1886 static int set_rss_table(struct net_device *dev,
1887                          const struct ethtool_rxfh_indir *p)
1888 {
1889         unsigned int i;
1890         struct port_info *pi = netdev_priv(dev);
1891
1892         if (p->size != pi->rss_size)
1893                 return -EINVAL;
1894         for (i = 0; i < p->size; i++)
1895                 if (p->ring_index[i] >= pi->nqsets)
1896                         return -EINVAL;
1897         for (i = 0; i < p->size; i++)
1898                 pi->rss[i] = p->ring_index[i];
1899         if (pi->adapter->flags & FULL_INIT_DONE)
1900                 return write_rss(pi, pi->rss);
1901         return 0;
1902 }
1903
1904 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1905                      u32 *rules)
1906 {
1907         const struct port_info *pi = netdev_priv(dev);
1908
1909         switch (info->cmd) {
1910         case ETHTOOL_GRXFH: {
1911                 unsigned int v = pi->rss_mode;
1912
1913                 info->data = 0;
1914                 switch (info->flow_type) {
1915                 case TCP_V4_FLOW:
1916                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1917                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1918                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1919                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1920                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1921                         break;
1922                 case UDP_V4_FLOW:
1923                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1924                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1925                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1926                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1927                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1928                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1929                         break;
1930                 case SCTP_V4_FLOW:
1931                 case AH_ESP_V4_FLOW:
1932                 case IPV4_FLOW:
1933                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1934                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1935                         break;
1936                 case TCP_V6_FLOW:
1937                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1938                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1939                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1940                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1941                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1942                         break;
1943                 case UDP_V6_FLOW:
1944                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1945                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1946                                 info->data = RXH_IP_SRC | RXH_IP_DST |
1947                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
1948                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1949                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1950                         break;
1951                 case SCTP_V6_FLOW:
1952                 case AH_ESP_V6_FLOW:
1953                 case IPV6_FLOW:
1954                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1955                                 info->data = RXH_IP_SRC | RXH_IP_DST;
1956                         break;
1957                 }
1958                 return 0;
1959         }
1960         case ETHTOOL_GRXRINGS:
1961                 info->data = pi->nqsets;
1962                 return 0;
1963         }
1964         return -EOPNOTSUPP;
1965 }
1966
1967 static struct ethtool_ops cxgb_ethtool_ops = {
1968         .get_settings      = get_settings,
1969         .set_settings      = set_settings,
1970         .get_drvinfo       = get_drvinfo,
1971         .get_msglevel      = get_msglevel,
1972         .set_msglevel      = set_msglevel,
1973         .get_ringparam     = get_sge_param,
1974         .set_ringparam     = set_sge_param,
1975         .get_coalesce      = get_coalesce,
1976         .set_coalesce      = set_coalesce,
1977         .get_eeprom_len    = get_eeprom_len,
1978         .get_eeprom        = get_eeprom,
1979         .set_eeprom        = set_eeprom,
1980         .get_pauseparam    = get_pauseparam,
1981         .set_pauseparam    = set_pauseparam,
1982         .get_link          = ethtool_op_get_link,
1983         .get_strings       = get_strings,
1984         .set_phys_id       = identify_port,
1985         .nway_reset        = restart_autoneg,
1986         .get_sset_count    = get_sset_count,
1987         .get_ethtool_stats = get_stats,
1988         .get_regs_len      = get_regs_len,
1989         .get_regs          = get_regs,
1990         .get_wol           = get_wol,
1991         .set_wol           = set_wol,
1992         .get_rxnfc         = get_rxnfc,
1993         .get_rxfh_indir    = get_rss_table,
1994         .set_rxfh_indir    = set_rss_table,
1995         .flash_device      = set_flash,
1996 };
1997
1998 /*
1999  * debugfs support
2000  */
2001
2002 static int mem_open(struct inode *inode, struct file *file)
2003 {
2004         file->private_data = inode->i_private;
2005         return 0;
2006 }
2007
2008 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2009                         loff_t *ppos)
2010 {
2011         loff_t pos = *ppos;
2012         loff_t avail = file->f_path.dentry->d_inode->i_size;
2013         unsigned int mem = (uintptr_t)file->private_data & 3;
2014         struct adapter *adap = file->private_data - mem;
2015
2016         if (pos < 0)
2017                 return -EINVAL;
2018         if (pos >= avail)
2019                 return 0;
2020         if (count > avail - pos)
2021                 count = avail - pos;
2022
2023         while (count) {
2024                 size_t len;
2025                 int ret, ofst;
2026                 __be32 data[16];
2027
2028                 if (mem == MEM_MC)
2029                         ret = t4_mc_read(adap, pos, data, NULL);
2030                 else
2031                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2032                 if (ret)
2033                         return ret;
2034
2035                 ofst = pos % sizeof(data);
2036                 len = min(count, sizeof(data) - ofst);
2037                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2038                         return -EFAULT;
2039
2040                 buf += len;
2041                 pos += len;
2042                 count -= len;
2043         }
2044         count = pos - *ppos;
2045         *ppos = pos;
2046         return count;
2047 }
2048
2049 static const struct file_operations mem_debugfs_fops = {
2050         .owner   = THIS_MODULE,
2051         .open    = mem_open,
2052         .read    = mem_read,
2053         .llseek  = default_llseek,
2054 };
2055
2056 static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2057                                       unsigned int idx, unsigned int size_mb)
2058 {
2059         struct dentry *de;
2060
2061         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2062                                  (void *)adap + idx, &mem_debugfs_fops);
2063         if (de && de->d_inode)
2064                 de->d_inode->i_size = size_mb << 20;
2065 }
2066
2067 static int __devinit setup_debugfs(struct adapter *adap)
2068 {
2069         int i;
2070
2071         if (IS_ERR_OR_NULL(adap->debugfs_root))
2072                 return -1;
2073
2074         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2075         if (i & EDRAM0_ENABLE)
2076                 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2077         if (i & EDRAM1_ENABLE)
2078                 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2079         if (i & EXT_MEM_ENABLE)
2080                 add_debugfs_mem(adap, "mc", MEM_MC,
2081                         EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2082         if (adap->l2t)
2083                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2084                                     &t4_l2t_fops);
2085         return 0;
2086 }
2087
2088 /*
2089  * upper-layer driver support
2090  */
2091
2092 /*
2093  * Allocate an active-open TID and set it to the supplied value.
2094  */
2095 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2096 {
2097         int atid = -1;
2098
2099         spin_lock_bh(&t->atid_lock);
2100         if (t->afree) {
2101                 union aopen_entry *p = t->afree;
2102
2103                 atid = p - t->atid_tab;
2104                 t->afree = p->next;
2105                 p->data = data;
2106                 t->atids_in_use++;
2107         }
2108         spin_unlock_bh(&t->atid_lock);
2109         return atid;
2110 }
2111 EXPORT_SYMBOL(cxgb4_alloc_atid);
2112
2113 /*
2114  * Release an active-open TID.
2115  */
2116 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2117 {
2118         union aopen_entry *p = &t->atid_tab[atid];
2119
2120         spin_lock_bh(&t->atid_lock);
2121         p->next = t->afree;
2122         t->afree = p;
2123         t->atids_in_use--;
2124         spin_unlock_bh(&t->atid_lock);
2125 }
2126 EXPORT_SYMBOL(cxgb4_free_atid);
2127
2128 /*
2129  * Allocate a server TID and set it to the supplied value.
2130  */
2131 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2132 {
2133         int stid;
2134
2135         spin_lock_bh(&t->stid_lock);
2136         if (family == PF_INET) {
2137                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2138                 if (stid < t->nstids)
2139                         __set_bit(stid, t->stid_bmap);
2140                 else
2141                         stid = -1;
2142         } else {
2143                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2144                 if (stid < 0)
2145                         stid = -1;
2146         }
2147         if (stid >= 0) {
2148                 t->stid_tab[stid].data = data;
2149                 stid += t->stid_base;
2150                 t->stids_in_use++;
2151         }
2152         spin_unlock_bh(&t->stid_lock);
2153         return stid;
2154 }
2155 EXPORT_SYMBOL(cxgb4_alloc_stid);
2156
2157 /*
2158  * Release a server TID.
2159  */
2160 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2161 {
2162         stid -= t->stid_base;
2163         spin_lock_bh(&t->stid_lock);
2164         if (family == PF_INET)
2165                 __clear_bit(stid, t->stid_bmap);
2166         else
2167                 bitmap_release_region(t->stid_bmap, stid, 2);
2168         t->stid_tab[stid].data = NULL;
2169         t->stids_in_use--;
2170         spin_unlock_bh(&t->stid_lock);
2171 }
2172 EXPORT_SYMBOL(cxgb4_free_stid);
2173
2174 /*
2175  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
2176  */
2177 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2178                            unsigned int tid)
2179 {
2180         struct cpl_tid_release *req;
2181
2182         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2183         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2184         INIT_TP_WR(req, tid);
2185         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2186 }
2187
2188 /*
2189  * Queue a TID release request and if necessary schedule a work queue to
2190  * process it.
2191  */
2192 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2193                                     unsigned int tid)
2194 {
2195         void **p = &t->tid_tab[tid];
2196         struct adapter *adap = container_of(t, struct adapter, tids);
2197
2198         spin_lock_bh(&adap->tid_release_lock);
2199         *p = adap->tid_release_head;
2200         /* Low 2 bits encode the Tx channel number */
2201         adap->tid_release_head = (void **)((uintptr_t)p | chan);
2202         if (!adap->tid_release_task_busy) {
2203                 adap->tid_release_task_busy = true;
2204                 schedule_work(&adap->tid_release_task);
2205         }
2206         spin_unlock_bh(&adap->tid_release_lock);
2207 }
2208
2209 /*
2210  * Process the list of pending TID release requests.
2211  */
2212 static void process_tid_release_list(struct work_struct *work)
2213 {
2214         struct sk_buff *skb;
2215         struct adapter *adap;
2216
2217         adap = container_of(work, struct adapter, tid_release_task);
2218
2219         spin_lock_bh(&adap->tid_release_lock);
2220         while (adap->tid_release_head) {
2221                 void **p = adap->tid_release_head;
2222                 unsigned int chan = (uintptr_t)p & 3;
2223                 p = (void *)p - chan;
2224
2225                 adap->tid_release_head = *p;
2226                 *p = NULL;
2227                 spin_unlock_bh(&adap->tid_release_lock);
2228
2229                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2230                                          GFP_KERNEL)))
2231                         schedule_timeout_uninterruptible(1);
2232
2233                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2234                 t4_ofld_send(adap, skb);
2235                 spin_lock_bh(&adap->tid_release_lock);
2236         }
2237         adap->tid_release_task_busy = false;
2238         spin_unlock_bh(&adap->tid_release_lock);
2239 }
2240
2241 /*
2242  * Release a TID and inform HW.  If we are unable to allocate the release
2243  * message we defer to a work queue.
2244  */
2245 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2246 {
2247         void *old;
2248         struct sk_buff *skb;
2249         struct adapter *adap = container_of(t, struct adapter, tids);
2250
2251         old = t->tid_tab[tid];
2252         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2253         if (likely(skb)) {
2254                 t->tid_tab[tid] = NULL;
2255                 mk_tid_release(skb, chan, tid);
2256                 t4_ofld_send(adap, skb);
2257         } else
2258                 cxgb4_queue_tid_release(t, chan, tid);
2259         if (old)
2260                 atomic_dec(&t->tids_in_use);
2261 }
2262 EXPORT_SYMBOL(cxgb4_remove_tid);
2263
2264 /*
2265  * Allocate and initialize the TID tables.  Returns 0 on success.
2266  */
2267 static int tid_init(struct tid_info *t)
2268 {
2269         size_t size;
2270         unsigned int natids = t->natids;
2271
2272         size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2273                t->nstids * sizeof(*t->stid_tab) +
2274                BITS_TO_LONGS(t->nstids) * sizeof(long);
2275         t->tid_tab = t4_alloc_mem(size);
2276         if (!t->tid_tab)
2277                 return -ENOMEM;
2278
2279         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2280         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2281         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2282         spin_lock_init(&t->stid_lock);
2283         spin_lock_init(&t->atid_lock);
2284
2285         t->stids_in_use = 0;
2286         t->afree = NULL;
2287         t->atids_in_use = 0;
2288         atomic_set(&t->tids_in_use, 0);
2289
2290         /* Setup the free list for atid_tab and clear the stid bitmap. */
2291         if (natids) {
2292                 while (--natids)
2293                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2294                 t->afree = t->atid_tab;
2295         }
2296         bitmap_zero(t->stid_bmap, t->nstids);
2297         return 0;
2298 }
2299
2300 /**
2301  *      cxgb4_create_server - create an IP server
2302  *      @dev: the device
2303  *      @stid: the server TID
2304  *      @sip: local IP address to bind server to
2305  *      @sport: the server's TCP port
2306  *      @queue: queue to direct messages from this server to
2307  *
2308  *      Create an IP server for the given port and address.
2309  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
2310  */
2311 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2312                         __be32 sip, __be16 sport, unsigned int queue)
2313 {
2314         unsigned int chan;
2315         struct sk_buff *skb;
2316         struct adapter *adap;
2317         struct cpl_pass_open_req *req;
2318
2319         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2320         if (!skb)
2321                 return -ENOMEM;
2322
2323         adap = netdev2adap(dev);
2324         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2325         INIT_TP_WR(req, 0);
2326         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2327         req->local_port = sport;
2328         req->peer_port = htons(0);
2329         req->local_ip = sip;
2330         req->peer_ip = htonl(0);
2331         chan = rxq_to_chan(&adap->sge, queue);
2332         req->opt0 = cpu_to_be64(TX_CHAN(chan));
2333         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2334                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2335         return t4_mgmt_tx(adap, skb);
2336 }
2337 EXPORT_SYMBOL(cxgb4_create_server);
2338
2339 /**
2340  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2341  *      @mtus: the HW MTU table
2342  *      @mtu: the target MTU
2343  *      @idx: index of selected entry in the MTU table
2344  *
2345  *      Returns the index and the value in the HW MTU table that is closest to
2346  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
2347  *      table, in which case that smallest available value is selected.
2348  */
2349 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2350                             unsigned int *idx)
2351 {
2352         unsigned int i = 0;
2353
2354         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2355                 ++i;
2356         if (idx)
2357                 *idx = i;
2358         return mtus[i];
2359 }
2360 EXPORT_SYMBOL(cxgb4_best_mtu);
2361
2362 /**
2363  *      cxgb4_port_chan - get the HW channel of a port
2364  *      @dev: the net device for the port
2365  *
2366  *      Return the HW Tx channel of the given port.
2367  */
2368 unsigned int cxgb4_port_chan(const struct net_device *dev)
2369 {
2370         return netdev2pinfo(dev)->tx_chan;
2371 }
2372 EXPORT_SYMBOL(cxgb4_port_chan);
2373
2374 /**
2375  *      cxgb4_port_viid - get the VI id of a port
2376  *      @dev: the net device for the port
2377  *
2378  *      Return the VI id of the given port.
2379  */
2380 unsigned int cxgb4_port_viid(const struct net_device *dev)
2381 {
2382         return netdev2pinfo(dev)->viid;
2383 }
2384 EXPORT_SYMBOL(cxgb4_port_viid);
2385
2386 /**
2387  *      cxgb4_port_idx - get the index of a port
2388  *      @dev: the net device for the port
2389  *
2390  *      Return the index of the given port.
2391  */
2392 unsigned int cxgb4_port_idx(const struct net_device *dev)
2393 {
2394         return netdev2pinfo(dev)->port_id;
2395 }
2396 EXPORT_SYMBOL(cxgb4_port_idx);
2397
2398 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2399                          struct tp_tcp_stats *v6)
2400 {
2401         struct adapter *adap = pci_get_drvdata(pdev);
2402
2403         spin_lock(&adap->stats_lock);
2404         t4_tp_get_tcp_stats(adap, v4, v6);
2405         spin_unlock(&adap->stats_lock);
2406 }
2407 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2408
2409 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2410                       const unsigned int *pgsz_order)
2411 {
2412         struct adapter *adap = netdev2adap(dev);
2413
2414         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2415         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2416                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2417                      HPZ3(pgsz_order[3]));
2418 }
2419 EXPORT_SYMBOL(cxgb4_iscsi_init);
2420
2421 static struct pci_driver cxgb4_driver;
2422
2423 static void check_neigh_update(struct neighbour *neigh)
2424 {
2425         const struct device *parent;
2426         const struct net_device *netdev = neigh->dev;
2427
2428         if (netdev->priv_flags & IFF_802_1Q_VLAN)
2429                 netdev = vlan_dev_real_dev(netdev);
2430         parent = netdev->dev.parent;
2431         if (parent && parent->driver == &cxgb4_driver.driver)
2432                 t4_l2t_update(dev_get_drvdata(parent), neigh);
2433 }
2434
2435 static int netevent_cb(struct notifier_block *nb, unsigned long event,
2436                        void *data)
2437 {
2438         switch (event) {
2439         case NETEVENT_NEIGH_UPDATE:
2440                 check_neigh_update(data);
2441                 break;
2442         case NETEVENT_REDIRECT:
2443         default:
2444                 break;
2445         }
2446         return 0;
2447 }
2448
2449 static bool netevent_registered;
2450 static struct notifier_block cxgb4_netevent_nb = {
2451         .notifier_call = netevent_cb
2452 };
2453
2454 static void uld_attach(struct adapter *adap, unsigned int uld)
2455 {
2456         void *handle;
2457         struct cxgb4_lld_info lli;
2458
2459         lli.pdev = adap->pdev;
2460         lli.l2t = adap->l2t;
2461         lli.tids = &adap->tids;
2462         lli.ports = adap->port;
2463         lli.vr = &adap->vres;
2464         lli.mtus = adap->params.mtus;
2465         if (uld == CXGB4_ULD_RDMA) {
2466                 lli.rxq_ids = adap->sge.rdma_rxq;
2467                 lli.nrxq = adap->sge.rdmaqs;
2468         } else if (uld == CXGB4_ULD_ISCSI) {
2469                 lli.rxq_ids = adap->sge.ofld_rxq;
2470                 lli.nrxq = adap->sge.ofldqsets;
2471         }
2472         lli.ntxq = adap->sge.ofldqsets;
2473         lli.nchan = adap->params.nports;
2474         lli.nports = adap->params.nports;
2475         lli.wr_cred = adap->params.ofldq_wr_cred;
2476         lli.adapter_type = adap->params.rev;
2477         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2478         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2479                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2480                         (adap->fn * 4));
2481         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2482                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2483                         (adap->fn * 4));
2484         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2485         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2486         lli.fw_vers = adap->params.fw_vers;
2487
2488         handle = ulds[uld].add(&lli);
2489         if (IS_ERR(handle)) {
2490                 dev_warn(adap->pdev_dev,
2491                          "could not attach to the %s driver, error %ld\n",
2492                          uld_str[uld], PTR_ERR(handle));
2493                 return;
2494         }
2495
2496         adap->uld_handle[uld] = handle;
2497
2498         if (!netevent_registered) {
2499                 register_netevent_notifier(&cxgb4_netevent_nb);
2500                 netevent_registered = true;
2501         }
2502
2503         if (adap->flags & FULL_INIT_DONE)
2504                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2505 }
2506
2507 static void attach_ulds(struct adapter *adap)
2508 {
2509         unsigned int i;
2510
2511         mutex_lock(&uld_mutex);
2512         list_add_tail(&adap->list_node, &adapter_list);
2513         for (i = 0; i < CXGB4_ULD_MAX; i++)
2514                 if (ulds[i].add)
2515                         uld_attach(adap, i);
2516         mutex_unlock(&uld_mutex);
2517 }
2518
2519 static void detach_ulds(struct adapter *adap)
2520 {
2521         unsigned int i;
2522
2523         mutex_lock(&uld_mutex);
2524         list_del(&adap->list_node);
2525         for (i = 0; i < CXGB4_ULD_MAX; i++)
2526                 if (adap->uld_handle[i]) {
2527                         ulds[i].state_change(adap->uld_handle[i],
2528                                              CXGB4_STATE_DETACH);
2529                         adap->uld_handle[i] = NULL;
2530                 }
2531         if (netevent_registered && list_empty(&adapter_list)) {
2532                 unregister_netevent_notifier(&cxgb4_netevent_nb);
2533                 netevent_registered = false;
2534         }
2535         mutex_unlock(&uld_mutex);
2536 }
2537
2538 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2539 {
2540         unsigned int i;
2541
2542         mutex_lock(&uld_mutex);
2543         for (i = 0; i < CXGB4_ULD_MAX; i++)
2544                 if (adap->uld_handle[i])
2545                         ulds[i].state_change(adap->uld_handle[i], new_state);
2546         mutex_unlock(&uld_mutex);
2547 }
2548
2549 /**
2550  *      cxgb4_register_uld - register an upper-layer driver
2551  *      @type: the ULD type
2552  *      @p: the ULD methods
2553  *
2554  *      Registers an upper-layer driver with this driver and notifies the ULD
2555  *      about any presently available devices that support its type.  Returns
2556  *      %-EBUSY if a ULD of the same type is already registered.
2557  */
2558 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2559 {
2560         int ret = 0;
2561         struct adapter *adap;
2562
2563         if (type >= CXGB4_ULD_MAX)
2564                 return -EINVAL;
2565         mutex_lock(&uld_mutex);
2566         if (ulds[type].add) {
2567                 ret = -EBUSY;
2568                 goto out;
2569         }
2570         ulds[type] = *p;
2571         list_for_each_entry(adap, &adapter_list, list_node)
2572                 uld_attach(adap, type);
2573 out:    mutex_unlock(&uld_mutex);
2574         return ret;
2575 }
2576 EXPORT_SYMBOL(cxgb4_register_uld);
2577
2578 /**
2579  *      cxgb4_unregister_uld - unregister an upper-layer driver
2580  *      @type: the ULD type
2581  *
2582  *      Unregisters an existing upper-layer driver.
2583  */
2584 int cxgb4_unregister_uld(enum cxgb4_uld type)
2585 {
2586         struct adapter *adap;
2587
2588         if (type >= CXGB4_ULD_MAX)
2589                 return -EINVAL;
2590         mutex_lock(&uld_mutex);
2591         list_for_each_entry(adap, &adapter_list, list_node)
2592                 adap->uld_handle[type] = NULL;
2593         ulds[type].add = NULL;
2594         mutex_unlock(&uld_mutex);
2595         return 0;
2596 }
2597 EXPORT_SYMBOL(cxgb4_unregister_uld);
2598
2599 /**
2600  *      cxgb_up - enable the adapter
2601  *      @adap: adapter being enabled
2602  *
2603  *      Called when the first port is enabled, this function performs the
2604  *      actions necessary to make an adapter operational, such as completing
2605  *      the initialization of HW modules, and enabling interrupts.
2606  *
2607  *      Must be called with the rtnl lock held.
2608  */
2609 static int cxgb_up(struct adapter *adap)
2610 {
2611         int err;
2612
2613         err = setup_sge_queues(adap);
2614         if (err)
2615                 goto out;
2616         err = setup_rss(adap);
2617         if (err)
2618                 goto freeq;
2619
2620         if (adap->flags & USING_MSIX) {
2621                 name_msix_vecs(adap);
2622                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2623                                   adap->msix_info[0].desc, adap);
2624                 if (err)
2625                         goto irq_err;
2626
2627                 err = request_msix_queue_irqs(adap);
2628                 if (err) {
2629                         free_irq(adap->msix_info[0].vec, adap);
2630                         goto irq_err;
2631                 }
2632         } else {
2633                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2634                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2635                                   adap->port[0]->name, adap);
2636                 if (err)
2637                         goto irq_err;
2638         }
2639         enable_rx(adap);
2640         t4_sge_start(adap);
2641         t4_intr_enable(adap);
2642         adap->flags |= FULL_INIT_DONE;
2643         notify_ulds(adap, CXGB4_STATE_UP);
2644  out:
2645         return err;
2646  irq_err:
2647         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2648  freeq:
2649         t4_free_sge_resources(adap);
2650         goto out;
2651 }
2652
2653 static void cxgb_down(struct adapter *adapter)
2654 {
2655         t4_intr_disable(adapter);
2656         cancel_work_sync(&adapter->tid_release_task);
2657         adapter->tid_release_task_busy = false;
2658         adapter->tid_release_head = NULL;
2659
2660         if (adapter->flags & USING_MSIX) {
2661                 free_msix_queue_irqs(adapter);
2662                 free_irq(adapter->msix_info[0].vec, adapter);
2663         } else
2664                 free_irq(adapter->pdev->irq, adapter);
2665         quiesce_rx(adapter);
2666         t4_sge_stop(adapter);
2667         t4_free_sge_resources(adapter);
2668         adapter->flags &= ~FULL_INIT_DONE;
2669 }
2670
2671 /*
2672  * net_device operations
2673  */
2674 static int cxgb_open(struct net_device *dev)
2675 {
2676         int err;
2677         struct port_info *pi = netdev_priv(dev);
2678         struct adapter *adapter = pi->adapter;
2679
2680         netif_carrier_off(dev);
2681
2682         if (!(adapter->flags & FULL_INIT_DONE)) {
2683                 err = cxgb_up(adapter);
2684                 if (err < 0)
2685                         return err;
2686         }
2687
2688         err = link_start(dev);
2689         if (!err)
2690                 netif_tx_start_all_queues(dev);
2691         return err;
2692 }
2693
2694 static int cxgb_close(struct net_device *dev)
2695 {
2696         struct port_info *pi = netdev_priv(dev);
2697         struct adapter *adapter = pi->adapter;
2698
2699         netif_tx_stop_all_queues(dev);
2700         netif_carrier_off(dev);
2701         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2702 }
2703
2704 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2705                                                 struct rtnl_link_stats64 *ns)
2706 {
2707         struct port_stats stats;
2708         struct port_info *p = netdev_priv(dev);
2709         struct adapter *adapter = p->adapter;
2710
2711         spin_lock(&adapter->stats_lock);
2712         t4_get_port_stats(adapter, p->tx_chan, &stats);
2713         spin_unlock(&adapter->stats_lock);
2714
2715         ns->tx_bytes   = stats.tx_octets;
2716         ns->tx_packets = stats.tx_frames;
2717         ns->rx_bytes   = stats.rx_octets;
2718         ns->rx_packets = stats.rx_frames;
2719         ns->multicast  = stats.rx_mcast_frames;
2720
2721         /* detailed rx_errors */
2722         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2723                                stats.rx_runt;
2724         ns->rx_over_errors   = 0;
2725         ns->rx_crc_errors    = stats.rx_fcs_err;
2726         ns->rx_frame_errors  = stats.rx_symbol_err;
2727         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
2728                                stats.rx_ovflow2 + stats.rx_ovflow3 +
2729                                stats.rx_trunc0 + stats.rx_trunc1 +
2730                                stats.rx_trunc2 + stats.rx_trunc3;
2731         ns->rx_missed_errors = 0;
2732
2733         /* detailed tx_errors */
2734         ns->tx_aborted_errors   = 0;
2735         ns->tx_carrier_errors   = 0;
2736         ns->tx_fifo_errors      = 0;
2737         ns->tx_heartbeat_errors = 0;
2738         ns->tx_window_errors    = 0;
2739
2740         ns->tx_errors = stats.tx_error_frames;
2741         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2742                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2743         return ns;
2744 }
2745
2746 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2747 {
2748         unsigned int mbox;
2749         int ret = 0, prtad, devad;
2750         struct port_info *pi = netdev_priv(dev);
2751         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2752
2753         switch (cmd) {
2754         case SIOCGMIIPHY:
2755                 if (pi->mdio_addr < 0)
2756                         return -EOPNOTSUPP;
2757                 data->phy_id = pi->mdio_addr;
2758                 break;
2759         case SIOCGMIIREG:
2760         case SIOCSMIIREG:
2761                 if (mdio_phy_id_is_c45(data->phy_id)) {
2762                         prtad = mdio_phy_id_prtad(data->phy_id);
2763                         devad = mdio_phy_id_devad(data->phy_id);
2764                 } else if (data->phy_id < 32) {
2765                         prtad = data->phy_id;
2766                         devad = 0;
2767                         data->reg_num &= 0x1f;
2768                 } else
2769                         return -EINVAL;
2770
2771                 mbox = pi->adapter->fn;
2772                 if (cmd == SIOCGMIIREG)
2773                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2774                                          data->reg_num, &data->val_out);
2775                 else
2776                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2777                                          data->reg_num, data->val_in);
2778                 break;
2779         default:
2780                 return -EOPNOTSUPP;
2781         }
2782         return ret;
2783 }
2784
2785 static void cxgb_set_rxmode(struct net_device *dev)
2786 {
2787         /* unfortunately we can't return errors to the stack */
2788         set_rxmode(dev, -1, false);
2789 }
2790
2791 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2792 {
2793         int ret;
2794         struct port_info *pi = netdev_priv(dev);
2795
2796         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
2797                 return -EINVAL;
2798         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2799                             -1, -1, -1, true);
2800         if (!ret)
2801                 dev->mtu = new_mtu;
2802         return ret;
2803 }
2804
2805 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2806 {
2807         int ret;
2808         struct sockaddr *addr = p;
2809         struct port_info *pi = netdev_priv(dev);
2810
2811         if (!is_valid_ether_addr(addr->sa_data))
2812                 return -EINVAL;
2813
2814         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2815                             pi->xact_addr_filt, addr->sa_data, true, true);
2816         if (ret < 0)
2817                 return ret;
2818
2819         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2820         pi->xact_addr_filt = ret;
2821         return 0;
2822 }
2823
2824 #ifdef CONFIG_NET_POLL_CONTROLLER
2825 static void cxgb_netpoll(struct net_device *dev)
2826 {
2827         struct port_info *pi = netdev_priv(dev);
2828         struct adapter *adap = pi->adapter;
2829
2830         if (adap->flags & USING_MSIX) {
2831                 int i;
2832                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2833
2834                 for (i = pi->nqsets; i; i--, rx++)
2835                         t4_sge_intr_msix(0, &rx->rspq);
2836         } else
2837                 t4_intr_handler(adap)(0, adap);
2838 }
2839 #endif
2840
2841 static const struct net_device_ops cxgb4_netdev_ops = {
2842         .ndo_open             = cxgb_open,
2843         .ndo_stop             = cxgb_close,
2844         .ndo_start_xmit       = t4_eth_xmit,
2845         .ndo_get_stats64      = cxgb_get_stats,
2846         .ndo_set_rx_mode      = cxgb_set_rxmode,
2847         .ndo_set_mac_address  = cxgb_set_mac_addr,
2848         .ndo_set_features     = cxgb_set_features,
2849         .ndo_validate_addr    = eth_validate_addr,
2850         .ndo_do_ioctl         = cxgb_ioctl,
2851         .ndo_change_mtu       = cxgb_change_mtu,
2852 #ifdef CONFIG_NET_POLL_CONTROLLER
2853         .ndo_poll_controller  = cxgb_netpoll,
2854 #endif
2855 };
2856
2857 void t4_fatal_err(struct adapter *adap)
2858 {
2859         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2860         t4_intr_disable(adap);
2861         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2862 }
2863
2864 static void setup_memwin(struct adapter *adap)
2865 {
2866         u32 bar0;
2867
2868         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
2869         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2870                      (bar0 + MEMWIN0_BASE) | BIR(0) |
2871                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2872         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2873                      (bar0 + MEMWIN1_BASE) | BIR(0) |
2874                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2875         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2876                      (bar0 + MEMWIN2_BASE) | BIR(0) |
2877                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2878         if (adap->vres.ocq.size) {
2879                 unsigned int start, sz_kb;
2880
2881                 start = pci_resource_start(adap->pdev, 2) +
2882                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2883                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2884                 t4_write_reg(adap,
2885                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2886                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
2887                 t4_write_reg(adap,
2888                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2889                              adap->vres.ocq.start);
2890                 t4_read_reg(adap,
2891                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2892         }
2893 }
2894
2895 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2896 {
2897         u32 v;
2898         int ret;
2899
2900         /* get device capabilities */
2901         memset(c, 0, sizeof(*c));
2902         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2903                                FW_CMD_REQUEST | FW_CMD_READ);
2904         c->retval_len16 = htonl(FW_LEN16(*c));
2905         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
2906         if (ret < 0)
2907                 return ret;
2908
2909         /* select capabilities we'll be using */
2910         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2911                 if (!vf_acls)
2912                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2913                 else
2914                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2915         } else if (vf_acls) {
2916                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2917                 return ret;
2918         }
2919         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2920                                FW_CMD_REQUEST | FW_CMD_WRITE);
2921         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
2922         if (ret < 0)
2923                 return ret;
2924
2925         ret = t4_config_glbl_rss(adap, adap->fn,
2926                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2927                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2928                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2929         if (ret < 0)
2930                 return ret;
2931
2932         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2933                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2934         if (ret < 0)
2935                 return ret;
2936
2937         t4_sge_init(adap);
2938
2939         /* tweak some settings */
2940         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2941         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2942         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2943         v = t4_read_reg(adap, TP_PIO_DATA);
2944         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2945
2946         /* get basic stuff going */
2947         return t4_early_init(adap, adap->fn);
2948 }
2949
2950 /*
2951  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
2952  */
2953 #define MAX_ATIDS 8192U
2954
2955 /*
2956  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2957  */
2958 static int adap_init0(struct adapter *adap)
2959 {
2960         int ret;
2961         u32 v, port_vec;
2962         enum dev_state state;
2963         u32 params[7], val[7];
2964         struct fw_caps_config_cmd c;
2965
2966         ret = t4_check_fw_version(adap);
2967         if (ret == -EINVAL || ret > 0) {
2968                 if (upgrade_fw(adap) >= 0)             /* recache FW version */
2969                         ret = t4_check_fw_version(adap);
2970         }
2971         if (ret < 0)
2972                 return ret;
2973
2974         /* contact FW, request master */
2975         ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
2976         if (ret < 0) {
2977                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2978                         ret);
2979                 return ret;
2980         }
2981
2982         /* reset device */
2983         ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
2984         if (ret < 0)
2985                 goto bye;
2986
2987         for (v = 0; v < SGE_NTIMERS - 1; v++)
2988                 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2989         adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2990         adap->sge.counter_val[0] = 1;
2991         for (v = 1; v < SGE_NCOUNTERS; v++)
2992                 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2993                                                THRESHOLD_3_MASK);
2994 #define FW_PARAM_DEV(param) \
2995         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2996          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2997
2998         params[0] = FW_PARAM_DEV(CCLK);
2999         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
3000         if (ret < 0)
3001                 goto bye;
3002         adap->params.vpd.cclk = val[0];
3003
3004         ret = adap_init1(adap, &c);
3005         if (ret < 0)
3006                 goto bye;
3007
3008 #define FW_PARAM_PFVF(param) \
3009         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3010          FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3011          FW_PARAMS_PARAM_Y(adap->fn))
3012
3013         params[0] = FW_PARAM_DEV(PORTVEC);
3014         params[1] = FW_PARAM_PFVF(L2T_START);
3015         params[2] = FW_PARAM_PFVF(L2T_END);
3016         params[3] = FW_PARAM_PFVF(FILTER_START);
3017         params[4] = FW_PARAM_PFVF(FILTER_END);
3018         params[5] = FW_PARAM_PFVF(IQFLINT_START);
3019         params[6] = FW_PARAM_PFVF(EQ_START);
3020         ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
3021         if (ret < 0)
3022                 goto bye;
3023         port_vec = val[0];
3024         adap->tids.ftid_base = val[3];
3025         adap->tids.nftids = val[4] - val[3] + 1;
3026         adap->sge.ingr_start = val[5];
3027         adap->sge.egr_start = val[6];
3028
3029         if (c.ofldcaps) {
3030                 /* query offload-related parameters */
3031                 params[0] = FW_PARAM_DEV(NTID);
3032                 params[1] = FW_PARAM_PFVF(SERVER_START);
3033                 params[2] = FW_PARAM_PFVF(SERVER_END);
3034                 params[3] = FW_PARAM_PFVF(TDDP_START);
3035                 params[4] = FW_PARAM_PFVF(TDDP_END);
3036                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3037                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3038                                       val);
3039                 if (ret < 0)
3040                         goto bye;
3041                 adap->tids.ntids = val[0];
3042                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3043                 adap->tids.stid_base = val[1];
3044                 adap->tids.nstids = val[2] - val[1] + 1;
3045                 adap->vres.ddp.start = val[3];
3046                 adap->vres.ddp.size = val[4] - val[3] + 1;
3047                 adap->params.ofldq_wr_cred = val[5];
3048                 adap->params.offload = 1;
3049         }
3050         if (c.rdmacaps) {
3051                 params[0] = FW_PARAM_PFVF(STAG_START);
3052                 params[1] = FW_PARAM_PFVF(STAG_END);
3053                 params[2] = FW_PARAM_PFVF(RQ_START);
3054                 params[3] = FW_PARAM_PFVF(RQ_END);
3055                 params[4] = FW_PARAM_PFVF(PBL_START);
3056                 params[5] = FW_PARAM_PFVF(PBL_END);
3057                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3058                                       val);
3059                 if (ret < 0)
3060                         goto bye;
3061                 adap->vres.stag.start = val[0];
3062                 adap->vres.stag.size = val[1] - val[0] + 1;
3063                 adap->vres.rq.start = val[2];
3064                 adap->vres.rq.size = val[3] - val[2] + 1;
3065                 adap->vres.pbl.start = val[4];
3066                 adap->vres.pbl.size = val[5] - val[4] + 1;
3067
3068                 params[0] = FW_PARAM_PFVF(SQRQ_START);
3069                 params[1] = FW_PARAM_PFVF(SQRQ_END);
3070                 params[2] = FW_PARAM_PFVF(CQ_START);
3071                 params[3] = FW_PARAM_PFVF(CQ_END);
3072                 params[4] = FW_PARAM_PFVF(OCQ_START);
3073                 params[5] = FW_PARAM_PFVF(OCQ_END);
3074                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3075                                       val);
3076                 if (ret < 0)
3077                         goto bye;
3078                 adap->vres.qp.start = val[0];
3079                 adap->vres.qp.size = val[1] - val[0] + 1;
3080                 adap->vres.cq.start = val[2];
3081                 adap->vres.cq.size = val[3] - val[2] + 1;
3082                 adap->vres.ocq.start = val[4];
3083                 adap->vres.ocq.size = val[5] - val[4] + 1;
3084         }
3085         if (c.iscsicaps) {
3086                 params[0] = FW_PARAM_PFVF(ISCSI_START);
3087                 params[1] = FW_PARAM_PFVF(ISCSI_END);
3088                 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3089                                       val);
3090                 if (ret < 0)
3091                         goto bye;
3092                 adap->vres.iscsi.start = val[0];
3093                 adap->vres.iscsi.size = val[1] - val[0] + 1;
3094         }
3095 #undef FW_PARAM_PFVF
3096 #undef FW_PARAM_DEV
3097
3098         adap->params.nports = hweight32(port_vec);
3099         adap->params.portvec = port_vec;
3100         adap->flags |= FW_OK;
3101
3102         /* These are finalized by FW initialization, load their values now */
3103         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3104         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3105         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3106         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3107                      adap->params.b_wnd);
3108
3109 #ifdef CONFIG_PCI_IOV
3110         /*
3111          * Provision resource limits for Virtual Functions.  We currently
3112          * grant them all the same static resource limits except for the Port
3113          * Access Rights Mask which we're assigning based on the PF.  All of
3114          * the static provisioning stuff for both the PF and VF really needs
3115          * to be managed in a persistent manner for each device which the
3116          * firmware controls.
3117          */
3118         {
3119                 int pf, vf;
3120
3121                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3122                         if (num_vf[pf] <= 0)
3123                                 continue;
3124
3125                         /* VF numbering starts at 1! */
3126                         for (vf = 1; vf <= num_vf[pf]; vf++) {
3127                                 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3128                                                   VFRES_NEQ, VFRES_NETHCTRL,
3129                                                   VFRES_NIQFLINT, VFRES_NIQ,
3130                                                   VFRES_TC, VFRES_NVI,
3131                                                   FW_PFVF_CMD_CMASK_MASK,
3132                                                   pfvfres_pmask(adap, pf, vf),
3133                                                   VFRES_NEXACTF,
3134                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
3135                                 if (ret < 0)
3136                                         dev_warn(adap->pdev_dev, "failed to "
3137                                                  "provision pf/vf=%d/%d; "
3138                                                  "err=%d\n", pf, vf, ret);
3139                         }
3140                 }
3141         }
3142 #endif
3143
3144         setup_memwin(adap);
3145         return 0;
3146
3147         /*
3148          * If a command timed out or failed with EIO FW does not operate within
3149          * its spec or something catastrophic happened to HW/FW, stop issuing
3150          * commands.
3151          */
3152 bye:    if (ret != -ETIMEDOUT && ret != -EIO)
3153                 t4_fw_bye(adap, adap->fn);
3154         return ret;
3155 }
3156
3157 /* EEH callbacks */
3158
3159 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3160                                          pci_channel_state_t state)
3161 {
3162         int i;
3163         struct adapter *adap = pci_get_drvdata(pdev);
3164
3165         if (!adap)
3166                 goto out;
3167
3168         rtnl_lock();
3169         adap->flags &= ~FW_OK;
3170         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3171         for_each_port(adap, i) {
3172                 struct net_device *dev = adap->port[i];
3173
3174                 netif_device_detach(dev);
3175                 netif_carrier_off(dev);
3176         }
3177         if (adap->flags & FULL_INIT_DONE)
3178                 cxgb_down(adap);
3179         rtnl_unlock();
3180         pci_disable_device(pdev);
3181 out:    return state == pci_channel_io_perm_failure ?
3182                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3183 }
3184
3185 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3186 {
3187         int i, ret;
3188         struct fw_caps_config_cmd c;
3189         struct adapter *adap = pci_get_drvdata(pdev);
3190
3191         if (!adap) {
3192                 pci_restore_state(pdev);
3193                 pci_save_state(pdev);
3194                 return PCI_ERS_RESULT_RECOVERED;
3195         }
3196
3197         if (pci_enable_device(pdev)) {
3198                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3199                 return PCI_ERS_RESULT_DISCONNECT;
3200         }
3201
3202         pci_set_master(pdev);
3203         pci_restore_state(pdev);
3204         pci_save_state(pdev);
3205         pci_cleanup_aer_uncorrect_error_status(pdev);
3206
3207         if (t4_wait_dev_ready(adap) < 0)
3208                 return PCI_ERS_RESULT_DISCONNECT;
3209         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3210                 return PCI_ERS_RESULT_DISCONNECT;
3211         adap->flags |= FW_OK;
3212         if (adap_init1(adap, &c))
3213                 return PCI_ERS_RESULT_DISCONNECT;
3214
3215         for_each_port(adap, i) {
3216                 struct port_info *p = adap2pinfo(adap, i);
3217
3218                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3219                                   NULL, NULL);
3220                 if (ret < 0)
3221                         return PCI_ERS_RESULT_DISCONNECT;
3222                 p->viid = ret;
3223                 p->xact_addr_filt = -1;
3224         }
3225
3226         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3227                      adap->params.b_wnd);
3228         setup_memwin(adap);
3229         if (cxgb_up(adap))
3230                 return PCI_ERS_RESULT_DISCONNECT;
3231         return PCI_ERS_RESULT_RECOVERED;
3232 }
3233
3234 static void eeh_resume(struct pci_dev *pdev)
3235 {
3236         int i;
3237         struct adapter *adap = pci_get_drvdata(pdev);
3238
3239         if (!adap)
3240                 return;
3241
3242         rtnl_lock();
3243         for_each_port(adap, i) {
3244                 struct net_device *dev = adap->port[i];
3245
3246                 if (netif_running(dev)) {
3247                         link_start(dev);
3248                         cxgb_set_rxmode(dev);
3249                 }
3250                 netif_device_attach(dev);
3251         }
3252         rtnl_unlock();
3253 }
3254
3255 static struct pci_error_handlers cxgb4_eeh = {
3256         .error_detected = eeh_err_detected,
3257         .slot_reset     = eeh_slot_reset,
3258         .resume         = eeh_resume,
3259 };
3260
3261 static inline bool is_10g_port(const struct link_config *lc)
3262 {
3263         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3264 }
3265
3266 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3267                              unsigned int size, unsigned int iqe_size)
3268 {
3269         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3270                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3271         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3272         q->iqe_len = iqe_size;
3273         q->size = size;
3274 }
3275
3276 /*
3277  * Perform default configuration of DMA queues depending on the number and type
3278  * of ports we found and the number of available CPUs.  Most settings can be
3279  * modified by the admin prior to actual use.
3280  */
3281 static void __devinit cfg_queues(struct adapter *adap)
3282 {
3283         struct sge *s = &adap->sge;
3284         int i, q10g = 0, n10g = 0, qidx = 0;
3285
3286         for_each_port(adap, i)
3287                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3288
3289         /*
3290          * We default to 1 queue per non-10G port and up to # of cores queues
3291          * per 10G port.
3292          */
3293         if (n10g)
3294                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3295         if (q10g > num_online_cpus())
3296                 q10g = num_online_cpus();
3297
3298         for_each_port(adap, i) {
3299                 struct port_info *pi = adap2pinfo(adap, i);
3300
3301                 pi->first_qset = qidx;
3302                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3303                 qidx += pi->nqsets;
3304         }
3305
3306         s->ethqsets = qidx;
3307         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
3308
3309         if (is_offload(adap)) {
3310                 /*
3311                  * For offload we use 1 queue/channel if all ports are up to 1G,
3312                  * otherwise we divide all available queues amongst the channels
3313                  * capped by the number of available cores.
3314                  */
3315                 if (n10g) {
3316                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3317                                   num_online_cpus());
3318                         s->ofldqsets = roundup(i, adap->params.nports);
3319                 } else
3320                         s->ofldqsets = adap->params.nports;
3321                 /* For RDMA one Rx queue per channel suffices */
3322                 s->rdmaqs = adap->params.nports;
3323         }
3324
3325         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3326                 struct sge_eth_rxq *r = &s->ethrxq[i];
3327
3328                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3329                 r->fl.size = 72;
3330         }
3331
3332         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3333                 s->ethtxq[i].q.size = 1024;
3334
3335         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3336                 s->ctrlq[i].q.size = 512;
3337
3338         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3339                 s->ofldtxq[i].q.size = 1024;
3340
3341         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3342                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3343
3344                 init_rspq(&r->rspq, 0, 0, 1024, 64);
3345                 r->rspq.uld = CXGB4_ULD_ISCSI;
3346                 r->fl.size = 72;
3347         }
3348
3349         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3350                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3351
3352                 init_rspq(&r->rspq, 0, 0, 511, 64);
3353                 r->rspq.uld = CXGB4_ULD_RDMA;
3354                 r->fl.size = 72;
3355         }
3356
3357         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3358         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3359 }
3360
3361 /*
3362  * Reduce the number of Ethernet queues across all ports to at most n.
3363  * n provides at least one queue per port.
3364  */
3365 static void __devinit reduce_ethqs(struct adapter *adap, int n)
3366 {
3367         int i;
3368         struct port_info *pi;
3369
3370         while (n < adap->sge.ethqsets)
3371                 for_each_port(adap, i) {
3372                         pi = adap2pinfo(adap, i);
3373                         if (pi->nqsets > 1) {
3374                                 pi->nqsets--;
3375                                 adap->sge.ethqsets--;
3376                                 if (adap->sge.ethqsets <= n)
3377                                         break;
3378                         }
3379                 }
3380
3381         n = 0;
3382         for_each_port(adap, i) {
3383                 pi = adap2pinfo(adap, i);
3384                 pi->first_qset = n;
3385                 n += pi->nqsets;
3386         }
3387 }
3388
3389 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3390 #define EXTRA_VECS 2
3391
3392 static int __devinit enable_msix(struct adapter *adap)
3393 {
3394         int ofld_need = 0;
3395         int i, err, want, need;
3396         struct sge *s = &adap->sge;
3397         unsigned int nchan = adap->params.nports;
3398         struct msix_entry entries[MAX_INGQ + 1];
3399
3400         for (i = 0; i < ARRAY_SIZE(entries); ++i)
3401                 entries[i].entry = i;
3402
3403         want = s->max_ethqsets + EXTRA_VECS;
3404         if (is_offload(adap)) {
3405                 want += s->rdmaqs + s->ofldqsets;
3406                 /* need nchan for each possible ULD */
3407                 ofld_need = 2 * nchan;
3408         }
3409         need = adap->params.nports + EXTRA_VECS + ofld_need;
3410
3411         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3412                 want = err;
3413
3414         if (!err) {
3415                 /*
3416                  * Distribute available vectors to the various queue groups.
3417                  * Every group gets its minimum requirement and NIC gets top
3418                  * priority for leftovers.
3419                  */
3420                 i = want - EXTRA_VECS - ofld_need;
3421                 if (i < s->max_ethqsets) {
3422                         s->max_ethqsets = i;
3423                         if (i < s->ethqsets)
3424                                 reduce_ethqs(adap, i);
3425                 }
3426                 if (is_offload(adap)) {
3427                         i = want - EXTRA_VECS - s->max_ethqsets;
3428                         i -= ofld_need - nchan;
3429                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
3430                 }
3431                 for (i = 0; i < want; ++i)
3432                         adap->msix_info[i].vec = entries[i].vector;
3433         } else if (err > 0)
3434                 dev_info(adap->pdev_dev,
3435                          "only %d MSI-X vectors left, not using MSI-X\n", err);
3436         return err;
3437 }
3438
3439 #undef EXTRA_VECS
3440
3441 static int __devinit init_rss(struct adapter *adap)
3442 {
3443         unsigned int i, j;
3444
3445         for_each_port(adap, i) {
3446                 struct port_info *pi = adap2pinfo(adap, i);
3447
3448                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3449                 if (!pi->rss)
3450                         return -ENOMEM;
3451                 for (j = 0; j < pi->rss_size; j++)
3452                         pi->rss[j] = j % pi->nqsets;
3453         }
3454         return 0;
3455 }
3456
3457 static void __devinit print_port_info(const struct net_device *dev)
3458 {
3459         static const char *base[] = {
3460                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3461                 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
3462         };
3463
3464         char buf[80];
3465         char *bufp = buf;
3466         const char *spd = "";
3467         const struct port_info *pi = netdev_priv(dev);
3468         const struct adapter *adap = pi->adapter;
3469
3470         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3471                 spd = " 2.5 GT/s";
3472         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3473                 spd = " 5 GT/s";
3474
3475         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3476                 bufp += sprintf(bufp, "100/");
3477         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3478                 bufp += sprintf(bufp, "1000/");
3479         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3480                 bufp += sprintf(bufp, "10G/");
3481         if (bufp != buf)
3482                 --bufp;
3483         sprintf(bufp, "BASE-%s", base[pi->port_type]);
3484
3485         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3486                     adap->params.vpd.id, adap->params.rev, buf,
3487                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
3488                     (adap->flags & USING_MSIX) ? " MSI-X" :
3489                     (adap->flags & USING_MSI) ? " MSI" : "");
3490         netdev_info(dev, "S/N: %s, E/C: %s\n",
3491                     adap->params.vpd.sn, adap->params.vpd.ec);
3492 }
3493
3494 static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
3495 {
3496         u16 v;
3497         int pos;
3498
3499         pos = pci_pcie_cap(dev);
3500         if (pos > 0) {
3501                 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
3502                 v |= PCI_EXP_DEVCTL_RELAX_EN;
3503                 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
3504         }
3505 }
3506
3507 /*
3508  * Free the following resources:
3509  * - memory used for tables
3510  * - MSI/MSI-X
3511  * - net devices
3512  * - resources FW is holding for us
3513  */
3514 static void free_some_resources(struct adapter *adapter)
3515 {
3516         unsigned int i;
3517
3518         t4_free_mem(adapter->l2t);
3519         t4_free_mem(adapter->tids.tid_tab);
3520         disable_msi(adapter);
3521
3522         for_each_port(adapter, i)
3523                 if (adapter->port[i]) {
3524                         kfree(adap2pinfo(adapter, i)->rss);
3525                         free_netdev(adapter->port[i]);
3526                 }
3527         if (adapter->flags & FW_OK)
3528                 t4_fw_bye(adapter, adapter->fn);
3529 }
3530
3531 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3532 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3533                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3534
3535 static int __devinit init_one(struct pci_dev *pdev,
3536                               const struct pci_device_id *ent)
3537 {
3538         int func, i, err;
3539         struct port_info *pi;
3540         unsigned int highdma = 0;
3541         struct adapter *adapter = NULL;
3542
3543         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3544
3545         err = pci_request_regions(pdev, KBUILD_MODNAME);
3546         if (err) {
3547                 /* Just info, some other driver may have claimed the device. */
3548                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3549                 return err;
3550         }
3551
3552         /* We control everything through one PF */
3553         func = PCI_FUNC(pdev->devfn);
3554         if (func != ent->driver_data) {
3555                 pci_save_state(pdev);        /* to restore SR-IOV later */
3556                 goto sriov;
3557         }
3558
3559         err = pci_enable_device(pdev);
3560         if (err) {
3561                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3562                 goto out_release_regions;
3563         }
3564
3565         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3566                 highdma = NETIF_F_HIGHDMA;
3567                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3568                 if (err) {
3569                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3570                                 "coherent allocations\n");
3571                         goto out_disable_device;
3572                 }
3573         } else {
3574                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3575                 if (err) {
3576                         dev_err(&pdev->dev, "no usable DMA configuration\n");
3577                         goto out_disable_device;
3578                 }
3579         }
3580
3581         pci_enable_pcie_error_reporting(pdev);
3582         enable_pcie_relaxed_ordering(pdev);
3583         pci_set_master(pdev);
3584         pci_save_state(pdev);
3585
3586         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3587         if (!adapter) {
3588                 err = -ENOMEM;
3589                 goto out_disable_device;
3590         }
3591
3592         adapter->regs = pci_ioremap_bar(pdev, 0);
3593         if (!adapter->regs) {
3594                 dev_err(&pdev->dev, "cannot map device registers\n");
3595                 err = -ENOMEM;
3596                 goto out_free_adapter;
3597         }
3598
3599         adapter->pdev = pdev;
3600         adapter->pdev_dev = &pdev->dev;
3601         adapter->fn = func;
3602         adapter->msg_enable = dflt_msg_enable;
3603         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3604
3605         spin_lock_init(&adapter->stats_lock);
3606         spin_lock_init(&adapter->tid_release_lock);
3607
3608         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3609
3610         err = t4_prep_adapter(adapter);
3611         if (err)
3612                 goto out_unmap_bar;
3613         err = adap_init0(adapter);
3614         if (err)
3615                 goto out_unmap_bar;
3616
3617         for_each_port(adapter, i) {
3618                 struct net_device *netdev;
3619
3620                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3621                                            MAX_ETH_QSETS);
3622                 if (!netdev) {
3623                         err = -ENOMEM;
3624                         goto out_free_dev;
3625                 }
3626
3627                 SET_NETDEV_DEV(netdev, &pdev->dev);
3628
3629                 adapter->port[i] = netdev;
3630                 pi = netdev_priv(netdev);
3631                 pi->adapter = adapter;
3632                 pi->xact_addr_filt = -1;
3633                 pi->port_id = i;
3634                 netdev->irq = pdev->irq;
3635
3636                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3637                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3638                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
3639                         NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3640                 netdev->features |= netdev->hw_features | highdma;
3641                 netdev->vlan_features = netdev->features & VLAN_FEAT;
3642
3643                 netdev->priv_flags |= IFF_UNICAST_FLT;
3644
3645                 netdev->netdev_ops = &cxgb4_netdev_ops;
3646                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3647         }
3648
3649         pci_set_drvdata(pdev, adapter);
3650
3651         if (adapter->flags & FW_OK) {
3652                 err = t4_port_init(adapter, func, func, 0);
3653                 if (err)
3654                         goto out_free_dev;
3655         }
3656
3657         /*
3658          * Configure queues and allocate tables now, they can be needed as
3659          * soon as the first register_netdev completes.
3660          */
3661         cfg_queues(adapter);
3662
3663         adapter->l2t = t4_init_l2t();
3664         if (!adapter->l2t) {
3665                 /* We tolerate a lack of L2T, giving up some functionality */
3666                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3667                 adapter->params.offload = 0;
3668         }
3669
3670         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3671                 dev_warn(&pdev->dev, "could not allocate TID table, "
3672                          "continuing\n");
3673                 adapter->params.offload = 0;
3674         }
3675
3676         /* See what interrupts we'll be using */
3677         if (msi > 1 && enable_msix(adapter) == 0)
3678                 adapter->flags |= USING_MSIX;
3679         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3680                 adapter->flags |= USING_MSI;
3681
3682         err = init_rss(adapter);
3683         if (err)
3684                 goto out_free_dev;
3685
3686         /*
3687          * The card is now ready to go.  If any errors occur during device
3688          * registration we do not fail the whole card but rather proceed only
3689          * with the ports we manage to register successfully.  However we must
3690          * register at least one net device.
3691          */
3692         for_each_port(adapter, i) {
3693                 pi = adap2pinfo(adapter, i);
3694                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
3695                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
3696
3697                 err = register_netdev(adapter->port[i]);
3698                 if (err)
3699                         break;
3700                 adapter->chan_map[pi->tx_chan] = i;
3701                 print_port_info(adapter->port[i]);
3702         }
3703         if (i == 0) {
3704                 dev_err(&pdev->dev, "could not register any net devices\n");
3705                 goto out_free_dev;
3706         }
3707         if (err) {
3708                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
3709                 err = 0;
3710         }
3711
3712         if (cxgb4_debugfs_root) {
3713                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3714                                                            cxgb4_debugfs_root);
3715                 setup_debugfs(adapter);
3716         }
3717
3718         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
3719         pdev->needs_freset = 1;
3720
3721         if (is_offload(adapter))
3722                 attach_ulds(adapter);
3723
3724 sriov:
3725 #ifdef CONFIG_PCI_IOV
3726         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3727                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3728                         dev_info(&pdev->dev,
3729                                  "instantiated %u virtual functions\n",
3730                                  num_vf[func]);
3731 #endif
3732         return 0;
3733
3734  out_free_dev:
3735         free_some_resources(adapter);
3736  out_unmap_bar:
3737         iounmap(adapter->regs);
3738  out_free_adapter:
3739         kfree(adapter);
3740  out_disable_device:
3741         pci_disable_pcie_error_reporting(pdev);
3742         pci_disable_device(pdev);
3743  out_release_regions:
3744         pci_release_regions(pdev);
3745         pci_set_drvdata(pdev, NULL);
3746         return err;
3747 }
3748
3749 static void __devexit remove_one(struct pci_dev *pdev)
3750 {
3751         struct adapter *adapter = pci_get_drvdata(pdev);
3752
3753         pci_disable_sriov(pdev);
3754
3755         if (adapter) {
3756                 int i;
3757
3758                 if (is_offload(adapter))
3759                         detach_ulds(adapter);
3760
3761                 for_each_port(adapter, i)
3762                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
3763                                 unregister_netdev(adapter->port[i]);
3764
3765                 if (adapter->debugfs_root)
3766                         debugfs_remove_recursive(adapter->debugfs_root);
3767
3768                 if (adapter->flags & FULL_INIT_DONE)
3769                         cxgb_down(adapter);
3770
3771                 free_some_resources(adapter);
3772                 iounmap(adapter->regs);
3773                 kfree(adapter);
3774                 pci_disable_pcie_error_reporting(pdev);
3775                 pci_disable_device(pdev);
3776                 pci_release_regions(pdev);
3777                 pci_set_drvdata(pdev, NULL);
3778         } else
3779                 pci_release_regions(pdev);
3780 }
3781
3782 static struct pci_driver cxgb4_driver = {
3783         .name     = KBUILD_MODNAME,
3784         .id_table = cxgb4_pci_tbl,
3785         .probe    = init_one,
3786         .remove   = __devexit_p(remove_one),
3787         .err_handler = &cxgb4_eeh,
3788 };
3789
3790 static int __init cxgb4_init_module(void)
3791 {
3792         int ret;
3793
3794         /* Debugfs support is optional, just warn if this fails */
3795         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3796         if (!cxgb4_debugfs_root)
3797                 pr_warning("could not create debugfs entry, continuing\n");
3798
3799         ret = pci_register_driver(&cxgb4_driver);
3800         if (ret < 0)
3801                 debugfs_remove(cxgb4_debugfs_root);
3802         return ret;
3803 }
3804
3805 static void __exit cxgb4_cleanup_module(void)
3806 {
3807         pci_unregister_driver(&cxgb4_driver);
3808         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
3809 }
3810
3811 module_init(cxgb4_init_module);
3812 module_exit(cxgb4_cleanup_module);