drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 /**
 212  *      t3_os_phymod_changed - handle PHY module changes
 213  *      @phy: the PHY reporting the module change
 214  *      @mod_type: new module type
 215  *
 216  *      This is the OS-dependent handler for PHY module changes.  It is
 217  *      invoked when a PHY module is removed or inserted for any OS-specific
 218  *      processing.
 219  */
 220 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 221 {
 222         static const char *mod_str[] = {
 223                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 224         };
 225
 226         const struct net_device *dev = adap->port[port_id];
 227         const struct port_info *pi = netdev_priv(dev);
 228
 229         if (pi->phy.modtype == phy_modtype_none)
 230                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 231         else
 232                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 233                        mod_str[pi->phy.modtype]);
 234 }
 235
 236 static void cxgb_set_rxmode(struct net_device *dev)
 237 {
 238         struct t3_rx_mode rm;
 239         struct port_info *pi = netdev_priv(dev);
 240
 241         init_rx_mode(&rm, dev, dev->mc_list);
 242         t3_mac_set_rx_mode(&pi->mac, &rm);
 243 }
 244
 245 /**
 246  *      link_start - enable a port
 247  *      @dev: the device to enable
 248  *
 249  *      Performs the MAC and PHY actions needed to enable a port.
 250  */
 251 static void link_start(struct net_device *dev)
 252 {
 253         struct t3_rx_mode rm;
 254         struct port_info *pi = netdev_priv(dev);
 255         struct cmac *mac = &pi->mac;
 256
 257         init_rx_mode(&rm, dev, dev->mc_list);
 258         t3_mac_reset(mac);
 259         t3_mac_set_mtu(mac, dev->mtu);
 260         t3_mac_set_address(mac, 0, dev->dev_addr);
 261         t3_mac_set_rx_mode(mac, &rm);
 262         t3_link_start(&pi->phy, mac, &pi->link_config);
 263         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 264 }
 265
 266 static inline void cxgb_disable_msi(struct adapter *adapter)
 267 {
 268         if (adapter->flags & USING_MSIX) {
 269                 pci_disable_msix(adapter->pdev);
 270                 adapter->flags &= ~USING_MSIX;
 271         } else if (adapter->flags & USING_MSI) {
 272                 pci_disable_msi(adapter->pdev);
 273                 adapter->flags &= ~USING_MSI;
 274         }
 275 }
 276
 277 /*
 278  * Interrupt handler for asynchronous events used with MSI-X.
 279  */
 280 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 281 {
 282         t3_slow_intr_handler(cookie);
 283         return IRQ_HANDLED;
 284 }
 285
 286 /*
 287  * Name the MSI-X interrupts.
 288  */
 289 static void name_msix_vecs(struct adapter *adap)
 290 {
 291         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 292
 293         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 294         adap->msix_info[0].desc[n] = 0;
 295
 296         for_each_port(adap, j) {
 297                 struct net_device *d = adap->port[j];
 298                 const struct port_info *pi = netdev_priv(d);
 299
 300                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 301                         snprintf(adap->msix_info[msi_idx].desc, n,
 302                                  "%s-%d", d->name, pi->first_qset + i);
 303                         adap->msix_info[msi_idx].desc[n] = 0;
 304                 }
 305         }
 306 }
 307
 308 static int request_msix_data_irqs(struct adapter *adap)
 309 {
 310         int i, j, err, qidx = 0;
 311
 312         for_each_port(adap, i) {
 313                 int nqsets = adap2pinfo(adap, i)->nqsets;
 314
 315                 for (j = 0; j < nqsets; ++j) {
 316                         err = request_irq(adap->msix_info[qidx + 1].vec,
 317                                           t3_intr_handler(adap,
 318                                                           adap->sge.qs[qidx].
 319                                                           rspq.polling), 0,
 320                                           adap->msix_info[qidx + 1].desc,
 321                                           &adap->sge.qs[qidx]);
 322                         if (err) {
 323                                 while (--qidx >= 0)
 324                                         free_irq(adap->msix_info[qidx + 1].vec,
 325                                                  &adap->sge.qs[qidx]);
 326                                 return err;
 327                         }
 328                         qidx++;
 329                 }
 330         }
 331         return 0;
 332 }
 333
 334 static void free_irq_resources(struct adapter *adapter)
 335 {
 336         if (adapter->flags & USING_MSIX) {
 337                 int i, n = 0;
 338
 339                 free_irq(adapter->msix_info[0].vec, adapter);
 340                 for_each_port(adapter, i)
 341                     n += adap2pinfo(adapter, i)->nqsets;
 342
 343                 for (i = 0; i < n; ++i)
 344                         free_irq(adapter->msix_info[i + 1].vec,
 345                                  &adapter->sge.qs[i]);
 346         } else
 347                 free_irq(adapter->pdev->irq, adapter);
 348 }
 349
 350 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 351                               unsigned long n)
 352 {
 353         int attempts = 5;
 354
 355         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 356                 if (!--attempts)
 357                         return -ETIMEDOUT;
 358                 msleep(10);
 359         }
 360         return 0;
 361 }
 362
 363 static int init_tp_parity(struct adapter *adap)
 364 {
 365         int i;
 366         struct sk_buff *skb;
 367         struct cpl_set_tcb_field *greq;
 368         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 369
 370         t3_tp_set_offload_mode(adap, 1);
 371
 372         for (i = 0; i < 16; i++) {
 373                 struct cpl_smt_write_req *req;
 374
 375                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 376                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 377                 memset(req, 0, sizeof(*req));
 378                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 379                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 380                 req->iff = i;
 381                 t3_mgmt_tx(adap, skb);
 382         }
 383
 384         for (i = 0; i < 2048; i++) {
 385                 struct cpl_l2t_write_req *req;
 386
 387                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 388                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 389                 memset(req, 0, sizeof(*req));
 390                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 391                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 392                 req->params = htonl(V_L2T_W_IDX(i));
 393                 t3_mgmt_tx(adap, skb);
 394         }
 395
 396         for (i = 0; i < 2048; i++) {
 397                 struct cpl_rte_write_req *req;
 398
 399                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 400                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 401                 memset(req, 0, sizeof(*req));
 402                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 403                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 404                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 405                 t3_mgmt_tx(adap, skb);
 406         }
 407
 408         skb = alloc_skb(sizeof(*greq), GFP_KERNEL | __GFP_NOFAIL);
 409         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 410         memset(greq, 0, sizeof(*greq));
 411         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 412         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 413         greq->mask = cpu_to_be64(1);
 414         t3_mgmt_tx(adap, skb);
 415
 416         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 417         t3_tp_set_offload_mode(adap, 0);
 418         return i;
 419 }
 420
 421 /**
 422  *      setup_rss - configure RSS
 423  *      @adap: the adapter
 424  *
 425  *      Sets up RSS to distribute packets to multiple receive queues.  We
 426  *      configure the RSS CPU lookup table to distribute to the number of HW
 427  *      receive queues, and the response queue lookup table to narrow that
 428  *      down to the response queues actually configured for each port.
 429  *      We always configure the RSS mapping for two ports since the mapping
 430  *      table has plenty of entries.
 431  */
 432 static void setup_rss(struct adapter *adap)
 433 {
 434         int i;
 435         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 436         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 437         u8 cpus[SGE_QSETS + 1];
 438         u16 rspq_map[RSS_TABLE_SIZE];
 439
 440         for (i = 0; i < SGE_QSETS; ++i)
 441                 cpus[i] = i;
 442         cpus[SGE_QSETS] = 0xff; /* terminator */
 443
 444         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 445                 rspq_map[i] = i % nq0;
 446                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 447         }
 448
 449         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 450                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 451                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 452 }
 453
 454 static void init_napi(struct adapter *adap)
 455 {
 456         int i;
 457
 458         for (i = 0; i < SGE_QSETS; i++) {
 459                 struct sge_qset *qs = &adap->sge.qs[i];
 460
 461                 if (qs->adap)
 462                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 463                                        64);
 464         }
 465
 466         /*
 467          * netif_napi_add() can be called only once per napi_struct because it
 468          * adds each new napi_struct to a list.  Be careful not to call it a
 469          * second time, e.g., during EEH recovery, by making a note of it.
 470          */
 471         adap->flags |= NAPI_INIT;
 472 }
 473
 474 /*
 475  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 476  * both netdevices representing interfaces and the dummy ones for the extra
 477  * queues.
 478  */
 479 static void quiesce_rx(struct adapter *adap)
 480 {
 481         int i;
 482
 483         for (i = 0; i < SGE_QSETS; i++)
 484                 if (adap->sge.qs[i].adap)
 485                         napi_disable(&adap->sge.qs[i].napi);
 486 }
 487
 488 static void enable_all_napi(struct adapter *adap)
 489 {
 490         int i;
 491         for (i = 0; i < SGE_QSETS; i++)
 492                 if (adap->sge.qs[i].adap)
 493                         napi_enable(&adap->sge.qs[i].napi);
 494 }
 495
 496 /**
 497  *      set_qset_lro - Turn a queue set's LRO capability on and off
 498  *      @dev: the device the qset is attached to
 499  *      @qset_idx: the queue set index
 500  *      @val: the LRO switch
 501  *
 502  *      Sets LRO on or off for a particular queue set.
 503  *      the device's features flag is updated to reflect the LRO
 504  *      capability when all queues belonging to the device are
 505  *      in the same state.
 506  */
 507 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 508 {
 509         struct port_info *pi = netdev_priv(dev);
 510         struct adapter *adapter = pi->adapter;
 511         int i, lro_on = 1;
 512
 513         adapter->params.sge.qset[qset_idx].lro = !!val;
 514         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 515
 516         /* let ethtool report LRO on only if all queues are LRO enabled */
 517         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; ++i)
 518                 lro_on &= adapter->params.sge.qset[i].lro;
 519
 520         if (lro_on)
 521                 dev->features |= NETIF_F_LRO;
 522         else
 523                 dev->features &= ~NETIF_F_LRO;
 524 }
 525
 526 /**
 527  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 528  *      @adap: the adapter
 529  *
 530  *      Determines how many sets of SGE queues to use and initializes them.
 531  *      We support multiple queue sets per port if we have MSI-X, otherwise
 532  *      just one queue set per port.
 533  */
 534 static int setup_sge_qsets(struct adapter *adap)
 535 {
 536         int i, j, err, irq_idx = 0, qset_idx = 0;
 537         unsigned int ntxq = SGE_TXQ_PER_SET;
 538
 539         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 540                 irq_idx = -1;
 541
 542         for_each_port(adap, i) {
 543                 struct net_device *dev = adap->port[i];
 544                 struct port_info *pi = netdev_priv(dev);
 545
 546                 pi->qs = &adap->sge.qs[pi->first_qset];
 547                 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets;
 548                      ++j, ++qset_idx) {
 549                         set_qset_lro(dev, qset_idx, pi->rx_csum_offload);
 550                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 551                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 552                                                              irq_idx,
 553                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 554                                 netdev_get_tx_queue(dev, j));
 555                         if (err) {
 556                                 t3_stop_sge_timers(adap);
 557                                 t3_free_sge_resources(adap);
 558                                 return err;
 559                         }
 560                 }
 561         }
 562
 563         return 0;
 564 }
 565
 566 static ssize_t attr_show(struct device *d, char *buf,
 567                          ssize_t(*format) (struct net_device *, char *))
 568 {
 569         ssize_t len;
 570
 571         /* Synchronize with ioctls that may shut down the device */
 572         rtnl_lock();
 573         len = (*format) (to_net_dev(d), buf);
 574         rtnl_unlock();
 575         return len;
 576 }
 577
 578 static ssize_t attr_store(struct device *d,
 579                           const char *buf, size_t len,
 580                           ssize_t(*set) (struct net_device *, unsigned int),
 581                           unsigned int min_val, unsigned int max_val)
 582 {
 583         char *endp;
 584         ssize_t ret;
 585         unsigned int val;
 586
 587         if (!capable(CAP_NET_ADMIN))
 588                 return -EPERM;
 589
 590         val = simple_strtoul(buf, &endp, 0);
 591         if (endp == buf || val < min_val || val > max_val)
 592                 return -EINVAL;
 593
 594         rtnl_lock();
 595         ret = (*set) (to_net_dev(d), val);
 596         if (!ret)
 597                 ret = len;
 598         rtnl_unlock();
 599         return ret;
 600 }
 601
 602 #define CXGB3_SHOW(name, val_expr) \
 603 static ssize_t format_##name(struct net_device *dev, char *buf) \
 604 { \
 605         struct port_info *pi = netdev_priv(dev); \
 606         struct adapter *adap = pi->adapter; \
 607         return sprintf(buf, "%u\n", val_expr); \
 608 } \
 609 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 610                            char *buf) \
 611 { \
 612         return attr_show(d, buf, format_##name); \
 613 }
 614
 615 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 616 {
 617         struct port_info *pi = netdev_priv(dev);
 618         struct adapter *adap = pi->adapter;
 619         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 620
 621         if (adap->flags & FULL_INIT_DONE)
 622                 return -EBUSY;
 623         if (val && adap->params.rev == 0)
 624                 return -EINVAL;
 625         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 626             min_tids)
 627                 return -EINVAL;
 628         adap->params.mc5.nfilters = val;
 629         return 0;
 630 }
 631
 632 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 633                               const char *buf, size_t len)
 634 {
 635         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 636 }
 637
 638 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 639 {
 640         struct port_info *pi = netdev_priv(dev);
 641         struct adapter *adap = pi->adapter;
 642
 643         if (adap->flags & FULL_INIT_DONE)
 644                 return -EBUSY;
 645         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 646             MC5_MIN_TIDS)
 647                 return -EINVAL;
 648         adap->params.mc5.nservers = val;
 649         return 0;
 650 }
 651
 652 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 653                               const char *buf, size_t len)
 654 {
 655         return attr_store(d, buf, len, set_nservers, 0, ~0);
 656 }
 657
 658 #define CXGB3_ATTR_R(name, val_expr) \
 659 CXGB3_SHOW(name, val_expr) \
 660 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 661
 662 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 663 CXGB3_SHOW(name, val_expr) \
 664 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 665
 666 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 667 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 668 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 669
 670 static struct attribute *cxgb3_attrs[] = {
 671         &dev_attr_cam_size.attr,
 672         &dev_attr_nfilters.attr,
 673         &dev_attr_nservers.attr,
 674         NULL
 675 };
 676
 677 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 678
 679 static ssize_t tm_attr_show(struct device *d,
 680                             char *buf, int sched)
 681 {
 682         struct port_info *pi = netdev_priv(to_net_dev(d));
 683         struct adapter *adap = pi->adapter;
 684         unsigned int v, addr, bpt, cpt;
 685         ssize_t len;
 686
 687         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 688         rtnl_lock();
 689         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 690         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 691         if (sched & 1)
 692                 v >>= 16;
 693         bpt = (v >> 8) & 0xff;
 694         cpt = v & 0xff;
 695         if (!cpt)
 696                 len = sprintf(buf, "disabled\n");
 697         else {
 698                 v = (adap->params.vpd.cclk * 1000) / cpt;
 699                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 700         }
 701         rtnl_unlock();
 702         return len;
 703 }
 704
 705 static ssize_t tm_attr_store(struct device *d,
 706                              const char *buf, size_t len, int sched)
 707 {
 708         struct port_info *pi = netdev_priv(to_net_dev(d));
 709         struct adapter *adap = pi->adapter;
 710         unsigned int val;
 711         char *endp;
 712         ssize_t ret;
 713
 714         if (!capable(CAP_NET_ADMIN))
 715                 return -EPERM;
 716
 717         val = simple_strtoul(buf, &endp, 0);
 718         if (endp == buf || val > 10000000)
 719                 return -EINVAL;
 720
 721         rtnl_lock();
 722         ret = t3_config_sched(adap, val, sched);
 723         if (!ret)
 724                 ret = len;
 725         rtnl_unlock();
 726         return ret;
 727 }
 728
 729 #define TM_ATTR(name, sched) \
 730 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 731                            char *buf) \
 732 { \
 733         return tm_attr_show(d, buf, sched); \
 734 } \
 735 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 736                             const char *buf, size_t len) \
 737 { \
 738         return tm_attr_store(d, buf, len, sched); \
 739 } \
 740 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 741
 742 TM_ATTR(sched0, 0);
 743 TM_ATTR(sched1, 1);
 744 TM_ATTR(sched2, 2);
 745 TM_ATTR(sched3, 3);
 746 TM_ATTR(sched4, 4);
 747 TM_ATTR(sched5, 5);
 748 TM_ATTR(sched6, 6);
 749 TM_ATTR(sched7, 7);
 750
 751 static struct attribute *offload_attrs[] = {
 752         &dev_attr_sched0.attr,
 753         &dev_attr_sched1.attr,
 754         &dev_attr_sched2.attr,
 755         &dev_attr_sched3.attr,
 756         &dev_attr_sched4.attr,
 757         &dev_attr_sched5.attr,
 758         &dev_attr_sched6.attr,
 759         &dev_attr_sched7.attr,
 760         NULL
 761 };
 762
 763 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 764
 765 /*
 766  * Sends an sk_buff to an offload queue driver
 767  * after dealing with any active network taps.
 768  */
 769 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 770 {
 771         int ret;
 772
 773         local_bh_disable();
 774         ret = t3_offload_tx(tdev, skb);
 775         local_bh_enable();
 776         return ret;
 777 }
 778
 779 static int write_smt_entry(struct adapter *adapter, int idx)
 780 {
 781         struct cpl_smt_write_req *req;
 782         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 783
 784         if (!skb)
 785                 return -ENOMEM;
 786
 787         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 788         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 789         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 790         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 791         req->iff = idx;
 792         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 793         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 794         skb->priority = 1;
 795         offload_tx(&adapter->tdev, skb);
 796         return 0;
 797 }
 798
 799 static int init_smt(struct adapter *adapter)
 800 {
 801         int i;
 802
 803         for_each_port(adapter, i)
 804             write_smt_entry(adapter, i);
 805         return 0;
 806 }
 807
 808 static void init_port_mtus(struct adapter *adapter)
 809 {
 810         unsigned int mtus = adapter->port[0]->mtu;
 811
 812         if (adapter->port[1])
 813                 mtus |= adapter->port[1]->mtu << 16;
 814         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 815 }
 816
 817 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 818                               int hi, int port)
 819 {
 820         struct sk_buff *skb;
 821         struct mngt_pktsched_wr *req;
 822         int ret;
 823
 824         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 825         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 826         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 827         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 828         req->sched = sched;
 829         req->idx = qidx;
 830         req->min = lo;
 831         req->max = hi;
 832         req->binding = port;
 833         ret = t3_mgmt_tx(adap, skb);
 834
 835         return ret;
 836 }
 837
 838 static int bind_qsets(struct adapter *adap)
 839 {
 840         int i, j, err = 0;
 841
 842         for_each_port(adap, i) {
 843                 const struct port_info *pi = adap2pinfo(adap, i);
 844
 845                 for (j = 0; j < pi->nqsets; ++j) {
 846                         int ret = send_pktsched_cmd(adap, 1,
 847                                                     pi->first_qset + j, -1,
 848                                                     -1, i);
 849                         if (ret)
 850                                 err = ret;
 851                 }
 852         }
 853
 854         return err;
 855 }
 856
 857 #define FW_FNAME "cxgb3/t3fw-%d.%d.%d.bin"
 858 #define TPSRAM_NAME "cxgb3/t3%c_psram-%d.%d.%d.bin"
 859
 860 static int upgrade_fw(struct adapter *adap)
 861 {
 862         int ret;
 863         char buf[64];
 864         const struct firmware *fw;
 865         struct device *dev = &adap->pdev->dev;
 866
 867         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 868                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 869         ret = request_firmware(&fw, buf, dev);
 870         if (ret < 0) {
 871                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 872                         buf);
 873                 return ret;
 874         }
 875         ret = t3_load_fw(adap, fw->data, fw->size);
 876         release_firmware(fw);
 877
 878         if (ret == 0)
 879                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 880                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 881         else
 882                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 883                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 884
 885         return ret;
 886 }
 887
 888 static inline char t3rev2char(struct adapter *adapter)
 889 {
 890         char rev = 0;
 891
 892         switch(adapter->params.rev) {
 893         case T3_REV_B:
 894         case T3_REV_B2:
 895                 rev = 'b';
 896                 break;
 897         case T3_REV_C:
 898                 rev = 'c';
 899                 break;
 900         }
 901         return rev;
 902 }
 903
 904 static int update_tpsram(struct adapter *adap)
 905 {
 906         const struct firmware *tpsram;
 907         char buf[64];
 908         struct device *dev = &adap->pdev->dev;
 909         int ret;
 910         char rev;
 911
 912         rev = t3rev2char(adap);
 913         if (!rev)
 914                 return 0;
 915
 916         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 917                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 918
 919         ret = request_firmware(&tpsram, buf, dev);
 920         if (ret < 0) {
 921                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 922                         buf);
 923                 return ret;
 924         }
 925
 926         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 927         if (ret)
 928                 goto release_tpsram;
 929
 930         ret = t3_set_proto_sram(adap, tpsram->data);
 931         if (ret == 0)
 932                 dev_info(dev,
 933                          "successful update of protocol engine "
 934                          "to %d.%d.%d\n",
 935                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 936         else
 937                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 938                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 939         if (ret)
 940                 dev_err(dev, "loading protocol SRAM failed\n");
 941
 942 release_tpsram:
 943         release_firmware(tpsram);
 944
 945         return ret;
 946 }
 947
 948 /**
 949  *      cxgb_up - enable the adapter
 950  *      @adapter: adapter being enabled
 951  *
 952  *      Called when the first port is enabled, this function performs the
 953  *      actions necessary to make an adapter operational, such as completing
 954  *      the initialization of HW modules, and enabling interrupts.
 955  *
 956  *      Must be called with the rtnl lock held.
 957  */
 958 static int cxgb_up(struct adapter *adap)
 959 {
 960         int err;
 961         int must_load;
 962
 963         if (!(adap->flags & FULL_INIT_DONE)) {
 964                 err = t3_check_fw_version(adap, &must_load);
 965                 if (err == -EINVAL) {
 966                         err = upgrade_fw(adap);
 967                         if (err && must_load)
 968                                 goto out;
 969                 }
 970
 971                 err = t3_check_tpsram_version(adap, &must_load);
 972                 if (err == -EINVAL) {
 973                         err = update_tpsram(adap);
 974                         if (err && must_load)
 975                                 goto out;
 976                 }
 977
 978                 /*
 979                  * Clear interrupts now to catch errors if t3_init_hw fails.
 980                  * We clear them again later as initialization may trigger
 981                  * conditions that can interrupt.
 982                  */
 983                 t3_intr_clear(adap);
 984
 985                 err = t3_init_hw(adap, 0);
 986                 if (err)
 987                         goto out;
 988
 989                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 990                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 991
 992                 err = setup_sge_qsets(adap);
 993                 if (err)
 994                         goto out;
 995
 996                 setup_rss(adap);
 997                 if (!(adap->flags & NAPI_INIT))
 998                         init_napi(adap);
 999                 adap->flags |= FULL_INIT_DONE;
1000         }
1001
1002         t3_intr_clear(adap);
1003
1004         if (adap->flags & USING_MSIX) {
1005                 name_msix_vecs(adap);
1006                 err = request_irq(adap->msix_info[0].vec,
1007                                   t3_async_intr_handler, 0,
1008                                   adap->msix_info[0].desc, adap);
1009                 if (err)
1010                         goto irq_err;
1011
1012                 err = request_msix_data_irqs(adap);
1013                 if (err) {
1014                         free_irq(adap->msix_info[0].vec, adap);
1015                         goto irq_err;
1016                 }
1017         } else if ((err = request_irq(adap->pdev->irq,
1018                                       t3_intr_handler(adap,
1019                                                       adap->sge.qs[0].rspq.
1020                                                       polling),
1021                                       (adap->flags & USING_MSI) ?
1022                                        0 : IRQF_SHARED,
1023                                       adap->name, adap)))
1024                 goto irq_err;
1025
1026         enable_all_napi(adap);
1027         t3_sge_start(adap);
1028         t3_intr_enable(adap);
1029
1030         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1031             is_offload(adap) && init_tp_parity(adap) == 0)
1032                 adap->flags |= TP_PARITY_INIT;
1033
1034         if (adap->flags & TP_PARITY_INIT) {
1035                 t3_write_reg(adap, A_TP_INT_CAUSE,
1036                              F_CMCACHEPERR | F_ARPLUTPERR);
1037                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1038         }
1039
1040         if (!(adap->flags & QUEUES_BOUND)) {
1041                 err = bind_qsets(adap);
1042                 if (err) {
1043                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1044                         t3_intr_disable(adap);
1045                         free_irq_resources(adap);
1046                         goto out;
1047                 }
1048                 adap->flags |= QUEUES_BOUND;
1049         }
1050
1051 out:
1052         return err;
1053 irq_err:
1054         CH_ERR(adap, "request_irq failed, err %d\n", err);
1055         goto out;
1056 }
1057
1058 /*
1059  * Release resources when all the ports and offloading have been stopped.
1060  */
1061 static void cxgb_down(struct adapter *adapter)
1062 {
1063         t3_sge_stop(adapter);
1064         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1065         t3_intr_disable(adapter);
1066         spin_unlock_irq(&adapter->work_lock);
1067
1068         free_irq_resources(adapter);
1069         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1070         quiesce_rx(adapter);
1071 }
1072
1073 static void schedule_chk_task(struct adapter *adap)
1074 {
1075         unsigned int timeo;
1076
1077         timeo = adap->params.linkpoll_period ?
1078             (HZ * adap->params.linkpoll_period) / 10 :
1079             adap->params.stats_update_period * HZ;
1080         if (timeo)
1081                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1082 }
1083
1084 static int offload_open(struct net_device *dev)
1085 {
1086         struct port_info *pi = netdev_priv(dev);
1087         struct adapter *adapter = pi->adapter;
1088         struct t3cdev *tdev = dev2t3cdev(dev);
1089         int adap_up = adapter->open_device_map & PORT_MASK;
1090         int err;
1091
1092         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1093                 return 0;
1094
1095         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1096                 goto out;
1097
1098         t3_tp_set_offload_mode(adapter, 1);
1099         tdev->lldev = adapter->port[0];
1100         err = cxgb3_offload_activate(adapter);
1101         if (err)
1102                 goto out;
1103
1104         init_port_mtus(adapter);
1105         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1106                      adapter->params.b_wnd,
1107                      adapter->params.rev == 0 ?
1108                      adapter->port[0]->mtu : 0xffff);
1109         init_smt(adapter);
1110
1111         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1112                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1113
1114         /* Call back all registered clients */
1115         cxgb3_add_clients(tdev);
1116
1117 out:
1118         /* restore them in case the offload module has changed them */
1119         if (err) {
1120                 t3_tp_set_offload_mode(adapter, 0);
1121                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1122                 cxgb3_set_dummy_ops(tdev);
1123         }
1124         return err;
1125 }
1126
1127 static int offload_close(struct t3cdev *tdev)
1128 {
1129         struct adapter *adapter = tdev2adap(tdev);
1130
1131         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1132                 return 0;
1133
1134         /* Call back all registered clients */
1135         cxgb3_remove_clients(tdev);
1136
1137         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1138
1139         tdev->lldev = NULL;
1140         cxgb3_set_dummy_ops(tdev);
1141         t3_tp_set_offload_mode(adapter, 0);
1142         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1143
1144         if (!adapter->open_device_map)
1145                 cxgb_down(adapter);
1146
1147         cxgb3_offload_deactivate(adapter);
1148         return 0;
1149 }
1150
1151 static int cxgb_open(struct net_device *dev)
1152 {
1153         struct port_info *pi = netdev_priv(dev);
1154         struct adapter *adapter = pi->adapter;
1155         int other_ports = adapter->open_device_map & PORT_MASK;
1156         int err;
1157
1158         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1159                 return err;
1160
1161         set_bit(pi->port_id, &adapter->open_device_map);
1162         if (is_offload(adapter) && !ofld_disable) {
1163                 err = offload_open(dev);
1164                 if (err)
1165                         printk(KERN_WARNING
1166                                "Could not initialize offload capabilities\n");
1167         }
1168
1169         dev->real_num_tx_queues = pi->nqsets;
1170         link_start(dev);
1171         t3_port_intr_enable(adapter, pi->port_id);
1172         netif_tx_start_all_queues(dev);
1173         if (!other_ports)
1174                 schedule_chk_task(adapter);
1175
1176         return 0;
1177 }
1178
1179 static int cxgb_close(struct net_device *dev)
1180 {
1181         struct port_info *pi = netdev_priv(dev);
1182         struct adapter *adapter = pi->adapter;
1183
1184         t3_port_intr_disable(adapter, pi->port_id);
1185         netif_tx_stop_all_queues(dev);
1186         pi->phy.ops->power_down(&pi->phy, 1);
1187         netif_carrier_off(dev);
1188         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1189
1190         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1191         clear_bit(pi->port_id, &adapter->open_device_map);
1192         spin_unlock_irq(&adapter->work_lock);
1193
1194         if (!(adapter->open_device_map & PORT_MASK))
1195                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1196                                                   &adapter->adap_check_task);
1197
1198         if (!adapter->open_device_map)
1199                 cxgb_down(adapter);
1200
1201         return 0;
1202 }
1203
1204 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1205 {
1206         struct port_info *pi = netdev_priv(dev);
1207         struct adapter *adapter = pi->adapter;
1208         struct net_device_stats *ns = &pi->netstats;
1209         const struct mac_stats *pstats;
1210
1211         spin_lock(&adapter->stats_lock);
1212         pstats = t3_mac_update_stats(&pi->mac);
1213         spin_unlock(&adapter->stats_lock);
1214
1215         ns->tx_bytes = pstats->tx_octets;
1216         ns->tx_packets = pstats->tx_frames;
1217         ns->rx_bytes = pstats->rx_octets;
1218         ns->rx_packets = pstats->rx_frames;
1219         ns->multicast = pstats->rx_mcast_frames;
1220
1221         ns->tx_errors = pstats->tx_underrun;
1222         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1223             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1224             pstats->rx_fifo_ovfl;
1225
1226         /* detailed rx_errors */
1227         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1228         ns->rx_over_errors = 0;
1229         ns->rx_crc_errors = pstats->rx_fcs_errs;
1230         ns->rx_frame_errors = pstats->rx_symbol_errs;
1231         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1232         ns->rx_missed_errors = pstats->rx_cong_drops;
1233
1234         /* detailed tx_errors */
1235         ns->tx_aborted_errors = 0;
1236         ns->tx_carrier_errors = 0;
1237         ns->tx_fifo_errors = pstats->tx_underrun;
1238         ns->tx_heartbeat_errors = 0;
1239         ns->tx_window_errors = 0;
1240         return ns;
1241 }
1242
1243 static u32 get_msglevel(struct net_device *dev)
1244 {
1245         struct port_info *pi = netdev_priv(dev);
1246         struct adapter *adapter = pi->adapter;
1247
1248         return adapter->msg_enable;
1249 }
1250
1251 static void set_msglevel(struct net_device *dev, u32 val)
1252 {
1253         struct port_info *pi = netdev_priv(dev);
1254         struct adapter *adapter = pi->adapter;
1255
1256         adapter->msg_enable = val;
1257 }
1258
1259 static char stats_strings[][ETH_GSTRING_LEN] = {
1260         "TxOctetsOK         ",
1261         "TxFramesOK         ",
1262         "TxMulticastFramesOK",
1263         "TxBroadcastFramesOK",
1264         "TxPauseFrames      ",
1265         "TxUnderrun         ",
1266         "TxExtUnderrun      ",
1267
1268         "TxFrames64         ",
1269         "TxFrames65To127    ",
1270         "TxFrames128To255   ",
1271         "TxFrames256To511   ",
1272         "TxFrames512To1023  ",
1273         "TxFrames1024To1518 ",
1274         "TxFrames1519ToMax  ",
1275
1276         "RxOctetsOK         ",
1277         "RxFramesOK         ",
1278         "RxMulticastFramesOK",
1279         "RxBroadcastFramesOK",
1280         "RxPauseFrames      ",
1281         "RxFCSErrors        ",
1282         "RxSymbolErrors     ",
1283         "RxShortErrors      ",
1284         "RxJabberErrors     ",
1285         "RxLengthErrors     ",
1286         "RxFIFOoverflow     ",
1287
1288         "RxFrames64         ",
1289         "RxFrames65To127    ",
1290         "RxFrames128To255   ",
1291         "RxFrames256To511   ",
1292         "RxFrames512To1023  ",
1293         "RxFrames1024To1518 ",
1294         "RxFrames1519ToMax  ",
1295
1296         "PhyFIFOErrors      ",
1297         "TSO                ",
1298         "VLANextractions    ",
1299         "VLANinsertions     ",
1300         "TxCsumOffload      ",
1301         "RxCsumGood         ",
1302         "LroAggregated      ",
1303         "LroFlushed         ",
1304         "LroNoDesc          ",
1305         "RxDrops            ",
1306
1307         "CheckTXEnToggled   ",
1308         "CheckResets        ",
1309
1310 };
1311
1312 static int get_sset_count(struct net_device *dev, int sset)
1313 {
1314         switch (sset) {
1315         case ETH_SS_STATS:
1316                 return ARRAY_SIZE(stats_strings);
1317         default:
1318                 return -EOPNOTSUPP;
1319         }
1320 }
1321
1322 #define T3_REGMAP_SIZE (3 * 1024)
1323
1324 static int get_regs_len(struct net_device *dev)
1325 {
1326         return T3_REGMAP_SIZE;
1327 }
1328
1329 static int get_eeprom_len(struct net_device *dev)
1330 {
1331         return EEPROMSIZE;
1332 }
1333
1334 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1335 {
1336         struct port_info *pi = netdev_priv(dev);
1337         struct adapter *adapter = pi->adapter;
1338         u32 fw_vers = 0;
1339         u32 tp_vers = 0;
1340
1341         spin_lock(&adapter->stats_lock);
1342         t3_get_fw_version(adapter, &fw_vers);
1343         t3_get_tp_version(adapter, &tp_vers);
1344         spin_unlock(&adapter->stats_lock);
1345
1346         strcpy(info->driver, DRV_NAME);
1347         strcpy(info->version, DRV_VERSION);
1348         strcpy(info->bus_info, pci_name(adapter->pdev));
1349         if (!fw_vers)
1350                 strcpy(info->fw_version, "N/A");
1351         else {
1352                 snprintf(info->fw_version, sizeof(info->fw_version),
1353                          "%s %u.%u.%u TP %u.%u.%u",
1354                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1355                          G_FW_VERSION_MAJOR(fw_vers),
1356                          G_FW_VERSION_MINOR(fw_vers),
1357                          G_FW_VERSION_MICRO(fw_vers),
1358                          G_TP_VERSION_MAJOR(tp_vers),
1359                          G_TP_VERSION_MINOR(tp_vers),
1360                          G_TP_VERSION_MICRO(tp_vers));
1361         }
1362 }
1363
1364 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1365 {
1366         if (stringset == ETH_SS_STATS)
1367                 memcpy(data, stats_strings, sizeof(stats_strings));
1368 }
1369
1370 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1371                                             struct port_info *p, int idx)
1372 {
1373         int i;
1374         unsigned long tot = 0;
1375
1376         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1377                 tot += adapter->sge.qs[i].port_stats[idx];
1378         return tot;
1379 }
1380
1381 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1382                       u64 *data)
1383 {
1384         struct port_info *pi = netdev_priv(dev);
1385         struct adapter *adapter = pi->adapter;
1386         const struct mac_stats *s;
1387
1388         spin_lock(&adapter->stats_lock);
1389         s = t3_mac_update_stats(&pi->mac);
1390         spin_unlock(&adapter->stats_lock);
1391
1392         *data++ = s->tx_octets;
1393         *data++ = s->tx_frames;
1394         *data++ = s->tx_mcast_frames;
1395         *data++ = s->tx_bcast_frames;
1396         *data++ = s->tx_pause;
1397         *data++ = s->tx_underrun;
1398         *data++ = s->tx_fifo_urun;
1399
1400         *data++ = s->tx_frames_64;
1401         *data++ = s->tx_frames_65_127;
1402         *data++ = s->tx_frames_128_255;
1403         *data++ = s->tx_frames_256_511;
1404         *data++ = s->tx_frames_512_1023;
1405         *data++ = s->tx_frames_1024_1518;
1406         *data++ = s->tx_frames_1519_max;
1407
1408         *data++ = s->rx_octets;
1409         *data++ = s->rx_frames;
1410         *data++ = s->rx_mcast_frames;
1411         *data++ = s->rx_bcast_frames;
1412         *data++ = s->rx_pause;
1413         *data++ = s->rx_fcs_errs;
1414         *data++ = s->rx_symbol_errs;
1415         *data++ = s->rx_short;
1416         *data++ = s->rx_jabber;
1417         *data++ = s->rx_too_long;
1418         *data++ = s->rx_fifo_ovfl;
1419
1420         *data++ = s->rx_frames_64;
1421         *data++ = s->rx_frames_65_127;
1422         *data++ = s->rx_frames_128_255;
1423         *data++ = s->rx_frames_256_511;
1424         *data++ = s->rx_frames_512_1023;
1425         *data++ = s->rx_frames_1024_1518;
1426         *data++ = s->rx_frames_1519_max;
1427
1428         *data++ = pi->phy.fifo_errors;
1429
1430         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1431         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1432         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1433         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1434         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1435         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_AGGR);
1436         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_FLUSHED);
1437         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_NO_DESC);
1438         *data++ = s->rx_cong_drops;
1439
1440         *data++ = s->num_toggled;
1441         *data++ = s->num_resets;
1442 }
1443
1444 static inline void reg_block_dump(struct adapter *ap, void *buf,
1445                                   unsigned int start, unsigned int end)
1446 {
1447         u32 *p = buf + start;
1448
1449         for (; start <= end; start += sizeof(u32))
1450                 *p++ = t3_read_reg(ap, start);
1451 }
1452
1453 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1454                      void *buf)
1455 {
1456         struct port_info *pi = netdev_priv(dev);
1457         struct adapter *ap = pi->adapter;
1458
1459         /*
1460          * Version scheme:
1461          * bits 0..9: chip version
1462          * bits 10..15: chip revision
1463          * bit 31: set for PCIe cards
1464          */
1465         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1466
1467         /*
1468          * We skip the MAC statistics registers because they are clear-on-read.
1469          * Also reading multi-register stats would need to synchronize with the
1470          * periodic mac stats accumulation.  Hard to justify the complexity.
1471          */
1472         memset(buf, 0, T3_REGMAP_SIZE);
1473         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1474         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1475         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1476         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1477         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1478         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1479                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1480         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1481                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1482 }
1483
1484 static int restart_autoneg(struct net_device *dev)
1485 {
1486         struct port_info *p = netdev_priv(dev);
1487
1488         if (!netif_running(dev))
1489                 return -EAGAIN;
1490         if (p->link_config.autoneg != AUTONEG_ENABLE)
1491                 return -EINVAL;
1492         p->phy.ops->autoneg_restart(&p->phy);
1493         return 0;
1494 }
1495
1496 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1497 {
1498         struct port_info *pi = netdev_priv(dev);
1499         struct adapter *adapter = pi->adapter;
1500         int i;
1501
1502         if (data == 0)
1503                 data = 2;
1504
1505         for (i = 0; i < data * 2; i++) {
1506                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1507                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1508                 if (msleep_interruptible(500))
1509                         break;
1510         }
1511         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1512                          F_GPIO0_OUT_VAL);
1513         return 0;
1514 }
1515
1516 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1517 {
1518         struct port_info *p = netdev_priv(dev);
1519
1520         cmd->supported = p->link_config.supported;
1521         cmd->advertising = p->link_config.advertising;
1522
1523         if (netif_carrier_ok(dev)) {
1524                 cmd->speed = p->link_config.speed;
1525                 cmd->duplex = p->link_config.duplex;
1526         } else {
1527                 cmd->speed = -1;
1528                 cmd->duplex = -1;
1529         }
1530
1531         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1532         cmd->phy_address = p->phy.addr;
1533         cmd->transceiver = XCVR_EXTERNAL;
1534         cmd->autoneg = p->link_config.autoneg;
1535         cmd->maxtxpkt = 0;
1536         cmd->maxrxpkt = 0;
1537         return 0;
1538 }
1539
1540 static int speed_duplex_to_caps(int speed, int duplex)
1541 {
1542         int cap = 0;
1543
1544         switch (speed) {
1545         case SPEED_10:
1546                 if (duplex == DUPLEX_FULL)
1547                         cap = SUPPORTED_10baseT_Full;
1548                 else
1549                         cap = SUPPORTED_10baseT_Half;
1550                 break;
1551         case SPEED_100:
1552                 if (duplex == DUPLEX_FULL)
1553                         cap = SUPPORTED_100baseT_Full;
1554                 else
1555                         cap = SUPPORTED_100baseT_Half;
1556                 break;
1557         case SPEED_1000:
1558                 if (duplex == DUPLEX_FULL)
1559                         cap = SUPPORTED_1000baseT_Full;
1560                 else
1561                         cap = SUPPORTED_1000baseT_Half;
1562                 break;
1563         case SPEED_10000:
1564                 if (duplex == DUPLEX_FULL)
1565                         cap = SUPPORTED_10000baseT_Full;
1566         }
1567         return cap;
1568 }
1569
1570 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1571                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1572                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1573                       ADVERTISED_10000baseT_Full)
1574
1575 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1576 {
1577         int cap;
1578         struct port_info *p = netdev_priv(dev);
1579         struct link_config *lc = &p->link_config;
1580
1581         if (!(lc->supported & SUPPORTED_Autoneg)) {
1582                 /*
1583                  * PHY offers a single speed/duplex.  See if that's what's
1584                  * being requested.
1585                  */
1586                 if (cmd->autoneg == AUTONEG_DISABLE) {
1587                         cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1588                         if (lc->supported & cap)
1589                                 return 0;
1590                 }
1591                 return -EINVAL;
1592         }
1593
1594         if (cmd->autoneg == AUTONEG_DISABLE) {
1595                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1596
1597                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1598                         return -EINVAL;
1599                 lc->requested_speed = cmd->speed;
1600                 lc->requested_duplex = cmd->duplex;
1601                 lc->advertising = 0;
1602         } else {
1603                 cmd->advertising &= ADVERTISED_MASK;
1604                 cmd->advertising &= lc->supported;
1605                 if (!cmd->advertising)
1606                         return -EINVAL;
1607                 lc->requested_speed = SPEED_INVALID;
1608                 lc->requested_duplex = DUPLEX_INVALID;
1609                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1610         }
1611         lc->autoneg = cmd->autoneg;
1612         if (netif_running(dev))
1613                 t3_link_start(&p->phy, &p->mac, lc);
1614         return 0;
1615 }
1616
1617 static void get_pauseparam(struct net_device *dev,
1618                            struct ethtool_pauseparam *epause)
1619 {
1620         struct port_info *p = netdev_priv(dev);
1621
1622         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1623         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1624         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1625 }
1626
1627 static int set_pauseparam(struct net_device *dev,
1628                           struct ethtool_pauseparam *epause)
1629 {
1630         struct port_info *p = netdev_priv(dev);
1631         struct link_config *lc = &p->link_config;
1632
1633         if (epause->autoneg == AUTONEG_DISABLE)
1634                 lc->requested_fc = 0;
1635         else if (lc->supported & SUPPORTED_Autoneg)
1636                 lc->requested_fc = PAUSE_AUTONEG;
1637         else
1638                 return -EINVAL;
1639
1640         if (epause->rx_pause)
1641                 lc->requested_fc |= PAUSE_RX;
1642         if (epause->tx_pause)
1643                 lc->requested_fc |= PAUSE_TX;
1644         if (lc->autoneg == AUTONEG_ENABLE) {
1645                 if (netif_running(dev))
1646                         t3_link_start(&p->phy, &p->mac, lc);
1647         } else {
1648                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1649                 if (netif_running(dev))
1650                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1651         }
1652         return 0;
1653 }
1654
1655 static u32 get_rx_csum(struct net_device *dev)
1656 {
1657         struct port_info *p = netdev_priv(dev);
1658
1659         return p->rx_csum_offload;
1660 }
1661
1662 static int set_rx_csum(struct net_device *dev, u32 data)
1663 {
1664         struct port_info *p = netdev_priv(dev);
1665
1666         p->rx_csum_offload = data;
1667         if (!data) {
1668                 int i;
1669
1670                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1671                         set_qset_lro(dev, i, 0);
1672         }
1673         return 0;
1674 }
1675
1676 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1677 {
1678         struct port_info *pi = netdev_priv(dev);
1679         struct adapter *adapter = pi->adapter;
1680         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1681
1682         e->rx_max_pending = MAX_RX_BUFFERS;
1683         e->rx_mini_max_pending = 0;
1684         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1685         e->tx_max_pending = MAX_TXQ_ENTRIES;
1686
1687         e->rx_pending = q->fl_size;
1688         e->rx_mini_pending = q->rspq_size;
1689         e->rx_jumbo_pending = q->jumbo_size;
1690         e->tx_pending = q->txq_size[0];
1691 }
1692
1693 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1694 {
1695         struct port_info *pi = netdev_priv(dev);
1696         struct adapter *adapter = pi->adapter;
1697         struct qset_params *q;
1698         int i;
1699
1700         if (e->rx_pending > MAX_RX_BUFFERS ||
1701             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1702             e->tx_pending > MAX_TXQ_ENTRIES ||
1703             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1704             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1705             e->rx_pending < MIN_FL_ENTRIES ||
1706             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1707             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1708                 return -EINVAL;
1709
1710         if (adapter->flags & FULL_INIT_DONE)
1711                 return -EBUSY;
1712
1713         q = &adapter->params.sge.qset[pi->first_qset];
1714         for (i = 0; i < pi->nqsets; ++i, ++q) {
1715                 q->rspq_size = e->rx_mini_pending;
1716                 q->fl_size = e->rx_pending;
1717                 q->jumbo_size = e->rx_jumbo_pending;
1718                 q->txq_size[0] = e->tx_pending;
1719                 q->txq_size[1] = e->tx_pending;
1720                 q->txq_size[2] = e->tx_pending;
1721         }
1722         return 0;
1723 }
1724
1725 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1726 {
1727         struct port_info *pi = netdev_priv(dev);
1728         struct adapter *adapter = pi->adapter;
1729         struct qset_params *qsp = &adapter->params.sge.qset[0];
1730         struct sge_qset *qs = &adapter->sge.qs[0];
1731
1732         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1733                 return -EINVAL;
1734
1735         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1736         t3_update_qset_coalesce(qs, qsp);
1737         return 0;
1738 }
1739
1740 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1741 {
1742         struct port_info *pi = netdev_priv(dev);
1743         struct adapter *adapter = pi->adapter;
1744         struct qset_params *q = adapter->params.sge.qset;
1745
1746         c->rx_coalesce_usecs = q->coalesce_usecs;
1747         return 0;
1748 }
1749
1750 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1751                       u8 * data)
1752 {
1753         struct port_info *pi = netdev_priv(dev);
1754         struct adapter *adapter = pi->adapter;
1755         int i, err = 0;
1756
1757         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1758         if (!buf)
1759                 return -ENOMEM;
1760
1761         e->magic = EEPROM_MAGIC;
1762         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1763                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1764
1765         if (!err)
1766                 memcpy(data, buf + e->offset, e->len);
1767         kfree(buf);
1768         return err;
1769 }
1770
1771 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1772                       u8 * data)
1773 {
1774         struct port_info *pi = netdev_priv(dev);
1775         struct adapter *adapter = pi->adapter;
1776         u32 aligned_offset, aligned_len;
1777         __le32 *p;
1778         u8 *buf;
1779         int err;
1780
1781         if (eeprom->magic != EEPROM_MAGIC)
1782                 return -EINVAL;
1783
1784         aligned_offset = eeprom->offset & ~3;
1785         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1786
1787         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1788                 buf = kmalloc(aligned_len, GFP_KERNEL);
1789                 if (!buf)
1790                         return -ENOMEM;
1791                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
1792                 if (!err && aligned_len > 4)
1793                         err = t3_seeprom_read(adapter,
1794                                               aligned_offset + aligned_len - 4,
1795                                               (__le32 *) & buf[aligned_len - 4]);
1796                 if (err)
1797                         goto out;
1798                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1799         } else
1800                 buf = data;
1801
1802         err = t3_seeprom_wp(adapter, 0);
1803         if (err)
1804                 goto out;
1805
1806         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1807                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1808                 aligned_offset += 4;
1809         }
1810
1811         if (!err)
1812                 err = t3_seeprom_wp(adapter, 1);
1813 out:
1814         if (buf != data)
1815                 kfree(buf);
1816         return err;
1817 }
1818
1819 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1820 {
1821         wol->supported = 0;
1822         wol->wolopts = 0;
1823         memset(&wol->sopass, 0, sizeof(wol->sopass));
1824 }
1825
1826 static int cxgb3_set_flags(struct net_device *dev, u32 data)
1827 {
1828         struct port_info *pi = netdev_priv(dev);
1829         int i;
1830
1831         if (data & ETH_FLAG_LRO) {
1832                 if (!pi->rx_csum_offload)
1833                         return -EINVAL;
1834
1835                 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
1836                         set_qset_lro(dev, i, 1);
1837
1838         } else
1839                 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
1840                         set_qset_lro(dev, i, 0);
1841
1842         return 0;
1843 }
1844
1845 static const struct ethtool_ops cxgb_ethtool_ops = {
1846         .get_settings = get_settings,
1847         .set_settings = set_settings,
1848         .get_drvinfo = get_drvinfo,
1849         .get_msglevel = get_msglevel,
1850         .set_msglevel = set_msglevel,
1851         .get_ringparam = get_sge_param,
1852         .set_ringparam = set_sge_param,
1853         .get_coalesce = get_coalesce,
1854         .set_coalesce = set_coalesce,
1855         .get_eeprom_len = get_eeprom_len,
1856         .get_eeprom = get_eeprom,
1857         .set_eeprom = set_eeprom,
1858         .get_pauseparam = get_pauseparam,
1859         .set_pauseparam = set_pauseparam,
1860         .get_rx_csum = get_rx_csum,
1861         .set_rx_csum = set_rx_csum,
1862         .set_tx_csum = ethtool_op_set_tx_csum,
1863         .set_sg = ethtool_op_set_sg,
1864         .get_link = ethtool_op_get_link,
1865         .get_strings = get_strings,
1866         .phys_id = cxgb3_phys_id,
1867         .nway_reset = restart_autoneg,
1868         .get_sset_count = get_sset_count,
1869         .get_ethtool_stats = get_stats,
1870         .get_regs_len = get_regs_len,
1871         .get_regs = get_regs,
1872         .get_wol = get_wol,
1873         .set_tso = ethtool_op_set_tso,
1874         .get_flags = ethtool_op_get_flags,
1875         .set_flags = cxgb3_set_flags,
1876 };
1877
1878 static int in_range(int val, int lo, int hi)
1879 {
1880         return val < 0 || (val <= hi && val >= lo);
1881 }
1882
1883 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1884 {
1885         struct port_info *pi = netdev_priv(dev);
1886         struct adapter *adapter = pi->adapter;
1887         u32 cmd;
1888         int ret;
1889
1890         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1891                 return -EFAULT;
1892
1893         switch (cmd) {
1894         case CHELSIO_SET_QSET_PARAMS:{
1895                 int i;
1896                 struct qset_params *q;
1897                 struct ch_qset_params t;
1898                 int q1 = pi->first_qset;
1899                 int nqsets = pi->nqsets;
1900
1901                 if (!capable(CAP_NET_ADMIN))
1902                         return -EPERM;
1903                 if (copy_from_user(&t, useraddr, sizeof(t)))
1904                         return -EFAULT;
1905                 if (t.qset_idx >= SGE_QSETS)
1906                         return -EINVAL;
1907                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1908                         !in_range(t.cong_thres, 0, 255) ||
1909                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1910                                 MAX_TXQ_ENTRIES) ||
1911                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1912                                 MAX_TXQ_ENTRIES) ||
1913                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1914                                 MAX_CTRL_TXQ_ENTRIES) ||
1915                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1916                                 MAX_RX_BUFFERS)
1917                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1918                                         MAX_RX_JUMBO_BUFFERS)
1919                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1920                                         MAX_RSPQ_ENTRIES))
1921                         return -EINVAL;
1922
1923                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
1924                         for_each_port(adapter, i) {
1925                                 pi = adap2pinfo(adapter, i);
1926                                 if (t.qset_idx >= pi->first_qset &&
1927                                     t.qset_idx < pi->first_qset + pi->nqsets &&
1928                                     !pi->rx_csum_offload)
1929                                         return -EINVAL;
1930                         }
1931
1932                 if ((adapter->flags & FULL_INIT_DONE) &&
1933                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1934                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1935                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1936                         t.polling >= 0 || t.cong_thres >= 0))
1937                         return -EBUSY;
1938
1939                 /* Allow setting of any available qset when offload enabled */
1940                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1941                         q1 = 0;
1942                         for_each_port(adapter, i) {
1943                                 pi = adap2pinfo(adapter, i);
1944                                 nqsets += pi->first_qset + pi->nqsets;
1945                         }
1946                 }
1947
1948                 if (t.qset_idx < q1)
1949                         return -EINVAL;
1950                 if (t.qset_idx > q1 + nqsets - 1)
1951                         return -EINVAL;
1952
1953                 q = &adapter->params.sge.qset[t.qset_idx];
1954
1955                 if (t.rspq_size >= 0)
1956                         q->rspq_size = t.rspq_size;
1957                 if (t.fl_size[0] >= 0)
1958                         q->fl_size = t.fl_size[0];
1959                 if (t.fl_size[1] >= 0)
1960                         q->jumbo_size = t.fl_size[1];
1961                 if (t.txq_size[0] >= 0)
1962                         q->txq_size[0] = t.txq_size[0];
1963                 if (t.txq_size[1] >= 0)
1964                         q->txq_size[1] = t.txq_size[1];
1965                 if (t.txq_size[2] >= 0)
1966                         q->txq_size[2] = t.txq_size[2];
1967                 if (t.cong_thres >= 0)
1968                         q->cong_thres = t.cong_thres;
1969                 if (t.intr_lat >= 0) {
1970                         struct sge_qset *qs =
1971                                 &adapter->sge.qs[t.qset_idx];
1972
1973                         q->coalesce_usecs = t.intr_lat;
1974                         t3_update_qset_coalesce(qs, q);
1975                 }
1976                 if (t.polling >= 0) {
1977                         if (adapter->flags & USING_MSIX)
1978                                 q->polling = t.polling;
1979                         else {
1980                                 /* No polling with INTx for T3A */
1981                                 if (adapter->params.rev == 0 &&
1982                                         !(adapter->flags & USING_MSI))
1983                                         t.polling = 0;
1984
1985                                 for (i = 0; i < SGE_QSETS; i++) {
1986                                         q = &adapter->params.sge.
1987                                                 qset[i];
1988                                         q->polling = t.polling;
1989                                 }
1990                         }
1991                 }
1992                 if (t.lro >= 0)
1993                         set_qset_lro(dev, t.qset_idx, t.lro);
1994
1995                 break;
1996         }
1997         case CHELSIO_GET_QSET_PARAMS:{
1998                 struct qset_params *q;
1999                 struct ch_qset_params t;
2000                 int q1 = pi->first_qset;
2001                 int nqsets = pi->nqsets;
2002                 int i;
2003
2004                 if (copy_from_user(&t, useraddr, sizeof(t)))
2005                         return -EFAULT;
2006
2007                 /* Display qsets for all ports when offload enabled */
2008                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2009                         q1 = 0;
2010                         for_each_port(adapter, i) {
2011                                 pi = adap2pinfo(adapter, i);
2012                                 nqsets = pi->first_qset + pi->nqsets;
2013                         }
2014                 }
2015
2016                 if (t.qset_idx >= nqsets)
2017                         return -EINVAL;
2018
2019                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2020                 t.rspq_size = q->rspq_size;
2021                 t.txq_size[0] = q->txq_size[0];
2022                 t.txq_size[1] = q->txq_size[1];
2023                 t.txq_size[2] = q->txq_size[2];
2024                 t.fl_size[0] = q->fl_size;
2025                 t.fl_size[1] = q->jumbo_size;
2026                 t.polling = q->polling;
2027                 t.lro = q->lro;
2028                 t.intr_lat = q->coalesce_usecs;
2029                 t.cong_thres = q->cong_thres;
2030                 t.qnum = q1;
2031
2032                 if (adapter->flags & USING_MSIX)
2033                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2034                 else
2035                         t.vector = adapter->pdev->irq;
2036
2037                 if (copy_to_user(useraddr, &t, sizeof(t)))
2038                         return -EFAULT;
2039                 break;
2040         }
2041         case CHELSIO_SET_QSET_NUM:{
2042                 struct ch_reg edata;
2043                 unsigned int i, first_qset = 0, other_qsets = 0;
2044
2045                 if (!capable(CAP_NET_ADMIN))
2046                         return -EPERM;
2047                 if (adapter->flags & FULL_INIT_DONE)
2048                         return -EBUSY;
2049                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2050                         return -EFAULT;
2051                 if (edata.val < 1 ||
2052                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2053                         return -EINVAL;
2054
2055                 for_each_port(adapter, i)
2056                         if (adapter->port[i] && adapter->port[i] != dev)
2057                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2058
2059                 if (edata.val + other_qsets > SGE_QSETS)
2060                         return -EINVAL;
2061
2062                 pi->nqsets = edata.val;
2063
2064                 for_each_port(adapter, i)
2065                         if (adapter->port[i]) {
2066                                 pi = adap2pinfo(adapter, i);
2067                                 pi->first_qset = first_qset;
2068                                 first_qset += pi->nqsets;
2069                         }
2070                 break;
2071         }
2072         case CHELSIO_GET_QSET_NUM:{
2073                 struct ch_reg edata;
2074
2075                 edata.cmd = CHELSIO_GET_QSET_NUM;
2076                 edata.val = pi->nqsets;
2077                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2078                         return -EFAULT;
2079                 break;
2080         }
2081         case CHELSIO_LOAD_FW:{
2082                 u8 *fw_data;
2083                 struct ch_mem_range t;
2084
2085                 if (!capable(CAP_SYS_RAWIO))
2086                         return -EPERM;
2087                 if (copy_from_user(&t, useraddr, sizeof(t)))
2088                         return -EFAULT;
2089                 /* Check t.len sanity ? */
2090                 fw_data = kmalloc(t.len, GFP_KERNEL);
2091                 if (!fw_data)
2092                         return -ENOMEM;
2093
2094                 if (copy_from_user
2095                         (fw_data, useraddr + sizeof(t), t.len)) {
2096                         kfree(fw_data);
2097                         return -EFAULT;
2098                 }
2099
2100                 ret = t3_load_fw(adapter, fw_data, t.len);
2101                 kfree(fw_data);
2102                 if (ret)
2103                         return ret;
2104                 break;
2105         }
2106         case CHELSIO_SETMTUTAB:{
2107                 struct ch_mtus m;
2108                 int i;
2109
2110                 if (!is_offload(adapter))
2111                         return -EOPNOTSUPP;
2112                 if (!capable(CAP_NET_ADMIN))
2113                         return -EPERM;
2114                 if (offload_running(adapter))
2115                         return -EBUSY;
2116                 if (copy_from_user(&m, useraddr, sizeof(m)))
2117                         return -EFAULT;
2118                 if (m.nmtus != NMTUS)
2119                         return -EINVAL;
2120                 if (m.mtus[0] < 81)     /* accommodate SACK */
2121                         return -EINVAL;
2122
2123                 /* MTUs must be in ascending order */
2124                 for (i = 1; i < NMTUS; ++i)
2125                         if (m.mtus[i] < m.mtus[i - 1])
2126                                 return -EINVAL;
2127
2128                 memcpy(adapter->params.mtus, m.mtus,
2129                         sizeof(adapter->params.mtus));
2130                 break;
2131         }
2132         case CHELSIO_GET_PM:{
2133                 struct tp_params *p = &adapter->params.tp;
2134                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2135
2136                 if (!is_offload(adapter))
2137                         return -EOPNOTSUPP;
2138                 m.tx_pg_sz = p->tx_pg_size;
2139                 m.tx_num_pg = p->tx_num_pgs;
2140                 m.rx_pg_sz = p->rx_pg_size;
2141                 m.rx_num_pg = p->rx_num_pgs;
2142                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2143                 if (copy_to_user(useraddr, &m, sizeof(m)))
2144                         return -EFAULT;
2145                 break;
2146         }
2147         case CHELSIO_SET_PM:{
2148                 struct ch_pm m;
2149                 struct tp_params *p = &adapter->params.tp;
2150
2151                 if (!is_offload(adapter))
2152                         return -EOPNOTSUPP;
2153                 if (!capable(CAP_NET_ADMIN))
2154                         return -EPERM;
2155                 if (adapter->flags & FULL_INIT_DONE)
2156                         return -EBUSY;
2157                 if (copy_from_user(&m, useraddr, sizeof(m)))
2158                         return -EFAULT;
2159                 if (!is_power_of_2(m.rx_pg_sz) ||
2160                         !is_power_of_2(m.tx_pg_sz))
2161                         return -EINVAL; /* not power of 2 */
2162                 if (!(m.rx_pg_sz & 0x14000))
2163                         return -EINVAL; /* not 16KB or 64KB */
2164                 if (!(m.tx_pg_sz & 0x1554000))
2165                         return -EINVAL;
2166                 if (m.tx_num_pg == -1)
2167                         m.tx_num_pg = p->tx_num_pgs;
2168                 if (m.rx_num_pg == -1)
2169                         m.rx_num_pg = p->rx_num_pgs;
2170                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2171                         return -EINVAL;
2172                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2173                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2174                         return -EINVAL;
2175                 p->rx_pg_size = m.rx_pg_sz;
2176                 p->tx_pg_size = m.tx_pg_sz;
2177                 p->rx_num_pgs = m.rx_num_pg;
2178                 p->tx_num_pgs = m.tx_num_pg;
2179                 break;
2180         }
2181         case CHELSIO_GET_MEM:{
2182                 struct ch_mem_range t;
2183                 struct mc7 *mem;
2184                 u64 buf[32];
2185
2186                 if (!is_offload(adapter))
2187                         return -EOPNOTSUPP;
2188                 if (!(adapter->flags & FULL_INIT_DONE))
2189                         return -EIO;    /* need the memory controllers */
2190                 if (copy_from_user(&t, useraddr, sizeof(t)))
2191                         return -EFAULT;
2192                 if ((t.addr & 7) || (t.len & 7))
2193                         return -EINVAL;
2194                 if (t.mem_id == MEM_CM)
2195                         mem = &adapter->cm;
2196                 else if (t.mem_id == MEM_PMRX)
2197                         mem = &adapter->pmrx;
2198                 else if (t.mem_id == MEM_PMTX)
2199                         mem = &adapter->pmtx;
2200                 else
2201                         return -EINVAL;
2202
2203                 /*
2204                  * Version scheme:
2205                  * bits 0..9: chip version
2206                  * bits 10..15: chip revision
2207                  */
2208                 t.version = 3 | (adapter->params.rev << 10);
2209                 if (copy_to_user(useraddr, &t, sizeof(t)))
2210                         return -EFAULT;
2211
2212                 /*
2213                  * Read 256 bytes at a time as len can be large and we don't
2214                  * want to use huge intermediate buffers.
2215                  */
2216                 useraddr += sizeof(t);  /* advance to start of buffer */
2217                 while (t.len) {
2218                         unsigned int chunk =
2219                                 min_t(unsigned int, t.len, sizeof(buf));
2220
2221                         ret =
2222                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2223                                                 buf);
2224                         if (ret)
2225                                 return ret;
2226                         if (copy_to_user(useraddr, buf, chunk))
2227                                 return -EFAULT;
2228                         useraddr += chunk;
2229                         t.addr += chunk;
2230                         t.len -= chunk;
2231                 }
2232                 break;
2233         }
2234         case CHELSIO_SET_TRACE_FILTER:{
2235                 struct ch_trace t;
2236                 const struct trace_params *tp;
2237
2238                 if (!capable(CAP_NET_ADMIN))
2239                         return -EPERM;
2240                 if (!offload_running(adapter))
2241                         return -EAGAIN;
2242                 if (copy_from_user(&t, useraddr, sizeof(t)))
2243                         return -EFAULT;
2244
2245                 tp = (const struct trace_params *)&t.sip;
2246                 if (t.config_tx)
2247                         t3_config_trace_filter(adapter, tp, 0,
2248                                                 t.invert_match,
2249                                                 t.trace_tx);
2250                 if (t.config_rx)
2251                         t3_config_trace_filter(adapter, tp, 1,
2252                                                 t.invert_match,
2253                                                 t.trace_rx);
2254                 break;
2255         }
2256         default:
2257                 return -EOPNOTSUPP;
2258         }
2259         return 0;
2260 }
2261
2262 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2263 {
2264         struct mii_ioctl_data *data = if_mii(req);
2265         struct port_info *pi = netdev_priv(dev);
2266         struct adapter *adapter = pi->adapter;
2267         int ret, mmd;
2268
2269         switch (cmd) {
2270         case SIOCGMIIPHY:
2271                 data->phy_id = pi->phy.addr;
2272                 /* FALLTHRU */
2273         case SIOCGMIIREG:{
2274                 u32 val;
2275                 struct cphy *phy = &pi->phy;
2276
2277                 if (!phy->mdio_read)
2278                         return -EOPNOTSUPP;
2279                 if (is_10G(adapter)) {
2280                         mmd = data->phy_id >> 8;
2281                         if (!mmd)
2282                                 mmd = MDIO_DEV_PCS;
2283                         else if (mmd > MDIO_DEV_VEND2)
2284                                 return -EINVAL;
2285
2286                         ret =
2287                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2288                                                 mmd, data->reg_num, &val);
2289                 } else
2290                         ret =
2291                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2292                                                 0, data->reg_num & 0x1f,
2293                                                 &val);
2294                 if (!ret)
2295                         data->val_out = val;
2296                 break;
2297         }
2298         case SIOCSMIIREG:{
2299                 struct cphy *phy = &pi->phy;
2300
2301                 if (!capable(CAP_NET_ADMIN))
2302                         return -EPERM;
2303                 if (!phy->mdio_write)
2304                         return -EOPNOTSUPP;
2305                 if (is_10G(adapter)) {
2306                         mmd = data->phy_id >> 8;
2307                         if (!mmd)
2308                                 mmd = MDIO_DEV_PCS;
2309                         else if (mmd > MDIO_DEV_VEND2)
2310                                 return -EINVAL;
2311
2312                         ret =
2313                                 phy->mdio_write(adapter,
2314                                                 data->phy_id & 0x1f, mmd,
2315                                                 data->reg_num,
2316                                                 data->val_in);
2317                 } else
2318                         ret =
2319                                 phy->mdio_write(adapter,
2320                                                 data->phy_id & 0x1f, 0,
2321                                                 data->reg_num & 0x1f,
2322                                                 data->val_in);
2323                 break;
2324         }
2325         case SIOCCHIOCTL:
2326                 return cxgb_extension_ioctl(dev, req->ifr_data);
2327         default:
2328                 return -EOPNOTSUPP;
2329         }
2330         return ret;
2331 }
2332
2333 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2334 {
2335         struct port_info *pi = netdev_priv(dev);
2336         struct adapter *adapter = pi->adapter;
2337         int ret;
2338
2339         if (new_mtu < 81)       /* accommodate SACK */
2340                 return -EINVAL;
2341         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2342                 return ret;
2343         dev->mtu = new_mtu;
2344         init_port_mtus(adapter);
2345         if (adapter->params.rev == 0 && offload_running(adapter))
2346                 t3_load_mtus(adapter, adapter->params.mtus,
2347                              adapter->params.a_wnd, adapter->params.b_wnd,
2348                              adapter->port[0]->mtu);
2349         return 0;
2350 }
2351
2352 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2353 {
2354         struct port_info *pi = netdev_priv(dev);
2355         struct adapter *adapter = pi->adapter;
2356         struct sockaddr *addr = p;
2357
2358         if (!is_valid_ether_addr(addr->sa_data))
2359                 return -EINVAL;
2360
2361         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2362         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2363         if (offload_running(adapter))
2364                 write_smt_entry(adapter, pi->port_id);
2365         return 0;
2366 }
2367
2368 /**
2369  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2370  * @adap: the adapter
2371  * @p: the port
2372  *
2373  * Ensures that current Rx processing on any of the queues associated with
2374  * the given port completes before returning.  We do this by acquiring and
2375  * releasing the locks of the response queues associated with the port.
2376  */
2377 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2378 {
2379         int i;
2380
2381         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2382                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2383
2384                 spin_lock_irq(&q->lock);
2385                 spin_unlock_irq(&q->lock);
2386         }
2387 }
2388
2389 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2390 {
2391         struct port_info *pi = netdev_priv(dev);
2392         struct adapter *adapter = pi->adapter;
2393
2394         pi->vlan_grp = grp;
2395         if (adapter->params.rev > 0)
2396                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2397         else {
2398                 /* single control for all ports */
2399                 unsigned int i, have_vlans = 0;
2400                 for_each_port(adapter, i)
2401                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2402
2403                 t3_set_vlan_accel(adapter, 1, have_vlans);
2404         }
2405         t3_synchronize_rx(adapter, pi);
2406 }
2407
2408 #ifdef CONFIG_NET_POLL_CONTROLLER
2409 static void cxgb_netpoll(struct net_device *dev)
2410 {
2411         struct port_info *pi = netdev_priv(dev);
2412         struct adapter *adapter = pi->adapter;
2413         int qidx;
2414
2415         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2416                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2417                 void *source;
2418
2419                 if (adapter->flags & USING_MSIX)
2420                         source = qs;
2421                 else
2422                         source = adapter;
2423
2424                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2425         }
2426 }
2427 #endif
2428
2429 /*
2430  * Periodic accumulation of MAC statistics.
2431  */
2432 static void mac_stats_update(struct adapter *adapter)
2433 {
2434         int i;
2435
2436         for_each_port(adapter, i) {
2437                 struct net_device *dev = adapter->port[i];
2438                 struct port_info *p = netdev_priv(dev);
2439
2440                 if (netif_running(dev)) {
2441                         spin_lock(&adapter->stats_lock);
2442                         t3_mac_update_stats(&p->mac);
2443                         spin_unlock(&adapter->stats_lock);
2444                 }
2445         }
2446 }
2447
2448 static void check_link_status(struct adapter *adapter)
2449 {
2450         int i;
2451
2452         for_each_port(adapter, i) {
2453                 struct net_device *dev = adapter->port[i];
2454                 struct port_info *p = netdev_priv(dev);
2455
2456                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev))
2457                         t3_link_changed(adapter, i);
2458         }
2459 }
2460
2461 static void check_t3b2_mac(struct adapter *adapter)
2462 {
2463         int i;
2464
2465         if (!rtnl_trylock())    /* synchronize with ifdown */
2466                 return;
2467
2468         for_each_port(adapter, i) {
2469                 struct net_device *dev = adapter->port[i];
2470                 struct port_info *p = netdev_priv(dev);
2471                 int status;
2472
2473                 if (!netif_running(dev))
2474                         continue;
2475
2476                 status = 0;
2477                 if (netif_running(dev) && netif_carrier_ok(dev))
2478                         status = t3b2_mac_watchdog_task(&p->mac);
2479                 if (status == 1)
2480                         p->mac.stats.num_toggled++;
2481                 else if (status == 2) {
2482                         struct cmac *mac = &p->mac;
2483
2484                         t3_mac_set_mtu(mac, dev->mtu);
2485                         t3_mac_set_address(mac, 0, dev->dev_addr);
2486                         cxgb_set_rxmode(dev);
2487                         t3_link_start(&p->phy, mac, &p->link_config);
2488                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2489                         t3_port_intr_enable(adapter, p->port_id);
2490                         p->mac.stats.num_resets++;
2491                 }
2492         }
2493         rtnl_unlock();
2494 }
2495
2496
2497 static void t3_adap_check_task(struct work_struct *work)
2498 {
2499         struct adapter *adapter = container_of(work, struct adapter,
2500                                                adap_check_task.work);
2501         const struct adapter_params *p = &adapter->params;
2502
2503         adapter->check_task_cnt++;
2504
2505         /* Check link status for PHYs without interrupts */
2506         if (p->linkpoll_period)
2507                 check_link_status(adapter);
2508
2509         /* Accumulate MAC stats if needed */
2510         if (!p->linkpoll_period ||
2511             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2512             p->stats_update_period) {
2513                 mac_stats_update(adapter);
2514                 adapter->check_task_cnt = 0;
2515         }
2516
2517         if (p->rev == T3_REV_B2)
2518                 check_t3b2_mac(adapter);
2519
2520         /* Schedule the next check update if any port is active. */
2521         spin_lock_irq(&adapter->work_lock);
2522         if (adapter->open_device_map & PORT_MASK)
2523                 schedule_chk_task(adapter);
2524         spin_unlock_irq(&adapter->work_lock);
2525 }
2526
2527 /*
2528  * Processes external (PHY) interrupts in process context.
2529  */
2530 static void ext_intr_task(struct work_struct *work)
2531 {
2532         struct adapter *adapter = container_of(work, struct adapter,
2533                                                ext_intr_handler_task);
2534
2535         t3_phy_intr_handler(adapter);
2536
2537         /* Now reenable external interrupts */
2538         spin_lock_irq(&adapter->work_lock);
2539         if (adapter->slow_intr_mask) {
2540                 adapter->slow_intr_mask |= F_T3DBG;
2541                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2542                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2543                              adapter->slow_intr_mask);
2544         }
2545         spin_unlock_irq(&adapter->work_lock);
2546 }
2547
2548 /*
2549  * Interrupt-context handler for external (PHY) interrupts.
2550  */
2551 void t3_os_ext_intr_handler(struct adapter *adapter)
2552 {
2553         /*
2554          * Schedule a task to handle external interrupts as they may be slow
2555          * and we use a mutex to protect MDIO registers.  We disable PHY
2556          * interrupts in the meantime and let the task reenable them when
2557          * it's done.
2558          */
2559         spin_lock(&adapter->work_lock);
2560         if (adapter->slow_intr_mask) {
2561                 adapter->slow_intr_mask &= ~F_T3DBG;
2562                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2563                              adapter->slow_intr_mask);
2564                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2565         }
2566         spin_unlock(&adapter->work_lock);
2567 }
2568
2569 static int t3_adapter_error(struct adapter *adapter, int reset)
2570 {
2571         int i, ret = 0;
2572
2573         /* Stop all ports */
2574         for_each_port(adapter, i) {
2575                 struct net_device *netdev = adapter->port[i];
2576
2577                 if (netif_running(netdev))
2578                         cxgb_close(netdev);
2579         }
2580
2581         if (is_offload(adapter) &&
2582             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
2583                 offload_close(&adapter->tdev);
2584
2585         /* Stop SGE timers */
2586         t3_stop_sge_timers(adapter);
2587
2588         adapter->flags &= ~FULL_INIT_DONE;
2589
2590         if (reset)
2591                 ret = t3_reset_adapter(adapter);
2592
2593         pci_disable_device(adapter->pdev);
2594
2595         return ret;
2596 }
2597
2598 static int t3_reenable_adapter(struct adapter *adapter)
2599 {
2600         if (pci_enable_device(adapter->pdev)) {
2601                 dev_err(&adapter->pdev->dev,
2602                         "Cannot re-enable PCI device after reset.\n");
2603                 goto err;
2604         }
2605         pci_set_master(adapter->pdev);
2606         pci_restore_state(adapter->pdev);
2607
2608         /* Free sge resources */
2609         t3_free_sge_resources(adapter);
2610
2611         if (t3_replay_prep_adapter(adapter))
2612                 goto err;
2613
2614         return 0;
2615 err:
2616         return -1;
2617 }
2618
2619 static void t3_resume_ports(struct adapter *adapter)
2620 {
2621         int i;
2622
2623         /* Restart the ports */
2624         for_each_port(adapter, i) {
2625                 struct net_device *netdev = adapter->port[i];
2626
2627                 if (netif_running(netdev)) {
2628                         if (cxgb_open(netdev)) {
2629                                 dev_err(&adapter->pdev->dev,
2630                                         "can't bring device back up"
2631                                         " after reset\n");
2632                                 continue;
2633                         }
2634                 }
2635         }
2636 }
2637
2638 /*
2639  * processes a fatal error.
2640  * Bring the ports down, reset the chip, bring the ports back up.
2641  */
2642 static void fatal_error_task(struct work_struct *work)
2643 {
2644         struct adapter *adapter = container_of(work, struct adapter,
2645                                                fatal_error_handler_task);
2646         int err = 0;
2647
2648         rtnl_lock();
2649         err = t3_adapter_error(adapter, 1);
2650         if (!err)
2651                 err = t3_reenable_adapter(adapter);
2652         if (!err)
2653                 t3_resume_ports(adapter);
2654
2655         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2656         rtnl_unlock();
2657 }
2658
2659 void t3_fatal_err(struct adapter *adapter)
2660 {
2661         unsigned int fw_status[4];
2662
2663         if (adapter->flags & FULL_INIT_DONE) {
2664                 t3_sge_stop(adapter);
2665                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2666                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2667                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2668                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2669
2670                 spin_lock(&adapter->work_lock);
2671                 t3_intr_disable(adapter);
2672                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2673                 spin_unlock(&adapter->work_lock);
2674         }
2675         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2676         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2677                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2678                          fw_status[0], fw_status[1],
2679                          fw_status[2], fw_status[3]);
2680
2681 }
2682
2683 /**
2684  * t3_io_error_detected - called when PCI error is detected
2685  * @pdev: Pointer to PCI device
2686  * @state: The current pci connection state
2687  *
2688  * This function is called after a PCI bus error affecting
2689  * this device has been detected.
2690  */
2691 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2692                                              pci_channel_state_t state)
2693 {
2694         struct adapter *adapter = pci_get_drvdata(pdev);
2695         int ret;
2696
2697         ret = t3_adapter_error(adapter, 0);
2698
2699         /* Request a slot reset. */
2700         return PCI_ERS_RESULT_NEED_RESET;
2701 }
2702
2703 /**
2704  * t3_io_slot_reset - called after the pci bus has been reset.
2705  * @pdev: Pointer to PCI device
2706  *
2707  * Restart the card from scratch, as if from a cold-boot.
2708  */
2709 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2710 {
2711         struct adapter *adapter = pci_get_drvdata(pdev);
2712
2713         if (!t3_reenable_adapter(adapter))
2714                 return PCI_ERS_RESULT_RECOVERED;
2715
2716         return PCI_ERS_RESULT_DISCONNECT;
2717 }
2718
2719 /**
2720  * t3_io_resume - called when traffic can start flowing again.
2721  * @pdev: Pointer to PCI device
2722  *
2723  * This callback is called when the error recovery driver tells us that
2724  * its OK to resume normal operation.
2725  */
2726 static void t3_io_resume(struct pci_dev *pdev)
2727 {
2728         struct adapter *adapter = pci_get_drvdata(pdev);
2729
2730         t3_resume_ports(adapter);
2731 }
2732
2733 static struct pci_error_handlers t3_err_handler = {
2734         .error_detected = t3_io_error_detected,
2735         .slot_reset = t3_io_slot_reset,
2736         .resume = t3_io_resume,
2737 };
2738
2739 /*
2740  * Set the number of qsets based on the number of CPUs and the number of ports,
2741  * not to exceed the number of available qsets, assuming there are enough qsets
2742  * per port in HW.
2743  */
2744 static void set_nqsets(struct adapter *adap)
2745 {
2746         int i, j = 0;
2747         int num_cpus = num_online_cpus();
2748         int hwports = adap->params.nports;
2749         int nqsets = SGE_QSETS;
2750
2751         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
2752                 if (hwports == 2 &&
2753                     (hwports * nqsets > SGE_QSETS ||
2754                      num_cpus >= nqsets / hwports))
2755                         nqsets /= hwports;
2756                 if (nqsets > num_cpus)
2757                         nqsets = num_cpus;
2758                 if (nqsets < 1 || hwports == 4)
2759                         nqsets = 1;
2760         } else
2761                 nqsets = 1;
2762
2763         for_each_port(adap, i) {
2764                 struct port_info *pi = adap2pinfo(adap, i);
2765
2766                 pi->first_qset = j;
2767                 pi->nqsets = nqsets;
2768                 j = pi->first_qset + nqsets;
2769
2770                 dev_info(&adap->pdev->dev,
2771                          "Port %d using %d queue sets.\n", i, nqsets);
2772         }
2773 }
2774
2775 static int __devinit cxgb_enable_msix(struct adapter *adap)
2776 {
2777         struct msix_entry entries[SGE_QSETS + 1];
2778         int i, err;
2779
2780         for (i = 0; i < ARRAY_SIZE(entries); ++i)
2781                 entries[i].entry = i;
2782
2783         err = pci_enable_msix(adap->pdev, entries, ARRAY_SIZE(entries));
2784         if (!err) {
2785                 for (i = 0; i < ARRAY_SIZE(entries); ++i)
2786                         adap->msix_info[i].vec = entries[i].vector;
2787         } else if (err > 0)
2788                 dev_info(&adap->pdev->dev,
2789                        "only %d MSI-X vectors left, not using MSI-X\n", err);
2790         return err;
2791 }
2792
2793 static void __devinit print_port_info(struct adapter *adap,
2794                                       const struct adapter_info *ai)
2795 {
2796         static const char *pci_variant[] = {
2797                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2798         };
2799
2800         int i;
2801         char buf[80];
2802
2803         if (is_pcie(adap))
2804                 snprintf(buf, sizeof(buf), "%s x%d",
2805                          pci_variant[adap->params.pci.variant],
2806                          adap->params.pci.width);
2807         else
2808                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2809                          pci_variant[adap->params.pci.variant],
2810                          adap->params.pci.speed, adap->params.pci.width);
2811
2812         for_each_port(adap, i) {
2813                 struct net_device *dev = adap->port[i];
2814                 const struct port_info *pi = netdev_priv(dev);
2815
2816                 if (!test_bit(i, &adap->registered_device_map))
2817                         continue;
2818                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2819                        dev->name, ai->desc, pi->phy.desc,
2820                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2821                        (adap->flags & USING_MSIX) ? " MSI-X" :
2822                        (adap->flags & USING_MSI) ? " MSI" : "");
2823                 if (adap->name == dev->name && adap->params.vpd.mclk)
2824                         printk(KERN_INFO
2825                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
2826                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2827                                t3_mc7_size(&adap->pmtx) >> 20,
2828                                t3_mc7_size(&adap->pmrx) >> 20,
2829                                adap->params.vpd.sn);
2830         }
2831 }
2832
2833 static const struct net_device_ops cxgb_netdev_ops = {
2834         .ndo_open               = cxgb_open,
2835         .ndo_stop               = cxgb_close,
2836         .ndo_start_xmit         = t3_eth_xmit,
2837         .ndo_get_stats          = cxgb_get_stats,
2838         .ndo_validate_addr      = eth_validate_addr,
2839         .ndo_set_multicast_list = cxgb_set_rxmode,
2840         .ndo_do_ioctl           = cxgb_ioctl,
2841         .ndo_change_mtu         = cxgb_change_mtu,
2842         .ndo_set_mac_address    = cxgb_set_mac_addr,
2843         .ndo_vlan_rx_register   = vlan_rx_register,
2844 #ifdef CONFIG_NET_POLL_CONTROLLER
2845         .ndo_poll_controller    = cxgb_netpoll,
2846 #endif
2847 };
2848
2849 static int __devinit init_one(struct pci_dev *pdev,
2850                               const struct pci_device_id *ent)
2851 {
2852         static int version_printed;
2853
2854         int i, err, pci_using_dac = 0;
2855         unsigned long mmio_start, mmio_len;
2856         const struct adapter_info *ai;
2857         struct adapter *adapter = NULL;
2858         struct port_info *pi;
2859
2860         if (!version_printed) {
2861                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2862                 ++version_printed;
2863         }
2864
2865         if (!cxgb3_wq) {
2866                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2867                 if (!cxgb3_wq) {
2868                         printk(KERN_ERR DRV_NAME
2869                                ": cannot initialize work queue\n");
2870                         return -ENOMEM;
2871                 }
2872         }
2873
2874         err = pci_request_regions(pdev, DRV_NAME);
2875         if (err) {
2876                 /* Just info, some other driver may have claimed the device. */
2877                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2878                 return err;
2879         }
2880
2881         err = pci_enable_device(pdev);
2882         if (err) {
2883                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2884                 goto out_release_regions;
2885         }
2886
2887         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2888                 pci_using_dac = 1;
2889                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2890                 if (err) {
2891                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2892                                "coherent allocations\n");
2893                         goto out_disable_device;
2894                 }
2895         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2896                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2897                 goto out_disable_device;
2898         }
2899
2900         pci_set_master(pdev);
2901         pci_save_state(pdev);
2902
2903         mmio_start = pci_resource_start(pdev, 0);
2904         mmio_len = pci_resource_len(pdev, 0);
2905         ai = t3_get_adapter_info(ent->driver_data);
2906
2907         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2908         if (!adapter) {
2909                 err = -ENOMEM;
2910                 goto out_disable_device;
2911         }
2912
2913         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2914         if (!adapter->regs) {
2915                 dev_err(&pdev->dev, "cannot map device registers\n");
2916                 err = -ENOMEM;
2917                 goto out_free_adapter;
2918         }
2919
2920         adapter->pdev = pdev;
2921         adapter->name = pci_name(pdev);
2922         adapter->msg_enable = dflt_msg_enable;
2923         adapter->mmio_len = mmio_len;
2924
2925         mutex_init(&adapter->mdio_lock);
2926         spin_lock_init(&adapter->work_lock);
2927         spin_lock_init(&adapter->stats_lock);
2928
2929         INIT_LIST_HEAD(&adapter->adapter_list);
2930         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2931         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
2932         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2933
2934         for (i = 0; i < ai->nports; ++i) {
2935                 struct net_device *netdev;
2936
2937                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
2938                 if (!netdev) {
2939                         err = -ENOMEM;
2940                         goto out_free_dev;
2941                 }
2942
2943                 SET_NETDEV_DEV(netdev, &pdev->dev);
2944
2945                 adapter->port[i] = netdev;
2946                 pi = netdev_priv(netdev);
2947                 pi->adapter = adapter;
2948                 pi->rx_csum_offload = 1;
2949                 pi->port_id = i;
2950                 netif_carrier_off(netdev);
2951                 netif_tx_stop_all_queues(netdev);
2952                 netdev->irq = pdev->irq;
2953                 netdev->mem_start = mmio_start;
2954                 netdev->mem_end = mmio_start + mmio_len - 1;
2955                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2956                 netdev->features |= NETIF_F_LLTX;
2957                 if (pci_using_dac)
2958                         netdev->features |= NETIF_F_HIGHDMA;
2959
2960                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2961                 netdev->netdev_ops = &cxgb_netdev_ops;
2962                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2963         }
2964
2965         pci_set_drvdata(pdev, adapter);
2966         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2967                 err = -ENODEV;
2968                 goto out_free_dev;
2969         }
2970
2971         /*
2972          * The card is now ready to go.  If any errors occur during device
2973          * registration we do not fail the whole card but rather proceed only
2974          * with the ports we manage to register successfully.  However we must
2975          * register at least one net device.
2976          */
2977         for_each_port(adapter, i) {
2978                 err = register_netdev(adapter->port[i]);
2979                 if (err)
2980                         dev_warn(&pdev->dev,
2981                                  "cannot register net device %s, skipping\n",
2982                                  adapter->port[i]->name);
2983                 else {
2984                         /*
2985                          * Change the name we use for messages to the name of
2986                          * the first successfully registered interface.
2987                          */
2988                         if (!adapter->registered_device_map)
2989                                 adapter->name = adapter->port[i]->name;
2990
2991                         __set_bit(i, &adapter->registered_device_map);
2992                 }
2993         }
2994         if (!adapter->registered_device_map) {
2995                 dev_err(&pdev->dev, "could not register any net devices\n");
2996                 goto out_free_dev;
2997         }
2998
2999         /* Driver's ready. Reflect it on LEDs */
3000         t3_led_ready(adapter);
3001
3002         if (is_offload(adapter)) {
3003                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3004                 cxgb3_adapter_ofld(adapter);
3005         }
3006
3007         /* See what interrupts we'll be using */
3008         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3009                 adapter->flags |= USING_MSIX;
3010         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3011                 adapter->flags |= USING_MSI;
3012
3013         set_nqsets(adapter);
3014
3015         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3016                                  &cxgb3_attr_group);
3017
3018         print_port_info(adapter, ai);
3019         return 0;
3020
3021 out_free_dev:
3022         iounmap(adapter->regs);
3023         for (i = ai->nports - 1; i >= 0; --i)
3024                 if (adapter->port[i])
3025                         free_netdev(adapter->port[i]);
3026
3027 out_free_adapter:
3028         kfree(adapter);
3029
3030 out_disable_device:
3031         pci_disable_device(pdev);
3032 out_release_regions:
3033         pci_release_regions(pdev);
3034         pci_set_drvdata(pdev, NULL);
3035         return err;
3036 }
3037
3038 static void __devexit remove_one(struct pci_dev *pdev)
3039 {
3040         struct adapter *adapter = pci_get_drvdata(pdev);
3041
3042         if (adapter) {
3043                 int i;
3044
3045                 t3_sge_stop(adapter);
3046                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3047                                    &cxgb3_attr_group);
3048
3049                 if (is_offload(adapter)) {
3050                         cxgb3_adapter_unofld(adapter);
3051                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3052                                      &adapter->open_device_map))
3053                                 offload_close(&adapter->tdev);
3054                 }
3055
3056                 for_each_port(adapter, i)
3057                     if (test_bit(i, &adapter->registered_device_map))
3058                         unregister_netdev(adapter->port[i]);
3059
3060                 t3_stop_sge_timers(adapter);
3061                 t3_free_sge_resources(adapter);
3062                 cxgb_disable_msi(adapter);
3063
3064                 for_each_port(adapter, i)
3065                         if (adapter->port[i])
3066                                 free_netdev(adapter->port[i]);
3067
3068                 iounmap(adapter->regs);
3069                 kfree(adapter);
3070                 pci_release_regions(pdev);
3071                 pci_disable_device(pdev);
3072                 pci_set_drvdata(pdev, NULL);
3073         }
3074 }
3075
3076 static struct pci_driver driver = {
3077         .name = DRV_NAME,
3078         .id_table = cxgb3_pci_tbl,
3079         .probe = init_one,
3080         .remove = __devexit_p(remove_one),
3081         .err_handler = &t3_err_handler,
3082 };
3083
3084 static int __init cxgb3_init_module(void)
3085 {
3086         int ret;
3087
3088         cxgb3_offload_init();
3089
3090         ret = pci_register_driver(&driver);
3091         return ret;
3092 }
3093
3094 static void __exit cxgb3_cleanup_module(void)
3095 {
3096         pci_unregister_driver(&driver);
3097         if (cxgb3_wq)
3098                 destroy_workqueue(cxgb3_wq);
3099 }
3100
3101 module_init(cxgb3_init_module);
3102 module_exit(cxgb3_cleanup_module);