drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 /**
 212  *      t3_os_phymod_changed - handle PHY module changes
 213  *      @phy: the PHY reporting the module change
 214  *      @mod_type: new module type
 215  *
 216  *      This is the OS-dependent handler for PHY module changes.  It is
 217  *      invoked when a PHY module is removed or inserted for any OS-specific
 218  *      processing.
 219  */
 220 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 221 {
 222         static const char *mod_str[] = {
 223                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 224         };
 225
 226         const struct net_device *dev = adap->port[port_id];
 227         const struct port_info *pi = netdev_priv(dev);
 228
 229         if (pi->phy.modtype == phy_modtype_none)
 230                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 231         else
 232                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 233                        mod_str[pi->phy.modtype]);
 234 }
 235
 236 static void cxgb_set_rxmode(struct net_device *dev)
 237 {
 238         struct t3_rx_mode rm;
 239         struct port_info *pi = netdev_priv(dev);
 240
 241         init_rx_mode(&rm, dev, dev->mc_list);
 242         t3_mac_set_rx_mode(&pi->mac, &rm);
 243 }
 244
 245 /**
 246  *      link_start - enable a port
 247  *      @dev: the device to enable
 248  *
 249  *      Performs the MAC and PHY actions needed to enable a port.
 250  */
 251 static void link_start(struct net_device *dev)
 252 {
 253         struct t3_rx_mode rm;
 254         struct port_info *pi = netdev_priv(dev);
 255         struct cmac *mac = &pi->mac;
 256
 257         init_rx_mode(&rm, dev, dev->mc_list);
 258         t3_mac_reset(mac);
 259         t3_mac_set_mtu(mac, dev->mtu);
 260         t3_mac_set_address(mac, 0, dev->dev_addr);
 261         t3_mac_set_rx_mode(mac, &rm);
 262         t3_link_start(&pi->phy, mac, &pi->link_config);
 263         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 264 }
 265
 266 static inline void cxgb_disable_msi(struct adapter *adapter)
 267 {
 268         if (adapter->flags & USING_MSIX) {
 269                 pci_disable_msix(adapter->pdev);
 270                 adapter->flags &= ~USING_MSIX;
 271         } else if (adapter->flags & USING_MSI) {
 272                 pci_disable_msi(adapter->pdev);
 273                 adapter->flags &= ~USING_MSI;
 274         }
 275 }
 276
 277 /*
 278  * Interrupt handler for asynchronous events used with MSI-X.
 279  */
 280 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 281 {
 282         t3_slow_intr_handler(cookie);
 283         return IRQ_HANDLED;
 284 }
 285
 286 /*
 287  * Name the MSI-X interrupts.
 288  */
 289 static void name_msix_vecs(struct adapter *adap)
 290 {
 291         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 292
 293         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 294         adap->msix_info[0].desc[n] = 0;
 295
 296         for_each_port(adap, j) {
 297                 struct net_device *d = adap->port[j];
 298                 const struct port_info *pi = netdev_priv(d);
 299
 300                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 301                         snprintf(adap->msix_info[msi_idx].desc, n,
 302                                  "%s-%d", d->name, pi->first_qset + i);
 303                         adap->msix_info[msi_idx].desc[n] = 0;
 304                 }
 305         }
 306 }
 307
 308 static int request_msix_data_irqs(struct adapter *adap)
 309 {
 310         int i, j, err, qidx = 0;
 311
 312         for_each_port(adap, i) {
 313                 int nqsets = adap2pinfo(adap, i)->nqsets;
 314
 315                 for (j = 0; j < nqsets; ++j) {
 316                         err = request_irq(adap->msix_info[qidx + 1].vec,
 317                                           t3_intr_handler(adap,
 318                                                           adap->sge.qs[qidx].
 319                                                           rspq.polling), 0,
 320                                           adap->msix_info[qidx + 1].desc,
 321                                           &adap->sge.qs[qidx]);
 322                         if (err) {
 323                                 while (--qidx >= 0)
 324                                         free_irq(adap->msix_info[qidx + 1].vec,
 325                                                  &adap->sge.qs[qidx]);
 326                                 return err;
 327                         }
 328                         qidx++;
 329                 }
 330         }
 331         return 0;
 332 }
 333
 334 static void free_irq_resources(struct adapter *adapter)
 335 {
 336         if (adapter->flags & USING_MSIX) {
 337                 int i, n = 0;
 338
 339                 free_irq(adapter->msix_info[0].vec, adapter);
 340                 for_each_port(adapter, i)
 341                     n += adap2pinfo(adapter, i)->nqsets;
 342
 343                 for (i = 0; i < n; ++i)
 344                         free_irq(adapter->msix_info[i + 1].vec,
 345                                  &adapter->sge.qs[i]);
 346         } else
 347                 free_irq(adapter->pdev->irq, adapter);
 348 }
 349
 350 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 351                               unsigned long n)
 352 {
 353         int attempts = 5;
 354
 355         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 356                 if (!--attempts)
 357                         return -ETIMEDOUT;
 358                 msleep(10);
 359         }
 360         return 0;
 361 }
 362
 363 static int init_tp_parity(struct adapter *adap)
 364 {
 365         int i;
 366         struct sk_buff *skb;
 367         struct cpl_set_tcb_field *greq;
 368         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 369
 370         t3_tp_set_offload_mode(adap, 1);
 371
 372         for (i = 0; i < 16; i++) {
 373                 struct cpl_smt_write_req *req;
 374
 375                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 376                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 377                 memset(req, 0, sizeof(*req));
 378                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 379                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 380                 req->iff = i;
 381                 t3_mgmt_tx(adap, skb);
 382         }
 383
 384         for (i = 0; i < 2048; i++) {
 385                 struct cpl_l2t_write_req *req;
 386
 387                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 388                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 389                 memset(req, 0, sizeof(*req));
 390                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 391                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 392                 req->params = htonl(V_L2T_W_IDX(i));
 393                 t3_mgmt_tx(adap, skb);
 394         }
 395
 396         for (i = 0; i < 2048; i++) {
 397                 struct cpl_rte_write_req *req;
 398
 399                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 400                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 401                 memset(req, 0, sizeof(*req));
 402                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 403                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 404                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 405                 t3_mgmt_tx(adap, skb);
 406         }
 407
 408         skb = alloc_skb(sizeof(*greq), GFP_KERNEL | __GFP_NOFAIL);
 409         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 410         memset(greq, 0, sizeof(*greq));
 411         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 412         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 413         greq->mask = cpu_to_be64(1);
 414         t3_mgmt_tx(adap, skb);
 415
 416         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 417         t3_tp_set_offload_mode(adap, 0);
 418         return i;
 419 }
 420
 421 /**
 422  *      setup_rss - configure RSS
 423  *      @adap: the adapter
 424  *
 425  *      Sets up RSS to distribute packets to multiple receive queues.  We
 426  *      configure the RSS CPU lookup table to distribute to the number of HW
 427  *      receive queues, and the response queue lookup table to narrow that
 428  *      down to the response queues actually configured for each port.
 429  *      We always configure the RSS mapping for two ports since the mapping
 430  *      table has plenty of entries.
 431  */
 432 static void setup_rss(struct adapter *adap)
 433 {
 434         int i;
 435         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 436         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 437         u8 cpus[SGE_QSETS + 1];
 438         u16 rspq_map[RSS_TABLE_SIZE];
 439
 440         for (i = 0; i < SGE_QSETS; ++i)
 441                 cpus[i] = i;
 442         cpus[SGE_QSETS] = 0xff; /* terminator */
 443
 444         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 445                 rspq_map[i] = i % nq0;
 446                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 447         }
 448
 449         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 450                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 451                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 452 }
 453
 454 static void init_napi(struct adapter *adap)
 455 {
 456         int i;
 457
 458         for (i = 0; i < SGE_QSETS; i++) {
 459                 struct sge_qset *qs = &adap->sge.qs[i];
 460
 461                 if (qs->adap)
 462                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 463                                        64);
 464         }
 465
 466         /*
 467          * netif_napi_add() can be called only once per napi_struct because it
 468          * adds each new napi_struct to a list.  Be careful not to call it a
 469          * second time, e.g., during EEH recovery, by making a note of it.
 470          */
 471         adap->flags |= NAPI_INIT;
 472 }
 473
 474 /*
 475  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 476  * both netdevices representing interfaces and the dummy ones for the extra
 477  * queues.
 478  */
 479 static void quiesce_rx(struct adapter *adap)
 480 {
 481         int i;
 482
 483         for (i = 0; i < SGE_QSETS; i++)
 484                 if (adap->sge.qs[i].adap)
 485                         napi_disable(&adap->sge.qs[i].napi);
 486 }
 487
 488 static void enable_all_napi(struct adapter *adap)
 489 {
 490         int i;
 491         for (i = 0; i < SGE_QSETS; i++)
 492                 if (adap->sge.qs[i].adap)
 493                         napi_enable(&adap->sge.qs[i].napi);
 494 }
 495
 496 /**
 497  *      set_qset_lro - Turn a queue set's LRO capability on and off
 498  *      @dev: the device the qset is attached to
 499  *      @qset_idx: the queue set index
 500  *      @val: the LRO switch
 501  *
 502  *      Sets LRO on or off for a particular queue set.
 503  *      the device's features flag is updated to reflect the LRO
 504  *      capability when all queues belonging to the device are
 505  *      in the same state.
 506  */
 507 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 508 {
 509         struct port_info *pi = netdev_priv(dev);
 510         struct adapter *adapter = pi->adapter;
 511         int i, lro_on = 1;
 512
 513         adapter->params.sge.qset[qset_idx].lro = !!val;
 514         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 515
 516         /* let ethtool report LRO on only if all queues are LRO enabled */
 517         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; ++i)
 518                 lro_on &= adapter->params.sge.qset[i].lro;
 519
 520         if (lro_on)
 521                 dev->features |= NETIF_F_LRO;
 522         else
 523                 dev->features &= ~NETIF_F_LRO;
 524 }
 525
 526 /**
 527  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 528  *      @adap: the adapter
 529  *
 530  *      Determines how many sets of SGE queues to use and initializes them.
 531  *      We support multiple queue sets per port if we have MSI-X, otherwise
 532  *      just one queue set per port.
 533  */
 534 static int setup_sge_qsets(struct adapter *adap)
 535 {
 536         int i, j, err, irq_idx = 0, qset_idx = 0;
 537         unsigned int ntxq = SGE_TXQ_PER_SET;
 538
 539         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 540                 irq_idx = -1;
 541
 542         for_each_port(adap, i) {
 543                 struct net_device *dev = adap->port[i];
 544                 struct port_info *pi = netdev_priv(dev);
 545
 546                 pi->qs = &adap->sge.qs[pi->first_qset];
 547                 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets;
 548                      ++j, ++qset_idx) {
 549                         set_qset_lro(dev, qset_idx, pi->rx_csum_offload);
 550                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 551                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 552                                                              irq_idx,
 553                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 554                                 netdev_get_tx_queue(dev, j));
 555                         if (err) {
 556                                 t3_stop_sge_timers(adap);
 557                                 t3_free_sge_resources(adap);
 558                                 return err;
 559                         }
 560                 }
 561         }
 562
 563         return 0;
 564 }
 565
 566 static ssize_t attr_show(struct device *d, char *buf,
 567                          ssize_t(*format) (struct net_device *, char *))
 568 {
 569         ssize_t len;
 570
 571         /* Synchronize with ioctls that may shut down the device */
 572         rtnl_lock();
 573         len = (*format) (to_net_dev(d), buf);
 574         rtnl_unlock();
 575         return len;
 576 }
 577
 578 static ssize_t attr_store(struct device *d,
 579                           const char *buf, size_t len,
 580                           ssize_t(*set) (struct net_device *, unsigned int),
 581                           unsigned int min_val, unsigned int max_val)
 582 {
 583         char *endp;
 584         ssize_t ret;
 585         unsigned int val;
 586
 587         if (!capable(CAP_NET_ADMIN))
 588                 return -EPERM;
 589
 590         val = simple_strtoul(buf, &endp, 0);
 591         if (endp == buf || val < min_val || val > max_val)
 592                 return -EINVAL;
 593
 594         rtnl_lock();
 595         ret = (*set) (to_net_dev(d), val);
 596         if (!ret)
 597                 ret = len;
 598         rtnl_unlock();
 599         return ret;
 600 }
 601
 602 #define CXGB3_SHOW(name, val_expr) \
 603 static ssize_t format_##name(struct net_device *dev, char *buf) \
 604 { \
 605         struct port_info *pi = netdev_priv(dev); \
 606         struct adapter *adap = pi->adapter; \
 607         return sprintf(buf, "%u\n", val_expr); \
 608 } \
 609 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 610                            char *buf) \
 611 { \
 612         return attr_show(d, buf, format_##name); \
 613 }
 614
 615 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 616 {
 617         struct port_info *pi = netdev_priv(dev);
 618         struct adapter *adap = pi->adapter;
 619         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 620
 621         if (adap->flags & FULL_INIT_DONE)
 622                 return -EBUSY;
 623         if (val && adap->params.rev == 0)
 624                 return -EINVAL;
 625         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 626             min_tids)
 627                 return -EINVAL;
 628         adap->params.mc5.nfilters = val;
 629         return 0;
 630 }
 631
 632 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 633                               const char *buf, size_t len)
 634 {
 635         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 636 }
 637
 638 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 639 {
 640         struct port_info *pi = netdev_priv(dev);
 641         struct adapter *adap = pi->adapter;
 642
 643         if (adap->flags & FULL_INIT_DONE)
 644                 return -EBUSY;
 645         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 646             MC5_MIN_TIDS)
 647                 return -EINVAL;
 648         adap->params.mc5.nservers = val;
 649         return 0;
 650 }
 651
 652 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 653                               const char *buf, size_t len)
 654 {
 655         return attr_store(d, buf, len, set_nservers, 0, ~0);
 656 }
 657
 658 #define CXGB3_ATTR_R(name, val_expr) \
 659 CXGB3_SHOW(name, val_expr) \
 660 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 661
 662 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 663 CXGB3_SHOW(name, val_expr) \
 664 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 665
 666 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 667 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 668 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 669
 670 static struct attribute *cxgb3_attrs[] = {
 671         &dev_attr_cam_size.attr,
 672         &dev_attr_nfilters.attr,
 673         &dev_attr_nservers.attr,
 674         NULL
 675 };
 676
 677 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 678
 679 static ssize_t tm_attr_show(struct device *d,
 680                             char *buf, int sched)
 681 {
 682         struct port_info *pi = netdev_priv(to_net_dev(d));
 683         struct adapter *adap = pi->adapter;
 684         unsigned int v, addr, bpt, cpt;
 685         ssize_t len;
 686
 687         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 688         rtnl_lock();
 689         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 690         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 691         if (sched & 1)
 692                 v >>= 16;
 693         bpt = (v >> 8) & 0xff;
 694         cpt = v & 0xff;
 695         if (!cpt)
 696                 len = sprintf(buf, "disabled\n");
 697         else {
 698                 v = (adap->params.vpd.cclk * 1000) / cpt;
 699                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 700         }
 701         rtnl_unlock();
 702         return len;
 703 }
 704
 705 static ssize_t tm_attr_store(struct device *d,
 706                              const char *buf, size_t len, int sched)
 707 {
 708         struct port_info *pi = netdev_priv(to_net_dev(d));
 709         struct adapter *adap = pi->adapter;
 710         unsigned int val;
 711         char *endp;
 712         ssize_t ret;
 713
 714         if (!capable(CAP_NET_ADMIN))
 715                 return -EPERM;
 716
 717         val = simple_strtoul(buf, &endp, 0);
 718         if (endp == buf || val > 10000000)
 719                 return -EINVAL;
 720
 721         rtnl_lock();
 722         ret = t3_config_sched(adap, val, sched);
 723         if (!ret)
 724                 ret = len;
 725         rtnl_unlock();
 726         return ret;
 727 }
 728
 729 #define TM_ATTR(name, sched) \
 730 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 731                            char *buf) \
 732 { \
 733         return tm_attr_show(d, buf, sched); \
 734 } \
 735 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 736                             const char *buf, size_t len) \
 737 { \
 738         return tm_attr_store(d, buf, len, sched); \
 739 } \
 740 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 741
 742 TM_ATTR(sched0, 0);
 743 TM_ATTR(sched1, 1);
 744 TM_ATTR(sched2, 2);
 745 TM_ATTR(sched3, 3);
 746 TM_ATTR(sched4, 4);
 747 TM_ATTR(sched5, 5);
 748 TM_ATTR(sched6, 6);
 749 TM_ATTR(sched7, 7);
 750
 751 static struct attribute *offload_attrs[] = {
 752         &dev_attr_sched0.attr,
 753         &dev_attr_sched1.attr,
 754         &dev_attr_sched2.attr,
 755         &dev_attr_sched3.attr,
 756         &dev_attr_sched4.attr,
 757         &dev_attr_sched5.attr,
 758         &dev_attr_sched6.attr,
 759         &dev_attr_sched7.attr,
 760         NULL
 761 };
 762
 763 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 764
 765 /*
 766  * Sends an sk_buff to an offload queue driver
 767  * after dealing with any active network taps.
 768  */
 769 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 770 {
 771         int ret;
 772
 773         local_bh_disable();
 774         ret = t3_offload_tx(tdev, skb);
 775         local_bh_enable();
 776         return ret;
 777 }
 778
 779 static int write_smt_entry(struct adapter *adapter, int idx)
 780 {
 781         struct cpl_smt_write_req *req;
 782         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 783
 784         if (!skb)
 785                 return -ENOMEM;
 786
 787         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 788         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 789         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 790         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 791         req->iff = idx;
 792         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 793         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 794         skb->priority = 1;
 795         offload_tx(&adapter->tdev, skb);
 796         return 0;
 797 }
 798
 799 static int init_smt(struct adapter *adapter)
 800 {
 801         int i;
 802
 803         for_each_port(adapter, i)
 804             write_smt_entry(adapter, i);
 805         return 0;
 806 }
 807
 808 static void init_port_mtus(struct adapter *adapter)
 809 {
 810         unsigned int mtus = adapter->port[0]->mtu;
 811
 812         if (adapter->port[1])
 813                 mtus |= adapter->port[1]->mtu << 16;
 814         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 815 }
 816
 817 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 818                               int hi, int port)
 819 {
 820         struct sk_buff *skb;
 821         struct mngt_pktsched_wr *req;
 822         int ret;
 823
 824         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 825         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 826         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 827         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 828         req->sched = sched;
 829         req->idx = qidx;
 830         req->min = lo;
 831         req->max = hi;
 832         req->binding = port;
 833         ret = t3_mgmt_tx(adap, skb);
 834
 835         return ret;
 836 }
 837
 838 static int bind_qsets(struct adapter *adap)
 839 {
 840         int i, j, err = 0;
 841
 842         for_each_port(adap, i) {
 843                 const struct port_info *pi = adap2pinfo(adap, i);
 844
 845                 for (j = 0; j < pi->nqsets; ++j) {
 846                         int ret = send_pktsched_cmd(adap, 1,
 847                                                     pi->first_qset + j, -1,
 848                                                     -1, i);
 849                         if (ret)
 850                                 err = ret;
 851                 }
 852         }
 853
 854         return err;
 855 }
 856
 857 #define FW_FNAME "cxgb3/t3fw-%d.%d.%d.bin"
 858 #define TPSRAM_NAME "cxgb3/t3%c_psram-%d.%d.%d.bin"
 859
 860 static int upgrade_fw(struct adapter *adap)
 861 {
 862         int ret;
 863         char buf[64];
 864         const struct firmware *fw;
 865         struct device *dev = &adap->pdev->dev;
 866
 867         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 868                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 869         ret = request_firmware(&fw, buf, dev);
 870         if (ret < 0) {
 871                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 872                         buf);
 873                 return ret;
 874         }
 875         ret = t3_load_fw(adap, fw->data, fw->size);
 876         release_firmware(fw);
 877
 878         if (ret == 0)
 879                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 880                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 881         else
 882                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 883                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 884
 885         return ret;
 886 }
 887
 888 static inline char t3rev2char(struct adapter *adapter)
 889 {
 890         char rev = 0;
 891
 892         switch(adapter->params.rev) {
 893         case T3_REV_B:
 894         case T3_REV_B2:
 895                 rev = 'b';
 896                 break;
 897         case T3_REV_C:
 898                 rev = 'c';
 899                 break;
 900         }
 901         return rev;
 902 }
 903
 904 static int update_tpsram(struct adapter *adap)
 905 {
 906         const struct firmware *tpsram;
 907         char buf[64];
 908         struct device *dev = &adap->pdev->dev;
 909         int ret;
 910         char rev;
 911
 912         rev = t3rev2char(adap);
 913         if (!rev)
 914                 return 0;
 915
 916         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 917                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 918
 919         ret = request_firmware(&tpsram, buf, dev);
 920         if (ret < 0) {
 921                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 922                         buf);
 923                 return ret;
 924         }
 925
 926         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 927         if (ret)
 928                 goto release_tpsram;
 929
 930         ret = t3_set_proto_sram(adap, tpsram->data);
 931         if (ret == 0)
 932                 dev_info(dev,
 933                          "successful update of protocol engine "
 934                          "to %d.%d.%d\n",
 935                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 936         else
 937                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 938                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 939         if (ret)
 940                 dev_err(dev, "loading protocol SRAM failed\n");
 941
 942 release_tpsram:
 943         release_firmware(tpsram);
 944
 945         return ret;
 946 }
 947
 948 /**
 949  *      cxgb_up - enable the adapter
 950  *      @adapter: adapter being enabled
 951  *
 952  *      Called when the first port is enabled, this function performs the
 953  *      actions necessary to make an adapter operational, such as completing
 954  *      the initialization of HW modules, and enabling interrupts.
 955  *
 956  *      Must be called with the rtnl lock held.
 957  */
 958 static int cxgb_up(struct adapter *adap)
 959 {
 960         int err;
 961
 962         if (!(adap->flags & FULL_INIT_DONE)) {
 963                 err = t3_check_fw_version(adap);
 964                 if (err == -EINVAL) {
 965                         err = upgrade_fw(adap);
 966                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
 967                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
 968                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
 969                 }
 970
 971                 err = t3_check_tpsram_version(adap);
 972                 if (err == -EINVAL) {
 973                         err = update_tpsram(adap);
 974                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
 975                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
 976                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
 977                 }
 978
 979                 /*
 980                  * Clear interrupts now to catch errors if t3_init_hw fails.
 981                  * We clear them again later as initialization may trigger
 982                  * conditions that can interrupt.
 983                  */
 984                 t3_intr_clear(adap);
 985
 986                 err = t3_init_hw(adap, 0);
 987                 if (err)
 988                         goto out;
 989
 990                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 991                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 992
 993                 err = setup_sge_qsets(adap);
 994                 if (err)
 995                         goto out;
 996
 997                 setup_rss(adap);
 998                 if (!(adap->flags & NAPI_INIT))
 999                         init_napi(adap);
1000                 adap->flags |= FULL_INIT_DONE;
1001         }
1002
1003         t3_intr_clear(adap);
1004
1005         if (adap->flags & USING_MSIX) {
1006                 name_msix_vecs(adap);
1007                 err = request_irq(adap->msix_info[0].vec,
1008                                   t3_async_intr_handler, 0,
1009                                   adap->msix_info[0].desc, adap);
1010                 if (err)
1011                         goto irq_err;
1012
1013                 err = request_msix_data_irqs(adap);
1014                 if (err) {
1015                         free_irq(adap->msix_info[0].vec, adap);
1016                         goto irq_err;
1017                 }
1018         } else if ((err = request_irq(adap->pdev->irq,
1019                                       t3_intr_handler(adap,
1020                                                       adap->sge.qs[0].rspq.
1021                                                       polling),
1022                                       (adap->flags & USING_MSI) ?
1023                                        0 : IRQF_SHARED,
1024                                       adap->name, adap)))
1025                 goto irq_err;
1026
1027         enable_all_napi(adap);
1028         t3_sge_start(adap);
1029         t3_intr_enable(adap);
1030
1031         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1032             is_offload(adap) && init_tp_parity(adap) == 0)
1033                 adap->flags |= TP_PARITY_INIT;
1034
1035         if (adap->flags & TP_PARITY_INIT) {
1036                 t3_write_reg(adap, A_TP_INT_CAUSE,
1037                              F_CMCACHEPERR | F_ARPLUTPERR);
1038                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1039         }
1040
1041         if (!(adap->flags & QUEUES_BOUND)) {
1042                 err = bind_qsets(adap);
1043                 if (err) {
1044                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1045                         t3_intr_disable(adap);
1046                         free_irq_resources(adap);
1047                         goto out;
1048                 }
1049                 adap->flags |= QUEUES_BOUND;
1050         }
1051
1052 out:
1053         return err;
1054 irq_err:
1055         CH_ERR(adap, "request_irq failed, err %d\n", err);
1056         goto out;
1057 }
1058
1059 /*
1060  * Release resources when all the ports and offloading have been stopped.
1061  */
1062 static void cxgb_down(struct adapter *adapter)
1063 {
1064         t3_sge_stop(adapter);
1065         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1066         t3_intr_disable(adapter);
1067         spin_unlock_irq(&adapter->work_lock);
1068
1069         free_irq_resources(adapter);
1070         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1071         quiesce_rx(adapter);
1072 }
1073
1074 static void schedule_chk_task(struct adapter *adap)
1075 {
1076         unsigned int timeo;
1077
1078         timeo = adap->params.linkpoll_period ?
1079             (HZ * adap->params.linkpoll_period) / 10 :
1080             adap->params.stats_update_period * HZ;
1081         if (timeo)
1082                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1083 }
1084
1085 static int offload_open(struct net_device *dev)
1086 {
1087         struct port_info *pi = netdev_priv(dev);
1088         struct adapter *adapter = pi->adapter;
1089         struct t3cdev *tdev = dev2t3cdev(dev);
1090         int adap_up = adapter->open_device_map & PORT_MASK;
1091         int err;
1092
1093         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1094                 return 0;
1095
1096         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1097                 goto out;
1098
1099         t3_tp_set_offload_mode(adapter, 1);
1100         tdev->lldev = adapter->port[0];
1101         err = cxgb3_offload_activate(adapter);
1102         if (err)
1103                 goto out;
1104
1105         init_port_mtus(adapter);
1106         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1107                      adapter->params.b_wnd,
1108                      adapter->params.rev == 0 ?
1109                      adapter->port[0]->mtu : 0xffff);
1110         init_smt(adapter);
1111
1112         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1113                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1114
1115         /* Call back all registered clients */
1116         cxgb3_add_clients(tdev);
1117
1118 out:
1119         /* restore them in case the offload module has changed them */
1120         if (err) {
1121                 t3_tp_set_offload_mode(adapter, 0);
1122                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1123                 cxgb3_set_dummy_ops(tdev);
1124         }
1125         return err;
1126 }
1127
1128 static int offload_close(struct t3cdev *tdev)
1129 {
1130         struct adapter *adapter = tdev2adap(tdev);
1131
1132         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1133                 return 0;
1134
1135         /* Call back all registered clients */
1136         cxgb3_remove_clients(tdev);
1137
1138         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1139
1140         tdev->lldev = NULL;
1141         cxgb3_set_dummy_ops(tdev);
1142         t3_tp_set_offload_mode(adapter, 0);
1143         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1144
1145         if (!adapter->open_device_map)
1146                 cxgb_down(adapter);
1147
1148         cxgb3_offload_deactivate(adapter);
1149         return 0;
1150 }
1151
1152 static int cxgb_open(struct net_device *dev)
1153 {
1154         struct port_info *pi = netdev_priv(dev);
1155         struct adapter *adapter = pi->adapter;
1156         int other_ports = adapter->open_device_map & PORT_MASK;
1157         int err;
1158
1159         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1160                 return err;
1161
1162         set_bit(pi->port_id, &adapter->open_device_map);
1163         if (is_offload(adapter) && !ofld_disable) {
1164                 err = offload_open(dev);
1165                 if (err)
1166                         printk(KERN_WARNING
1167                                "Could not initialize offload capabilities\n");
1168         }
1169
1170         dev->real_num_tx_queues = pi->nqsets;
1171         link_start(dev);
1172         t3_port_intr_enable(adapter, pi->port_id);
1173         netif_tx_start_all_queues(dev);
1174         if (!other_ports)
1175                 schedule_chk_task(adapter);
1176
1177         return 0;
1178 }
1179
1180 static int cxgb_close(struct net_device *dev)
1181 {
1182         struct port_info *pi = netdev_priv(dev);
1183         struct adapter *adapter = pi->adapter;
1184
1185         t3_port_intr_disable(adapter, pi->port_id);
1186         netif_tx_stop_all_queues(dev);
1187         pi->phy.ops->power_down(&pi->phy, 1);
1188         netif_carrier_off(dev);
1189         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1190
1191         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1192         clear_bit(pi->port_id, &adapter->open_device_map);
1193         spin_unlock_irq(&adapter->work_lock);
1194
1195         if (!(adapter->open_device_map & PORT_MASK))
1196                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1197                                                   &adapter->adap_check_task);
1198
1199         if (!adapter->open_device_map)
1200                 cxgb_down(adapter);
1201
1202         return 0;
1203 }
1204
1205 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1206 {
1207         struct port_info *pi = netdev_priv(dev);
1208         struct adapter *adapter = pi->adapter;
1209         struct net_device_stats *ns = &pi->netstats;
1210         const struct mac_stats *pstats;
1211
1212         spin_lock(&adapter->stats_lock);
1213         pstats = t3_mac_update_stats(&pi->mac);
1214         spin_unlock(&adapter->stats_lock);
1215
1216         ns->tx_bytes = pstats->tx_octets;
1217         ns->tx_packets = pstats->tx_frames;
1218         ns->rx_bytes = pstats->rx_octets;
1219         ns->rx_packets = pstats->rx_frames;
1220         ns->multicast = pstats->rx_mcast_frames;
1221
1222         ns->tx_errors = pstats->tx_underrun;
1223         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1224             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1225             pstats->rx_fifo_ovfl;
1226
1227         /* detailed rx_errors */
1228         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1229         ns->rx_over_errors = 0;
1230         ns->rx_crc_errors = pstats->rx_fcs_errs;
1231         ns->rx_frame_errors = pstats->rx_symbol_errs;
1232         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1233         ns->rx_missed_errors = pstats->rx_cong_drops;
1234
1235         /* detailed tx_errors */
1236         ns->tx_aborted_errors = 0;
1237         ns->tx_carrier_errors = 0;
1238         ns->tx_fifo_errors = pstats->tx_underrun;
1239         ns->tx_heartbeat_errors = 0;
1240         ns->tx_window_errors = 0;
1241         return ns;
1242 }
1243
1244 static u32 get_msglevel(struct net_device *dev)
1245 {
1246         struct port_info *pi = netdev_priv(dev);
1247         struct adapter *adapter = pi->adapter;
1248
1249         return adapter->msg_enable;
1250 }
1251
1252 static void set_msglevel(struct net_device *dev, u32 val)
1253 {
1254         struct port_info *pi = netdev_priv(dev);
1255         struct adapter *adapter = pi->adapter;
1256
1257         adapter->msg_enable = val;
1258 }
1259
1260 static char stats_strings[][ETH_GSTRING_LEN] = {
1261         "TxOctetsOK         ",
1262         "TxFramesOK         ",
1263         "TxMulticastFramesOK",
1264         "TxBroadcastFramesOK",
1265         "TxPauseFrames      ",
1266         "TxUnderrun         ",
1267         "TxExtUnderrun      ",
1268
1269         "TxFrames64         ",
1270         "TxFrames65To127    ",
1271         "TxFrames128To255   ",
1272         "TxFrames256To511   ",
1273         "TxFrames512To1023  ",
1274         "TxFrames1024To1518 ",
1275         "TxFrames1519ToMax  ",
1276
1277         "RxOctetsOK         ",
1278         "RxFramesOK         ",
1279         "RxMulticastFramesOK",
1280         "RxBroadcastFramesOK",
1281         "RxPauseFrames      ",
1282         "RxFCSErrors        ",
1283         "RxSymbolErrors     ",
1284         "RxShortErrors      ",
1285         "RxJabberErrors     ",
1286         "RxLengthErrors     ",
1287         "RxFIFOoverflow     ",
1288
1289         "RxFrames64         ",
1290         "RxFrames65To127    ",
1291         "RxFrames128To255   ",
1292         "RxFrames256To511   ",
1293         "RxFrames512To1023  ",
1294         "RxFrames1024To1518 ",
1295         "RxFrames1519ToMax  ",
1296
1297         "PhyFIFOErrors      ",
1298         "TSO                ",
1299         "VLANextractions    ",
1300         "VLANinsertions     ",
1301         "TxCsumOffload      ",
1302         "RxCsumGood         ",
1303         "LroAggregated      ",
1304         "LroFlushed         ",
1305         "LroNoDesc          ",
1306         "RxDrops            ",
1307
1308         "CheckTXEnToggled   ",
1309         "CheckResets        ",
1310
1311 };
1312
1313 static int get_sset_count(struct net_device *dev, int sset)
1314 {
1315         switch (sset) {
1316         case ETH_SS_STATS:
1317                 return ARRAY_SIZE(stats_strings);
1318         default:
1319                 return -EOPNOTSUPP;
1320         }
1321 }
1322
1323 #define T3_REGMAP_SIZE (3 * 1024)
1324
1325 static int get_regs_len(struct net_device *dev)
1326 {
1327         return T3_REGMAP_SIZE;
1328 }
1329
1330 static int get_eeprom_len(struct net_device *dev)
1331 {
1332         return EEPROMSIZE;
1333 }
1334
1335 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1336 {
1337         struct port_info *pi = netdev_priv(dev);
1338         struct adapter *adapter = pi->adapter;
1339         u32 fw_vers = 0;
1340         u32 tp_vers = 0;
1341
1342         spin_lock(&adapter->stats_lock);
1343         t3_get_fw_version(adapter, &fw_vers);
1344         t3_get_tp_version(adapter, &tp_vers);
1345         spin_unlock(&adapter->stats_lock);
1346
1347         strcpy(info->driver, DRV_NAME);
1348         strcpy(info->version, DRV_VERSION);
1349         strcpy(info->bus_info, pci_name(adapter->pdev));
1350         if (!fw_vers)
1351                 strcpy(info->fw_version, "N/A");
1352         else {
1353                 snprintf(info->fw_version, sizeof(info->fw_version),
1354                          "%s %u.%u.%u TP %u.%u.%u",
1355                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1356                          G_FW_VERSION_MAJOR(fw_vers),
1357                          G_FW_VERSION_MINOR(fw_vers),
1358                          G_FW_VERSION_MICRO(fw_vers),
1359                          G_TP_VERSION_MAJOR(tp_vers),
1360                          G_TP_VERSION_MINOR(tp_vers),
1361                          G_TP_VERSION_MICRO(tp_vers));
1362         }
1363 }
1364
1365 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1366 {
1367         if (stringset == ETH_SS_STATS)
1368                 memcpy(data, stats_strings, sizeof(stats_strings));
1369 }
1370
1371 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1372                                             struct port_info *p, int idx)
1373 {
1374         int i;
1375         unsigned long tot = 0;
1376
1377         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1378                 tot += adapter->sge.qs[i].port_stats[idx];
1379         return tot;
1380 }
1381
1382 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1383                       u64 *data)
1384 {
1385         struct port_info *pi = netdev_priv(dev);
1386         struct adapter *adapter = pi->adapter;
1387         const struct mac_stats *s;
1388
1389         spin_lock(&adapter->stats_lock);
1390         s = t3_mac_update_stats(&pi->mac);
1391         spin_unlock(&adapter->stats_lock);
1392
1393         *data++ = s->tx_octets;
1394         *data++ = s->tx_frames;
1395         *data++ = s->tx_mcast_frames;
1396         *data++ = s->tx_bcast_frames;
1397         *data++ = s->tx_pause;
1398         *data++ = s->tx_underrun;
1399         *data++ = s->tx_fifo_urun;
1400
1401         *data++ = s->tx_frames_64;
1402         *data++ = s->tx_frames_65_127;
1403         *data++ = s->tx_frames_128_255;
1404         *data++ = s->tx_frames_256_511;
1405         *data++ = s->tx_frames_512_1023;
1406         *data++ = s->tx_frames_1024_1518;
1407         *data++ = s->tx_frames_1519_max;
1408
1409         *data++ = s->rx_octets;
1410         *data++ = s->rx_frames;
1411         *data++ = s->rx_mcast_frames;
1412         *data++ = s->rx_bcast_frames;
1413         *data++ = s->rx_pause;
1414         *data++ = s->rx_fcs_errs;
1415         *data++ = s->rx_symbol_errs;
1416         *data++ = s->rx_short;
1417         *data++ = s->rx_jabber;
1418         *data++ = s->rx_too_long;
1419         *data++ = s->rx_fifo_ovfl;
1420
1421         *data++ = s->rx_frames_64;
1422         *data++ = s->rx_frames_65_127;
1423         *data++ = s->rx_frames_128_255;
1424         *data++ = s->rx_frames_256_511;
1425         *data++ = s->rx_frames_512_1023;
1426         *data++ = s->rx_frames_1024_1518;
1427         *data++ = s->rx_frames_1519_max;
1428
1429         *data++ = pi->phy.fifo_errors;
1430
1431         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1432         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1433         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1434         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1435         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1436         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_AGGR);
1437         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_FLUSHED);
1438         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_NO_DESC);
1439         *data++ = s->rx_cong_drops;
1440
1441         *data++ = s->num_toggled;
1442         *data++ = s->num_resets;
1443 }
1444
1445 static inline void reg_block_dump(struct adapter *ap, void *buf,
1446                                   unsigned int start, unsigned int end)
1447 {
1448         u32 *p = buf + start;
1449
1450         for (; start <= end; start += sizeof(u32))
1451                 *p++ = t3_read_reg(ap, start);
1452 }
1453
1454 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1455                      void *buf)
1456 {
1457         struct port_info *pi = netdev_priv(dev);
1458         struct adapter *ap = pi->adapter;
1459
1460         /*
1461          * Version scheme:
1462          * bits 0..9: chip version
1463          * bits 10..15: chip revision
1464          * bit 31: set for PCIe cards
1465          */
1466         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1467
1468         /*
1469          * We skip the MAC statistics registers because they are clear-on-read.
1470          * Also reading multi-register stats would need to synchronize with the
1471          * periodic mac stats accumulation.  Hard to justify the complexity.
1472          */
1473         memset(buf, 0, T3_REGMAP_SIZE);
1474         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1475         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1476         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1477         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1478         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1479         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1480                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1481         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1482                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1483 }
1484
1485 static int restart_autoneg(struct net_device *dev)
1486 {
1487         struct port_info *p = netdev_priv(dev);
1488
1489         if (!netif_running(dev))
1490                 return -EAGAIN;
1491         if (p->link_config.autoneg != AUTONEG_ENABLE)
1492                 return -EINVAL;
1493         p->phy.ops->autoneg_restart(&p->phy);
1494         return 0;
1495 }
1496
1497 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1498 {
1499         struct port_info *pi = netdev_priv(dev);
1500         struct adapter *adapter = pi->adapter;
1501         int i;
1502
1503         if (data == 0)
1504                 data = 2;
1505
1506         for (i = 0; i < data * 2; i++) {
1507                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1508                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1509                 if (msleep_interruptible(500))
1510                         break;
1511         }
1512         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1513                          F_GPIO0_OUT_VAL);
1514         return 0;
1515 }
1516
1517 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1518 {
1519         struct port_info *p = netdev_priv(dev);
1520
1521         cmd->supported = p->link_config.supported;
1522         cmd->advertising = p->link_config.advertising;
1523
1524         if (netif_carrier_ok(dev)) {
1525                 cmd->speed = p->link_config.speed;
1526                 cmd->duplex = p->link_config.duplex;
1527         } else {
1528                 cmd->speed = -1;
1529                 cmd->duplex = -1;
1530         }
1531
1532         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1533         cmd->phy_address = p->phy.addr;
1534         cmd->transceiver = XCVR_EXTERNAL;
1535         cmd->autoneg = p->link_config.autoneg;
1536         cmd->maxtxpkt = 0;
1537         cmd->maxrxpkt = 0;
1538         return 0;
1539 }
1540
1541 static int speed_duplex_to_caps(int speed, int duplex)
1542 {
1543         int cap = 0;
1544
1545         switch (speed) {
1546         case SPEED_10:
1547                 if (duplex == DUPLEX_FULL)
1548                         cap = SUPPORTED_10baseT_Full;
1549                 else
1550                         cap = SUPPORTED_10baseT_Half;
1551                 break;
1552         case SPEED_100:
1553                 if (duplex == DUPLEX_FULL)
1554                         cap = SUPPORTED_100baseT_Full;
1555                 else
1556                         cap = SUPPORTED_100baseT_Half;
1557                 break;
1558         case SPEED_1000:
1559                 if (duplex == DUPLEX_FULL)
1560                         cap = SUPPORTED_1000baseT_Full;
1561                 else
1562                         cap = SUPPORTED_1000baseT_Half;
1563                 break;
1564         case SPEED_10000:
1565                 if (duplex == DUPLEX_FULL)
1566                         cap = SUPPORTED_10000baseT_Full;
1567         }
1568         return cap;
1569 }
1570
1571 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1572                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1573                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1574                       ADVERTISED_10000baseT_Full)
1575
1576 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1577 {
1578         int cap;
1579         struct port_info *p = netdev_priv(dev);
1580         struct link_config *lc = &p->link_config;
1581
1582         if (!(lc->supported & SUPPORTED_Autoneg)) {
1583                 /*
1584                  * PHY offers a single speed/duplex.  See if that's what's
1585                  * being requested.
1586                  */
1587                 if (cmd->autoneg == AUTONEG_DISABLE) {
1588                         cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1589                         if (lc->supported & cap)
1590                                 return 0;
1591                 }
1592                 return -EINVAL;
1593         }
1594
1595         if (cmd->autoneg == AUTONEG_DISABLE) {
1596                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1597
1598                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1599                         return -EINVAL;
1600                 lc->requested_speed = cmd->speed;
1601                 lc->requested_duplex = cmd->duplex;
1602                 lc->advertising = 0;
1603         } else {
1604                 cmd->advertising &= ADVERTISED_MASK;
1605                 cmd->advertising &= lc->supported;
1606                 if (!cmd->advertising)
1607                         return -EINVAL;
1608                 lc->requested_speed = SPEED_INVALID;
1609                 lc->requested_duplex = DUPLEX_INVALID;
1610                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1611         }
1612         lc->autoneg = cmd->autoneg;
1613         if (netif_running(dev))
1614                 t3_link_start(&p->phy, &p->mac, lc);
1615         return 0;
1616 }
1617
1618 static void get_pauseparam(struct net_device *dev,
1619                            struct ethtool_pauseparam *epause)
1620 {
1621         struct port_info *p = netdev_priv(dev);
1622
1623         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1624         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1625         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1626 }
1627
1628 static int set_pauseparam(struct net_device *dev,
1629                           struct ethtool_pauseparam *epause)
1630 {
1631         struct port_info *p = netdev_priv(dev);
1632         struct link_config *lc = &p->link_config;
1633
1634         if (epause->autoneg == AUTONEG_DISABLE)
1635                 lc->requested_fc = 0;
1636         else if (lc->supported & SUPPORTED_Autoneg)
1637                 lc->requested_fc = PAUSE_AUTONEG;
1638         else
1639                 return -EINVAL;
1640
1641         if (epause->rx_pause)
1642                 lc->requested_fc |= PAUSE_RX;
1643         if (epause->tx_pause)
1644                 lc->requested_fc |= PAUSE_TX;
1645         if (lc->autoneg == AUTONEG_ENABLE) {
1646                 if (netif_running(dev))
1647                         t3_link_start(&p->phy, &p->mac, lc);
1648         } else {
1649                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1650                 if (netif_running(dev))
1651                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1652         }
1653         return 0;
1654 }
1655
1656 static u32 get_rx_csum(struct net_device *dev)
1657 {
1658         struct port_info *p = netdev_priv(dev);
1659
1660         return p->rx_csum_offload;
1661 }
1662
1663 static int set_rx_csum(struct net_device *dev, u32 data)
1664 {
1665         struct port_info *p = netdev_priv(dev);
1666
1667         p->rx_csum_offload = data;
1668         if (!data) {
1669                 int i;
1670
1671                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1672                         set_qset_lro(dev, i, 0);
1673         }
1674         return 0;
1675 }
1676
1677 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1678 {
1679         struct port_info *pi = netdev_priv(dev);
1680         struct adapter *adapter = pi->adapter;
1681         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1682
1683         e->rx_max_pending = MAX_RX_BUFFERS;
1684         e->rx_mini_max_pending = 0;
1685         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1686         e->tx_max_pending = MAX_TXQ_ENTRIES;
1687
1688         e->rx_pending = q->fl_size;
1689         e->rx_mini_pending = q->rspq_size;
1690         e->rx_jumbo_pending = q->jumbo_size;
1691         e->tx_pending = q->txq_size[0];
1692 }
1693
1694 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1695 {
1696         struct port_info *pi = netdev_priv(dev);
1697         struct adapter *adapter = pi->adapter;
1698         struct qset_params *q;
1699         int i;
1700
1701         if (e->rx_pending > MAX_RX_BUFFERS ||
1702             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1703             e->tx_pending > MAX_TXQ_ENTRIES ||
1704             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1705             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1706             e->rx_pending < MIN_FL_ENTRIES ||
1707             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1708             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1709                 return -EINVAL;
1710
1711         if (adapter->flags & FULL_INIT_DONE)
1712                 return -EBUSY;
1713
1714         q = &adapter->params.sge.qset[pi->first_qset];
1715         for (i = 0; i < pi->nqsets; ++i, ++q) {
1716                 q->rspq_size = e->rx_mini_pending;
1717                 q->fl_size = e->rx_pending;
1718                 q->jumbo_size = e->rx_jumbo_pending;
1719                 q->txq_size[0] = e->tx_pending;
1720                 q->txq_size[1] = e->tx_pending;
1721                 q->txq_size[2] = e->tx_pending;
1722         }
1723         return 0;
1724 }
1725
1726 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1727 {
1728         struct port_info *pi = netdev_priv(dev);
1729         struct adapter *adapter = pi->adapter;
1730         struct qset_params *qsp = &adapter->params.sge.qset[0];
1731         struct sge_qset *qs = &adapter->sge.qs[0];
1732
1733         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1734                 return -EINVAL;
1735
1736         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1737         t3_update_qset_coalesce(qs, qsp);
1738         return 0;
1739 }
1740
1741 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1742 {
1743         struct port_info *pi = netdev_priv(dev);
1744         struct adapter *adapter = pi->adapter;
1745         struct qset_params *q = adapter->params.sge.qset;
1746
1747         c->rx_coalesce_usecs = q->coalesce_usecs;
1748         return 0;
1749 }
1750
1751 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1752                       u8 * data)
1753 {
1754         struct port_info *pi = netdev_priv(dev);
1755         struct adapter *adapter = pi->adapter;
1756         int i, err = 0;
1757
1758         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1759         if (!buf)
1760                 return -ENOMEM;
1761
1762         e->magic = EEPROM_MAGIC;
1763         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1764                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1765
1766         if (!err)
1767                 memcpy(data, buf + e->offset, e->len);
1768         kfree(buf);
1769         return err;
1770 }
1771
1772 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1773                       u8 * data)
1774 {
1775         struct port_info *pi = netdev_priv(dev);
1776         struct adapter *adapter = pi->adapter;
1777         u32 aligned_offset, aligned_len;
1778         __le32 *p;
1779         u8 *buf;
1780         int err;
1781
1782         if (eeprom->magic != EEPROM_MAGIC)
1783                 return -EINVAL;
1784
1785         aligned_offset = eeprom->offset & ~3;
1786         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1787
1788         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1789                 buf = kmalloc(aligned_len, GFP_KERNEL);
1790                 if (!buf)
1791                         return -ENOMEM;
1792                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
1793                 if (!err && aligned_len > 4)
1794                         err = t3_seeprom_read(adapter,
1795                                               aligned_offset + aligned_len - 4,
1796                                               (__le32 *) & buf[aligned_len - 4]);
1797                 if (err)
1798                         goto out;
1799                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1800         } else
1801                 buf = data;
1802
1803         err = t3_seeprom_wp(adapter, 0);
1804         if (err)
1805                 goto out;
1806
1807         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1808                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1809                 aligned_offset += 4;
1810         }
1811
1812         if (!err)
1813                 err = t3_seeprom_wp(adapter, 1);
1814 out:
1815         if (buf != data)
1816                 kfree(buf);
1817         return err;
1818 }
1819
1820 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1821 {
1822         wol->supported = 0;
1823         wol->wolopts = 0;
1824         memset(&wol->sopass, 0, sizeof(wol->sopass));
1825 }
1826
1827 static int cxgb3_set_flags(struct net_device *dev, u32 data)
1828 {
1829         struct port_info *pi = netdev_priv(dev);
1830         int i;
1831
1832         if (data & ETH_FLAG_LRO) {
1833                 if (!pi->rx_csum_offload)
1834                         return -EINVAL;
1835
1836                 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
1837                         set_qset_lro(dev, i, 1);
1838
1839         } else
1840                 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
1841                         set_qset_lro(dev, i, 0);
1842
1843         return 0;
1844 }
1845
1846 static const struct ethtool_ops cxgb_ethtool_ops = {
1847         .get_settings = get_settings,
1848         .set_settings = set_settings,
1849         .get_drvinfo = get_drvinfo,
1850         .get_msglevel = get_msglevel,
1851         .set_msglevel = set_msglevel,
1852         .get_ringparam = get_sge_param,
1853         .set_ringparam = set_sge_param,
1854         .get_coalesce = get_coalesce,
1855         .set_coalesce = set_coalesce,
1856         .get_eeprom_len = get_eeprom_len,
1857         .get_eeprom = get_eeprom,
1858         .set_eeprom = set_eeprom,
1859         .get_pauseparam = get_pauseparam,
1860         .set_pauseparam = set_pauseparam,
1861         .get_rx_csum = get_rx_csum,
1862         .set_rx_csum = set_rx_csum,
1863         .set_tx_csum = ethtool_op_set_tx_csum,
1864         .set_sg = ethtool_op_set_sg,
1865         .get_link = ethtool_op_get_link,
1866         .get_strings = get_strings,
1867         .phys_id = cxgb3_phys_id,
1868         .nway_reset = restart_autoneg,
1869         .get_sset_count = get_sset_count,
1870         .get_ethtool_stats = get_stats,
1871         .get_regs_len = get_regs_len,
1872         .get_regs = get_regs,
1873         .get_wol = get_wol,
1874         .set_tso = ethtool_op_set_tso,
1875         .get_flags = ethtool_op_get_flags,
1876         .set_flags = cxgb3_set_flags,
1877 };
1878
1879 static int in_range(int val, int lo, int hi)
1880 {
1881         return val < 0 || (val <= hi && val >= lo);
1882 }
1883
1884 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1885 {
1886         struct port_info *pi = netdev_priv(dev);
1887         struct adapter *adapter = pi->adapter;
1888         u32 cmd;
1889         int ret;
1890
1891         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1892                 return -EFAULT;
1893
1894         switch (cmd) {
1895         case CHELSIO_SET_QSET_PARAMS:{
1896                 int i;
1897                 struct qset_params *q;
1898                 struct ch_qset_params t;
1899                 int q1 = pi->first_qset;
1900                 int nqsets = pi->nqsets;
1901
1902                 if (!capable(CAP_NET_ADMIN))
1903                         return -EPERM;
1904                 if (copy_from_user(&t, useraddr, sizeof(t)))
1905                         return -EFAULT;
1906                 if (t.qset_idx >= SGE_QSETS)
1907                         return -EINVAL;
1908                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1909                         !in_range(t.cong_thres, 0, 255) ||
1910                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1911                                 MAX_TXQ_ENTRIES) ||
1912                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1913                                 MAX_TXQ_ENTRIES) ||
1914                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1915                                 MAX_CTRL_TXQ_ENTRIES) ||
1916                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1917                                 MAX_RX_BUFFERS)
1918                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1919                                         MAX_RX_JUMBO_BUFFERS)
1920                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1921                                         MAX_RSPQ_ENTRIES))
1922                         return -EINVAL;
1923
1924                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
1925                         for_each_port(adapter, i) {
1926                                 pi = adap2pinfo(adapter, i);
1927                                 if (t.qset_idx >= pi->first_qset &&
1928                                     t.qset_idx < pi->first_qset + pi->nqsets &&
1929                                     !pi->rx_csum_offload)
1930                                         return -EINVAL;
1931                         }
1932
1933                 if ((adapter->flags & FULL_INIT_DONE) &&
1934                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1935                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1936                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1937                         t.polling >= 0 || t.cong_thres >= 0))
1938                         return -EBUSY;
1939
1940                 /* Allow setting of any available qset when offload enabled */
1941                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1942                         q1 = 0;
1943                         for_each_port(adapter, i) {
1944                                 pi = adap2pinfo(adapter, i);
1945                                 nqsets += pi->first_qset + pi->nqsets;
1946                         }
1947                 }
1948
1949                 if (t.qset_idx < q1)
1950                         return -EINVAL;
1951                 if (t.qset_idx > q1 + nqsets - 1)
1952                         return -EINVAL;
1953
1954                 q = &adapter->params.sge.qset[t.qset_idx];
1955
1956                 if (t.rspq_size >= 0)
1957                         q->rspq_size = t.rspq_size;
1958                 if (t.fl_size[0] >= 0)
1959                         q->fl_size = t.fl_size[0];
1960                 if (t.fl_size[1] >= 0)
1961                         q->jumbo_size = t.fl_size[1];
1962                 if (t.txq_size[0] >= 0)
1963                         q->txq_size[0] = t.txq_size[0];
1964                 if (t.txq_size[1] >= 0)
1965                         q->txq_size[1] = t.txq_size[1];
1966                 if (t.txq_size[2] >= 0)
1967                         q->txq_size[2] = t.txq_size[2];
1968                 if (t.cong_thres >= 0)
1969                         q->cong_thres = t.cong_thres;
1970                 if (t.intr_lat >= 0) {
1971                         struct sge_qset *qs =
1972                                 &adapter->sge.qs[t.qset_idx];
1973
1974                         q->coalesce_usecs = t.intr_lat;
1975                         t3_update_qset_coalesce(qs, q);
1976                 }
1977                 if (t.polling >= 0) {
1978                         if (adapter->flags & USING_MSIX)
1979                                 q->polling = t.polling;
1980                         else {
1981                                 /* No polling with INTx for T3A */
1982                                 if (adapter->params.rev == 0 &&
1983                                         !(adapter->flags & USING_MSI))
1984                                         t.polling = 0;
1985
1986                                 for (i = 0; i < SGE_QSETS; i++) {
1987                                         q = &adapter->params.sge.
1988                                                 qset[i];
1989                                         q->polling = t.polling;
1990                                 }
1991                         }
1992                 }
1993                 if (t.lro >= 0)
1994                         set_qset_lro(dev, t.qset_idx, t.lro);
1995
1996                 break;
1997         }
1998         case CHELSIO_GET_QSET_PARAMS:{
1999                 struct qset_params *q;
2000                 struct ch_qset_params t;
2001                 int q1 = pi->first_qset;
2002                 int nqsets = pi->nqsets;
2003                 int i;
2004
2005                 if (copy_from_user(&t, useraddr, sizeof(t)))
2006                         return -EFAULT;
2007
2008                 /* Display qsets for all ports when offload enabled */
2009                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2010                         q1 = 0;
2011                         for_each_port(adapter, i) {
2012                                 pi = adap2pinfo(adapter, i);
2013                                 nqsets = pi->first_qset + pi->nqsets;
2014                         }
2015                 }
2016
2017                 if (t.qset_idx >= nqsets)
2018                         return -EINVAL;
2019
2020                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2021                 t.rspq_size = q->rspq_size;
2022                 t.txq_size[0] = q->txq_size[0];
2023                 t.txq_size[1] = q->txq_size[1];
2024                 t.txq_size[2] = q->txq_size[2];
2025                 t.fl_size[0] = q->fl_size;
2026                 t.fl_size[1] = q->jumbo_size;
2027                 t.polling = q->polling;
2028                 t.lro = q->lro;
2029                 t.intr_lat = q->coalesce_usecs;
2030                 t.cong_thres = q->cong_thres;
2031                 t.qnum = q1;
2032
2033                 if (adapter->flags & USING_MSIX)
2034                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2035                 else
2036                         t.vector = adapter->pdev->irq;
2037
2038                 if (copy_to_user(useraddr, &t, sizeof(t)))
2039                         return -EFAULT;
2040                 break;
2041         }
2042         case CHELSIO_SET_QSET_NUM:{
2043                 struct ch_reg edata;
2044                 unsigned int i, first_qset = 0, other_qsets = 0;
2045
2046                 if (!capable(CAP_NET_ADMIN))
2047                         return -EPERM;
2048                 if (adapter->flags & FULL_INIT_DONE)
2049                         return -EBUSY;
2050                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2051                         return -EFAULT;
2052                 if (edata.val < 1 ||
2053                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2054                         return -EINVAL;
2055
2056                 for_each_port(adapter, i)
2057                         if (adapter->port[i] && adapter->port[i] != dev)
2058                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2059
2060                 if (edata.val + other_qsets > SGE_QSETS)
2061                         return -EINVAL;
2062
2063                 pi->nqsets = edata.val;
2064
2065                 for_each_port(adapter, i)
2066                         if (adapter->port[i]) {
2067                                 pi = adap2pinfo(adapter, i);
2068                                 pi->first_qset = first_qset;
2069                                 first_qset += pi->nqsets;
2070                         }
2071                 break;
2072         }
2073         case CHELSIO_GET_QSET_NUM:{
2074                 struct ch_reg edata;
2075
2076                 edata.cmd = CHELSIO_GET_QSET_NUM;
2077                 edata.val = pi->nqsets;
2078                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2079                         return -EFAULT;
2080                 break;
2081         }
2082         case CHELSIO_LOAD_FW:{
2083                 u8 *fw_data;
2084                 struct ch_mem_range t;
2085
2086                 if (!capable(CAP_SYS_RAWIO))
2087                         return -EPERM;
2088                 if (copy_from_user(&t, useraddr, sizeof(t)))
2089                         return -EFAULT;
2090                 /* Check t.len sanity ? */
2091                 fw_data = kmalloc(t.len, GFP_KERNEL);
2092                 if (!fw_data)
2093                         return -ENOMEM;
2094
2095                 if (copy_from_user
2096                         (fw_data, useraddr + sizeof(t), t.len)) {
2097                         kfree(fw_data);
2098                         return -EFAULT;
2099                 }
2100
2101                 ret = t3_load_fw(adapter, fw_data, t.len);
2102                 kfree(fw_data);
2103                 if (ret)
2104                         return ret;
2105                 break;
2106         }
2107         case CHELSIO_SETMTUTAB:{
2108                 struct ch_mtus m;
2109                 int i;
2110
2111                 if (!is_offload(adapter))
2112                         return -EOPNOTSUPP;
2113                 if (!capable(CAP_NET_ADMIN))
2114                         return -EPERM;
2115                 if (offload_running(adapter))
2116                         return -EBUSY;
2117                 if (copy_from_user(&m, useraddr, sizeof(m)))
2118                         return -EFAULT;
2119                 if (m.nmtus != NMTUS)
2120                         return -EINVAL;
2121                 if (m.mtus[0] < 81)     /* accommodate SACK */
2122                         return -EINVAL;
2123
2124                 /* MTUs must be in ascending order */
2125                 for (i = 1; i < NMTUS; ++i)
2126                         if (m.mtus[i] < m.mtus[i - 1])
2127                                 return -EINVAL;
2128
2129                 memcpy(adapter->params.mtus, m.mtus,
2130                         sizeof(adapter->params.mtus));
2131                 break;
2132         }
2133         case CHELSIO_GET_PM:{
2134                 struct tp_params *p = &adapter->params.tp;
2135                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2136
2137                 if (!is_offload(adapter))
2138                         return -EOPNOTSUPP;
2139                 m.tx_pg_sz = p->tx_pg_size;
2140                 m.tx_num_pg = p->tx_num_pgs;
2141                 m.rx_pg_sz = p->rx_pg_size;
2142                 m.rx_num_pg = p->rx_num_pgs;
2143                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2144                 if (copy_to_user(useraddr, &m, sizeof(m)))
2145                         return -EFAULT;
2146                 break;
2147         }
2148         case CHELSIO_SET_PM:{
2149                 struct ch_pm m;
2150                 struct tp_params *p = &adapter->params.tp;
2151
2152                 if (!is_offload(adapter))
2153                         return -EOPNOTSUPP;
2154                 if (!capable(CAP_NET_ADMIN))
2155                         return -EPERM;
2156                 if (adapter->flags & FULL_INIT_DONE)
2157                         return -EBUSY;
2158                 if (copy_from_user(&m, useraddr, sizeof(m)))
2159                         return -EFAULT;
2160                 if (!is_power_of_2(m.rx_pg_sz) ||
2161                         !is_power_of_2(m.tx_pg_sz))
2162                         return -EINVAL; /* not power of 2 */
2163                 if (!(m.rx_pg_sz & 0x14000))
2164                         return -EINVAL; /* not 16KB or 64KB */
2165                 if (!(m.tx_pg_sz & 0x1554000))
2166                         return -EINVAL;
2167                 if (m.tx_num_pg == -1)
2168                         m.tx_num_pg = p->tx_num_pgs;
2169                 if (m.rx_num_pg == -1)
2170                         m.rx_num_pg = p->rx_num_pgs;
2171                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2172                         return -EINVAL;
2173                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2174                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2175                         return -EINVAL;
2176                 p->rx_pg_size = m.rx_pg_sz;
2177                 p->tx_pg_size = m.tx_pg_sz;
2178                 p->rx_num_pgs = m.rx_num_pg;
2179                 p->tx_num_pgs = m.tx_num_pg;
2180                 break;
2181         }
2182         case CHELSIO_GET_MEM:{
2183                 struct ch_mem_range t;
2184                 struct mc7 *mem;
2185                 u64 buf[32];
2186
2187                 if (!is_offload(adapter))
2188                         return -EOPNOTSUPP;
2189                 if (!(adapter->flags & FULL_INIT_DONE))
2190                         return -EIO;    /* need the memory controllers */
2191                 if (copy_from_user(&t, useraddr, sizeof(t)))
2192                         return -EFAULT;
2193                 if ((t.addr & 7) || (t.len & 7))
2194                         return -EINVAL;
2195                 if (t.mem_id == MEM_CM)
2196                         mem = &adapter->cm;
2197                 else if (t.mem_id == MEM_PMRX)
2198                         mem = &adapter->pmrx;
2199                 else if (t.mem_id == MEM_PMTX)
2200                         mem = &adapter->pmtx;
2201                 else
2202                         return -EINVAL;
2203
2204                 /*
2205                  * Version scheme:
2206                  * bits 0..9: chip version
2207                  * bits 10..15: chip revision
2208                  */
2209                 t.version = 3 | (adapter->params.rev << 10);
2210                 if (copy_to_user(useraddr, &t, sizeof(t)))
2211                         return -EFAULT;
2212
2213                 /*
2214                  * Read 256 bytes at a time as len can be large and we don't
2215                  * want to use huge intermediate buffers.
2216                  */
2217                 useraddr += sizeof(t);  /* advance to start of buffer */
2218                 while (t.len) {
2219                         unsigned int chunk =
2220                                 min_t(unsigned int, t.len, sizeof(buf));
2221
2222                         ret =
2223                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2224                                                 buf);
2225                         if (ret)
2226                                 return ret;
2227                         if (copy_to_user(useraddr, buf, chunk))
2228                                 return -EFAULT;
2229                         useraddr += chunk;
2230                         t.addr += chunk;
2231                         t.len -= chunk;
2232                 }
2233                 break;
2234         }
2235         case CHELSIO_SET_TRACE_FILTER:{
2236                 struct ch_trace t;
2237                 const struct trace_params *tp;
2238
2239                 if (!capable(CAP_NET_ADMIN))
2240                         return -EPERM;
2241                 if (!offload_running(adapter))
2242                         return -EAGAIN;
2243                 if (copy_from_user(&t, useraddr, sizeof(t)))
2244                         return -EFAULT;
2245
2246                 tp = (const struct trace_params *)&t.sip;
2247                 if (t.config_tx)
2248                         t3_config_trace_filter(adapter, tp, 0,
2249                                                 t.invert_match,
2250                                                 t.trace_tx);
2251                 if (t.config_rx)
2252                         t3_config_trace_filter(adapter, tp, 1,
2253                                                 t.invert_match,
2254                                                 t.trace_rx);
2255                 break;
2256         }
2257         default:
2258                 return -EOPNOTSUPP;
2259         }
2260         return 0;
2261 }
2262
2263 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2264 {
2265         struct mii_ioctl_data *data = if_mii(req);
2266         struct port_info *pi = netdev_priv(dev);
2267         struct adapter *adapter = pi->adapter;
2268         int ret, mmd;
2269
2270         switch (cmd) {
2271         case SIOCGMIIPHY:
2272                 data->phy_id = pi->phy.addr;
2273                 /* FALLTHRU */
2274         case SIOCGMIIREG:{
2275                 u32 val;
2276                 struct cphy *phy = &pi->phy;
2277
2278                 if (!phy->mdio_read)
2279                         return -EOPNOTSUPP;
2280                 if (is_10G(adapter)) {
2281                         mmd = data->phy_id >> 8;
2282                         if (!mmd)
2283                                 mmd = MDIO_DEV_PCS;
2284                         else if (mmd > MDIO_DEV_VEND2)
2285                                 return -EINVAL;
2286
2287                         ret =
2288                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2289                                                 mmd, data->reg_num, &val);
2290                 } else
2291                         ret =
2292                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2293                                                 0, data->reg_num & 0x1f,
2294                                                 &val);
2295                 if (!ret)
2296                         data->val_out = val;
2297                 break;
2298         }
2299         case SIOCSMIIREG:{
2300                 struct cphy *phy = &pi->phy;
2301
2302                 if (!capable(CAP_NET_ADMIN))
2303                         return -EPERM;
2304                 if (!phy->mdio_write)
2305                         return -EOPNOTSUPP;
2306                 if (is_10G(adapter)) {
2307                         mmd = data->phy_id >> 8;
2308                         if (!mmd)
2309                                 mmd = MDIO_DEV_PCS;
2310                         else if (mmd > MDIO_DEV_VEND2)
2311                                 return -EINVAL;
2312
2313                         ret =
2314                                 phy->mdio_write(adapter,
2315                                                 data->phy_id & 0x1f, mmd,
2316                                                 data->reg_num,
2317                                                 data->val_in);
2318                 } else
2319                         ret =
2320                                 phy->mdio_write(adapter,
2321                                                 data->phy_id & 0x1f, 0,
2322                                                 data->reg_num & 0x1f,
2323                                                 data->val_in);
2324                 break;
2325         }
2326         case SIOCCHIOCTL:
2327                 return cxgb_extension_ioctl(dev, req->ifr_data);
2328         default:
2329                 return -EOPNOTSUPP;
2330         }
2331         return ret;
2332 }
2333
2334 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2335 {
2336         struct port_info *pi = netdev_priv(dev);
2337         struct adapter *adapter = pi->adapter;
2338         int ret;
2339
2340         if (new_mtu < 81)       /* accommodate SACK */
2341                 return -EINVAL;
2342         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2343                 return ret;
2344         dev->mtu = new_mtu;
2345         init_port_mtus(adapter);
2346         if (adapter->params.rev == 0 && offload_running(adapter))
2347                 t3_load_mtus(adapter, adapter->params.mtus,
2348                              adapter->params.a_wnd, adapter->params.b_wnd,
2349                              adapter->port[0]->mtu);
2350         return 0;
2351 }
2352
2353 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2354 {
2355         struct port_info *pi = netdev_priv(dev);
2356         struct adapter *adapter = pi->adapter;
2357         struct sockaddr *addr = p;
2358
2359         if (!is_valid_ether_addr(addr->sa_data))
2360                 return -EINVAL;
2361
2362         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2363         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2364         if (offload_running(adapter))
2365                 write_smt_entry(adapter, pi->port_id);
2366         return 0;
2367 }
2368
2369 /**
2370  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2371  * @adap: the adapter
2372  * @p: the port
2373  *
2374  * Ensures that current Rx processing on any of the queues associated with
2375  * the given port completes before returning.  We do this by acquiring and
2376  * releasing the locks of the response queues associated with the port.
2377  */
2378 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2379 {
2380         int i;
2381
2382         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2383                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2384
2385                 spin_lock_irq(&q->lock);
2386                 spin_unlock_irq(&q->lock);
2387         }
2388 }
2389
2390 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2391 {
2392         struct port_info *pi = netdev_priv(dev);
2393         struct adapter *adapter = pi->adapter;
2394
2395         pi->vlan_grp = grp;
2396         if (adapter->params.rev > 0)
2397                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2398         else {
2399                 /* single control for all ports */
2400                 unsigned int i, have_vlans = 0;
2401                 for_each_port(adapter, i)
2402                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2403
2404                 t3_set_vlan_accel(adapter, 1, have_vlans);
2405         }
2406         t3_synchronize_rx(adapter, pi);
2407 }
2408
2409 #ifdef CONFIG_NET_POLL_CONTROLLER
2410 static void cxgb_netpoll(struct net_device *dev)
2411 {
2412         struct port_info *pi = netdev_priv(dev);
2413         struct adapter *adapter = pi->adapter;
2414         int qidx;
2415
2416         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2417                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2418                 void *source;
2419
2420                 if (adapter->flags & USING_MSIX)
2421                         source = qs;
2422                 else
2423                         source = adapter;
2424
2425                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2426         }
2427 }
2428 #endif
2429
2430 /*
2431  * Periodic accumulation of MAC statistics.
2432  */
2433 static void mac_stats_update(struct adapter *adapter)
2434 {
2435         int i;
2436
2437         for_each_port(adapter, i) {
2438                 struct net_device *dev = adapter->port[i];
2439                 struct port_info *p = netdev_priv(dev);
2440
2441                 if (netif_running(dev)) {
2442                         spin_lock(&adapter->stats_lock);
2443                         t3_mac_update_stats(&p->mac);
2444                         spin_unlock(&adapter->stats_lock);
2445                 }
2446         }
2447 }
2448
2449 static void check_link_status(struct adapter *adapter)
2450 {
2451         int i;
2452
2453         for_each_port(adapter, i) {
2454                 struct net_device *dev = adapter->port[i];
2455                 struct port_info *p = netdev_priv(dev);
2456
2457                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev))
2458                         t3_link_changed(adapter, i);
2459         }
2460 }
2461
2462 static void check_t3b2_mac(struct adapter *adapter)
2463 {
2464         int i;
2465
2466         if (!rtnl_trylock())    /* synchronize with ifdown */
2467                 return;
2468
2469         for_each_port(adapter, i) {
2470                 struct net_device *dev = adapter->port[i];
2471                 struct port_info *p = netdev_priv(dev);
2472                 int status;
2473
2474                 if (!netif_running(dev))
2475                         continue;
2476
2477                 status = 0;
2478                 if (netif_running(dev) && netif_carrier_ok(dev))
2479                         status = t3b2_mac_watchdog_task(&p->mac);
2480                 if (status == 1)
2481                         p->mac.stats.num_toggled++;
2482                 else if (status == 2) {
2483                         struct cmac *mac = &p->mac;
2484
2485                         t3_mac_set_mtu(mac, dev->mtu);
2486                         t3_mac_set_address(mac, 0, dev->dev_addr);
2487                         cxgb_set_rxmode(dev);
2488                         t3_link_start(&p->phy, mac, &p->link_config);
2489                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2490                         t3_port_intr_enable(adapter, p->port_id);
2491                         p->mac.stats.num_resets++;
2492                 }
2493         }
2494         rtnl_unlock();
2495 }
2496
2497
2498 static void t3_adap_check_task(struct work_struct *work)
2499 {
2500         struct adapter *adapter = container_of(work, struct adapter,
2501                                                adap_check_task.work);
2502         const struct adapter_params *p = &adapter->params;
2503
2504         adapter->check_task_cnt++;
2505
2506         /* Check link status for PHYs without interrupts */
2507         if (p->linkpoll_period)
2508                 check_link_status(adapter);
2509
2510         /* Accumulate MAC stats if needed */
2511         if (!p->linkpoll_period ||
2512             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2513             p->stats_update_period) {
2514                 mac_stats_update(adapter);
2515                 adapter->check_task_cnt = 0;
2516         }
2517
2518         if (p->rev == T3_REV_B2)
2519                 check_t3b2_mac(adapter);
2520
2521         /* Schedule the next check update if any port is active. */
2522         spin_lock_irq(&adapter->work_lock);
2523         if (adapter->open_device_map & PORT_MASK)
2524                 schedule_chk_task(adapter);
2525         spin_unlock_irq(&adapter->work_lock);
2526 }
2527
2528 /*
2529  * Processes external (PHY) interrupts in process context.
2530  */
2531 static void ext_intr_task(struct work_struct *work)
2532 {
2533         struct adapter *adapter = container_of(work, struct adapter,
2534                                                ext_intr_handler_task);
2535
2536         t3_phy_intr_handler(adapter);
2537
2538         /* Now reenable external interrupts */
2539         spin_lock_irq(&adapter->work_lock);
2540         if (adapter->slow_intr_mask) {
2541                 adapter->slow_intr_mask |= F_T3DBG;
2542                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2543                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2544                              adapter->slow_intr_mask);
2545         }
2546         spin_unlock_irq(&adapter->work_lock);
2547 }
2548
2549 /*
2550  * Interrupt-context handler for external (PHY) interrupts.
2551  */
2552 void t3_os_ext_intr_handler(struct adapter *adapter)
2553 {
2554         /*
2555          * Schedule a task to handle external interrupts as they may be slow
2556          * and we use a mutex to protect MDIO registers.  We disable PHY
2557          * interrupts in the meantime and let the task reenable them when
2558          * it's done.
2559          */
2560         spin_lock(&adapter->work_lock);
2561         if (adapter->slow_intr_mask) {
2562                 adapter->slow_intr_mask &= ~F_T3DBG;
2563                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2564                              adapter->slow_intr_mask);
2565                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2566         }
2567         spin_unlock(&adapter->work_lock);
2568 }
2569
2570 static int t3_adapter_error(struct adapter *adapter, int reset)
2571 {
2572         int i, ret = 0;
2573
2574         /* Stop all ports */
2575         for_each_port(adapter, i) {
2576                 struct net_device *netdev = adapter->port[i];
2577
2578                 if (netif_running(netdev))
2579                         cxgb_close(netdev);
2580         }
2581
2582         if (is_offload(adapter) &&
2583             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
2584                 offload_close(&adapter->tdev);
2585
2586         /* Stop SGE timers */
2587         t3_stop_sge_timers(adapter);
2588
2589         adapter->flags &= ~FULL_INIT_DONE;
2590
2591         if (reset)
2592                 ret = t3_reset_adapter(adapter);
2593
2594         pci_disable_device(adapter->pdev);
2595
2596         return ret;
2597 }
2598
2599 static int t3_reenable_adapter(struct adapter *adapter)
2600 {
2601         if (pci_enable_device(adapter->pdev)) {
2602                 dev_err(&adapter->pdev->dev,
2603                         "Cannot re-enable PCI device after reset.\n");
2604                 goto err;
2605         }
2606         pci_set_master(adapter->pdev);
2607         pci_restore_state(adapter->pdev);
2608
2609         /* Free sge resources */
2610         t3_free_sge_resources(adapter);
2611
2612         if (t3_replay_prep_adapter(adapter))
2613                 goto err;
2614
2615         return 0;
2616 err:
2617         return -1;
2618 }
2619
2620 static void t3_resume_ports(struct adapter *adapter)
2621 {
2622         int i;
2623
2624         /* Restart the ports */
2625         for_each_port(adapter, i) {
2626                 struct net_device *netdev = adapter->port[i];
2627
2628                 if (netif_running(netdev)) {
2629                         if (cxgb_open(netdev)) {
2630                                 dev_err(&adapter->pdev->dev,
2631                                         "can't bring device back up"
2632                                         " after reset\n");
2633                                 continue;
2634                         }
2635                 }
2636         }
2637 }
2638
2639 /*
2640  * processes a fatal error.
2641  * Bring the ports down, reset the chip, bring the ports back up.
2642  */
2643 static void fatal_error_task(struct work_struct *work)
2644 {
2645         struct adapter *adapter = container_of(work, struct adapter,
2646                                                fatal_error_handler_task);
2647         int err = 0;
2648
2649         rtnl_lock();
2650         err = t3_adapter_error(adapter, 1);
2651         if (!err)
2652                 err = t3_reenable_adapter(adapter);
2653         if (!err)
2654                 t3_resume_ports(adapter);
2655
2656         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2657         rtnl_unlock();
2658 }
2659
2660 void t3_fatal_err(struct adapter *adapter)
2661 {
2662         unsigned int fw_status[4];
2663
2664         if (adapter->flags & FULL_INIT_DONE) {
2665                 t3_sge_stop(adapter);
2666                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2667                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2668                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2669                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2670
2671                 spin_lock(&adapter->work_lock);
2672                 t3_intr_disable(adapter);
2673                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2674                 spin_unlock(&adapter->work_lock);
2675         }
2676         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2677         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2678                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2679                          fw_status[0], fw_status[1],
2680                          fw_status[2], fw_status[3]);
2681
2682 }
2683
2684 /**
2685  * t3_io_error_detected - called when PCI error is detected
2686  * @pdev: Pointer to PCI device
2687  * @state: The current pci connection state
2688  *
2689  * This function is called after a PCI bus error affecting
2690  * this device has been detected.
2691  */
2692 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2693                                              pci_channel_state_t state)
2694 {
2695         struct adapter *adapter = pci_get_drvdata(pdev);
2696         int ret;
2697
2698         ret = t3_adapter_error(adapter, 0);
2699
2700         /* Request a slot reset. */
2701         return PCI_ERS_RESULT_NEED_RESET;
2702 }
2703
2704 /**
2705  * t3_io_slot_reset - called after the pci bus has been reset.
2706  * @pdev: Pointer to PCI device
2707  *
2708  * Restart the card from scratch, as if from a cold-boot.
2709  */
2710 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2711 {
2712         struct adapter *adapter = pci_get_drvdata(pdev);
2713
2714         if (!t3_reenable_adapter(adapter))
2715                 return PCI_ERS_RESULT_RECOVERED;
2716
2717         return PCI_ERS_RESULT_DISCONNECT;
2718 }
2719
2720 /**
2721  * t3_io_resume - called when traffic can start flowing again.
2722  * @pdev: Pointer to PCI device
2723  *
2724  * This callback is called when the error recovery driver tells us that
2725  * its OK to resume normal operation.
2726  */
2727 static void t3_io_resume(struct pci_dev *pdev)
2728 {
2729         struct adapter *adapter = pci_get_drvdata(pdev);
2730
2731         t3_resume_ports(adapter);
2732 }
2733
2734 static struct pci_error_handlers t3_err_handler = {
2735         .error_detected = t3_io_error_detected,
2736         .slot_reset = t3_io_slot_reset,
2737         .resume = t3_io_resume,
2738 };
2739
2740 /*
2741  * Set the number of qsets based on the number of CPUs and the number of ports,
2742  * not to exceed the number of available qsets, assuming there are enough qsets
2743  * per port in HW.
2744  */
2745 static void set_nqsets(struct adapter *adap)
2746 {
2747         int i, j = 0;
2748         int num_cpus = num_online_cpus();
2749         int hwports = adap->params.nports;
2750         int nqsets = SGE_QSETS;
2751
2752         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
2753                 if (hwports == 2 &&
2754                     (hwports * nqsets > SGE_QSETS ||
2755                      num_cpus >= nqsets / hwports))
2756                         nqsets /= hwports;
2757                 if (nqsets > num_cpus)
2758                         nqsets = num_cpus;
2759                 if (nqsets < 1 || hwports == 4)
2760                         nqsets = 1;
2761         } else
2762                 nqsets = 1;
2763
2764         for_each_port(adap, i) {
2765                 struct port_info *pi = adap2pinfo(adap, i);
2766
2767                 pi->first_qset = j;
2768                 pi->nqsets = nqsets;
2769                 j = pi->first_qset + nqsets;
2770
2771                 dev_info(&adap->pdev->dev,
2772                          "Port %d using %d queue sets.\n", i, nqsets);
2773         }
2774 }
2775
2776 static int __devinit cxgb_enable_msix(struct adapter *adap)
2777 {
2778         struct msix_entry entries[SGE_QSETS + 1];
2779         int i, err;
2780
2781         for (i = 0; i < ARRAY_SIZE(entries); ++i)
2782                 entries[i].entry = i;
2783
2784         err = pci_enable_msix(adap->pdev, entries, ARRAY_SIZE(entries));
2785         if (!err) {
2786                 for (i = 0; i < ARRAY_SIZE(entries); ++i)
2787                         adap->msix_info[i].vec = entries[i].vector;
2788         } else if (err > 0)
2789                 dev_info(&adap->pdev->dev,
2790                        "only %d MSI-X vectors left, not using MSI-X\n", err);
2791         return err;
2792 }
2793
2794 static void __devinit print_port_info(struct adapter *adap,
2795                                       const struct adapter_info *ai)
2796 {
2797         static const char *pci_variant[] = {
2798                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2799         };
2800
2801         int i;
2802         char buf[80];
2803
2804         if (is_pcie(adap))
2805                 snprintf(buf, sizeof(buf), "%s x%d",
2806                          pci_variant[adap->params.pci.variant],
2807                          adap->params.pci.width);
2808         else
2809                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2810                          pci_variant[adap->params.pci.variant],
2811                          adap->params.pci.speed, adap->params.pci.width);
2812
2813         for_each_port(adap, i) {
2814                 struct net_device *dev = adap->port[i];
2815                 const struct port_info *pi = netdev_priv(dev);
2816
2817                 if (!test_bit(i, &adap->registered_device_map))
2818                         continue;
2819                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2820                        dev->name, ai->desc, pi->phy.desc,
2821                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2822                        (adap->flags & USING_MSIX) ? " MSI-X" :
2823                        (adap->flags & USING_MSI) ? " MSI" : "");
2824                 if (adap->name == dev->name && adap->params.vpd.mclk)
2825                         printk(KERN_INFO
2826                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
2827                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2828                                t3_mc7_size(&adap->pmtx) >> 20,
2829                                t3_mc7_size(&adap->pmrx) >> 20,
2830                                adap->params.vpd.sn);
2831         }
2832 }
2833
2834 static const struct net_device_ops cxgb_netdev_ops = {
2835         .ndo_open               = cxgb_open,
2836         .ndo_stop               = cxgb_close,
2837         .ndo_start_xmit         = t3_eth_xmit,
2838         .ndo_get_stats          = cxgb_get_stats,
2839         .ndo_validate_addr      = eth_validate_addr,
2840         .ndo_set_multicast_list = cxgb_set_rxmode,
2841         .ndo_do_ioctl           = cxgb_ioctl,
2842         .ndo_change_mtu         = cxgb_change_mtu,
2843         .ndo_set_mac_address    = cxgb_set_mac_addr,
2844         .ndo_vlan_rx_register   = vlan_rx_register,
2845 #ifdef CONFIG_NET_POLL_CONTROLLER
2846         .ndo_poll_controller    = cxgb_netpoll,
2847 #endif
2848 };
2849
2850 static int __devinit init_one(struct pci_dev *pdev,
2851                               const struct pci_device_id *ent)
2852 {
2853         static int version_printed;
2854
2855         int i, err, pci_using_dac = 0;
2856         unsigned long mmio_start, mmio_len;
2857         const struct adapter_info *ai;
2858         struct adapter *adapter = NULL;
2859         struct port_info *pi;
2860
2861         if (!version_printed) {
2862                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2863                 ++version_printed;
2864         }
2865
2866         if (!cxgb3_wq) {
2867                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2868                 if (!cxgb3_wq) {
2869                         printk(KERN_ERR DRV_NAME
2870                                ": cannot initialize work queue\n");
2871                         return -ENOMEM;
2872                 }
2873         }
2874
2875         err = pci_request_regions(pdev, DRV_NAME);
2876         if (err) {
2877                 /* Just info, some other driver may have claimed the device. */
2878                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2879                 return err;
2880         }
2881
2882         err = pci_enable_device(pdev);
2883         if (err) {
2884                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2885                 goto out_release_regions;
2886         }
2887
2888         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2889                 pci_using_dac = 1;
2890                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2891                 if (err) {
2892                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2893                                "coherent allocations\n");
2894                         goto out_disable_device;
2895                 }
2896         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2897                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2898                 goto out_disable_device;
2899         }
2900
2901         pci_set_master(pdev);
2902         pci_save_state(pdev);
2903
2904         mmio_start = pci_resource_start(pdev, 0);
2905         mmio_len = pci_resource_len(pdev, 0);
2906         ai = t3_get_adapter_info(ent->driver_data);
2907
2908         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2909         if (!adapter) {
2910                 err = -ENOMEM;
2911                 goto out_disable_device;
2912         }
2913
2914         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2915         if (!adapter->regs) {
2916                 dev_err(&pdev->dev, "cannot map device registers\n");
2917                 err = -ENOMEM;
2918                 goto out_free_adapter;
2919         }
2920
2921         adapter->pdev = pdev;
2922         adapter->name = pci_name(pdev);
2923         adapter->msg_enable = dflt_msg_enable;
2924         adapter->mmio_len = mmio_len;
2925
2926         mutex_init(&adapter->mdio_lock);
2927         spin_lock_init(&adapter->work_lock);
2928         spin_lock_init(&adapter->stats_lock);
2929
2930         INIT_LIST_HEAD(&adapter->adapter_list);
2931         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2932         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
2933         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2934
2935         for (i = 0; i < ai->nports; ++i) {
2936                 struct net_device *netdev;
2937
2938                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
2939                 if (!netdev) {
2940                         err = -ENOMEM;
2941                         goto out_free_dev;
2942                 }
2943
2944                 SET_NETDEV_DEV(netdev, &pdev->dev);
2945
2946                 adapter->port[i] = netdev;
2947                 pi = netdev_priv(netdev);
2948                 pi->adapter = adapter;
2949                 pi->rx_csum_offload = 1;
2950                 pi->port_id = i;
2951                 netif_carrier_off(netdev);
2952                 netif_tx_stop_all_queues(netdev);
2953                 netdev->irq = pdev->irq;
2954                 netdev->mem_start = mmio_start;
2955                 netdev->mem_end = mmio_start + mmio_len - 1;
2956                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2957                 netdev->features |= NETIF_F_LLTX;
2958                 if (pci_using_dac)
2959                         netdev->features |= NETIF_F_HIGHDMA;
2960
2961                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2962                 netdev->netdev_ops = &cxgb_netdev_ops;
2963                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2964         }
2965
2966         pci_set_drvdata(pdev, adapter);
2967         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2968                 err = -ENODEV;
2969                 goto out_free_dev;
2970         }
2971
2972         /*
2973          * The card is now ready to go.  If any errors occur during device
2974          * registration we do not fail the whole card but rather proceed only
2975          * with the ports we manage to register successfully.  However we must
2976          * register at least one net device.
2977          */
2978         for_each_port(adapter, i) {
2979                 err = register_netdev(adapter->port[i]);
2980                 if (err)
2981                         dev_warn(&pdev->dev,
2982                                  "cannot register net device %s, skipping\n",
2983                                  adapter->port[i]->name);
2984                 else {
2985                         /*
2986                          * Change the name we use for messages to the name of
2987                          * the first successfully registered interface.
2988                          */
2989                         if (!adapter->registered_device_map)
2990                                 adapter->name = adapter->port[i]->name;
2991
2992                         __set_bit(i, &adapter->registered_device_map);
2993                 }
2994         }
2995         if (!adapter->registered_device_map) {
2996                 dev_err(&pdev->dev, "could not register any net devices\n");
2997                 goto out_free_dev;
2998         }
2999
3000         /* Driver's ready. Reflect it on LEDs */
3001         t3_led_ready(adapter);
3002
3003         if (is_offload(adapter)) {
3004                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3005                 cxgb3_adapter_ofld(adapter);
3006         }
3007
3008         /* See what interrupts we'll be using */
3009         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3010                 adapter->flags |= USING_MSIX;
3011         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3012                 adapter->flags |= USING_MSI;
3013
3014         set_nqsets(adapter);
3015
3016         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3017                                  &cxgb3_attr_group);
3018
3019         print_port_info(adapter, ai);
3020         return 0;
3021
3022 out_free_dev:
3023         iounmap(adapter->regs);
3024         for (i = ai->nports - 1; i >= 0; --i)
3025                 if (adapter->port[i])
3026                         free_netdev(adapter->port[i]);
3027
3028 out_free_adapter:
3029         kfree(adapter);
3030
3031 out_disable_device:
3032         pci_disable_device(pdev);
3033 out_release_regions:
3034         pci_release_regions(pdev);
3035         pci_set_drvdata(pdev, NULL);
3036         return err;
3037 }
3038
3039 static void __devexit remove_one(struct pci_dev *pdev)
3040 {
3041         struct adapter *adapter = pci_get_drvdata(pdev);
3042
3043         if (adapter) {
3044                 int i;
3045
3046                 t3_sge_stop(adapter);
3047                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3048                                    &cxgb3_attr_group);
3049
3050                 if (is_offload(adapter)) {
3051                         cxgb3_adapter_unofld(adapter);
3052                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3053                                      &adapter->open_device_map))
3054                                 offload_close(&adapter->tdev);
3055                 }
3056
3057                 for_each_port(adapter, i)
3058                     if (test_bit(i, &adapter->registered_device_map))
3059                         unregister_netdev(adapter->port[i]);
3060
3061                 t3_stop_sge_timers(adapter);
3062                 t3_free_sge_resources(adapter);
3063                 cxgb_disable_msi(adapter);
3064
3065                 for_each_port(adapter, i)
3066                         if (adapter->port[i])
3067                                 free_netdev(adapter->port[i]);
3068
3069                 iounmap(adapter->regs);
3070                 kfree(adapter);
3071                 pci_release_regions(pdev);
3072                 pci_disable_device(pdev);
3073                 pci_set_drvdata(pdev, NULL);
3074         }
3075 }
3076
3077 static struct pci_driver driver = {
3078         .name = DRV_NAME,
3079         .id_table = cxgb3_pci_tbl,
3080         .probe = init_one,
3081         .remove = __devexit_p(remove_one),
3082         .err_handler = &t3_err_handler,
3083 };
3084
3085 static int __init cxgb3_init_module(void)
3086 {
3087         int ret;
3088
3089         cxgb3_offload_init();
3090
3091         ret = pci_register_driver(&driver);
3092         return ret;
3093 }
3094
3095 static void __exit cxgb3_cleanup_module(void)
3096 {
3097         pci_unregister_driver(&driver);
3098         if (cxgb3_wq)
3099                 destroy_workqueue(cxgb3_wq);
3100 }
3101
3102 module_init(cxgb3_init_module);
3103 module_exit(cxgb3_cleanup_module);