drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mdio.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
  94         CH_DEVICE(0x36, 3),     /* S320E-CR */
  95         CH_DEVICE(0x37, 7),     /* N320E-G2 */
  96         {0,}
  97 };
  98
  99 MODULE_DESCRIPTION(DRV_DESC);
 100 MODULE_AUTHOR("Chelsio Communications");
 101 MODULE_LICENSE("Dual BSD/GPL");
 102 MODULE_VERSION(DRV_VERSION);
 103 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 104
 105 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 106
 107 module_param(dflt_msg_enable, int, 0644);
 108 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 109
 110 /*
 111  * The driver uses the best interrupt scheme available on a platform in the
 112  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 113  * of these schemes the driver may consider as follows:
 114  *
 115  * msi = 2: choose from among all three options
 116  * msi = 1: only consider MSI and pin interrupts
 117  * msi = 0: force pin interrupts
 118  */
 119 static int msi = 2;
 120
 121 module_param(msi, int, 0644);
 122 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 123
 124 /*
 125  * The driver enables offload as a default.
 126  * To disable it, use ofld_disable = 1.
 127  */
 128
 129 static int ofld_disable = 0;
 130
 131 module_param(ofld_disable, int, 0644);
 132 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 133
 134 /*
 135  * We have work elements that we need to cancel when an interface is taken
 136  * down.  Normally the work elements would be executed by keventd but that
 137  * can deadlock because of linkwatch.  If our close method takes the rtnl
 138  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 139  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 140  * for our work to complete.  Get our own work queue to solve this.
 141  */
 142 static struct workqueue_struct *cxgb3_wq;
 143
 144 /**
 145  *      link_report - show link status and link speed/duplex
 146  *      @p: the port whose settings are to be reported
 147  *
 148  *      Shows the link status, speed, and duplex of a port.
 149  */
 150 static void link_report(struct net_device *dev)
 151 {
 152         if (!netif_carrier_ok(dev))
 153                 printk(KERN_INFO "%s: link down\n", dev->name);
 154         else {
 155                 const char *s = "10Mbps";
 156                 const struct port_info *p = netdev_priv(dev);
 157
 158                 switch (p->link_config.speed) {
 159                 case SPEED_10000:
 160                         s = "10Gbps";
 161                         break;
 162                 case SPEED_1000:
 163                         s = "1000Mbps";
 164                         break;
 165                 case SPEED_100:
 166                         s = "100Mbps";
 167                         break;
 168                 }
 169
 170                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 171                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 172         }
 173 }
 174
 175 static void enable_tx_fifo_drain(struct adapter *adapter,
 176                                  struct port_info *pi)
 177 {
 178         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 179                          F_ENDROPPKT);
 180         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 181         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 182         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 183 }
 184
 185 static void disable_tx_fifo_drain(struct adapter *adapter,
 186                                   struct port_info *pi)
 187 {
 188         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 189                          F_ENDROPPKT, 0);
 190 }
 191
 192 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 193 {
 194         struct net_device *dev = adap->port[port_id];
 195         struct port_info *pi = netdev_priv(dev);
 196
 197         if (state == netif_carrier_ok(dev))
 198                 return;
 199
 200         if (state) {
 201                 struct cmac *mac = &pi->mac;
 202
 203                 netif_carrier_on(dev);
 204
 205                 disable_tx_fifo_drain(adap, pi);
 206
 207                 /* Clear local faults */
 208                 t3_xgm_intr_disable(adap, pi->port_id);
 209                 t3_read_reg(adap, A_XGM_INT_STATUS +
 210                                     pi->mac.offset);
 211                 t3_write_reg(adap,
 212                              A_XGM_INT_CAUSE + pi->mac.offset,
 213                              F_XGM_INT);
 214
 215                 t3_set_reg_field(adap,
 216                                  A_XGM_INT_ENABLE +
 217                                  pi->mac.offset,
 218                                  F_XGM_INT, F_XGM_INT);
 219                 t3_xgm_intr_enable(adap, pi->port_id);
 220
 221                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 222         } else {
 223                 netif_carrier_off(dev);
 224
 225                 /* Flush TX FIFO */
 226                 enable_tx_fifo_drain(adap, pi);
 227         }
 228         link_report(dev);
 229 }
 230
 231 /**
 232  *      t3_os_link_changed - handle link status changes
 233  *      @adapter: the adapter associated with the link change
 234  *      @port_id: the port index whose limk status has changed
 235  *      @link_stat: the new status of the link
 236  *      @speed: the new speed setting
 237  *      @duplex: the new duplex setting
 238  *      @pause: the new flow-control setting
 239  *
 240  *      This is the OS-dependent handler for link status changes.  The OS
 241  *      neutral handler takes care of most of the processing for these events,
 242  *      then calls this handler for any OS-specific processing.
 243  */
 244 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 245                         int speed, int duplex, int pause)
 246 {
 247         struct net_device *dev = adapter->port[port_id];
 248         struct port_info *pi = netdev_priv(dev);
 249         struct cmac *mac = &pi->mac;
 250
 251         /* Skip changes from disabled ports. */
 252         if (!netif_running(dev))
 253                 return;
 254
 255         if (link_stat != netif_carrier_ok(dev)) {
 256                 if (link_stat) {
 257                         disable_tx_fifo_drain(adapter, pi);
 258
 259                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 260
 261                         /* Clear local faults */
 262                         t3_xgm_intr_disable(adapter, pi->port_id);
 263                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 264                                     pi->mac.offset);
 265                         t3_write_reg(adapter,
 266                                      A_XGM_INT_CAUSE + pi->mac.offset,
 267                                      F_XGM_INT);
 268
 269                         t3_set_reg_field(adapter,
 270                                          A_XGM_INT_ENABLE + pi->mac.offset,
 271                                          F_XGM_INT, F_XGM_INT);
 272                         t3_xgm_intr_enable(adapter, pi->port_id);
 273
 274                         netif_carrier_on(dev);
 275                 } else {
 276                         netif_carrier_off(dev);
 277
 278                         t3_xgm_intr_disable(adapter, pi->port_id);
 279                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 280                         t3_set_reg_field(adapter,
 281                                          A_XGM_INT_ENABLE + pi->mac.offset,
 282                                          F_XGM_INT, 0);
 283
 284                         if (is_10G(adapter))
 285                                 pi->phy.ops->power_down(&pi->phy, 1);
 286
 287                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 288                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 289                         t3_link_start(&pi->phy, mac, &pi->link_config);
 290
 291                         /* Flush TX FIFO */
 292                         enable_tx_fifo_drain(adapter, pi);
 293                 }
 294
 295                 link_report(dev);
 296         }
 297 }
 298
 299 /**
 300  *      t3_os_phymod_changed - handle PHY module changes
 301  *      @phy: the PHY reporting the module change
 302  *      @mod_type: new module type
 303  *
 304  *      This is the OS-dependent handler for PHY module changes.  It is
 305  *      invoked when a PHY module is removed or inserted for any OS-specific
 306  *      processing.
 307  */
 308 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 309 {
 310         static const char *mod_str[] = {
 311                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 312         };
 313
 314         const struct net_device *dev = adap->port[port_id];
 315         const struct port_info *pi = netdev_priv(dev);
 316
 317         if (pi->phy.modtype == phy_modtype_none)
 318                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 319         else
 320                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 321                        mod_str[pi->phy.modtype]);
 322 }
 323
 324 static void cxgb_set_rxmode(struct net_device *dev)
 325 {
 326         struct t3_rx_mode rm;
 327         struct port_info *pi = netdev_priv(dev);
 328
 329         init_rx_mode(&rm, dev, dev->mc_list);
 330         t3_mac_set_rx_mode(&pi->mac, &rm);
 331 }
 332
 333 /**
 334  *      link_start - enable a port
 335  *      @dev: the device to enable
 336  *
 337  *      Performs the MAC and PHY actions needed to enable a port.
 338  */
 339 static void link_start(struct net_device *dev)
 340 {
 341         struct t3_rx_mode rm;
 342         struct port_info *pi = netdev_priv(dev);
 343         struct cmac *mac = &pi->mac;
 344
 345         init_rx_mode(&rm, dev, dev->mc_list);
 346         t3_mac_reset(mac);
 347         t3_mac_set_mtu(mac, dev->mtu);
 348         t3_mac_set_address(mac, 0, dev->dev_addr);
 349         t3_mac_set_rx_mode(mac, &rm);
 350         t3_link_start(&pi->phy, mac, &pi->link_config);
 351         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 352 }
 353
 354 static inline void cxgb_disable_msi(struct adapter *adapter)
 355 {
 356         if (adapter->flags & USING_MSIX) {
 357                 pci_disable_msix(adapter->pdev);
 358                 adapter->flags &= ~USING_MSIX;
 359         } else if (adapter->flags & USING_MSI) {
 360                 pci_disable_msi(adapter->pdev);
 361                 adapter->flags &= ~USING_MSI;
 362         }
 363 }
 364
 365 /*
 366  * Interrupt handler for asynchronous events used with MSI-X.
 367  */
 368 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 369 {
 370         t3_slow_intr_handler(cookie);
 371         return IRQ_HANDLED;
 372 }
 373
 374 /*
 375  * Name the MSI-X interrupts.
 376  */
 377 static void name_msix_vecs(struct adapter *adap)
 378 {
 379         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 380
 381         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 382         adap->msix_info[0].desc[n] = 0;
 383
 384         for_each_port(adap, j) {
 385                 struct net_device *d = adap->port[j];
 386                 const struct port_info *pi = netdev_priv(d);
 387
 388                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 389                         snprintf(adap->msix_info[msi_idx].desc, n,
 390                                  "%s-%d", d->name, pi->first_qset + i);
 391                         adap->msix_info[msi_idx].desc[n] = 0;
 392                 }
 393         }
 394 }
 395
 396 static int request_msix_data_irqs(struct adapter *adap)
 397 {
 398         int i, j, err, qidx = 0;
 399
 400         for_each_port(adap, i) {
 401                 int nqsets = adap2pinfo(adap, i)->nqsets;
 402
 403                 for (j = 0; j < nqsets; ++j) {
 404                         err = request_irq(adap->msix_info[qidx + 1].vec,
 405                                           t3_intr_handler(adap,
 406                                                           adap->sge.qs[qidx].
 407                                                           rspq.polling), 0,
 408                                           adap->msix_info[qidx + 1].desc,
 409                                           &adap->sge.qs[qidx]);
 410                         if (err) {
 411                                 while (--qidx >= 0)
 412                                         free_irq(adap->msix_info[qidx + 1].vec,
 413                                                  &adap->sge.qs[qidx]);
 414                                 return err;
 415                         }
 416                         qidx++;
 417                 }
 418         }
 419         return 0;
 420 }
 421
 422 static void free_irq_resources(struct adapter *adapter)
 423 {
 424         if (adapter->flags & USING_MSIX) {
 425                 int i, n = 0;
 426
 427                 free_irq(adapter->msix_info[0].vec, adapter);
 428                 for_each_port(adapter, i)
 429                         n += adap2pinfo(adapter, i)->nqsets;
 430
 431                 for (i = 0; i < n; ++i)
 432                         free_irq(adapter->msix_info[i + 1].vec,
 433                                  &adapter->sge.qs[i]);
 434         } else
 435                 free_irq(adapter->pdev->irq, adapter);
 436 }
 437
 438 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 439                               unsigned long n)
 440 {
 441         int attempts = 5;
 442
 443         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 444                 if (!--attempts)
 445                         return -ETIMEDOUT;
 446                 msleep(10);
 447         }
 448         return 0;
 449 }
 450
 451 static int init_tp_parity(struct adapter *adap)
 452 {
 453         int i;
 454         struct sk_buff *skb;
 455         struct cpl_set_tcb_field *greq;
 456         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 457
 458         t3_tp_set_offload_mode(adap, 1);
 459
 460         for (i = 0; i < 16; i++) {
 461                 struct cpl_smt_write_req *req;
 462
 463                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 464                 if (!skb)
 465                         skb = adap->nofail_skb;
 466                 if (!skb)
 467                         goto alloc_skb_fail;
 468
 469                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 470                 memset(req, 0, sizeof(*req));
 471                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 472                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 473                 req->mtu_idx = NMTUS - 1;
 474                 req->iff = i;
 475                 t3_mgmt_tx(adap, skb);
 476                 if (skb == adap->nofail_skb) {
 477                         await_mgmt_replies(adap, cnt, i + 1);
 478                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 479                         if (!adap->nofail_skb)
 480                                 goto alloc_skb_fail;
 481                 }
 482         }
 483
 484         for (i = 0; i < 2048; i++) {
 485                 struct cpl_l2t_write_req *req;
 486
 487                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 488                 if (!skb)
 489                         skb = adap->nofail_skb;
 490                 if (!skb)
 491                         goto alloc_skb_fail;
 492
 493                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 494                 memset(req, 0, sizeof(*req));
 495                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 496                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 497                 req->params = htonl(V_L2T_W_IDX(i));
 498                 t3_mgmt_tx(adap, skb);
 499                 if (skb == adap->nofail_skb) {
 500                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 501                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 502                         if (!adap->nofail_skb)
 503                                 goto alloc_skb_fail;
 504                 }
 505         }
 506
 507         for (i = 0; i < 2048; i++) {
 508                 struct cpl_rte_write_req *req;
 509
 510                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 511                 if (!skb)
 512                         skb = adap->nofail_skb;
 513                 if (!skb)
 514                         goto alloc_skb_fail;
 515
 516                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 517                 memset(req, 0, sizeof(*req));
 518                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 519                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 520                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 521                 t3_mgmt_tx(adap, skb);
 522                 if (skb == adap->nofail_skb) {
 523                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 524                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 525                         if (!adap->nofail_skb)
 526                                 goto alloc_skb_fail;
 527                 }
 528         }
 529
 530         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 531         if (!skb)
 532                 skb = adap->nofail_skb;
 533         if (!skb)
 534                 goto alloc_skb_fail;
 535
 536         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 537         memset(greq, 0, sizeof(*greq));
 538         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 539         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 540         greq->mask = cpu_to_be64(1);
 541         t3_mgmt_tx(adap, skb);
 542
 543         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 544         if (skb == adap->nofail_skb) {
 545                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 546                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 547         }
 548
 549         t3_tp_set_offload_mode(adap, 0);
 550         return i;
 551
 552 alloc_skb_fail:
 553         t3_tp_set_offload_mode(adap, 0);
 554         return -ENOMEM;
 555 }
 556
 557 /**
 558  *      setup_rss - configure RSS
 559  *      @adap: the adapter
 560  *
 561  *      Sets up RSS to distribute packets to multiple receive queues.  We
 562  *      configure the RSS CPU lookup table to distribute to the number of HW
 563  *      receive queues, and the response queue lookup table to narrow that
 564  *      down to the response queues actually configured for each port.
 565  *      We always configure the RSS mapping for two ports since the mapping
 566  *      table has plenty of entries.
 567  */
 568 static void setup_rss(struct adapter *adap)
 569 {
 570         int i;
 571         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 572         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 573         u8 cpus[SGE_QSETS + 1];
 574         u16 rspq_map[RSS_TABLE_SIZE];
 575
 576         for (i = 0; i < SGE_QSETS; ++i)
 577                 cpus[i] = i;
 578         cpus[SGE_QSETS] = 0xff; /* terminator */
 579
 580         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 581                 rspq_map[i] = i % nq0;
 582                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 583         }
 584
 585         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 586                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 587                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 588 }
 589
 590 static void init_napi(struct adapter *adap)
 591 {
 592         int i;
 593
 594         for (i = 0; i < SGE_QSETS; i++) {
 595                 struct sge_qset *qs = &adap->sge.qs[i];
 596
 597                 if (qs->adap)
 598                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 599                                        64);
 600         }
 601
 602         /*
 603          * netif_napi_add() can be called only once per napi_struct because it
 604          * adds each new napi_struct to a list.  Be careful not to call it a
 605          * second time, e.g., during EEH recovery, by making a note of it.
 606          */
 607         adap->flags |= NAPI_INIT;
 608 }
 609
 610 /*
 611  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 612  * both netdevices representing interfaces and the dummy ones for the extra
 613  * queues.
 614  */
 615 static void quiesce_rx(struct adapter *adap)
 616 {
 617         int i;
 618
 619         for (i = 0; i < SGE_QSETS; i++)
 620                 if (adap->sge.qs[i].adap)
 621                         napi_disable(&adap->sge.qs[i].napi);
 622 }
 623
 624 static void enable_all_napi(struct adapter *adap)
 625 {
 626         int i;
 627         for (i = 0; i < SGE_QSETS; i++)
 628                 if (adap->sge.qs[i].adap)
 629                         napi_enable(&adap->sge.qs[i].napi);
 630 }
 631
 632 /**
 633  *      set_qset_lro - Turn a queue set's LRO capability on and off
 634  *      @dev: the device the qset is attached to
 635  *      @qset_idx: the queue set index
 636  *      @val: the LRO switch
 637  *
 638  *      Sets LRO on or off for a particular queue set.
 639  *      the device's features flag is updated to reflect the LRO
 640  *      capability when all queues belonging to the device are
 641  *      in the same state.
 642  */
 643 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 644 {
 645         struct port_info *pi = netdev_priv(dev);
 646         struct adapter *adapter = pi->adapter;
 647
 648         adapter->params.sge.qset[qset_idx].lro = !!val;
 649         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 650 }
 651
 652 /**
 653  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 654  *      @adap: the adapter
 655  *
 656  *      Determines how many sets of SGE queues to use and initializes them.
 657  *      We support multiple queue sets per port if we have MSI-X, otherwise
 658  *      just one queue set per port.
 659  */
 660 static int setup_sge_qsets(struct adapter *adap)
 661 {
 662         int i, j, err, irq_idx = 0, qset_idx = 0;
 663         unsigned int ntxq = SGE_TXQ_PER_SET;
 664
 665         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 666                 irq_idx = -1;
 667
 668         for_each_port(adap, i) {
 669                 struct net_device *dev = adap->port[i];
 670                 struct port_info *pi = netdev_priv(dev);
 671
 672                 pi->qs = &adap->sge.qs[pi->first_qset];
 673                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 674                         set_qset_lro(dev, qset_idx, pi->rx_offload & T3_LRO);
 675                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 676                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 677                                                              irq_idx,
 678                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 679                                 netdev_get_tx_queue(dev, j));
 680                         if (err) {
 681                                 t3_free_sge_resources(adap);
 682                                 return err;
 683                         }
 684                 }
 685         }
 686
 687         return 0;
 688 }
 689
 690 static ssize_t attr_show(struct device *d, char *buf,
 691                          ssize_t(*format) (struct net_device *, char *))
 692 {
 693         ssize_t len;
 694
 695         /* Synchronize with ioctls that may shut down the device */
 696         rtnl_lock();
 697         len = (*format) (to_net_dev(d), buf);
 698         rtnl_unlock();
 699         return len;
 700 }
 701
 702 static ssize_t attr_store(struct device *d,
 703                           const char *buf, size_t len,
 704                           ssize_t(*set) (struct net_device *, unsigned int),
 705                           unsigned int min_val, unsigned int max_val)
 706 {
 707         char *endp;
 708         ssize_t ret;
 709         unsigned int val;
 710
 711         if (!capable(CAP_NET_ADMIN))
 712                 return -EPERM;
 713
 714         val = simple_strtoul(buf, &endp, 0);
 715         if (endp == buf || val < min_val || val > max_val)
 716                 return -EINVAL;
 717
 718         rtnl_lock();
 719         ret = (*set) (to_net_dev(d), val);
 720         if (!ret)
 721                 ret = len;
 722         rtnl_unlock();
 723         return ret;
 724 }
 725
 726 #define CXGB3_SHOW(name, val_expr) \
 727 static ssize_t format_##name(struct net_device *dev, char *buf) \
 728 { \
 729         struct port_info *pi = netdev_priv(dev); \
 730         struct adapter *adap = pi->adapter; \
 731         return sprintf(buf, "%u\n", val_expr); \
 732 } \
 733 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 734                            char *buf) \
 735 { \
 736         return attr_show(d, buf, format_##name); \
 737 }
 738
 739 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 740 {
 741         struct port_info *pi = netdev_priv(dev);
 742         struct adapter *adap = pi->adapter;
 743         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 744
 745         if (adap->flags & FULL_INIT_DONE)
 746                 return -EBUSY;
 747         if (val && adap->params.rev == 0)
 748                 return -EINVAL;
 749         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 750             min_tids)
 751                 return -EINVAL;
 752         adap->params.mc5.nfilters = val;
 753         return 0;
 754 }
 755
 756 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 757                               const char *buf, size_t len)
 758 {
 759         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 760 }
 761
 762 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 763 {
 764         struct port_info *pi = netdev_priv(dev);
 765         struct adapter *adap = pi->adapter;
 766
 767         if (adap->flags & FULL_INIT_DONE)
 768                 return -EBUSY;
 769         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 770             MC5_MIN_TIDS)
 771                 return -EINVAL;
 772         adap->params.mc5.nservers = val;
 773         return 0;
 774 }
 775
 776 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 777                               const char *buf, size_t len)
 778 {
 779         return attr_store(d, buf, len, set_nservers, 0, ~0);
 780 }
 781
 782 #define CXGB3_ATTR_R(name, val_expr) \
 783 CXGB3_SHOW(name, val_expr) \
 784 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 785
 786 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 787 CXGB3_SHOW(name, val_expr) \
 788 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 789
 790 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 791 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 792 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 793
 794 static struct attribute *cxgb3_attrs[] = {
 795         &dev_attr_cam_size.attr,
 796         &dev_attr_nfilters.attr,
 797         &dev_attr_nservers.attr,
 798         NULL
 799 };
 800
 801 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 802
 803 static ssize_t tm_attr_show(struct device *d,
 804                             char *buf, int sched)
 805 {
 806         struct port_info *pi = netdev_priv(to_net_dev(d));
 807         struct adapter *adap = pi->adapter;
 808         unsigned int v, addr, bpt, cpt;
 809         ssize_t len;
 810
 811         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 812         rtnl_lock();
 813         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 814         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 815         if (sched & 1)
 816                 v >>= 16;
 817         bpt = (v >> 8) & 0xff;
 818         cpt = v & 0xff;
 819         if (!cpt)
 820                 len = sprintf(buf, "disabled\n");
 821         else {
 822                 v = (adap->params.vpd.cclk * 1000) / cpt;
 823                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 824         }
 825         rtnl_unlock();
 826         return len;
 827 }
 828
 829 static ssize_t tm_attr_store(struct device *d,
 830                              const char *buf, size_t len, int sched)
 831 {
 832         struct port_info *pi = netdev_priv(to_net_dev(d));
 833         struct adapter *adap = pi->adapter;
 834         unsigned int val;
 835         char *endp;
 836         ssize_t ret;
 837
 838         if (!capable(CAP_NET_ADMIN))
 839                 return -EPERM;
 840
 841         val = simple_strtoul(buf, &endp, 0);
 842         if (endp == buf || val > 10000000)
 843                 return -EINVAL;
 844
 845         rtnl_lock();
 846         ret = t3_config_sched(adap, val, sched);
 847         if (!ret)
 848                 ret = len;
 849         rtnl_unlock();
 850         return ret;
 851 }
 852
 853 #define TM_ATTR(name, sched) \
 854 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 855                            char *buf) \
 856 { \
 857         return tm_attr_show(d, buf, sched); \
 858 } \
 859 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 860                             const char *buf, size_t len) \
 861 { \
 862         return tm_attr_store(d, buf, len, sched); \
 863 } \
 864 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 865
 866 TM_ATTR(sched0, 0);
 867 TM_ATTR(sched1, 1);
 868 TM_ATTR(sched2, 2);
 869 TM_ATTR(sched3, 3);
 870 TM_ATTR(sched4, 4);
 871 TM_ATTR(sched5, 5);
 872 TM_ATTR(sched6, 6);
 873 TM_ATTR(sched7, 7);
 874
 875 static struct attribute *offload_attrs[] = {
 876         &dev_attr_sched0.attr,
 877         &dev_attr_sched1.attr,
 878         &dev_attr_sched2.attr,
 879         &dev_attr_sched3.attr,
 880         &dev_attr_sched4.attr,
 881         &dev_attr_sched5.attr,
 882         &dev_attr_sched6.attr,
 883         &dev_attr_sched7.attr,
 884         NULL
 885 };
 886
 887 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 888
 889 /*
 890  * Sends an sk_buff to an offload queue driver
 891  * after dealing with any active network taps.
 892  */
 893 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 894 {
 895         int ret;
 896
 897         local_bh_disable();
 898         ret = t3_offload_tx(tdev, skb);
 899         local_bh_enable();
 900         return ret;
 901 }
 902
 903 static int write_smt_entry(struct adapter *adapter, int idx)
 904 {
 905         struct cpl_smt_write_req *req;
 906         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 907
 908         if (!skb)
 909                 return -ENOMEM;
 910
 911         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 912         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 913         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 914         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 915         req->iff = idx;
 916         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 917         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 918         skb->priority = 1;
 919         offload_tx(&adapter->tdev, skb);
 920         return 0;
 921 }
 922
 923 static int init_smt(struct adapter *adapter)
 924 {
 925         int i;
 926
 927         for_each_port(adapter, i)
 928             write_smt_entry(adapter, i);
 929         return 0;
 930 }
 931
 932 static void init_port_mtus(struct adapter *adapter)
 933 {
 934         unsigned int mtus = adapter->port[0]->mtu;
 935
 936         if (adapter->port[1])
 937                 mtus |= adapter->port[1]->mtu << 16;
 938         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 939 }
 940
 941 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 942                               int hi, int port)
 943 {
 944         struct sk_buff *skb;
 945         struct mngt_pktsched_wr *req;
 946         int ret;
 947
 948         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 949         if (!skb)
 950                 skb = adap->nofail_skb;
 951         if (!skb)
 952                 return -ENOMEM;
 953
 954         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 955         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 956         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 957         req->sched = sched;
 958         req->idx = qidx;
 959         req->min = lo;
 960         req->max = hi;
 961         req->binding = port;
 962         ret = t3_mgmt_tx(adap, skb);
 963         if (skb == adap->nofail_skb) {
 964                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 965                                              GFP_KERNEL);
 966                 if (!adap->nofail_skb)
 967                         ret = -ENOMEM;
 968         }
 969
 970         return ret;
 971 }
 972
 973 static int bind_qsets(struct adapter *adap)
 974 {
 975         int i, j, err = 0;
 976
 977         for_each_port(adap, i) {
 978                 const struct port_info *pi = adap2pinfo(adap, i);
 979
 980                 for (j = 0; j < pi->nqsets; ++j) {
 981                         int ret = send_pktsched_cmd(adap, 1,
 982                                                     pi->first_qset + j, -1,
 983                                                     -1, i);
 984                         if (ret)
 985                                 err = ret;
 986                 }
 987         }
 988
 989         return err;
 990 }
 991
 992 #define FW_FNAME "cxgb3/t3fw-%d.%d.%d.bin"
 993 #define TPSRAM_NAME "cxgb3/t3%c_psram-%d.%d.%d.bin"
 994 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
 995 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
 996 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
 997
 998 static inline const char *get_edc_fw_name(int edc_idx)
 999 {
1000         const char *fw_name = NULL;
1001
1002         switch (edc_idx) {
1003         case EDC_OPT_AEL2005:
1004                 fw_name = AEL2005_OPT_EDC_NAME;
1005                 break;
1006         case EDC_TWX_AEL2005:
1007                 fw_name = AEL2005_TWX_EDC_NAME;
1008                 break;
1009         case EDC_TWX_AEL2020:
1010                 fw_name = AEL2020_TWX_EDC_NAME;
1011                 break;
1012         }
1013         return fw_name;
1014 }
1015
1016 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1017 {
1018         struct adapter *adapter = phy->adapter;
1019         const struct firmware *fw;
1020         char buf[64];
1021         u32 csum;
1022         const __be32 *p;
1023         u16 *cache = phy->phy_cache;
1024         int i, ret;
1025
1026         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1027
1028         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1029         if (ret < 0) {
1030                 dev_err(&adapter->pdev->dev,
1031                         "could not upgrade firmware: unable to load %s\n",
1032                         buf);
1033                 return ret;
1034         }
1035
1036         /* check size, take checksum in account */
1037         if (fw->size > size + 4) {
1038                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1039                        (unsigned int)fw->size, size + 4);
1040                 ret = -EINVAL;
1041         }
1042
1043         /* compute checksum */
1044         p = (const __be32 *)fw->data;
1045         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1046                 csum += ntohl(p[i]);
1047
1048         if (csum != 0xffffffff) {
1049                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1050                        csum);
1051                 ret = -EINVAL;
1052         }
1053
1054         for (i = 0; i < size / 4 ; i++) {
1055                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1056                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1057         }
1058
1059         release_firmware(fw);
1060
1061         return ret;
1062 }
1063
1064 static int upgrade_fw(struct adapter *adap)
1065 {
1066         int ret;
1067         char buf[64];
1068         const struct firmware *fw;
1069         struct device *dev = &adap->pdev->dev;
1070
1071         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
1072                  FW_VERSION_MINOR, FW_VERSION_MICRO);
1073         ret = request_firmware(&fw, buf, dev);
1074         if (ret < 0) {
1075                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1076                         buf);
1077                 return ret;
1078         }
1079         ret = t3_load_fw(adap, fw->data, fw->size);
1080         release_firmware(fw);
1081
1082         if (ret == 0)
1083                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1084                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1085         else
1086                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1087                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1088
1089         return ret;
1090 }
1091
1092 static inline char t3rev2char(struct adapter *adapter)
1093 {
1094         char rev = 0;
1095
1096         switch(adapter->params.rev) {
1097         case T3_REV_B:
1098         case T3_REV_B2:
1099                 rev = 'b';
1100                 break;
1101         case T3_REV_C:
1102                 rev = 'c';
1103                 break;
1104         }
1105         return rev;
1106 }
1107
1108 static int update_tpsram(struct adapter *adap)
1109 {
1110         const struct firmware *tpsram;
1111         char buf[64];
1112         struct device *dev = &adap->pdev->dev;
1113         int ret;
1114         char rev;
1115
1116         rev = t3rev2char(adap);
1117         if (!rev)
1118                 return 0;
1119
1120         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
1121                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1122
1123         ret = request_firmware(&tpsram, buf, dev);
1124         if (ret < 0) {
1125                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1126                         buf);
1127                 return ret;
1128         }
1129
1130         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1131         if (ret)
1132                 goto release_tpsram;
1133
1134         ret = t3_set_proto_sram(adap, tpsram->data);
1135         if (ret == 0)
1136                 dev_info(dev,
1137                          "successful update of protocol engine "
1138                          "to %d.%d.%d\n",
1139                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1140         else
1141                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1142                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1143         if (ret)
1144                 dev_err(dev, "loading protocol SRAM failed\n");
1145
1146 release_tpsram:
1147         release_firmware(tpsram);
1148
1149         return ret;
1150 }
1151
1152 /**
1153  *      cxgb_up - enable the adapter
1154  *      @adapter: adapter being enabled
1155  *
1156  *      Called when the first port is enabled, this function performs the
1157  *      actions necessary to make an adapter operational, such as completing
1158  *      the initialization of HW modules, and enabling interrupts.
1159  *
1160  *      Must be called with the rtnl lock held.
1161  */
1162 static int cxgb_up(struct adapter *adap)
1163 {
1164         int err;
1165
1166         if (!(adap->flags & FULL_INIT_DONE)) {
1167                 err = t3_check_fw_version(adap);
1168                 if (err == -EINVAL) {
1169                         err = upgrade_fw(adap);
1170                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1171                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1172                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1173                 }
1174
1175                 err = t3_check_tpsram_version(adap);
1176                 if (err == -EINVAL) {
1177                         err = update_tpsram(adap);
1178                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1179                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1180                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1181                 }
1182
1183                 /*
1184                  * Clear interrupts now to catch errors if t3_init_hw fails.
1185                  * We clear them again later as initialization may trigger
1186                  * conditions that can interrupt.
1187                  */
1188                 t3_intr_clear(adap);
1189
1190                 err = t3_init_hw(adap, 0);
1191                 if (err)
1192                         goto out;
1193
1194                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1195                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1196
1197                 err = setup_sge_qsets(adap);
1198                 if (err)
1199                         goto out;
1200
1201                 setup_rss(adap);
1202                 if (!(adap->flags & NAPI_INIT))
1203                         init_napi(adap);
1204
1205                 t3_start_sge_timers(adap);
1206                 adap->flags |= FULL_INIT_DONE;
1207         }
1208
1209         t3_intr_clear(adap);
1210
1211         if (adap->flags & USING_MSIX) {
1212                 name_msix_vecs(adap);
1213                 err = request_irq(adap->msix_info[0].vec,
1214                                   t3_async_intr_handler, 0,
1215                                   adap->msix_info[0].desc, adap);
1216                 if (err)
1217                         goto irq_err;
1218
1219                 err = request_msix_data_irqs(adap);
1220                 if (err) {
1221                         free_irq(adap->msix_info[0].vec, adap);
1222                         goto irq_err;
1223                 }
1224         } else if ((err = request_irq(adap->pdev->irq,
1225                                       t3_intr_handler(adap,
1226                                                       adap->sge.qs[0].rspq.
1227                                                       polling),
1228                                       (adap->flags & USING_MSI) ?
1229                                        0 : IRQF_SHARED,
1230                                       adap->name, adap)))
1231                 goto irq_err;
1232
1233         enable_all_napi(adap);
1234         t3_sge_start(adap);
1235         t3_intr_enable(adap);
1236
1237         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1238             is_offload(adap) && init_tp_parity(adap) == 0)
1239                 adap->flags |= TP_PARITY_INIT;
1240
1241         if (adap->flags & TP_PARITY_INIT) {
1242                 t3_write_reg(adap, A_TP_INT_CAUSE,
1243                              F_CMCACHEPERR | F_ARPLUTPERR);
1244                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1245         }
1246
1247         if (!(adap->flags & QUEUES_BOUND)) {
1248                 err = bind_qsets(adap);
1249                 if (err) {
1250                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1251                         t3_intr_disable(adap);
1252                         free_irq_resources(adap);
1253                         goto out;
1254                 }
1255                 adap->flags |= QUEUES_BOUND;
1256         }
1257
1258 out:
1259         return err;
1260 irq_err:
1261         CH_ERR(adap, "request_irq failed, err %d\n", err);
1262         goto out;
1263 }
1264
1265 /*
1266  * Release resources when all the ports and offloading have been stopped.
1267  */
1268 static void cxgb_down(struct adapter *adapter)
1269 {
1270         t3_sge_stop(adapter);
1271         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1272         t3_intr_disable(adapter);
1273         spin_unlock_irq(&adapter->work_lock);
1274
1275         free_irq_resources(adapter);
1276         quiesce_rx(adapter);
1277         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1278 }
1279
1280 static void schedule_chk_task(struct adapter *adap)
1281 {
1282         unsigned int timeo;
1283
1284         timeo = adap->params.linkpoll_period ?
1285             (HZ * adap->params.linkpoll_period) / 10 :
1286             adap->params.stats_update_period * HZ;
1287         if (timeo)
1288                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1289 }
1290
1291 static int offload_open(struct net_device *dev)
1292 {
1293         struct port_info *pi = netdev_priv(dev);
1294         struct adapter *adapter = pi->adapter;
1295         struct t3cdev *tdev = dev2t3cdev(dev);
1296         int adap_up = adapter->open_device_map & PORT_MASK;
1297         int err;
1298
1299         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1300                 return 0;
1301
1302         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1303                 goto out;
1304
1305         t3_tp_set_offload_mode(adapter, 1);
1306         tdev->lldev = adapter->port[0];
1307         err = cxgb3_offload_activate(adapter);
1308         if (err)
1309                 goto out;
1310
1311         init_port_mtus(adapter);
1312         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1313                      adapter->params.b_wnd,
1314                      adapter->params.rev == 0 ?
1315                      adapter->port[0]->mtu : 0xffff);
1316         init_smt(adapter);
1317
1318         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1319                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1320
1321         /* Call back all registered clients */
1322         cxgb3_add_clients(tdev);
1323
1324 out:
1325         /* restore them in case the offload module has changed them */
1326         if (err) {
1327                 t3_tp_set_offload_mode(adapter, 0);
1328                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1329                 cxgb3_set_dummy_ops(tdev);
1330         }
1331         return err;
1332 }
1333
1334 static int offload_close(struct t3cdev *tdev)
1335 {
1336         struct adapter *adapter = tdev2adap(tdev);
1337
1338         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1339                 return 0;
1340
1341         /* Call back all registered clients */
1342         cxgb3_remove_clients(tdev);
1343
1344         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1345
1346         /* Flush work scheduled while releasing TIDs */
1347         flush_scheduled_work();
1348
1349         tdev->lldev = NULL;
1350         cxgb3_set_dummy_ops(tdev);
1351         t3_tp_set_offload_mode(adapter, 0);
1352         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1353
1354         if (!adapter->open_device_map)
1355                 cxgb_down(adapter);
1356
1357         cxgb3_offload_deactivate(adapter);
1358         return 0;
1359 }
1360
1361 static int cxgb_open(struct net_device *dev)
1362 {
1363         struct port_info *pi = netdev_priv(dev);
1364         struct adapter *adapter = pi->adapter;
1365         int other_ports = adapter->open_device_map & PORT_MASK;
1366         int err;
1367
1368         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1369                 return err;
1370
1371         set_bit(pi->port_id, &adapter->open_device_map);
1372         if (is_offload(adapter) && !ofld_disable) {
1373                 err = offload_open(dev);
1374                 if (err)
1375                         printk(KERN_WARNING
1376                                "Could not initialize offload capabilities\n");
1377         }
1378
1379         dev->real_num_tx_queues = pi->nqsets;
1380         link_start(dev);
1381         t3_port_intr_enable(adapter, pi->port_id);
1382         netif_tx_start_all_queues(dev);
1383         if (!other_ports)
1384                 schedule_chk_task(adapter);
1385
1386         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1387         return 0;
1388 }
1389
1390 static int cxgb_close(struct net_device *dev)
1391 {
1392         struct port_info *pi = netdev_priv(dev);
1393         struct adapter *adapter = pi->adapter;
1394
1395
1396         if (!adapter->open_device_map)
1397                 return 0;
1398
1399         /* Stop link fault interrupts */
1400         t3_xgm_intr_disable(adapter, pi->port_id);
1401         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1402
1403         t3_port_intr_disable(adapter, pi->port_id);
1404         netif_tx_stop_all_queues(dev);
1405         pi->phy.ops->power_down(&pi->phy, 1);
1406         netif_carrier_off(dev);
1407         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1408
1409         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1410         clear_bit(pi->port_id, &adapter->open_device_map);
1411         spin_unlock_irq(&adapter->work_lock);
1412
1413         if (!(adapter->open_device_map & PORT_MASK))
1414                 cancel_delayed_work_sync(&adapter->adap_check_task);
1415
1416         if (!adapter->open_device_map)
1417                 cxgb_down(adapter);
1418
1419         cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1420         return 0;
1421 }
1422
1423 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1424 {
1425         struct port_info *pi = netdev_priv(dev);
1426         struct adapter *adapter = pi->adapter;
1427         struct net_device_stats *ns = &pi->netstats;
1428         const struct mac_stats *pstats;
1429
1430         spin_lock(&adapter->stats_lock);
1431         pstats = t3_mac_update_stats(&pi->mac);
1432         spin_unlock(&adapter->stats_lock);
1433
1434         ns->tx_bytes = pstats->tx_octets;
1435         ns->tx_packets = pstats->tx_frames;
1436         ns->rx_bytes = pstats->rx_octets;
1437         ns->rx_packets = pstats->rx_frames;
1438         ns->multicast = pstats->rx_mcast_frames;
1439
1440         ns->tx_errors = pstats->tx_underrun;
1441         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1442             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1443             pstats->rx_fifo_ovfl;
1444
1445         /* detailed rx_errors */
1446         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1447         ns->rx_over_errors = 0;
1448         ns->rx_crc_errors = pstats->rx_fcs_errs;
1449         ns->rx_frame_errors = pstats->rx_symbol_errs;
1450         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1451         ns->rx_missed_errors = pstats->rx_cong_drops;
1452
1453         /* detailed tx_errors */
1454         ns->tx_aborted_errors = 0;
1455         ns->tx_carrier_errors = 0;
1456         ns->tx_fifo_errors = pstats->tx_underrun;
1457         ns->tx_heartbeat_errors = 0;
1458         ns->tx_window_errors = 0;
1459         return ns;
1460 }
1461
1462 static u32 get_msglevel(struct net_device *dev)
1463 {
1464         struct port_info *pi = netdev_priv(dev);
1465         struct adapter *adapter = pi->adapter;
1466
1467         return adapter->msg_enable;
1468 }
1469
1470 static void set_msglevel(struct net_device *dev, u32 val)
1471 {
1472         struct port_info *pi = netdev_priv(dev);
1473         struct adapter *adapter = pi->adapter;
1474
1475         adapter->msg_enable = val;
1476 }
1477
1478 static char stats_strings[][ETH_GSTRING_LEN] = {
1479         "TxOctetsOK         ",
1480         "TxFramesOK         ",
1481         "TxMulticastFramesOK",
1482         "TxBroadcastFramesOK",
1483         "TxPauseFrames      ",
1484         "TxUnderrun         ",
1485         "TxExtUnderrun      ",
1486
1487         "TxFrames64         ",
1488         "TxFrames65To127    ",
1489         "TxFrames128To255   ",
1490         "TxFrames256To511   ",
1491         "TxFrames512To1023  ",
1492         "TxFrames1024To1518 ",
1493         "TxFrames1519ToMax  ",
1494
1495         "RxOctetsOK         ",
1496         "RxFramesOK         ",
1497         "RxMulticastFramesOK",
1498         "RxBroadcastFramesOK",
1499         "RxPauseFrames      ",
1500         "RxFCSErrors        ",
1501         "RxSymbolErrors     ",
1502         "RxShortErrors      ",
1503         "RxJabberErrors     ",
1504         "RxLengthErrors     ",
1505         "RxFIFOoverflow     ",
1506
1507         "RxFrames64         ",
1508         "RxFrames65To127    ",
1509         "RxFrames128To255   ",
1510         "RxFrames256To511   ",
1511         "RxFrames512To1023  ",
1512         "RxFrames1024To1518 ",
1513         "RxFrames1519ToMax  ",
1514
1515         "PhyFIFOErrors      ",
1516         "TSO                ",
1517         "VLANextractions    ",
1518         "VLANinsertions     ",
1519         "TxCsumOffload      ",
1520         "RxCsumGood         ",
1521         "LroAggregated      ",
1522         "LroFlushed         ",
1523         "LroNoDesc          ",
1524         "RxDrops            ",
1525
1526         "CheckTXEnToggled   ",
1527         "CheckResets        ",
1528
1529         "LinkFaults         ",
1530 };
1531
1532 static int get_sset_count(struct net_device *dev, int sset)
1533 {
1534         switch (sset) {
1535         case ETH_SS_STATS:
1536                 return ARRAY_SIZE(stats_strings);
1537         default:
1538                 return -EOPNOTSUPP;
1539         }
1540 }
1541
1542 #define T3_REGMAP_SIZE (3 * 1024)
1543
1544 static int get_regs_len(struct net_device *dev)
1545 {
1546         return T3_REGMAP_SIZE;
1547 }
1548
1549 static int get_eeprom_len(struct net_device *dev)
1550 {
1551         return EEPROMSIZE;
1552 }
1553
1554 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1555 {
1556         struct port_info *pi = netdev_priv(dev);
1557         struct adapter *adapter = pi->adapter;
1558         u32 fw_vers = 0;
1559         u32 tp_vers = 0;
1560
1561         spin_lock(&adapter->stats_lock);
1562         t3_get_fw_version(adapter, &fw_vers);
1563         t3_get_tp_version(adapter, &tp_vers);
1564         spin_unlock(&adapter->stats_lock);
1565
1566         strcpy(info->driver, DRV_NAME);
1567         strcpy(info->version, DRV_VERSION);
1568         strcpy(info->bus_info, pci_name(adapter->pdev));
1569         if (!fw_vers)
1570                 strcpy(info->fw_version, "N/A");
1571         else {
1572                 snprintf(info->fw_version, sizeof(info->fw_version),
1573                          "%s %u.%u.%u TP %u.%u.%u",
1574                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1575                          G_FW_VERSION_MAJOR(fw_vers),
1576                          G_FW_VERSION_MINOR(fw_vers),
1577                          G_FW_VERSION_MICRO(fw_vers),
1578                          G_TP_VERSION_MAJOR(tp_vers),
1579                          G_TP_VERSION_MINOR(tp_vers),
1580                          G_TP_VERSION_MICRO(tp_vers));
1581         }
1582 }
1583
1584 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1585 {
1586         if (stringset == ETH_SS_STATS)
1587                 memcpy(data, stats_strings, sizeof(stats_strings));
1588 }
1589
1590 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1591                                             struct port_info *p, int idx)
1592 {
1593         int i;
1594         unsigned long tot = 0;
1595
1596         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1597                 tot += adapter->sge.qs[i].port_stats[idx];
1598         return tot;
1599 }
1600
1601 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1602                       u64 *data)
1603 {
1604         struct port_info *pi = netdev_priv(dev);
1605         struct adapter *adapter = pi->adapter;
1606         const struct mac_stats *s;
1607
1608         spin_lock(&adapter->stats_lock);
1609         s = t3_mac_update_stats(&pi->mac);
1610         spin_unlock(&adapter->stats_lock);
1611
1612         *data++ = s->tx_octets;
1613         *data++ = s->tx_frames;
1614         *data++ = s->tx_mcast_frames;
1615         *data++ = s->tx_bcast_frames;
1616         *data++ = s->tx_pause;
1617         *data++ = s->tx_underrun;
1618         *data++ = s->tx_fifo_urun;
1619
1620         *data++ = s->tx_frames_64;
1621         *data++ = s->tx_frames_65_127;
1622         *data++ = s->tx_frames_128_255;
1623         *data++ = s->tx_frames_256_511;
1624         *data++ = s->tx_frames_512_1023;
1625         *data++ = s->tx_frames_1024_1518;
1626         *data++ = s->tx_frames_1519_max;
1627
1628         *data++ = s->rx_octets;
1629         *data++ = s->rx_frames;
1630         *data++ = s->rx_mcast_frames;
1631         *data++ = s->rx_bcast_frames;
1632         *data++ = s->rx_pause;
1633         *data++ = s->rx_fcs_errs;
1634         *data++ = s->rx_symbol_errs;
1635         *data++ = s->rx_short;
1636         *data++ = s->rx_jabber;
1637         *data++ = s->rx_too_long;
1638         *data++ = s->rx_fifo_ovfl;
1639
1640         *data++ = s->rx_frames_64;
1641         *data++ = s->rx_frames_65_127;
1642         *data++ = s->rx_frames_128_255;
1643         *data++ = s->rx_frames_256_511;
1644         *data++ = s->rx_frames_512_1023;
1645         *data++ = s->rx_frames_1024_1518;
1646         *data++ = s->rx_frames_1519_max;
1647
1648         *data++ = pi->phy.fifo_errors;
1649
1650         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1651         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1652         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1653         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1654         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1655         *data++ = 0;
1656         *data++ = 0;
1657         *data++ = 0;
1658         *data++ = s->rx_cong_drops;
1659
1660         *data++ = s->num_toggled;
1661         *data++ = s->num_resets;
1662
1663         *data++ = s->link_faults;
1664 }
1665
1666 static inline void reg_block_dump(struct adapter *ap, void *buf,
1667                                   unsigned int start, unsigned int end)
1668 {
1669         u32 *p = buf + start;
1670
1671         for (; start <= end; start += sizeof(u32))
1672                 *p++ = t3_read_reg(ap, start);
1673 }
1674
1675 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1676                      void *buf)
1677 {
1678         struct port_info *pi = netdev_priv(dev);
1679         struct adapter *ap = pi->adapter;
1680
1681         /*
1682          * Version scheme:
1683          * bits 0..9: chip version
1684          * bits 10..15: chip revision
1685          * bit 31: set for PCIe cards
1686          */
1687         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1688
1689         /*
1690          * We skip the MAC statistics registers because they are clear-on-read.
1691          * Also reading multi-register stats would need to synchronize with the
1692          * periodic mac stats accumulation.  Hard to justify the complexity.
1693          */
1694         memset(buf, 0, T3_REGMAP_SIZE);
1695         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1696         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1697         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1698         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1699         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1700         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1701                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1702         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1703                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1704 }
1705
1706 static int restart_autoneg(struct net_device *dev)
1707 {
1708         struct port_info *p = netdev_priv(dev);
1709
1710         if (!netif_running(dev))
1711                 return -EAGAIN;
1712         if (p->link_config.autoneg != AUTONEG_ENABLE)
1713                 return -EINVAL;
1714         p->phy.ops->autoneg_restart(&p->phy);
1715         return 0;
1716 }
1717
1718 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1719 {
1720         struct port_info *pi = netdev_priv(dev);
1721         struct adapter *adapter = pi->adapter;
1722         int i;
1723
1724         if (data == 0)
1725                 data = 2;
1726
1727         for (i = 0; i < data * 2; i++) {
1728                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1729                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1730                 if (msleep_interruptible(500))
1731                         break;
1732         }
1733         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1734                          F_GPIO0_OUT_VAL);
1735         return 0;
1736 }
1737
1738 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1739 {
1740         struct port_info *p = netdev_priv(dev);
1741
1742         cmd->supported = p->link_config.supported;
1743         cmd->advertising = p->link_config.advertising;
1744
1745         if (netif_carrier_ok(dev)) {
1746                 cmd->speed = p->link_config.speed;
1747                 cmd->duplex = p->link_config.duplex;
1748         } else {
1749                 cmd->speed = -1;
1750                 cmd->duplex = -1;
1751         }
1752
1753         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1754         cmd->phy_address = p->phy.mdio.prtad;
1755         cmd->transceiver = XCVR_EXTERNAL;
1756         cmd->autoneg = p->link_config.autoneg;
1757         cmd->maxtxpkt = 0;
1758         cmd->maxrxpkt = 0;
1759         return 0;
1760 }
1761
1762 static int speed_duplex_to_caps(int speed, int duplex)
1763 {
1764         int cap = 0;
1765
1766         switch (speed) {
1767         case SPEED_10:
1768                 if (duplex == DUPLEX_FULL)
1769                         cap = SUPPORTED_10baseT_Full;
1770                 else
1771                         cap = SUPPORTED_10baseT_Half;
1772                 break;
1773         case SPEED_100:
1774                 if (duplex == DUPLEX_FULL)
1775                         cap = SUPPORTED_100baseT_Full;
1776                 else
1777                         cap = SUPPORTED_100baseT_Half;
1778                 break;
1779         case SPEED_1000:
1780                 if (duplex == DUPLEX_FULL)
1781                         cap = SUPPORTED_1000baseT_Full;
1782                 else
1783                         cap = SUPPORTED_1000baseT_Half;
1784                 break;
1785         case SPEED_10000:
1786                 if (duplex == DUPLEX_FULL)
1787                         cap = SUPPORTED_10000baseT_Full;
1788         }
1789         return cap;
1790 }
1791
1792 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1793                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1794                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1795                       ADVERTISED_10000baseT_Full)
1796
1797 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1798 {
1799         struct port_info *p = netdev_priv(dev);
1800         struct link_config *lc = &p->link_config;
1801
1802         if (!(lc->supported & SUPPORTED_Autoneg)) {
1803                 /*
1804                  * PHY offers a single speed/duplex.  See if that's what's
1805                  * being requested.
1806                  */
1807                 if (cmd->autoneg == AUTONEG_DISABLE) {
1808                         int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1809                         if (lc->supported & cap)
1810                                 return 0;
1811                 }
1812                 return -EINVAL;
1813         }
1814
1815         if (cmd->autoneg == AUTONEG_DISABLE) {
1816                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1817
1818                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1819                         return -EINVAL;
1820                 lc->requested_speed = cmd->speed;
1821                 lc->requested_duplex = cmd->duplex;
1822                 lc->advertising = 0;
1823         } else {
1824                 cmd->advertising &= ADVERTISED_MASK;
1825                 cmd->advertising &= lc->supported;
1826                 if (!cmd->advertising)
1827                         return -EINVAL;
1828                 lc->requested_speed = SPEED_INVALID;
1829                 lc->requested_duplex = DUPLEX_INVALID;
1830                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1831         }
1832         lc->autoneg = cmd->autoneg;
1833         if (netif_running(dev))
1834                 t3_link_start(&p->phy, &p->mac, lc);
1835         return 0;
1836 }
1837
1838 static void get_pauseparam(struct net_device *dev,
1839                            struct ethtool_pauseparam *epause)
1840 {
1841         struct port_info *p = netdev_priv(dev);
1842
1843         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1844         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1845         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1846 }
1847
1848 static int set_pauseparam(struct net_device *dev,
1849                           struct ethtool_pauseparam *epause)
1850 {
1851         struct port_info *p = netdev_priv(dev);
1852         struct link_config *lc = &p->link_config;
1853
1854         if (epause->autoneg == AUTONEG_DISABLE)
1855                 lc->requested_fc = 0;
1856         else if (lc->supported & SUPPORTED_Autoneg)
1857                 lc->requested_fc = PAUSE_AUTONEG;
1858         else
1859                 return -EINVAL;
1860
1861         if (epause->rx_pause)
1862                 lc->requested_fc |= PAUSE_RX;
1863         if (epause->tx_pause)
1864                 lc->requested_fc |= PAUSE_TX;
1865         if (lc->autoneg == AUTONEG_ENABLE) {
1866                 if (netif_running(dev))
1867                         t3_link_start(&p->phy, &p->mac, lc);
1868         } else {
1869                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1870                 if (netif_running(dev))
1871                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1872         }
1873         return 0;
1874 }
1875
1876 static u32 get_rx_csum(struct net_device *dev)
1877 {
1878         struct port_info *p = netdev_priv(dev);
1879
1880         return p->rx_offload & T3_RX_CSUM;
1881 }
1882
1883 static int set_rx_csum(struct net_device *dev, u32 data)
1884 {
1885         struct port_info *p = netdev_priv(dev);
1886
1887         if (data) {
1888                 p->rx_offload |= T3_RX_CSUM;
1889         } else {
1890                 int i;
1891
1892                 p->rx_offload &= ~(T3_RX_CSUM | T3_LRO);
1893                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1894                         set_qset_lro(dev, i, 0);
1895         }
1896         return 0;
1897 }
1898
1899 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1900 {
1901         struct port_info *pi = netdev_priv(dev);
1902         struct adapter *adapter = pi->adapter;
1903         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1904
1905         e->rx_max_pending = MAX_RX_BUFFERS;
1906         e->rx_mini_max_pending = 0;
1907         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1908         e->tx_max_pending = MAX_TXQ_ENTRIES;
1909
1910         e->rx_pending = q->fl_size;
1911         e->rx_mini_pending = q->rspq_size;
1912         e->rx_jumbo_pending = q->jumbo_size;
1913         e->tx_pending = q->txq_size[0];
1914 }
1915
1916 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1917 {
1918         struct port_info *pi = netdev_priv(dev);
1919         struct adapter *adapter = pi->adapter;
1920         struct qset_params *q;
1921         int i;
1922
1923         if (e->rx_pending > MAX_RX_BUFFERS ||
1924             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1925             e->tx_pending > MAX_TXQ_ENTRIES ||
1926             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1927             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1928             e->rx_pending < MIN_FL_ENTRIES ||
1929             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1930             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1931                 return -EINVAL;
1932
1933         if (adapter->flags & FULL_INIT_DONE)
1934                 return -EBUSY;
1935
1936         q = &adapter->params.sge.qset[pi->first_qset];
1937         for (i = 0; i < pi->nqsets; ++i, ++q) {
1938                 q->rspq_size = e->rx_mini_pending;
1939                 q->fl_size = e->rx_pending;
1940                 q->jumbo_size = e->rx_jumbo_pending;
1941                 q->txq_size[0] = e->tx_pending;
1942                 q->txq_size[1] = e->tx_pending;
1943                 q->txq_size[2] = e->tx_pending;
1944         }
1945         return 0;
1946 }
1947
1948 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1949 {
1950         struct port_info *pi = netdev_priv(dev);
1951         struct adapter *adapter = pi->adapter;
1952         struct qset_params *qsp = &adapter->params.sge.qset[0];
1953         struct sge_qset *qs = &adapter->sge.qs[0];
1954
1955         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1956                 return -EINVAL;
1957
1958         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1959         t3_update_qset_coalesce(qs, qsp);
1960         return 0;
1961 }
1962
1963 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1964 {
1965         struct port_info *pi = netdev_priv(dev);
1966         struct adapter *adapter = pi->adapter;
1967         struct qset_params *q = adapter->params.sge.qset;
1968
1969         c->rx_coalesce_usecs = q->coalesce_usecs;
1970         return 0;
1971 }
1972
1973 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1974                       u8 * data)
1975 {
1976         struct port_info *pi = netdev_priv(dev);
1977         struct adapter *adapter = pi->adapter;
1978         int i, err = 0;
1979
1980         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1981         if (!buf)
1982                 return -ENOMEM;
1983
1984         e->magic = EEPROM_MAGIC;
1985         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1986                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1987
1988         if (!err)
1989                 memcpy(data, buf + e->offset, e->len);
1990         kfree(buf);
1991         return err;
1992 }
1993
1994 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1995                       u8 * data)
1996 {
1997         struct port_info *pi = netdev_priv(dev);
1998         struct adapter *adapter = pi->adapter;
1999         u32 aligned_offset, aligned_len;
2000         __le32 *p;
2001         u8 *buf;
2002         int err;
2003
2004         if (eeprom->magic != EEPROM_MAGIC)
2005                 return -EINVAL;
2006
2007         aligned_offset = eeprom->offset & ~3;
2008         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2009
2010         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2011                 buf = kmalloc(aligned_len, GFP_KERNEL);
2012                 if (!buf)
2013                         return -ENOMEM;
2014                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2015                 if (!err && aligned_len > 4)
2016                         err = t3_seeprom_read(adapter,
2017                                               aligned_offset + aligned_len - 4,
2018                                               (__le32 *) & buf[aligned_len - 4]);
2019                 if (err)
2020                         goto out;
2021                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2022         } else
2023                 buf = data;
2024
2025         err = t3_seeprom_wp(adapter, 0);
2026         if (err)
2027                 goto out;
2028
2029         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2030                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2031                 aligned_offset += 4;
2032         }
2033
2034         if (!err)
2035                 err = t3_seeprom_wp(adapter, 1);
2036 out:
2037         if (buf != data)
2038                 kfree(buf);
2039         return err;
2040 }
2041
2042 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2043 {
2044         wol->supported = 0;
2045         wol->wolopts = 0;
2046         memset(&wol->sopass, 0, sizeof(wol->sopass));
2047 }
2048
2049 static const struct ethtool_ops cxgb_ethtool_ops = {
2050         .get_settings = get_settings,
2051         .set_settings = set_settings,
2052         .get_drvinfo = get_drvinfo,
2053         .get_msglevel = get_msglevel,
2054         .set_msglevel = set_msglevel,
2055         .get_ringparam = get_sge_param,
2056         .set_ringparam = set_sge_param,
2057         .get_coalesce = get_coalesce,
2058         .set_coalesce = set_coalesce,
2059         .get_eeprom_len = get_eeprom_len,
2060         .get_eeprom = get_eeprom,
2061         .set_eeprom = set_eeprom,
2062         .get_pauseparam = get_pauseparam,
2063         .set_pauseparam = set_pauseparam,
2064         .get_rx_csum = get_rx_csum,
2065         .set_rx_csum = set_rx_csum,
2066         .set_tx_csum = ethtool_op_set_tx_csum,
2067         .set_sg = ethtool_op_set_sg,
2068         .get_link = ethtool_op_get_link,
2069         .get_strings = get_strings,
2070         .phys_id = cxgb3_phys_id,
2071         .nway_reset = restart_autoneg,
2072         .get_sset_count = get_sset_count,
2073         .get_ethtool_stats = get_stats,
2074         .get_regs_len = get_regs_len,
2075         .get_regs = get_regs,
2076         .get_wol = get_wol,
2077         .set_tso = ethtool_op_set_tso,
2078 };
2079
2080 static int in_range(int val, int lo, int hi)
2081 {
2082         return val < 0 || (val <= hi && val >= lo);
2083 }
2084
2085 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2086 {
2087         struct port_info *pi = netdev_priv(dev);
2088         struct adapter *adapter = pi->adapter;
2089         u32 cmd;
2090         int ret;
2091
2092         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2093                 return -EFAULT;
2094
2095         switch (cmd) {
2096         case CHELSIO_SET_QSET_PARAMS:{
2097                 int i;
2098                 struct qset_params *q;
2099                 struct ch_qset_params t;
2100                 int q1 = pi->first_qset;
2101                 int nqsets = pi->nqsets;
2102
2103                 if (!capable(CAP_NET_ADMIN))
2104                         return -EPERM;
2105                 if (copy_from_user(&t, useraddr, sizeof(t)))
2106                         return -EFAULT;
2107                 if (t.qset_idx >= SGE_QSETS)
2108                         return -EINVAL;
2109                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2110                         !in_range(t.cong_thres, 0, 255) ||
2111                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2112                                 MAX_TXQ_ENTRIES) ||
2113                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2114                                 MAX_TXQ_ENTRIES) ||
2115                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2116                                 MAX_CTRL_TXQ_ENTRIES) ||
2117                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2118                                 MAX_RX_BUFFERS)
2119                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2120                                         MAX_RX_JUMBO_BUFFERS)
2121                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2122                                         MAX_RSPQ_ENTRIES))
2123                         return -EINVAL;
2124
2125                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
2126                         for_each_port(adapter, i) {
2127                                 pi = adap2pinfo(adapter, i);
2128                                 if (t.qset_idx >= pi->first_qset &&
2129                                     t.qset_idx < pi->first_qset + pi->nqsets &&
2130                                     !(pi->rx_offload & T3_RX_CSUM))
2131                                         return -EINVAL;
2132                         }
2133
2134                 if ((adapter->flags & FULL_INIT_DONE) &&
2135                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2136                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2137                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2138                         t.polling >= 0 || t.cong_thres >= 0))
2139                         return -EBUSY;
2140
2141                 /* Allow setting of any available qset when offload enabled */
2142                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2143                         q1 = 0;
2144                         for_each_port(adapter, i) {
2145                                 pi = adap2pinfo(adapter, i);
2146                                 nqsets += pi->first_qset + pi->nqsets;
2147                         }
2148                 }
2149
2150                 if (t.qset_idx < q1)
2151                         return -EINVAL;
2152                 if (t.qset_idx > q1 + nqsets - 1)
2153                         return -EINVAL;
2154
2155                 q = &adapter->params.sge.qset[t.qset_idx];
2156
2157                 if (t.rspq_size >= 0)
2158                         q->rspq_size = t.rspq_size;
2159                 if (t.fl_size[0] >= 0)
2160                         q->fl_size = t.fl_size[0];
2161                 if (t.fl_size[1] >= 0)
2162                         q->jumbo_size = t.fl_size[1];
2163                 if (t.txq_size[0] >= 0)
2164                         q->txq_size[0] = t.txq_size[0];
2165                 if (t.txq_size[1] >= 0)
2166                         q->txq_size[1] = t.txq_size[1];
2167                 if (t.txq_size[2] >= 0)
2168                         q->txq_size[2] = t.txq_size[2];
2169                 if (t.cong_thres >= 0)
2170                         q->cong_thres = t.cong_thres;
2171                 if (t.intr_lat >= 0) {
2172                         struct sge_qset *qs =
2173                                 &adapter->sge.qs[t.qset_idx];
2174
2175                         q->coalesce_usecs = t.intr_lat;
2176                         t3_update_qset_coalesce(qs, q);
2177                 }
2178                 if (t.polling >= 0) {
2179                         if (adapter->flags & USING_MSIX)
2180                                 q->polling = t.polling;
2181                         else {
2182                                 /* No polling with INTx for T3A */
2183                                 if (adapter->params.rev == 0 &&
2184                                         !(adapter->flags & USING_MSI))
2185                                         t.polling = 0;
2186
2187                                 for (i = 0; i < SGE_QSETS; i++) {
2188                                         q = &adapter->params.sge.
2189                                                 qset[i];
2190                                         q->polling = t.polling;
2191                                 }
2192                         }
2193                 }
2194                 if (t.lro >= 0)
2195                         set_qset_lro(dev, t.qset_idx, t.lro);
2196
2197                 break;
2198         }
2199         case CHELSIO_GET_QSET_PARAMS:{
2200                 struct qset_params *q;
2201                 struct ch_qset_params t;
2202                 int q1 = pi->first_qset;
2203                 int nqsets = pi->nqsets;
2204                 int i;
2205
2206                 if (copy_from_user(&t, useraddr, sizeof(t)))
2207                         return -EFAULT;
2208
2209                 /* Display qsets for all ports when offload enabled */
2210                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2211                         q1 = 0;
2212                         for_each_port(adapter, i) {
2213                                 pi = adap2pinfo(adapter, i);
2214                                 nqsets = pi->first_qset + pi->nqsets;
2215                         }
2216                 }
2217
2218                 if (t.qset_idx >= nqsets)
2219                         return -EINVAL;
2220
2221                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2222                 t.rspq_size = q->rspq_size;
2223                 t.txq_size[0] = q->txq_size[0];
2224                 t.txq_size[1] = q->txq_size[1];
2225                 t.txq_size[2] = q->txq_size[2];
2226                 t.fl_size[0] = q->fl_size;
2227                 t.fl_size[1] = q->jumbo_size;
2228                 t.polling = q->polling;
2229                 t.lro = q->lro;
2230                 t.intr_lat = q->coalesce_usecs;
2231                 t.cong_thres = q->cong_thres;
2232                 t.qnum = q1;
2233
2234                 if (adapter->flags & USING_MSIX)
2235                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2236                 else
2237                         t.vector = adapter->pdev->irq;
2238
2239                 if (copy_to_user(useraddr, &t, sizeof(t)))
2240                         return -EFAULT;
2241                 break;
2242         }
2243         case CHELSIO_SET_QSET_NUM:{
2244                 struct ch_reg edata;
2245                 unsigned int i, first_qset = 0, other_qsets = 0;
2246
2247                 if (!capable(CAP_NET_ADMIN))
2248                         return -EPERM;
2249                 if (adapter->flags & FULL_INIT_DONE)
2250                         return -EBUSY;
2251                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2252                         return -EFAULT;
2253                 if (edata.val < 1 ||
2254                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2255                         return -EINVAL;
2256
2257                 for_each_port(adapter, i)
2258                         if (adapter->port[i] && adapter->port[i] != dev)
2259                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2260
2261                 if (edata.val + other_qsets > SGE_QSETS)
2262                         return -EINVAL;
2263
2264                 pi->nqsets = edata.val;
2265
2266                 for_each_port(adapter, i)
2267                         if (adapter->port[i]) {
2268                                 pi = adap2pinfo(adapter, i);
2269                                 pi->first_qset = first_qset;
2270                                 first_qset += pi->nqsets;
2271                         }
2272                 break;
2273         }
2274         case CHELSIO_GET_QSET_NUM:{
2275                 struct ch_reg edata;
2276
2277                 edata.cmd = CHELSIO_GET_QSET_NUM;
2278                 edata.val = pi->nqsets;
2279                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2280                         return -EFAULT;
2281                 break;
2282         }
2283         case CHELSIO_LOAD_FW:{
2284                 u8 *fw_data;
2285                 struct ch_mem_range t;
2286
2287                 if (!capable(CAP_SYS_RAWIO))
2288                         return -EPERM;
2289                 if (copy_from_user(&t, useraddr, sizeof(t)))
2290                         return -EFAULT;
2291                 /* Check t.len sanity ? */
2292                 fw_data = kmalloc(t.len, GFP_KERNEL);
2293                 if (!fw_data)
2294                         return -ENOMEM;
2295
2296                 if (copy_from_user
2297                         (fw_data, useraddr + sizeof(t), t.len)) {
2298                         kfree(fw_data);
2299                         return -EFAULT;
2300                 }
2301
2302                 ret = t3_load_fw(adapter, fw_data, t.len);
2303                 kfree(fw_data);
2304                 if (ret)
2305                         return ret;
2306                 break;
2307         }
2308         case CHELSIO_SETMTUTAB:{
2309                 struct ch_mtus m;
2310                 int i;
2311
2312                 if (!is_offload(adapter))
2313                         return -EOPNOTSUPP;
2314                 if (!capable(CAP_NET_ADMIN))
2315                         return -EPERM;
2316                 if (offload_running(adapter))
2317                         return -EBUSY;
2318                 if (copy_from_user(&m, useraddr, sizeof(m)))
2319                         return -EFAULT;
2320                 if (m.nmtus != NMTUS)
2321                         return -EINVAL;
2322                 if (m.mtus[0] < 81)     /* accommodate SACK */
2323                         return -EINVAL;
2324
2325                 /* MTUs must be in ascending order */
2326                 for (i = 1; i < NMTUS; ++i)
2327                         if (m.mtus[i] < m.mtus[i - 1])
2328                                 return -EINVAL;
2329
2330                 memcpy(adapter->params.mtus, m.mtus,
2331                         sizeof(adapter->params.mtus));
2332                 break;
2333         }
2334         case CHELSIO_GET_PM:{
2335                 struct tp_params *p = &adapter->params.tp;
2336                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2337
2338                 if (!is_offload(adapter))
2339                         return -EOPNOTSUPP;
2340                 m.tx_pg_sz = p->tx_pg_size;
2341                 m.tx_num_pg = p->tx_num_pgs;
2342                 m.rx_pg_sz = p->rx_pg_size;
2343                 m.rx_num_pg = p->rx_num_pgs;
2344                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2345                 if (copy_to_user(useraddr, &m, sizeof(m)))
2346                         return -EFAULT;
2347                 break;
2348         }
2349         case CHELSIO_SET_PM:{
2350                 struct ch_pm m;
2351                 struct tp_params *p = &adapter->params.tp;
2352
2353                 if (!is_offload(adapter))
2354                         return -EOPNOTSUPP;
2355                 if (!capable(CAP_NET_ADMIN))
2356                         return -EPERM;
2357                 if (adapter->flags & FULL_INIT_DONE)
2358                         return -EBUSY;
2359                 if (copy_from_user(&m, useraddr, sizeof(m)))
2360                         return -EFAULT;
2361                 if (!is_power_of_2(m.rx_pg_sz) ||
2362                         !is_power_of_2(m.tx_pg_sz))
2363                         return -EINVAL; /* not power of 2 */
2364                 if (!(m.rx_pg_sz & 0x14000))
2365                         return -EINVAL; /* not 16KB or 64KB */
2366                 if (!(m.tx_pg_sz & 0x1554000))
2367                         return -EINVAL;
2368                 if (m.tx_num_pg == -1)
2369                         m.tx_num_pg = p->tx_num_pgs;
2370                 if (m.rx_num_pg == -1)
2371                         m.rx_num_pg = p->rx_num_pgs;
2372                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2373                         return -EINVAL;
2374                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2375                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2376                         return -EINVAL;
2377                 p->rx_pg_size = m.rx_pg_sz;
2378                 p->tx_pg_size = m.tx_pg_sz;
2379                 p->rx_num_pgs = m.rx_num_pg;
2380                 p->tx_num_pgs = m.tx_num_pg;
2381                 break;
2382         }
2383         case CHELSIO_GET_MEM:{
2384                 struct ch_mem_range t;
2385                 struct mc7 *mem;
2386                 u64 buf[32];
2387
2388                 if (!is_offload(adapter))
2389                         return -EOPNOTSUPP;
2390                 if (!(adapter->flags & FULL_INIT_DONE))
2391                         return -EIO;    /* need the memory controllers */
2392                 if (copy_from_user(&t, useraddr, sizeof(t)))
2393                         return -EFAULT;
2394                 if ((t.addr & 7) || (t.len & 7))
2395                         return -EINVAL;
2396                 if (t.mem_id == MEM_CM)
2397                         mem = &adapter->cm;
2398                 else if (t.mem_id == MEM_PMRX)
2399                         mem = &adapter->pmrx;
2400                 else if (t.mem_id == MEM_PMTX)
2401                         mem = &adapter->pmtx;
2402                 else
2403                         return -EINVAL;
2404
2405                 /*
2406                  * Version scheme:
2407                  * bits 0..9: chip version
2408                  * bits 10..15: chip revision
2409                  */
2410                 t.version = 3 | (adapter->params.rev << 10);
2411                 if (copy_to_user(useraddr, &t, sizeof(t)))
2412                         return -EFAULT;
2413
2414                 /*
2415                  * Read 256 bytes at a time as len can be large and we don't
2416                  * want to use huge intermediate buffers.
2417                  */
2418                 useraddr += sizeof(t);  /* advance to start of buffer */
2419                 while (t.len) {
2420                         unsigned int chunk =
2421                                 min_t(unsigned int, t.len, sizeof(buf));
2422
2423                         ret =
2424                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2425                                                 buf);
2426                         if (ret)
2427                                 return ret;
2428                         if (copy_to_user(useraddr, buf, chunk))
2429                                 return -EFAULT;
2430                         useraddr += chunk;
2431                         t.addr += chunk;
2432                         t.len -= chunk;
2433                 }
2434                 break;
2435         }
2436         case CHELSIO_SET_TRACE_FILTER:{
2437                 struct ch_trace t;
2438                 const struct trace_params *tp;
2439
2440                 if (!capable(CAP_NET_ADMIN))
2441                         return -EPERM;
2442                 if (!offload_running(adapter))
2443                         return -EAGAIN;
2444                 if (copy_from_user(&t, useraddr, sizeof(t)))
2445                         return -EFAULT;
2446
2447                 tp = (const struct trace_params *)&t.sip;
2448                 if (t.config_tx)
2449                         t3_config_trace_filter(adapter, tp, 0,
2450                                                 t.invert_match,
2451                                                 t.trace_tx);
2452                 if (t.config_rx)
2453                         t3_config_trace_filter(adapter, tp, 1,
2454                                                 t.invert_match,
2455                                                 t.trace_rx);
2456                 break;
2457         }
2458         default:
2459                 return -EOPNOTSUPP;
2460         }
2461         return 0;
2462 }
2463
2464 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2465 {
2466         struct mii_ioctl_data *data = if_mii(req);
2467         struct port_info *pi = netdev_priv(dev);
2468         struct adapter *adapter = pi->adapter;
2469
2470         switch (cmd) {
2471         case SIOCGMIIREG:
2472         case SIOCSMIIREG:
2473                 /* Convert phy_id from older PRTAD/DEVAD format */
2474                 if (is_10G(adapter) &&
2475                     !mdio_phy_id_is_c45(data->phy_id) &&
2476                     (data->phy_id & 0x1f00) &&
2477                     !(data->phy_id & 0xe0e0))
2478                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2479                                                        data->phy_id & 0x1f);
2480                 /* FALLTHRU */
2481         case SIOCGMIIPHY:
2482                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2483         case SIOCCHIOCTL:
2484                 return cxgb_extension_ioctl(dev, req->ifr_data);
2485         default:
2486                 return -EOPNOTSUPP;
2487         }
2488 }
2489
2490 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2491 {
2492         struct port_info *pi = netdev_priv(dev);
2493         struct adapter *adapter = pi->adapter;
2494         int ret;
2495
2496         if (new_mtu < 81)       /* accommodate SACK */
2497                 return -EINVAL;
2498         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2499                 return ret;
2500         dev->mtu = new_mtu;
2501         init_port_mtus(adapter);
2502         if (adapter->params.rev == 0 && offload_running(adapter))
2503                 t3_load_mtus(adapter, adapter->params.mtus,
2504                              adapter->params.a_wnd, adapter->params.b_wnd,
2505                              adapter->port[0]->mtu);
2506         return 0;
2507 }
2508
2509 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2510 {
2511         struct port_info *pi = netdev_priv(dev);
2512         struct adapter *adapter = pi->adapter;
2513         struct sockaddr *addr = p;
2514
2515         if (!is_valid_ether_addr(addr->sa_data))
2516                 return -EINVAL;
2517
2518         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2519         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2520         if (offload_running(adapter))
2521                 write_smt_entry(adapter, pi->port_id);
2522         return 0;
2523 }
2524
2525 /**
2526  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2527  * @adap: the adapter
2528  * @p: the port
2529  *
2530  * Ensures that current Rx processing on any of the queues associated with
2531  * the given port completes before returning.  We do this by acquiring and
2532  * releasing the locks of the response queues associated with the port.
2533  */
2534 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2535 {
2536         int i;
2537
2538         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2539                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2540
2541                 spin_lock_irq(&q->lock);
2542                 spin_unlock_irq(&q->lock);
2543         }
2544 }
2545
2546 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2547 {
2548         struct port_info *pi = netdev_priv(dev);
2549         struct adapter *adapter = pi->adapter;
2550
2551         pi->vlan_grp = grp;
2552         if (adapter->params.rev > 0)
2553                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2554         else {
2555                 /* single control for all ports */
2556                 unsigned int i, have_vlans = 0;
2557                 for_each_port(adapter, i)
2558                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2559
2560                 t3_set_vlan_accel(adapter, 1, have_vlans);
2561         }
2562         t3_synchronize_rx(adapter, pi);
2563 }
2564
2565 #ifdef CONFIG_NET_POLL_CONTROLLER
2566 static void cxgb_netpoll(struct net_device *dev)
2567 {
2568         struct port_info *pi = netdev_priv(dev);
2569         struct adapter *adapter = pi->adapter;
2570         int qidx;
2571
2572         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2573                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2574                 void *source;
2575
2576                 if (adapter->flags & USING_MSIX)
2577                         source = qs;
2578                 else
2579                         source = adapter;
2580
2581                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2582         }
2583 }
2584 #endif
2585
2586 /*
2587  * Periodic accumulation of MAC statistics.
2588  */
2589 static void mac_stats_update(struct adapter *adapter)
2590 {
2591         int i;
2592
2593         for_each_port(adapter, i) {
2594                 struct net_device *dev = adapter->port[i];
2595                 struct port_info *p = netdev_priv(dev);
2596
2597                 if (netif_running(dev)) {
2598                         spin_lock(&adapter->stats_lock);
2599                         t3_mac_update_stats(&p->mac);
2600                         spin_unlock(&adapter->stats_lock);
2601                 }
2602         }
2603 }
2604
2605 static void check_link_status(struct adapter *adapter)
2606 {
2607         int i;
2608
2609         for_each_port(adapter, i) {
2610                 struct net_device *dev = adapter->port[i];
2611                 struct port_info *p = netdev_priv(dev);
2612                 int link_fault;
2613
2614                 spin_lock_irq(&adapter->work_lock);
2615                 link_fault = p->link_fault;
2616                 spin_unlock_irq(&adapter->work_lock);
2617
2618                 if (link_fault) {
2619                         t3_link_fault(adapter, i);
2620                         continue;
2621                 }
2622
2623                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2624                         t3_xgm_intr_disable(adapter, i);
2625                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2626
2627                         t3_link_changed(adapter, i);
2628                         t3_xgm_intr_enable(adapter, i);
2629                 }
2630         }
2631 }
2632
2633 static void check_t3b2_mac(struct adapter *adapter)
2634 {
2635         int i;
2636
2637         if (!rtnl_trylock())    /* synchronize with ifdown */
2638                 return;
2639
2640         for_each_port(adapter, i) {
2641                 struct net_device *dev = adapter->port[i];
2642                 struct port_info *p = netdev_priv(dev);
2643                 int status;
2644
2645                 if (!netif_running(dev))
2646                         continue;
2647
2648                 status = 0;
2649                 if (netif_running(dev) && netif_carrier_ok(dev))
2650                         status = t3b2_mac_watchdog_task(&p->mac);
2651                 if (status == 1)
2652                         p->mac.stats.num_toggled++;
2653                 else if (status == 2) {
2654                         struct cmac *mac = &p->mac;
2655
2656                         t3_mac_set_mtu(mac, dev->mtu);
2657                         t3_mac_set_address(mac, 0, dev->dev_addr);
2658                         cxgb_set_rxmode(dev);
2659                         t3_link_start(&p->phy, mac, &p->link_config);
2660                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2661                         t3_port_intr_enable(adapter, p->port_id);
2662                         p->mac.stats.num_resets++;
2663                 }
2664         }
2665         rtnl_unlock();
2666 }
2667
2668
2669 static void t3_adap_check_task(struct work_struct *work)
2670 {
2671         struct adapter *adapter = container_of(work, struct adapter,
2672                                                adap_check_task.work);
2673         const struct adapter_params *p = &adapter->params;
2674         int port;
2675         unsigned int v, status, reset;
2676
2677         adapter->check_task_cnt++;
2678
2679         check_link_status(adapter);
2680
2681         /* Accumulate MAC stats if needed */
2682         if (!p->linkpoll_period ||
2683             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2684             p->stats_update_period) {
2685                 mac_stats_update(adapter);
2686                 adapter->check_task_cnt = 0;
2687         }
2688
2689         if (p->rev == T3_REV_B2)
2690                 check_t3b2_mac(adapter);
2691
2692         /*
2693          * Scan the XGMAC's to check for various conditions which we want to
2694          * monitor in a periodic polling manner rather than via an interrupt
2695          * condition.  This is used for conditions which would otherwise flood
2696          * the system with interrupts and we only really need to know that the
2697          * conditions are "happening" ...  For each condition we count the
2698          * detection of the condition and reset it for the next polling loop.
2699          */
2700         for_each_port(adapter, port) {
2701                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2702                 u32 cause;
2703
2704                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2705                 reset = 0;
2706                 if (cause & F_RXFIFO_OVERFLOW) {
2707                         mac->stats.rx_fifo_ovfl++;
2708                         reset |= F_RXFIFO_OVERFLOW;
2709                 }
2710
2711                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2712         }
2713
2714         /*
2715          * We do the same as above for FL_EMPTY interrupts.
2716          */
2717         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2718         reset = 0;
2719
2720         if (status & F_FLEMPTY) {
2721                 struct sge_qset *qs = &adapter->sge.qs[0];
2722                 int i = 0;
2723
2724                 reset |= F_FLEMPTY;
2725
2726                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2727                     0xffff;
2728
2729                 while (v) {
2730                         qs->fl[i].empty += (v & 1);
2731                         if (i)
2732                                 qs++;
2733                         i ^= 1;
2734                         v >>= 1;
2735                 }
2736         }
2737
2738         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2739
2740         /* Schedule the next check update if any port is active. */
2741         spin_lock_irq(&adapter->work_lock);
2742         if (adapter->open_device_map & PORT_MASK)
2743                 schedule_chk_task(adapter);
2744         spin_unlock_irq(&adapter->work_lock);
2745 }
2746
2747 /*
2748  * Processes external (PHY) interrupts in process context.
2749  */
2750 static void ext_intr_task(struct work_struct *work)
2751 {
2752         struct adapter *adapter = container_of(work, struct adapter,
2753                                                ext_intr_handler_task);
2754         int i;
2755
2756         /* Disable link fault interrupts */
2757         for_each_port(adapter, i) {
2758                 struct net_device *dev = adapter->port[i];
2759                 struct port_info *p = netdev_priv(dev);
2760
2761                 t3_xgm_intr_disable(adapter, i);
2762                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2763         }
2764
2765         /* Re-enable link fault interrupts */
2766         t3_phy_intr_handler(adapter);
2767
2768         for_each_port(adapter, i)
2769                 t3_xgm_intr_enable(adapter, i);
2770
2771         /* Now reenable external interrupts */
2772         spin_lock_irq(&adapter->work_lock);
2773         if (adapter->slow_intr_mask) {
2774                 adapter->slow_intr_mask |= F_T3DBG;
2775                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2776                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2777                              adapter->slow_intr_mask);
2778         }
2779         spin_unlock_irq(&adapter->work_lock);
2780 }
2781
2782 /*
2783  * Interrupt-context handler for external (PHY) interrupts.
2784  */
2785 void t3_os_ext_intr_handler(struct adapter *adapter)
2786 {
2787         /*
2788          * Schedule a task to handle external interrupts as they may be slow
2789          * and we use a mutex to protect MDIO registers.  We disable PHY
2790          * interrupts in the meantime and let the task reenable them when
2791          * it's done.
2792          */
2793         spin_lock(&adapter->work_lock);
2794         if (adapter->slow_intr_mask) {
2795                 adapter->slow_intr_mask &= ~F_T3DBG;
2796                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2797                              adapter->slow_intr_mask);
2798                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2799         }
2800         spin_unlock(&adapter->work_lock);
2801 }
2802
2803 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2804 {
2805         struct net_device *netdev = adapter->port[port_id];
2806         struct port_info *pi = netdev_priv(netdev);
2807
2808         spin_lock(&adapter->work_lock);
2809         pi->link_fault = 1;
2810         spin_unlock(&adapter->work_lock);
2811 }
2812
2813 static int t3_adapter_error(struct adapter *adapter, int reset)
2814 {
2815         int i, ret = 0;
2816
2817         if (is_offload(adapter) &&
2818             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2819                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2820                 offload_close(&adapter->tdev);
2821         }
2822
2823         /* Stop all ports */
2824         for_each_port(adapter, i) {
2825                 struct net_device *netdev = adapter->port[i];
2826
2827                 if (netif_running(netdev))
2828                         cxgb_close(netdev);
2829         }
2830
2831         /* Stop SGE timers */
2832         t3_stop_sge_timers(adapter);
2833
2834         adapter->flags &= ~FULL_INIT_DONE;
2835
2836         if (reset)
2837                 ret = t3_reset_adapter(adapter);
2838
2839         pci_disable_device(adapter->pdev);
2840
2841         return ret;
2842 }
2843
2844 static int t3_reenable_adapter(struct adapter *adapter)
2845 {
2846         if (pci_enable_device(adapter->pdev)) {
2847                 dev_err(&adapter->pdev->dev,
2848                         "Cannot re-enable PCI device after reset.\n");
2849                 goto err;
2850         }
2851         pci_set_master(adapter->pdev);
2852         pci_restore_state(adapter->pdev);
2853
2854         /* Free sge resources */
2855         t3_free_sge_resources(adapter);
2856
2857         if (t3_replay_prep_adapter(adapter))
2858                 goto err;
2859
2860         return 0;
2861 err:
2862         return -1;
2863 }
2864
2865 static void t3_resume_ports(struct adapter *adapter)
2866 {
2867         int i;
2868
2869         /* Restart the ports */
2870         for_each_port(adapter, i) {
2871                 struct net_device *netdev = adapter->port[i];
2872
2873                 if (netif_running(netdev)) {
2874                         if (cxgb_open(netdev)) {
2875                                 dev_err(&adapter->pdev->dev,
2876                                         "can't bring device back up"
2877                                         " after reset\n");
2878                                 continue;
2879                         }
2880                 }
2881         }
2882
2883         if (is_offload(adapter) && !ofld_disable)
2884                 cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2885 }
2886
2887 /*
2888  * processes a fatal error.
2889  * Bring the ports down, reset the chip, bring the ports back up.
2890  */
2891 static void fatal_error_task(struct work_struct *work)
2892 {
2893         struct adapter *adapter = container_of(work, struct adapter,
2894                                                fatal_error_handler_task);
2895         int err = 0;
2896
2897         rtnl_lock();
2898         err = t3_adapter_error(adapter, 1);
2899         if (!err)
2900                 err = t3_reenable_adapter(adapter);
2901         if (!err)
2902                 t3_resume_ports(adapter);
2903
2904         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2905         rtnl_unlock();
2906 }
2907
2908 void t3_fatal_err(struct adapter *adapter)
2909 {
2910         unsigned int fw_status[4];
2911
2912         if (adapter->flags & FULL_INIT_DONE) {
2913                 t3_sge_stop(adapter);
2914                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2915                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2916                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2917                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2918
2919                 spin_lock(&adapter->work_lock);
2920                 t3_intr_disable(adapter);
2921                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2922                 spin_unlock(&adapter->work_lock);
2923         }
2924         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2925         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2926                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2927                          fw_status[0], fw_status[1],
2928                          fw_status[2], fw_status[3]);
2929 }
2930
2931 /**
2932  * t3_io_error_detected - called when PCI error is detected
2933  * @pdev: Pointer to PCI device
2934  * @state: The current pci connection state
2935  *
2936  * This function is called after a PCI bus error affecting
2937  * this device has been detected.
2938  */
2939 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2940                                              pci_channel_state_t state)
2941 {
2942         struct adapter *adapter = pci_get_drvdata(pdev);
2943         int ret;
2944
2945         if (state == pci_channel_io_perm_failure)
2946                 return PCI_ERS_RESULT_DISCONNECT;
2947
2948         ret = t3_adapter_error(adapter, 0);
2949
2950         /* Request a slot reset. */
2951         return PCI_ERS_RESULT_NEED_RESET;
2952 }
2953
2954 /**
2955  * t3_io_slot_reset - called after the pci bus has been reset.
2956  * @pdev: Pointer to PCI device
2957  *
2958  * Restart the card from scratch, as if from a cold-boot.
2959  */
2960 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2961 {
2962         struct adapter *adapter = pci_get_drvdata(pdev);
2963
2964         if (!t3_reenable_adapter(adapter))
2965                 return PCI_ERS_RESULT_RECOVERED;
2966
2967         return PCI_ERS_RESULT_DISCONNECT;
2968 }
2969
2970 /**
2971  * t3_io_resume - called when traffic can start flowing again.
2972  * @pdev: Pointer to PCI device
2973  *
2974  * This callback is called when the error recovery driver tells us that
2975  * its OK to resume normal operation.
2976  */
2977 static void t3_io_resume(struct pci_dev *pdev)
2978 {
2979         struct adapter *adapter = pci_get_drvdata(pdev);
2980
2981         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
2982                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
2983
2984         t3_resume_ports(adapter);
2985 }
2986
2987 static struct pci_error_handlers t3_err_handler = {
2988         .error_detected = t3_io_error_detected,
2989         .slot_reset = t3_io_slot_reset,
2990         .resume = t3_io_resume,
2991 };
2992
2993 /*
2994  * Set the number of qsets based on the number of CPUs and the number of ports,
2995  * not to exceed the number of available qsets, assuming there are enough qsets
2996  * per port in HW.
2997  */
2998 static void set_nqsets(struct adapter *adap)
2999 {
3000         int i, j = 0;
3001         int num_cpus = num_online_cpus();
3002         int hwports = adap->params.nports;
3003         int nqsets = adap->msix_nvectors - 1;
3004
3005         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3006                 if (hwports == 2 &&
3007                     (hwports * nqsets > SGE_QSETS ||
3008                      num_cpus >= nqsets / hwports))
3009                         nqsets /= hwports;
3010                 if (nqsets > num_cpus)
3011                         nqsets = num_cpus;
3012                 if (nqsets < 1 || hwports == 4)
3013                         nqsets = 1;
3014         } else
3015                 nqsets = 1;
3016
3017         for_each_port(adap, i) {
3018                 struct port_info *pi = adap2pinfo(adap, i);
3019
3020                 pi->first_qset = j;
3021                 pi->nqsets = nqsets;
3022                 j = pi->first_qset + nqsets;
3023
3024                 dev_info(&adap->pdev->dev,
3025                          "Port %d using %d queue sets.\n", i, nqsets);
3026         }
3027 }
3028
3029 static int __devinit cxgb_enable_msix(struct adapter *adap)
3030 {
3031         struct msix_entry entries[SGE_QSETS + 1];
3032         int vectors;
3033         int i, err;
3034
3035         vectors = ARRAY_SIZE(entries);
3036         for (i = 0; i < vectors; ++i)
3037                 entries[i].entry = i;
3038
3039         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
3040                 vectors = err;
3041
3042         if (err < 0)
3043                 pci_disable_msix(adap->pdev);
3044
3045         if (!err && vectors < (adap->params.nports + 1)) {
3046                 pci_disable_msix(adap->pdev);
3047                 err = -1;
3048         }
3049
3050         if (!err) {
3051                 for (i = 0; i < vectors; ++i)
3052                         adap->msix_info[i].vec = entries[i].vector;
3053                 adap->msix_nvectors = vectors;
3054         }
3055
3056         return err;
3057 }
3058
3059 static void __devinit print_port_info(struct adapter *adap,
3060                                       const struct adapter_info *ai)
3061 {
3062         static const char *pci_variant[] = {
3063                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3064         };
3065
3066         int i;
3067         char buf[80];
3068
3069         if (is_pcie(adap))
3070                 snprintf(buf, sizeof(buf), "%s x%d",
3071                          pci_variant[adap->params.pci.variant],
3072                          adap->params.pci.width);
3073         else
3074                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3075                          pci_variant[adap->params.pci.variant],
3076                          adap->params.pci.speed, adap->params.pci.width);
3077
3078         for_each_port(adap, i) {
3079                 struct net_device *dev = adap->port[i];
3080                 const struct port_info *pi = netdev_priv(dev);
3081
3082                 if (!test_bit(i, &adap->registered_device_map))
3083                         continue;
3084                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
3085                        dev->name, ai->desc, pi->phy.desc,
3086                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
3087                        (adap->flags & USING_MSIX) ? " MSI-X" :
3088                        (adap->flags & USING_MSI) ? " MSI" : "");
3089                 if (adap->name == dev->name && adap->params.vpd.mclk)
3090                         printk(KERN_INFO
3091                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3092                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3093                                t3_mc7_size(&adap->pmtx) >> 20,
3094                                t3_mc7_size(&adap->pmrx) >> 20,
3095                                adap->params.vpd.sn);
3096         }
3097 }
3098
3099 static const struct net_device_ops cxgb_netdev_ops = {
3100         .ndo_open               = cxgb_open,
3101         .ndo_stop               = cxgb_close,
3102         .ndo_start_xmit         = t3_eth_xmit,
3103         .ndo_get_stats          = cxgb_get_stats,
3104         .ndo_validate_addr      = eth_validate_addr,
3105         .ndo_set_multicast_list = cxgb_set_rxmode,
3106         .ndo_do_ioctl           = cxgb_ioctl,
3107         .ndo_change_mtu         = cxgb_change_mtu,
3108         .ndo_set_mac_address    = cxgb_set_mac_addr,
3109         .ndo_vlan_rx_register   = vlan_rx_register,
3110 #ifdef CONFIG_NET_POLL_CONTROLLER
3111         .ndo_poll_controller    = cxgb_netpoll,
3112 #endif
3113 };
3114
3115 static int __devinit init_one(struct pci_dev *pdev,
3116                               const struct pci_device_id *ent)
3117 {
3118         static int version_printed;
3119
3120         int i, err, pci_using_dac = 0;
3121         resource_size_t mmio_start, mmio_len;
3122         const struct adapter_info *ai;
3123         struct adapter *adapter = NULL;
3124         struct port_info *pi;
3125
3126         if (!version_printed) {
3127                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3128                 ++version_printed;
3129         }
3130
3131         if (!cxgb3_wq) {
3132                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3133                 if (!cxgb3_wq) {
3134                         printk(KERN_ERR DRV_NAME
3135                                ": cannot initialize work queue\n");
3136                         return -ENOMEM;
3137                 }
3138         }
3139
3140         err = pci_request_regions(pdev, DRV_NAME);
3141         if (err) {
3142                 /* Just info, some other driver may have claimed the device. */
3143                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3144                 return err;
3145         }
3146
3147         err = pci_enable_device(pdev);
3148         if (err) {
3149                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3150                 goto out_release_regions;
3151         }
3152
3153         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3154                 pci_using_dac = 1;
3155                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3156                 if (err) {
3157                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3158                                "coherent allocations\n");
3159                         goto out_disable_device;
3160                 }
3161         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3162                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3163                 goto out_disable_device;
3164         }
3165
3166         pci_set_master(pdev);
3167         pci_save_state(pdev);
3168
3169         mmio_start = pci_resource_start(pdev, 0);
3170         mmio_len = pci_resource_len(pdev, 0);
3171         ai = t3_get_adapter_info(ent->driver_data);
3172
3173         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3174         if (!adapter) {
3175                 err = -ENOMEM;
3176                 goto out_disable_device;
3177         }
3178
3179         adapter->nofail_skb =
3180                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3181         if (!adapter->nofail_skb) {
3182                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3183                 err = -ENOMEM;
3184                 goto out_free_adapter;
3185         }
3186
3187         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3188         if (!adapter->regs) {
3189                 dev_err(&pdev->dev, "cannot map device registers\n");
3190                 err = -ENOMEM;
3191                 goto out_free_adapter;
3192         }
3193
3194         adapter->pdev = pdev;
3195         adapter->name = pci_name(pdev);
3196         adapter->msg_enable = dflt_msg_enable;
3197         adapter->mmio_len = mmio_len;
3198
3199         mutex_init(&adapter->mdio_lock);
3200         spin_lock_init(&adapter->work_lock);
3201         spin_lock_init(&adapter->stats_lock);
3202
3203         INIT_LIST_HEAD(&adapter->adapter_list);
3204         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3205         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3206         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3207
3208         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3209                 struct net_device *netdev;
3210
3211                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3212                 if (!netdev) {
3213                         err = -ENOMEM;
3214                         goto out_free_dev;
3215                 }
3216
3217                 SET_NETDEV_DEV(netdev, &pdev->dev);
3218
3219                 adapter->port[i] = netdev;
3220                 pi = netdev_priv(netdev);
3221                 pi->adapter = adapter;
3222                 pi->rx_offload = T3_RX_CSUM | T3_LRO;
3223                 pi->port_id = i;
3224                 netif_carrier_off(netdev);
3225                 netif_tx_stop_all_queues(netdev);
3226                 netdev->irq = pdev->irq;
3227                 netdev->mem_start = mmio_start;
3228                 netdev->mem_end = mmio_start + mmio_len - 1;
3229                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
3230                 netdev->features |= NETIF_F_GRO;
3231                 if (pci_using_dac)
3232                         netdev->features |= NETIF_F_HIGHDMA;
3233
3234                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3235                 netdev->netdev_ops = &cxgb_netdev_ops;
3236                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3237         }
3238
3239         pci_set_drvdata(pdev, adapter);
3240         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3241                 err = -ENODEV;
3242                 goto out_free_dev;
3243         }
3244
3245         /*
3246          * The card is now ready to go.  If any errors occur during device
3247          * registration we do not fail the whole card but rather proceed only
3248          * with the ports we manage to register successfully.  However we must
3249          * register at least one net device.
3250          */
3251         for_each_port(adapter, i) {
3252                 err = register_netdev(adapter->port[i]);
3253                 if (err)
3254                         dev_warn(&pdev->dev,
3255                                  "cannot register net device %s, skipping\n",
3256                                  adapter->port[i]->name);
3257                 else {
3258                         /*
3259                          * Change the name we use for messages to the name of
3260                          * the first successfully registered interface.
3261                          */
3262                         if (!adapter->registered_device_map)
3263                                 adapter->name = adapter->port[i]->name;
3264
3265                         __set_bit(i, &adapter->registered_device_map);
3266                 }
3267         }
3268         if (!adapter->registered_device_map) {
3269                 dev_err(&pdev->dev, "could not register any net devices\n");
3270                 goto out_free_dev;
3271         }
3272
3273         /* Driver's ready. Reflect it on LEDs */
3274         t3_led_ready(adapter);
3275
3276         if (is_offload(adapter)) {
3277                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3278                 cxgb3_adapter_ofld(adapter);
3279         }
3280
3281         /* See what interrupts we'll be using */
3282         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3283                 adapter->flags |= USING_MSIX;
3284         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3285                 adapter->flags |= USING_MSI;
3286
3287         set_nqsets(adapter);
3288
3289         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3290                                  &cxgb3_attr_group);
3291
3292         print_port_info(adapter, ai);
3293         return 0;
3294
3295 out_free_dev:
3296         iounmap(adapter->regs);
3297         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3298                 if (adapter->port[i])
3299                         free_netdev(adapter->port[i]);
3300
3301 out_free_adapter:
3302         kfree(adapter);
3303
3304 out_disable_device:
3305         pci_disable_device(pdev);
3306 out_release_regions:
3307         pci_release_regions(pdev);
3308         pci_set_drvdata(pdev, NULL);
3309         return err;
3310 }
3311
3312 static void __devexit remove_one(struct pci_dev *pdev)
3313 {
3314         struct adapter *adapter = pci_get_drvdata(pdev);
3315
3316         if (adapter) {
3317                 int i;
3318
3319                 t3_sge_stop(adapter);
3320                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3321                                    &cxgb3_attr_group);
3322
3323                 if (is_offload(adapter)) {
3324                         cxgb3_adapter_unofld(adapter);
3325                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3326                                      &adapter->open_device_map))
3327                                 offload_close(&adapter->tdev);
3328                 }
3329
3330                 for_each_port(adapter, i)
3331                     if (test_bit(i, &adapter->registered_device_map))
3332                         unregister_netdev(adapter->port[i]);
3333
3334                 t3_stop_sge_timers(adapter);
3335                 t3_free_sge_resources(adapter);
3336                 cxgb_disable_msi(adapter);
3337
3338                 for_each_port(adapter, i)
3339                         if (adapter->port[i])
3340                                 free_netdev(adapter->port[i]);
3341
3342                 iounmap(adapter->regs);
3343                 if (adapter->nofail_skb)
3344                         kfree_skb(adapter->nofail_skb);
3345                 kfree(adapter);
3346                 pci_release_regions(pdev);
3347                 pci_disable_device(pdev);
3348                 pci_set_drvdata(pdev, NULL);
3349         }
3350 }
3351
3352 static struct pci_driver driver = {
3353         .name = DRV_NAME,
3354         .id_table = cxgb3_pci_tbl,
3355         .probe = init_one,
3356         .remove = __devexit_p(remove_one),
3357         .err_handler = &t3_err_handler,
3358 };
3359
3360 static int __init cxgb3_init_module(void)
3361 {
3362         int ret;
3363
3364         cxgb3_offload_init();
3365
3366         ret = pci_register_driver(&driver);
3367         return ret;
3368 }
3369
3370 static void __exit cxgb3_cleanup_module(void)
3371 {
3372         pci_unregister_driver(&driver);
3373         if (cxgb3_wq)
3374                 destroy_workqueue(cxgb3_wq);
3375 }
3376
3377 module_init(cxgb3_init_module);
3378 module_exit(cxgb3_cleanup_module);