drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mdio.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         CH_DEVICE(0x35, 6),     /* T3C20-derived T3C10 */
  94         CH_DEVICE(0x36, 3),     /* S320E-CR */
  95         CH_DEVICE(0x37, 7),     /* N320E-G2 */
  96         {0,}
  97 };
  98
  99 MODULE_DESCRIPTION(DRV_DESC);
 100 MODULE_AUTHOR("Chelsio Communications");
 101 MODULE_LICENSE("Dual BSD/GPL");
 102 MODULE_VERSION(DRV_VERSION);
 103 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 104
 105 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 106
 107 module_param(dflt_msg_enable, int, 0644);
 108 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 109
 110 /*
 111  * The driver uses the best interrupt scheme available on a platform in the
 112  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 113  * of these schemes the driver may consider as follows:
 114  *
 115  * msi = 2: choose from among all three options
 116  * msi = 1: only consider MSI and pin interrupts
 117  * msi = 0: force pin interrupts
 118  */
 119 static int msi = 2;
 120
 121 module_param(msi, int, 0644);
 122 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 123
 124 /*
 125  * The driver enables offload as a default.
 126  * To disable it, use ofld_disable = 1.
 127  */
 128
 129 static int ofld_disable = 0;
 130
 131 module_param(ofld_disable, int, 0644);
 132 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 133
 134 /*
 135  * We have work elements that we need to cancel when an interface is taken
 136  * down.  Normally the work elements would be executed by keventd but that
 137  * can deadlock because of linkwatch.  If our close method takes the rtnl
 138  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 139  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 140  * for our work to complete.  Get our own work queue to solve this.
 141  */
 142 static struct workqueue_struct *cxgb3_wq;
 143
 144 /**
 145  *      link_report - show link status and link speed/duplex
 146  *      @p: the port whose settings are to be reported
 147  *
 148  *      Shows the link status, speed, and duplex of a port.
 149  */
 150 static void link_report(struct net_device *dev)
 151 {
 152         if (!netif_carrier_ok(dev))
 153                 printk(KERN_INFO "%s: link down\n", dev->name);
 154         else {
 155                 const char *s = "10Mbps";
 156                 const struct port_info *p = netdev_priv(dev);
 157
 158                 switch (p->link_config.speed) {
 159                 case SPEED_10000:
 160                         s = "10Gbps";
 161                         break;
 162                 case SPEED_1000:
 163                         s = "1000Mbps";
 164                         break;
 165                 case SPEED_100:
 166                         s = "100Mbps";
 167                         break;
 168                 }
 169
 170                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 171                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 172         }
 173 }
 174
 175 static void enable_tx_fifo_drain(struct adapter *adapter,
 176                                  struct port_info *pi)
 177 {
 178         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
 179                          F_ENDROPPKT);
 180         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
 181         t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
 182         t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
 183 }
 184
 185 static void disable_tx_fifo_drain(struct adapter *adapter,
 186                                   struct port_info *pi)
 187 {
 188         t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
 189                          F_ENDROPPKT, 0);
 190 }
 191
 192 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
 193 {
 194         struct net_device *dev = adap->port[port_id];
 195         struct port_info *pi = netdev_priv(dev);
 196
 197         if (state == netif_carrier_ok(dev))
 198                 return;
 199
 200         if (state) {
 201                 struct cmac *mac = &pi->mac;
 202
 203                 netif_carrier_on(dev);
 204
 205                 disable_tx_fifo_drain(adap, pi);
 206
 207                 /* Clear local faults */
 208                 t3_xgm_intr_disable(adap, pi->port_id);
 209                 t3_read_reg(adap, A_XGM_INT_STATUS +
 210                                     pi->mac.offset);
 211                 t3_write_reg(adap,
 212                              A_XGM_INT_CAUSE + pi->mac.offset,
 213                              F_XGM_INT);
 214
 215                 t3_set_reg_field(adap,
 216                                  A_XGM_INT_ENABLE +
 217                                  pi->mac.offset,
 218                                  F_XGM_INT, F_XGM_INT);
 219                 t3_xgm_intr_enable(adap, pi->port_id);
 220
 221                 t3_mac_enable(mac, MAC_DIRECTION_TX);
 222         } else {
 223                 netif_carrier_off(dev);
 224
 225                 /* Flush TX FIFO */
 226                 enable_tx_fifo_drain(adap, pi);
 227         }
 228         link_report(dev);
 229 }
 230
 231 /**
 232  *      t3_os_link_changed - handle link status changes
 233  *      @adapter: the adapter associated with the link change
 234  *      @port_id: the port index whose limk status has changed
 235  *      @link_stat: the new status of the link
 236  *      @speed: the new speed setting
 237  *      @duplex: the new duplex setting
 238  *      @pause: the new flow-control setting
 239  *
 240  *      This is the OS-dependent handler for link status changes.  The OS
 241  *      neutral handler takes care of most of the processing for these events,
 242  *      then calls this handler for any OS-specific processing.
 243  */
 244 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 245                         int speed, int duplex, int pause)
 246 {
 247         struct net_device *dev = adapter->port[port_id];
 248         struct port_info *pi = netdev_priv(dev);
 249         struct cmac *mac = &pi->mac;
 250
 251         /* Skip changes from disabled ports. */
 252         if (!netif_running(dev))
 253                 return;
 254
 255         if (link_stat != netif_carrier_ok(dev)) {
 256                 if (link_stat) {
 257                         disable_tx_fifo_drain(adapter, pi);
 258
 259                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 260
 261                         /* Clear local faults */
 262                         t3_xgm_intr_disable(adapter, pi->port_id);
 263                         t3_read_reg(adapter, A_XGM_INT_STATUS +
 264                                     pi->mac.offset);
 265                         t3_write_reg(adapter,
 266                                      A_XGM_INT_CAUSE + pi->mac.offset,
 267                                      F_XGM_INT);
 268
 269                         t3_set_reg_field(adapter,
 270                                          A_XGM_INT_ENABLE + pi->mac.offset,
 271                                          F_XGM_INT, F_XGM_INT);
 272                         t3_xgm_intr_enable(adapter, pi->port_id);
 273
 274                         netif_carrier_on(dev);
 275                 } else {
 276                         netif_carrier_off(dev);
 277
 278                         t3_xgm_intr_disable(adapter, pi->port_id);
 279                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 280                         t3_set_reg_field(adapter,
 281                                          A_XGM_INT_ENABLE + pi->mac.offset,
 282                                          F_XGM_INT, 0);
 283
 284                         if (is_10G(adapter))
 285                                 pi->phy.ops->power_down(&pi->phy, 1);
 286
 287                         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
 288                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 289                         t3_link_start(&pi->phy, mac, &pi->link_config);
 290
 291                         /* Flush TX FIFO */
 292                         enable_tx_fifo_drain(adapter, pi);
 293                 }
 294
 295                 link_report(dev);
 296         }
 297 }
 298
 299 /**
 300  *      t3_os_phymod_changed - handle PHY module changes
 301  *      @phy: the PHY reporting the module change
 302  *      @mod_type: new module type
 303  *
 304  *      This is the OS-dependent handler for PHY module changes.  It is
 305  *      invoked when a PHY module is removed or inserted for any OS-specific
 306  *      processing.
 307  */
 308 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 309 {
 310         static const char *mod_str[] = {
 311                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 312         };
 313
 314         const struct net_device *dev = adap->port[port_id];
 315         const struct port_info *pi = netdev_priv(dev);
 316
 317         if (pi->phy.modtype == phy_modtype_none)
 318                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 319         else
 320                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 321                        mod_str[pi->phy.modtype]);
 322 }
 323
 324 static void cxgb_set_rxmode(struct net_device *dev)
 325 {
 326         struct t3_rx_mode rm;
 327         struct port_info *pi = netdev_priv(dev);
 328
 329         init_rx_mode(&rm, dev, dev->mc_list);
 330         t3_mac_set_rx_mode(&pi->mac, &rm);
 331 }
 332
 333 /**
 334  *      link_start - enable a port
 335  *      @dev: the device to enable
 336  *
 337  *      Performs the MAC and PHY actions needed to enable a port.
 338  */
 339 static void link_start(struct net_device *dev)
 340 {
 341         struct t3_rx_mode rm;
 342         struct port_info *pi = netdev_priv(dev);
 343         struct cmac *mac = &pi->mac;
 344
 345         init_rx_mode(&rm, dev, dev->mc_list);
 346         t3_mac_reset(mac);
 347         t3_mac_set_mtu(mac, dev->mtu);
 348         t3_mac_set_address(mac, 0, dev->dev_addr);
 349         t3_mac_set_rx_mode(mac, &rm);
 350         t3_link_start(&pi->phy, mac, &pi->link_config);
 351         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 352 }
 353
 354 static inline void cxgb_disable_msi(struct adapter *adapter)
 355 {
 356         if (adapter->flags & USING_MSIX) {
 357                 pci_disable_msix(adapter->pdev);
 358                 adapter->flags &= ~USING_MSIX;
 359         } else if (adapter->flags & USING_MSI) {
 360                 pci_disable_msi(adapter->pdev);
 361                 adapter->flags &= ~USING_MSI;
 362         }
 363 }
 364
 365 /*
 366  * Interrupt handler for asynchronous events used with MSI-X.
 367  */
 368 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 369 {
 370         t3_slow_intr_handler(cookie);
 371         return IRQ_HANDLED;
 372 }
 373
 374 /*
 375  * Name the MSI-X interrupts.
 376  */
 377 static void name_msix_vecs(struct adapter *adap)
 378 {
 379         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 380
 381         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 382         adap->msix_info[0].desc[n] = 0;
 383
 384         for_each_port(adap, j) {
 385                 struct net_device *d = adap->port[j];
 386                 const struct port_info *pi = netdev_priv(d);
 387
 388                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 389                         snprintf(adap->msix_info[msi_idx].desc, n,
 390                                  "%s-%d", d->name, pi->first_qset + i);
 391                         adap->msix_info[msi_idx].desc[n] = 0;
 392                 }
 393         }
 394 }
 395
 396 static int request_msix_data_irqs(struct adapter *adap)
 397 {
 398         int i, j, err, qidx = 0;
 399
 400         for_each_port(adap, i) {
 401                 int nqsets = adap2pinfo(adap, i)->nqsets;
 402
 403                 for (j = 0; j < nqsets; ++j) {
 404                         err = request_irq(adap->msix_info[qidx + 1].vec,
 405                                           t3_intr_handler(adap,
 406                                                           adap->sge.qs[qidx].
 407                                                           rspq.polling), 0,
 408                                           adap->msix_info[qidx + 1].desc,
 409                                           &adap->sge.qs[qidx]);
 410                         if (err) {
 411                                 while (--qidx >= 0)
 412                                         free_irq(adap->msix_info[qidx + 1].vec,
 413                                                  &adap->sge.qs[qidx]);
 414                                 return err;
 415                         }
 416                         qidx++;
 417                 }
 418         }
 419         return 0;
 420 }
 421
 422 static void free_irq_resources(struct adapter *adapter)
 423 {
 424         if (adapter->flags & USING_MSIX) {
 425                 int i, n = 0;
 426
 427                 free_irq(adapter->msix_info[0].vec, adapter);
 428                 for_each_port(adapter, i)
 429                         n += adap2pinfo(adapter, i)->nqsets;
 430
 431                 for (i = 0; i < n; ++i)
 432                         free_irq(adapter->msix_info[i + 1].vec,
 433                                  &adapter->sge.qs[i]);
 434         } else
 435                 free_irq(adapter->pdev->irq, adapter);
 436 }
 437
 438 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 439                               unsigned long n)
 440 {
 441         int attempts = 5;
 442
 443         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 444                 if (!--attempts)
 445                         return -ETIMEDOUT;
 446                 msleep(10);
 447         }
 448         return 0;
 449 }
 450
 451 static int init_tp_parity(struct adapter *adap)
 452 {
 453         int i;
 454         struct sk_buff *skb;
 455         struct cpl_set_tcb_field *greq;
 456         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 457
 458         t3_tp_set_offload_mode(adap, 1);
 459
 460         for (i = 0; i < 16; i++) {
 461                 struct cpl_smt_write_req *req;
 462
 463                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 464                 if (!skb)
 465                         skb = adap->nofail_skb;
 466                 if (!skb)
 467                         goto alloc_skb_fail;
 468
 469                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 470                 memset(req, 0, sizeof(*req));
 471                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 472                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 473                 req->mtu_idx = NMTUS - 1;
 474                 req->iff = i;
 475                 t3_mgmt_tx(adap, skb);
 476                 if (skb == adap->nofail_skb) {
 477                         await_mgmt_replies(adap, cnt, i + 1);
 478                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 479                         if (!adap->nofail_skb)
 480                                 goto alloc_skb_fail;
 481                 }
 482         }
 483
 484         for (i = 0; i < 2048; i++) {
 485                 struct cpl_l2t_write_req *req;
 486
 487                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 488                 if (!skb)
 489                         skb = adap->nofail_skb;
 490                 if (!skb)
 491                         goto alloc_skb_fail;
 492
 493                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 494                 memset(req, 0, sizeof(*req));
 495                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 496                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 497                 req->params = htonl(V_L2T_W_IDX(i));
 498                 t3_mgmt_tx(adap, skb);
 499                 if (skb == adap->nofail_skb) {
 500                         await_mgmt_replies(adap, cnt, 16 + i + 1);
 501                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 502                         if (!adap->nofail_skb)
 503                                 goto alloc_skb_fail;
 504                 }
 505         }
 506
 507         for (i = 0; i < 2048; i++) {
 508                 struct cpl_rte_write_req *req;
 509
 510                 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 511                 if (!skb)
 512                         skb = adap->nofail_skb;
 513                 if (!skb)
 514                         goto alloc_skb_fail;
 515
 516                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 517                 memset(req, 0, sizeof(*req));
 518                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 519                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 520                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 521                 t3_mgmt_tx(adap, skb);
 522                 if (skb == adap->nofail_skb) {
 523                         await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
 524                         adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 525                         if (!adap->nofail_skb)
 526                                 goto alloc_skb_fail;
 527                 }
 528         }
 529
 530         skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 531         if (!skb)
 532                 skb = adap->nofail_skb;
 533         if (!skb)
 534                 goto alloc_skb_fail;
 535
 536         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 537         memset(greq, 0, sizeof(*greq));
 538         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 539         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 540         greq->mask = cpu_to_be64(1);
 541         t3_mgmt_tx(adap, skb);
 542
 543         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 544         if (skb == adap->nofail_skb) {
 545                 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 546                 adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
 547         }
 548
 549         t3_tp_set_offload_mode(adap, 0);
 550         return i;
 551
 552 alloc_skb_fail:
 553         t3_tp_set_offload_mode(adap, 0);
 554         return -ENOMEM;
 555 }
 556
 557 /**
 558  *      setup_rss - configure RSS
 559  *      @adap: the adapter
 560  *
 561  *      Sets up RSS to distribute packets to multiple receive queues.  We
 562  *      configure the RSS CPU lookup table to distribute to the number of HW
 563  *      receive queues, and the response queue lookup table to narrow that
 564  *      down to the response queues actually configured for each port.
 565  *      We always configure the RSS mapping for two ports since the mapping
 566  *      table has plenty of entries.
 567  */
 568 static void setup_rss(struct adapter *adap)
 569 {
 570         int i;
 571         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 572         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 573         u8 cpus[SGE_QSETS + 1];
 574         u16 rspq_map[RSS_TABLE_SIZE];
 575
 576         for (i = 0; i < SGE_QSETS; ++i)
 577                 cpus[i] = i;
 578         cpus[SGE_QSETS] = 0xff; /* terminator */
 579
 580         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 581                 rspq_map[i] = i % nq0;
 582                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 583         }
 584
 585         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 586                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 587                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 588 }
 589
 590 static void init_napi(struct adapter *adap)
 591 {
 592         int i;
 593
 594         for (i = 0; i < SGE_QSETS; i++) {
 595                 struct sge_qset *qs = &adap->sge.qs[i];
 596
 597                 if (qs->adap)
 598                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 599                                        64);
 600         }
 601
 602         /*
 603          * netif_napi_add() can be called only once per napi_struct because it
 604          * adds each new napi_struct to a list.  Be careful not to call it a
 605          * second time, e.g., during EEH recovery, by making a note of it.
 606          */
 607         adap->flags |= NAPI_INIT;
 608 }
 609
 610 /*
 611  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 612  * both netdevices representing interfaces and the dummy ones for the extra
 613  * queues.
 614  */
 615 static void quiesce_rx(struct adapter *adap)
 616 {
 617         int i;
 618
 619         for (i = 0; i < SGE_QSETS; i++)
 620                 if (adap->sge.qs[i].adap)
 621                         napi_disable(&adap->sge.qs[i].napi);
 622 }
 623
 624 static void enable_all_napi(struct adapter *adap)
 625 {
 626         int i;
 627         for (i = 0; i < SGE_QSETS; i++)
 628                 if (adap->sge.qs[i].adap)
 629                         napi_enable(&adap->sge.qs[i].napi);
 630 }
 631
 632 /**
 633  *      set_qset_lro - Turn a queue set's LRO capability on and off
 634  *      @dev: the device the qset is attached to
 635  *      @qset_idx: the queue set index
 636  *      @val: the LRO switch
 637  *
 638  *      Sets LRO on or off for a particular queue set.
 639  *      the device's features flag is updated to reflect the LRO
 640  *      capability when all queues belonging to the device are
 641  *      in the same state.
 642  */
 643 static void set_qset_lro(struct net_device *dev, int qset_idx, int val)
 644 {
 645         struct port_info *pi = netdev_priv(dev);
 646         struct adapter *adapter = pi->adapter;
 647
 648         adapter->params.sge.qset[qset_idx].lro = !!val;
 649         adapter->sge.qs[qset_idx].lro_enabled = !!val;
 650 }
 651
 652 /**
 653  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 654  *      @adap: the adapter
 655  *
 656  *      Determines how many sets of SGE queues to use and initializes them.
 657  *      We support multiple queue sets per port if we have MSI-X, otherwise
 658  *      just one queue set per port.
 659  */
 660 static int setup_sge_qsets(struct adapter *adap)
 661 {
 662         int i, j, err, irq_idx = 0, qset_idx = 0;
 663         unsigned int ntxq = SGE_TXQ_PER_SET;
 664
 665         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 666                 irq_idx = -1;
 667
 668         for_each_port(adap, i) {
 669                 struct net_device *dev = adap->port[i];
 670                 struct port_info *pi = netdev_priv(dev);
 671
 672                 pi->qs = &adap->sge.qs[pi->first_qset];
 673                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 674                         set_qset_lro(dev, qset_idx, pi->rx_offload & T3_LRO);
 675                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 676                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 677                                                              irq_idx,
 678                                 &adap->params.sge.qset[qset_idx], ntxq, dev,
 679                                 netdev_get_tx_queue(dev, j));
 680                         if (err) {
 681                                 t3_free_sge_resources(adap);
 682                                 return err;
 683                         }
 684                 }
 685         }
 686
 687         return 0;
 688 }
 689
 690 static ssize_t attr_show(struct device *d, char *buf,
 691                          ssize_t(*format) (struct net_device *, char *))
 692 {
 693         ssize_t len;
 694
 695         /* Synchronize with ioctls that may shut down the device */
 696         rtnl_lock();
 697         len = (*format) (to_net_dev(d), buf);
 698         rtnl_unlock();
 699         return len;
 700 }
 701
 702 static ssize_t attr_store(struct device *d,
 703                           const char *buf, size_t len,
 704                           ssize_t(*set) (struct net_device *, unsigned int),
 705                           unsigned int min_val, unsigned int max_val)
 706 {
 707         char *endp;
 708         ssize_t ret;
 709         unsigned int val;
 710
 711         if (!capable(CAP_NET_ADMIN))
 712                 return -EPERM;
 713
 714         val = simple_strtoul(buf, &endp, 0);
 715         if (endp == buf || val < min_val || val > max_val)
 716                 return -EINVAL;
 717
 718         rtnl_lock();
 719         ret = (*set) (to_net_dev(d), val);
 720         if (!ret)
 721                 ret = len;
 722         rtnl_unlock();
 723         return ret;
 724 }
 725
 726 #define CXGB3_SHOW(name, val_expr) \
 727 static ssize_t format_##name(struct net_device *dev, char *buf) \
 728 { \
 729         struct port_info *pi = netdev_priv(dev); \
 730         struct adapter *adap = pi->adapter; \
 731         return sprintf(buf, "%u\n", val_expr); \
 732 } \
 733 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 734                            char *buf) \
 735 { \
 736         return attr_show(d, buf, format_##name); \
 737 }
 738
 739 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 740 {
 741         struct port_info *pi = netdev_priv(dev);
 742         struct adapter *adap = pi->adapter;
 743         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 744
 745         if (adap->flags & FULL_INIT_DONE)
 746                 return -EBUSY;
 747         if (val && adap->params.rev == 0)
 748                 return -EINVAL;
 749         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 750             min_tids)
 751                 return -EINVAL;
 752         adap->params.mc5.nfilters = val;
 753         return 0;
 754 }
 755
 756 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 757                               const char *buf, size_t len)
 758 {
 759         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 760 }
 761
 762 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 763 {
 764         struct port_info *pi = netdev_priv(dev);
 765         struct adapter *adap = pi->adapter;
 766
 767         if (adap->flags & FULL_INIT_DONE)
 768                 return -EBUSY;
 769         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 770             MC5_MIN_TIDS)
 771                 return -EINVAL;
 772         adap->params.mc5.nservers = val;
 773         return 0;
 774 }
 775
 776 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 777                               const char *buf, size_t len)
 778 {
 779         return attr_store(d, buf, len, set_nservers, 0, ~0);
 780 }
 781
 782 #define CXGB3_ATTR_R(name, val_expr) \
 783 CXGB3_SHOW(name, val_expr) \
 784 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 785
 786 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 787 CXGB3_SHOW(name, val_expr) \
 788 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 789
 790 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 791 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 792 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 793
 794 static struct attribute *cxgb3_attrs[] = {
 795         &dev_attr_cam_size.attr,
 796         &dev_attr_nfilters.attr,
 797         &dev_attr_nservers.attr,
 798         NULL
 799 };
 800
 801 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 802
 803 static ssize_t tm_attr_show(struct device *d,
 804                             char *buf, int sched)
 805 {
 806         struct port_info *pi = netdev_priv(to_net_dev(d));
 807         struct adapter *adap = pi->adapter;
 808         unsigned int v, addr, bpt, cpt;
 809         ssize_t len;
 810
 811         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 812         rtnl_lock();
 813         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 814         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 815         if (sched & 1)
 816                 v >>= 16;
 817         bpt = (v >> 8) & 0xff;
 818         cpt = v & 0xff;
 819         if (!cpt)
 820                 len = sprintf(buf, "disabled\n");
 821         else {
 822                 v = (adap->params.vpd.cclk * 1000) / cpt;
 823                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 824         }
 825         rtnl_unlock();
 826         return len;
 827 }
 828
 829 static ssize_t tm_attr_store(struct device *d,
 830                              const char *buf, size_t len, int sched)
 831 {
 832         struct port_info *pi = netdev_priv(to_net_dev(d));
 833         struct adapter *adap = pi->adapter;
 834         unsigned int val;
 835         char *endp;
 836         ssize_t ret;
 837
 838         if (!capable(CAP_NET_ADMIN))
 839                 return -EPERM;
 840
 841         val = simple_strtoul(buf, &endp, 0);
 842         if (endp == buf || val > 10000000)
 843                 return -EINVAL;
 844
 845         rtnl_lock();
 846         ret = t3_config_sched(adap, val, sched);
 847         if (!ret)
 848                 ret = len;
 849         rtnl_unlock();
 850         return ret;
 851 }
 852
 853 #define TM_ATTR(name, sched) \
 854 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 855                            char *buf) \
 856 { \
 857         return tm_attr_show(d, buf, sched); \
 858 } \
 859 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 860                             const char *buf, size_t len) \
 861 { \
 862         return tm_attr_store(d, buf, len, sched); \
 863 } \
 864 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 865
 866 TM_ATTR(sched0, 0);
 867 TM_ATTR(sched1, 1);
 868 TM_ATTR(sched2, 2);
 869 TM_ATTR(sched3, 3);
 870 TM_ATTR(sched4, 4);
 871 TM_ATTR(sched5, 5);
 872 TM_ATTR(sched6, 6);
 873 TM_ATTR(sched7, 7);
 874
 875 static struct attribute *offload_attrs[] = {
 876         &dev_attr_sched0.attr,
 877         &dev_attr_sched1.attr,
 878         &dev_attr_sched2.attr,
 879         &dev_attr_sched3.attr,
 880         &dev_attr_sched4.attr,
 881         &dev_attr_sched5.attr,
 882         &dev_attr_sched6.attr,
 883         &dev_attr_sched7.attr,
 884         NULL
 885 };
 886
 887 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 888
 889 /*
 890  * Sends an sk_buff to an offload queue driver
 891  * after dealing with any active network taps.
 892  */
 893 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 894 {
 895         int ret;
 896
 897         local_bh_disable();
 898         ret = t3_offload_tx(tdev, skb);
 899         local_bh_enable();
 900         return ret;
 901 }
 902
 903 static int write_smt_entry(struct adapter *adapter, int idx)
 904 {
 905         struct cpl_smt_write_req *req;
 906         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 907
 908         if (!skb)
 909                 return -ENOMEM;
 910
 911         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 912         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 913         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 914         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 915         req->iff = idx;
 916         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 917         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 918         skb->priority = 1;
 919         offload_tx(&adapter->tdev, skb);
 920         return 0;
 921 }
 922
 923 static int init_smt(struct adapter *adapter)
 924 {
 925         int i;
 926
 927         for_each_port(adapter, i)
 928             write_smt_entry(adapter, i);
 929         return 0;
 930 }
 931
 932 static void init_port_mtus(struct adapter *adapter)
 933 {
 934         unsigned int mtus = adapter->port[0]->mtu;
 935
 936         if (adapter->port[1])
 937                 mtus |= adapter->port[1]->mtu << 16;
 938         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 939 }
 940
 941 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 942                               int hi, int port)
 943 {
 944         struct sk_buff *skb;
 945         struct mngt_pktsched_wr *req;
 946         int ret;
 947
 948         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 949         if (!skb)
 950                 skb = adap->nofail_skb;
 951         if (!skb)
 952                 return -ENOMEM;
 953
 954         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 955         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 956         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 957         req->sched = sched;
 958         req->idx = qidx;
 959         req->min = lo;
 960         req->max = hi;
 961         req->binding = port;
 962         ret = t3_mgmt_tx(adap, skb);
 963         if (skb == adap->nofail_skb) {
 964                 adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
 965                                              GFP_KERNEL);
 966                 if (!adap->nofail_skb)
 967                         ret = -ENOMEM;
 968         }
 969
 970         return ret;
 971 }
 972
 973 static int bind_qsets(struct adapter *adap)
 974 {
 975         int i, j, err = 0;
 976
 977         for_each_port(adap, i) {
 978                 const struct port_info *pi = adap2pinfo(adap, i);
 979
 980                 for (j = 0; j < pi->nqsets; ++j) {
 981                         int ret = send_pktsched_cmd(adap, 1,
 982                                                     pi->first_qset + j, -1,
 983                                                     -1, i);
 984                         if (ret)
 985                                 err = ret;
 986                 }
 987         }
 988
 989         return err;
 990 }
 991
 992 #define FW_FNAME "cxgb3/t3fw-%d.%d.%d.bin"
 993 #define TPSRAM_NAME "cxgb3/t3%c_psram-%d.%d.%d.bin"
 994 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
 995 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
 996 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
 997
 998 static inline const char *get_edc_fw_name(int edc_idx)
 999 {
1000         const char *fw_name = NULL;
1001
1002         switch (edc_idx) {
1003         case EDC_OPT_AEL2005:
1004                 fw_name = AEL2005_OPT_EDC_NAME;
1005                 break;
1006         case EDC_TWX_AEL2005:
1007                 fw_name = AEL2005_TWX_EDC_NAME;
1008                 break;
1009         case EDC_TWX_AEL2020:
1010                 fw_name = AEL2020_TWX_EDC_NAME;
1011                 break;
1012         }
1013         return fw_name;
1014 }
1015
1016 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1017 {
1018         struct adapter *adapter = phy->adapter;
1019         const struct firmware *fw;
1020         char buf[64];
1021         u32 csum;
1022         const __be32 *p;
1023         u16 *cache = phy->phy_cache;
1024         int i, ret;
1025
1026         snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1027
1028         ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1029         if (ret < 0) {
1030                 dev_err(&adapter->pdev->dev,
1031                         "could not upgrade firmware: unable to load %s\n",
1032                         buf);
1033                 return ret;
1034         }
1035
1036         /* check size, take checksum in account */
1037         if (fw->size > size + 4) {
1038                 CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1039                        (unsigned int)fw->size, size + 4);
1040                 ret = -EINVAL;
1041         }
1042
1043         /* compute checksum */
1044         p = (const __be32 *)fw->data;
1045         for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1046                 csum += ntohl(p[i]);
1047
1048         if (csum != 0xffffffff) {
1049                 CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1050                        csum);
1051                 ret = -EINVAL;
1052         }
1053
1054         for (i = 0; i < size / 4 ; i++) {
1055                 *cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1056                 *cache++ = be32_to_cpu(p[i]) & 0xffff;
1057         }
1058
1059         release_firmware(fw);
1060
1061         return ret;
1062 }
1063
1064 static int upgrade_fw(struct adapter *adap)
1065 {
1066         int ret;
1067         char buf[64];
1068         const struct firmware *fw;
1069         struct device *dev = &adap->pdev->dev;
1070
1071         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
1072                  FW_VERSION_MINOR, FW_VERSION_MICRO);
1073         ret = request_firmware(&fw, buf, dev);
1074         if (ret < 0) {
1075                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1076                         buf);
1077                 return ret;
1078         }
1079         ret = t3_load_fw(adap, fw->data, fw->size);
1080         release_firmware(fw);
1081
1082         if (ret == 0)
1083                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1084                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1085         else
1086                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1087                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1088
1089         return ret;
1090 }
1091
1092 static inline char t3rev2char(struct adapter *adapter)
1093 {
1094         char rev = 0;
1095
1096         switch(adapter->params.rev) {
1097         case T3_REV_B:
1098         case T3_REV_B2:
1099                 rev = 'b';
1100                 break;
1101         case T3_REV_C:
1102                 rev = 'c';
1103                 break;
1104         }
1105         return rev;
1106 }
1107
1108 static int update_tpsram(struct adapter *adap)
1109 {
1110         const struct firmware *tpsram;
1111         char buf[64];
1112         struct device *dev = &adap->pdev->dev;
1113         int ret;
1114         char rev;
1115
1116         rev = t3rev2char(adap);
1117         if (!rev)
1118                 return 0;
1119
1120         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
1121                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1122
1123         ret = request_firmware(&tpsram, buf, dev);
1124         if (ret < 0) {
1125                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1126                         buf);
1127                 return ret;
1128         }
1129
1130         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1131         if (ret)
1132                 goto release_tpsram;
1133
1134         ret = t3_set_proto_sram(adap, tpsram->data);
1135         if (ret == 0)
1136                 dev_info(dev,
1137                          "successful update of protocol engine "
1138                          "to %d.%d.%d\n",
1139                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1140         else
1141                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1142                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1143         if (ret)
1144                 dev_err(dev, "loading protocol SRAM failed\n");
1145
1146 release_tpsram:
1147         release_firmware(tpsram);
1148
1149         return ret;
1150 }
1151
1152 /**
1153  *      cxgb_up - enable the adapter
1154  *      @adapter: adapter being enabled
1155  *
1156  *      Called when the first port is enabled, this function performs the
1157  *      actions necessary to make an adapter operational, such as completing
1158  *      the initialization of HW modules, and enabling interrupts.
1159  *
1160  *      Must be called with the rtnl lock held.
1161  */
1162 static int cxgb_up(struct adapter *adap)
1163 {
1164         int err;
1165
1166         if (!(adap->flags & FULL_INIT_DONE)) {
1167                 err = t3_check_fw_version(adap);
1168                 if (err == -EINVAL) {
1169                         err = upgrade_fw(adap);
1170                         CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1171                                 FW_VERSION_MAJOR, FW_VERSION_MINOR,
1172                                 FW_VERSION_MICRO, err ? "failed" : "succeeded");
1173                 }
1174
1175                 err = t3_check_tpsram_version(adap);
1176                 if (err == -EINVAL) {
1177                         err = update_tpsram(adap);
1178                         CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1179                                 TP_VERSION_MAJOR, TP_VERSION_MINOR,
1180                                 TP_VERSION_MICRO, err ? "failed" : "succeeded");
1181                 }
1182
1183                 /*
1184                  * Clear interrupts now to catch errors if t3_init_hw fails.
1185                  * We clear them again later as initialization may trigger
1186                  * conditions that can interrupt.
1187                  */
1188                 t3_intr_clear(adap);
1189
1190                 err = t3_init_hw(adap, 0);
1191                 if (err)
1192                         goto out;
1193
1194                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1195                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1196
1197                 err = setup_sge_qsets(adap);
1198                 if (err)
1199                         goto out;
1200
1201                 setup_rss(adap);
1202                 if (!(adap->flags & NAPI_INIT))
1203                         init_napi(adap);
1204
1205                 t3_start_sge_timers(adap);
1206                 adap->flags |= FULL_INIT_DONE;
1207         }
1208
1209         t3_intr_clear(adap);
1210
1211         if (adap->flags & USING_MSIX) {
1212                 name_msix_vecs(adap);
1213                 err = request_irq(adap->msix_info[0].vec,
1214                                   t3_async_intr_handler, 0,
1215                                   adap->msix_info[0].desc, adap);
1216                 if (err)
1217                         goto irq_err;
1218
1219                 err = request_msix_data_irqs(adap);
1220                 if (err) {
1221                         free_irq(adap->msix_info[0].vec, adap);
1222                         goto irq_err;
1223                 }
1224         } else if ((err = request_irq(adap->pdev->irq,
1225                                       t3_intr_handler(adap,
1226                                                       adap->sge.qs[0].rspq.
1227                                                       polling),
1228                                       (adap->flags & USING_MSI) ?
1229                                        0 : IRQF_SHARED,
1230                                       adap->name, adap)))
1231                 goto irq_err;
1232
1233         enable_all_napi(adap);
1234         t3_sge_start(adap);
1235         t3_intr_enable(adap);
1236
1237         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1238             is_offload(adap) && init_tp_parity(adap) == 0)
1239                 adap->flags |= TP_PARITY_INIT;
1240
1241         if (adap->flags & TP_PARITY_INIT) {
1242                 t3_write_reg(adap, A_TP_INT_CAUSE,
1243                              F_CMCACHEPERR | F_ARPLUTPERR);
1244                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1245         }
1246
1247         if (!(adap->flags & QUEUES_BOUND)) {
1248                 err = bind_qsets(adap);
1249                 if (err) {
1250                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1251                         t3_intr_disable(adap);
1252                         free_irq_resources(adap);
1253                         goto out;
1254                 }
1255                 adap->flags |= QUEUES_BOUND;
1256         }
1257
1258 out:
1259         return err;
1260 irq_err:
1261         CH_ERR(adap, "request_irq failed, err %d\n", err);
1262         goto out;
1263 }
1264
1265 /*
1266  * Release resources when all the ports and offloading have been stopped.
1267  */
1268 static void cxgb_down(struct adapter *adapter)
1269 {
1270         t3_sge_stop(adapter);
1271         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1272         t3_intr_disable(adapter);
1273         spin_unlock_irq(&adapter->work_lock);
1274
1275         free_irq_resources(adapter);
1276         quiesce_rx(adapter);
1277         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1278 }
1279
1280 static void schedule_chk_task(struct adapter *adap)
1281 {
1282         unsigned int timeo;
1283
1284         timeo = adap->params.linkpoll_period ?
1285             (HZ * adap->params.linkpoll_period) / 10 :
1286             adap->params.stats_update_period * HZ;
1287         if (timeo)
1288                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1289 }
1290
1291 static int offload_open(struct net_device *dev)
1292 {
1293         struct port_info *pi = netdev_priv(dev);
1294         struct adapter *adapter = pi->adapter;
1295         struct t3cdev *tdev = dev2t3cdev(dev);
1296         int adap_up = adapter->open_device_map & PORT_MASK;
1297         int err;
1298
1299         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1300                 return 0;
1301
1302         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1303                 goto out;
1304
1305         t3_tp_set_offload_mode(adapter, 1);
1306         tdev->lldev = adapter->port[0];
1307         err = cxgb3_offload_activate(adapter);
1308         if (err)
1309                 goto out;
1310
1311         init_port_mtus(adapter);
1312         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1313                      adapter->params.b_wnd,
1314                      adapter->params.rev == 0 ?
1315                      adapter->port[0]->mtu : 0xffff);
1316         init_smt(adapter);
1317
1318         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1319                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1320
1321         /* Call back all registered clients */
1322         cxgb3_add_clients(tdev);
1323
1324 out:
1325         /* restore them in case the offload module has changed them */
1326         if (err) {
1327                 t3_tp_set_offload_mode(adapter, 0);
1328                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1329                 cxgb3_set_dummy_ops(tdev);
1330         }
1331         return err;
1332 }
1333
1334 static int offload_close(struct t3cdev *tdev)
1335 {
1336         struct adapter *adapter = tdev2adap(tdev);
1337
1338         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1339                 return 0;
1340
1341         /* Call back all registered clients */
1342         cxgb3_remove_clients(tdev);
1343
1344         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1345
1346         /* Flush work scheduled while releasing TIDs */
1347         flush_scheduled_work();
1348
1349         tdev->lldev = NULL;
1350         cxgb3_set_dummy_ops(tdev);
1351         t3_tp_set_offload_mode(adapter, 0);
1352         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1353
1354         if (!adapter->open_device_map)
1355                 cxgb_down(adapter);
1356
1357         cxgb3_offload_deactivate(adapter);
1358         return 0;
1359 }
1360
1361 static int cxgb_open(struct net_device *dev)
1362 {
1363         struct port_info *pi = netdev_priv(dev);
1364         struct adapter *adapter = pi->adapter;
1365         int other_ports = adapter->open_device_map & PORT_MASK;
1366         int err;
1367
1368         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1369                 return err;
1370
1371         set_bit(pi->port_id, &adapter->open_device_map);
1372         if (is_offload(adapter) && !ofld_disable) {
1373                 err = offload_open(dev);
1374                 if (err)
1375                         printk(KERN_WARNING
1376                                "Could not initialize offload capabilities\n");
1377         }
1378
1379         dev->real_num_tx_queues = pi->nqsets;
1380         link_start(dev);
1381         t3_port_intr_enable(adapter, pi->port_id);
1382         netif_tx_start_all_queues(dev);
1383         if (!other_ports)
1384                 schedule_chk_task(adapter);
1385
1386         return 0;
1387 }
1388
1389 static int cxgb_close(struct net_device *dev)
1390 {
1391         struct port_info *pi = netdev_priv(dev);
1392         struct adapter *adapter = pi->adapter;
1393
1394
1395         if (!adapter->open_device_map)
1396                 return 0;
1397
1398         /* Stop link fault interrupts */
1399         t3_xgm_intr_disable(adapter, pi->port_id);
1400         t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1401
1402         t3_port_intr_disable(adapter, pi->port_id);
1403         netif_tx_stop_all_queues(dev);
1404         pi->phy.ops->power_down(&pi->phy, 1);
1405         netif_carrier_off(dev);
1406         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1407
1408         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1409         clear_bit(pi->port_id, &adapter->open_device_map);
1410         spin_unlock_irq(&adapter->work_lock);
1411
1412         if (!(adapter->open_device_map & PORT_MASK))
1413                 cancel_delayed_work_sync(&adapter->adap_check_task);
1414
1415         if (!adapter->open_device_map)
1416                 cxgb_down(adapter);
1417
1418         return 0;
1419 }
1420
1421 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1422 {
1423         struct port_info *pi = netdev_priv(dev);
1424         struct adapter *adapter = pi->adapter;
1425         struct net_device_stats *ns = &pi->netstats;
1426         const struct mac_stats *pstats;
1427
1428         spin_lock(&adapter->stats_lock);
1429         pstats = t3_mac_update_stats(&pi->mac);
1430         spin_unlock(&adapter->stats_lock);
1431
1432         ns->tx_bytes = pstats->tx_octets;
1433         ns->tx_packets = pstats->tx_frames;
1434         ns->rx_bytes = pstats->rx_octets;
1435         ns->rx_packets = pstats->rx_frames;
1436         ns->multicast = pstats->rx_mcast_frames;
1437
1438         ns->tx_errors = pstats->tx_underrun;
1439         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1440             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1441             pstats->rx_fifo_ovfl;
1442
1443         /* detailed rx_errors */
1444         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1445         ns->rx_over_errors = 0;
1446         ns->rx_crc_errors = pstats->rx_fcs_errs;
1447         ns->rx_frame_errors = pstats->rx_symbol_errs;
1448         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1449         ns->rx_missed_errors = pstats->rx_cong_drops;
1450
1451         /* detailed tx_errors */
1452         ns->tx_aborted_errors = 0;
1453         ns->tx_carrier_errors = 0;
1454         ns->tx_fifo_errors = pstats->tx_underrun;
1455         ns->tx_heartbeat_errors = 0;
1456         ns->tx_window_errors = 0;
1457         return ns;
1458 }
1459
1460 static u32 get_msglevel(struct net_device *dev)
1461 {
1462         struct port_info *pi = netdev_priv(dev);
1463         struct adapter *adapter = pi->adapter;
1464
1465         return adapter->msg_enable;
1466 }
1467
1468 static void set_msglevel(struct net_device *dev, u32 val)
1469 {
1470         struct port_info *pi = netdev_priv(dev);
1471         struct adapter *adapter = pi->adapter;
1472
1473         adapter->msg_enable = val;
1474 }
1475
1476 static char stats_strings[][ETH_GSTRING_LEN] = {
1477         "TxOctetsOK         ",
1478         "TxFramesOK         ",
1479         "TxMulticastFramesOK",
1480         "TxBroadcastFramesOK",
1481         "TxPauseFrames      ",
1482         "TxUnderrun         ",
1483         "TxExtUnderrun      ",
1484
1485         "TxFrames64         ",
1486         "TxFrames65To127    ",
1487         "TxFrames128To255   ",
1488         "TxFrames256To511   ",
1489         "TxFrames512To1023  ",
1490         "TxFrames1024To1518 ",
1491         "TxFrames1519ToMax  ",
1492
1493         "RxOctetsOK         ",
1494         "RxFramesOK         ",
1495         "RxMulticastFramesOK",
1496         "RxBroadcastFramesOK",
1497         "RxPauseFrames      ",
1498         "RxFCSErrors        ",
1499         "RxSymbolErrors     ",
1500         "RxShortErrors      ",
1501         "RxJabberErrors     ",
1502         "RxLengthErrors     ",
1503         "RxFIFOoverflow     ",
1504
1505         "RxFrames64         ",
1506         "RxFrames65To127    ",
1507         "RxFrames128To255   ",
1508         "RxFrames256To511   ",
1509         "RxFrames512To1023  ",
1510         "RxFrames1024To1518 ",
1511         "RxFrames1519ToMax  ",
1512
1513         "PhyFIFOErrors      ",
1514         "TSO                ",
1515         "VLANextractions    ",
1516         "VLANinsertions     ",
1517         "TxCsumOffload      ",
1518         "RxCsumGood         ",
1519         "LroAggregated      ",
1520         "LroFlushed         ",
1521         "LroNoDesc          ",
1522         "RxDrops            ",
1523
1524         "CheckTXEnToggled   ",
1525         "CheckResets        ",
1526
1527         "LinkFaults         ",
1528 };
1529
1530 static int get_sset_count(struct net_device *dev, int sset)
1531 {
1532         switch (sset) {
1533         case ETH_SS_STATS:
1534                 return ARRAY_SIZE(stats_strings);
1535         default:
1536                 return -EOPNOTSUPP;
1537         }
1538 }
1539
1540 #define T3_REGMAP_SIZE (3 * 1024)
1541
1542 static int get_regs_len(struct net_device *dev)
1543 {
1544         return T3_REGMAP_SIZE;
1545 }
1546
1547 static int get_eeprom_len(struct net_device *dev)
1548 {
1549         return EEPROMSIZE;
1550 }
1551
1552 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1553 {
1554         struct port_info *pi = netdev_priv(dev);
1555         struct adapter *adapter = pi->adapter;
1556         u32 fw_vers = 0;
1557         u32 tp_vers = 0;
1558
1559         spin_lock(&adapter->stats_lock);
1560         t3_get_fw_version(adapter, &fw_vers);
1561         t3_get_tp_version(adapter, &tp_vers);
1562         spin_unlock(&adapter->stats_lock);
1563
1564         strcpy(info->driver, DRV_NAME);
1565         strcpy(info->version, DRV_VERSION);
1566         strcpy(info->bus_info, pci_name(adapter->pdev));
1567         if (!fw_vers)
1568                 strcpy(info->fw_version, "N/A");
1569         else {
1570                 snprintf(info->fw_version, sizeof(info->fw_version),
1571                          "%s %u.%u.%u TP %u.%u.%u",
1572                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1573                          G_FW_VERSION_MAJOR(fw_vers),
1574                          G_FW_VERSION_MINOR(fw_vers),
1575                          G_FW_VERSION_MICRO(fw_vers),
1576                          G_TP_VERSION_MAJOR(tp_vers),
1577                          G_TP_VERSION_MINOR(tp_vers),
1578                          G_TP_VERSION_MICRO(tp_vers));
1579         }
1580 }
1581
1582 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1583 {
1584         if (stringset == ETH_SS_STATS)
1585                 memcpy(data, stats_strings, sizeof(stats_strings));
1586 }
1587
1588 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1589                                             struct port_info *p, int idx)
1590 {
1591         int i;
1592         unsigned long tot = 0;
1593
1594         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1595                 tot += adapter->sge.qs[i].port_stats[idx];
1596         return tot;
1597 }
1598
1599 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1600                       u64 *data)
1601 {
1602         struct port_info *pi = netdev_priv(dev);
1603         struct adapter *adapter = pi->adapter;
1604         const struct mac_stats *s;
1605
1606         spin_lock(&adapter->stats_lock);
1607         s = t3_mac_update_stats(&pi->mac);
1608         spin_unlock(&adapter->stats_lock);
1609
1610         *data++ = s->tx_octets;
1611         *data++ = s->tx_frames;
1612         *data++ = s->tx_mcast_frames;
1613         *data++ = s->tx_bcast_frames;
1614         *data++ = s->tx_pause;
1615         *data++ = s->tx_underrun;
1616         *data++ = s->tx_fifo_urun;
1617
1618         *data++ = s->tx_frames_64;
1619         *data++ = s->tx_frames_65_127;
1620         *data++ = s->tx_frames_128_255;
1621         *data++ = s->tx_frames_256_511;
1622         *data++ = s->tx_frames_512_1023;
1623         *data++ = s->tx_frames_1024_1518;
1624         *data++ = s->tx_frames_1519_max;
1625
1626         *data++ = s->rx_octets;
1627         *data++ = s->rx_frames;
1628         *data++ = s->rx_mcast_frames;
1629         *data++ = s->rx_bcast_frames;
1630         *data++ = s->rx_pause;
1631         *data++ = s->rx_fcs_errs;
1632         *data++ = s->rx_symbol_errs;
1633         *data++ = s->rx_short;
1634         *data++ = s->rx_jabber;
1635         *data++ = s->rx_too_long;
1636         *data++ = s->rx_fifo_ovfl;
1637
1638         *data++ = s->rx_frames_64;
1639         *data++ = s->rx_frames_65_127;
1640         *data++ = s->rx_frames_128_255;
1641         *data++ = s->rx_frames_256_511;
1642         *data++ = s->rx_frames_512_1023;
1643         *data++ = s->rx_frames_1024_1518;
1644         *data++ = s->rx_frames_1519_max;
1645
1646         *data++ = pi->phy.fifo_errors;
1647
1648         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1649         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1650         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1651         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1652         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1653         *data++ = 0;
1654         *data++ = 0;
1655         *data++ = 0;
1656         *data++ = s->rx_cong_drops;
1657
1658         *data++ = s->num_toggled;
1659         *data++ = s->num_resets;
1660
1661         *data++ = s->link_faults;
1662 }
1663
1664 static inline void reg_block_dump(struct adapter *ap, void *buf,
1665                                   unsigned int start, unsigned int end)
1666 {
1667         u32 *p = buf + start;
1668
1669         for (; start <= end; start += sizeof(u32))
1670                 *p++ = t3_read_reg(ap, start);
1671 }
1672
1673 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1674                      void *buf)
1675 {
1676         struct port_info *pi = netdev_priv(dev);
1677         struct adapter *ap = pi->adapter;
1678
1679         /*
1680          * Version scheme:
1681          * bits 0..9: chip version
1682          * bits 10..15: chip revision
1683          * bit 31: set for PCIe cards
1684          */
1685         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1686
1687         /*
1688          * We skip the MAC statistics registers because they are clear-on-read.
1689          * Also reading multi-register stats would need to synchronize with the
1690          * periodic mac stats accumulation.  Hard to justify the complexity.
1691          */
1692         memset(buf, 0, T3_REGMAP_SIZE);
1693         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1694         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1695         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1696         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1697         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1698         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1699                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1700         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1701                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1702 }
1703
1704 static int restart_autoneg(struct net_device *dev)
1705 {
1706         struct port_info *p = netdev_priv(dev);
1707
1708         if (!netif_running(dev))
1709                 return -EAGAIN;
1710         if (p->link_config.autoneg != AUTONEG_ENABLE)
1711                 return -EINVAL;
1712         p->phy.ops->autoneg_restart(&p->phy);
1713         return 0;
1714 }
1715
1716 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1717 {
1718         struct port_info *pi = netdev_priv(dev);
1719         struct adapter *adapter = pi->adapter;
1720         int i;
1721
1722         if (data == 0)
1723                 data = 2;
1724
1725         for (i = 0; i < data * 2; i++) {
1726                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1727                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1728                 if (msleep_interruptible(500))
1729                         break;
1730         }
1731         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1732                          F_GPIO0_OUT_VAL);
1733         return 0;
1734 }
1735
1736 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1737 {
1738         struct port_info *p = netdev_priv(dev);
1739
1740         cmd->supported = p->link_config.supported;
1741         cmd->advertising = p->link_config.advertising;
1742
1743         if (netif_carrier_ok(dev)) {
1744                 cmd->speed = p->link_config.speed;
1745                 cmd->duplex = p->link_config.duplex;
1746         } else {
1747                 cmd->speed = -1;
1748                 cmd->duplex = -1;
1749         }
1750
1751         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1752         cmd->phy_address = p->phy.mdio.prtad;
1753         cmd->transceiver = XCVR_EXTERNAL;
1754         cmd->autoneg = p->link_config.autoneg;
1755         cmd->maxtxpkt = 0;
1756         cmd->maxrxpkt = 0;
1757         return 0;
1758 }
1759
1760 static int speed_duplex_to_caps(int speed, int duplex)
1761 {
1762         int cap = 0;
1763
1764         switch (speed) {
1765         case SPEED_10:
1766                 if (duplex == DUPLEX_FULL)
1767                         cap = SUPPORTED_10baseT_Full;
1768                 else
1769                         cap = SUPPORTED_10baseT_Half;
1770                 break;
1771         case SPEED_100:
1772                 if (duplex == DUPLEX_FULL)
1773                         cap = SUPPORTED_100baseT_Full;
1774                 else
1775                         cap = SUPPORTED_100baseT_Half;
1776                 break;
1777         case SPEED_1000:
1778                 if (duplex == DUPLEX_FULL)
1779                         cap = SUPPORTED_1000baseT_Full;
1780                 else
1781                         cap = SUPPORTED_1000baseT_Half;
1782                 break;
1783         case SPEED_10000:
1784                 if (duplex == DUPLEX_FULL)
1785                         cap = SUPPORTED_10000baseT_Full;
1786         }
1787         return cap;
1788 }
1789
1790 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1791                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1792                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1793                       ADVERTISED_10000baseT_Full)
1794
1795 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1796 {
1797         struct port_info *p = netdev_priv(dev);
1798         struct link_config *lc = &p->link_config;
1799
1800         if (!(lc->supported & SUPPORTED_Autoneg)) {
1801                 /*
1802                  * PHY offers a single speed/duplex.  See if that's what's
1803                  * being requested.
1804                  */
1805                 if (cmd->autoneg == AUTONEG_DISABLE) {
1806                         int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1807                         if (lc->supported & cap)
1808                                 return 0;
1809                 }
1810                 return -EINVAL;
1811         }
1812
1813         if (cmd->autoneg == AUTONEG_DISABLE) {
1814                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1815
1816                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1817                         return -EINVAL;
1818                 lc->requested_speed = cmd->speed;
1819                 lc->requested_duplex = cmd->duplex;
1820                 lc->advertising = 0;
1821         } else {
1822                 cmd->advertising &= ADVERTISED_MASK;
1823                 cmd->advertising &= lc->supported;
1824                 if (!cmd->advertising)
1825                         return -EINVAL;
1826                 lc->requested_speed = SPEED_INVALID;
1827                 lc->requested_duplex = DUPLEX_INVALID;
1828                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1829         }
1830         lc->autoneg = cmd->autoneg;
1831         if (netif_running(dev))
1832                 t3_link_start(&p->phy, &p->mac, lc);
1833         return 0;
1834 }
1835
1836 static void get_pauseparam(struct net_device *dev,
1837                            struct ethtool_pauseparam *epause)
1838 {
1839         struct port_info *p = netdev_priv(dev);
1840
1841         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1842         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1843         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1844 }
1845
1846 static int set_pauseparam(struct net_device *dev,
1847                           struct ethtool_pauseparam *epause)
1848 {
1849         struct port_info *p = netdev_priv(dev);
1850         struct link_config *lc = &p->link_config;
1851
1852         if (epause->autoneg == AUTONEG_DISABLE)
1853                 lc->requested_fc = 0;
1854         else if (lc->supported & SUPPORTED_Autoneg)
1855                 lc->requested_fc = PAUSE_AUTONEG;
1856         else
1857                 return -EINVAL;
1858
1859         if (epause->rx_pause)
1860                 lc->requested_fc |= PAUSE_RX;
1861         if (epause->tx_pause)
1862                 lc->requested_fc |= PAUSE_TX;
1863         if (lc->autoneg == AUTONEG_ENABLE) {
1864                 if (netif_running(dev))
1865                         t3_link_start(&p->phy, &p->mac, lc);
1866         } else {
1867                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1868                 if (netif_running(dev))
1869                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1870         }
1871         return 0;
1872 }
1873
1874 static u32 get_rx_csum(struct net_device *dev)
1875 {
1876         struct port_info *p = netdev_priv(dev);
1877
1878         return p->rx_offload & T3_RX_CSUM;
1879 }
1880
1881 static int set_rx_csum(struct net_device *dev, u32 data)
1882 {
1883         struct port_info *p = netdev_priv(dev);
1884
1885         if (data) {
1886                 p->rx_offload |= T3_RX_CSUM;
1887         } else {
1888                 int i;
1889
1890                 p->rx_offload &= ~(T3_RX_CSUM | T3_LRO);
1891                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++)
1892                         set_qset_lro(dev, i, 0);
1893         }
1894         return 0;
1895 }
1896
1897 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1898 {
1899         struct port_info *pi = netdev_priv(dev);
1900         struct adapter *adapter = pi->adapter;
1901         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1902
1903         e->rx_max_pending = MAX_RX_BUFFERS;
1904         e->rx_mini_max_pending = 0;
1905         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1906         e->tx_max_pending = MAX_TXQ_ENTRIES;
1907
1908         e->rx_pending = q->fl_size;
1909         e->rx_mini_pending = q->rspq_size;
1910         e->rx_jumbo_pending = q->jumbo_size;
1911         e->tx_pending = q->txq_size[0];
1912 }
1913
1914 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1915 {
1916         struct port_info *pi = netdev_priv(dev);
1917         struct adapter *adapter = pi->adapter;
1918         struct qset_params *q;
1919         int i;
1920
1921         if (e->rx_pending > MAX_RX_BUFFERS ||
1922             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1923             e->tx_pending > MAX_TXQ_ENTRIES ||
1924             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1925             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1926             e->rx_pending < MIN_FL_ENTRIES ||
1927             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1928             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1929                 return -EINVAL;
1930
1931         if (adapter->flags & FULL_INIT_DONE)
1932                 return -EBUSY;
1933
1934         q = &adapter->params.sge.qset[pi->first_qset];
1935         for (i = 0; i < pi->nqsets; ++i, ++q) {
1936                 q->rspq_size = e->rx_mini_pending;
1937                 q->fl_size = e->rx_pending;
1938                 q->jumbo_size = e->rx_jumbo_pending;
1939                 q->txq_size[0] = e->tx_pending;
1940                 q->txq_size[1] = e->tx_pending;
1941                 q->txq_size[2] = e->tx_pending;
1942         }
1943         return 0;
1944 }
1945
1946 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1947 {
1948         struct port_info *pi = netdev_priv(dev);
1949         struct adapter *adapter = pi->adapter;
1950         struct qset_params *qsp = &adapter->params.sge.qset[0];
1951         struct sge_qset *qs = &adapter->sge.qs[0];
1952
1953         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1954                 return -EINVAL;
1955
1956         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1957         t3_update_qset_coalesce(qs, qsp);
1958         return 0;
1959 }
1960
1961 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1962 {
1963         struct port_info *pi = netdev_priv(dev);
1964         struct adapter *adapter = pi->adapter;
1965         struct qset_params *q = adapter->params.sge.qset;
1966
1967         c->rx_coalesce_usecs = q->coalesce_usecs;
1968         return 0;
1969 }
1970
1971 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1972                       u8 * data)
1973 {
1974         struct port_info *pi = netdev_priv(dev);
1975         struct adapter *adapter = pi->adapter;
1976         int i, err = 0;
1977
1978         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1979         if (!buf)
1980                 return -ENOMEM;
1981
1982         e->magic = EEPROM_MAGIC;
1983         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1984                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1985
1986         if (!err)
1987                 memcpy(data, buf + e->offset, e->len);
1988         kfree(buf);
1989         return err;
1990 }
1991
1992 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1993                       u8 * data)
1994 {
1995         struct port_info *pi = netdev_priv(dev);
1996         struct adapter *adapter = pi->adapter;
1997         u32 aligned_offset, aligned_len;
1998         __le32 *p;
1999         u8 *buf;
2000         int err;
2001
2002         if (eeprom->magic != EEPROM_MAGIC)
2003                 return -EINVAL;
2004
2005         aligned_offset = eeprom->offset & ~3;
2006         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2007
2008         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2009                 buf = kmalloc(aligned_len, GFP_KERNEL);
2010                 if (!buf)
2011                         return -ENOMEM;
2012                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2013                 if (!err && aligned_len > 4)
2014                         err = t3_seeprom_read(adapter,
2015                                               aligned_offset + aligned_len - 4,
2016                                               (__le32 *) & buf[aligned_len - 4]);
2017                 if (err)
2018                         goto out;
2019                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2020         } else
2021                 buf = data;
2022
2023         err = t3_seeprom_wp(adapter, 0);
2024         if (err)
2025                 goto out;
2026
2027         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2028                 err = t3_seeprom_write(adapter, aligned_offset, *p);
2029                 aligned_offset += 4;
2030         }
2031
2032         if (!err)
2033                 err = t3_seeprom_wp(adapter, 1);
2034 out:
2035         if (buf != data)
2036                 kfree(buf);
2037         return err;
2038 }
2039
2040 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2041 {
2042         wol->supported = 0;
2043         wol->wolopts = 0;
2044         memset(&wol->sopass, 0, sizeof(wol->sopass));
2045 }
2046
2047 static const struct ethtool_ops cxgb_ethtool_ops = {
2048         .get_settings = get_settings,
2049         .set_settings = set_settings,
2050         .get_drvinfo = get_drvinfo,
2051         .get_msglevel = get_msglevel,
2052         .set_msglevel = set_msglevel,
2053         .get_ringparam = get_sge_param,
2054         .set_ringparam = set_sge_param,
2055         .get_coalesce = get_coalesce,
2056         .set_coalesce = set_coalesce,
2057         .get_eeprom_len = get_eeprom_len,
2058         .get_eeprom = get_eeprom,
2059         .set_eeprom = set_eeprom,
2060         .get_pauseparam = get_pauseparam,
2061         .set_pauseparam = set_pauseparam,
2062         .get_rx_csum = get_rx_csum,
2063         .set_rx_csum = set_rx_csum,
2064         .set_tx_csum = ethtool_op_set_tx_csum,
2065         .set_sg = ethtool_op_set_sg,
2066         .get_link = ethtool_op_get_link,
2067         .get_strings = get_strings,
2068         .phys_id = cxgb3_phys_id,
2069         .nway_reset = restart_autoneg,
2070         .get_sset_count = get_sset_count,
2071         .get_ethtool_stats = get_stats,
2072         .get_regs_len = get_regs_len,
2073         .get_regs = get_regs,
2074         .get_wol = get_wol,
2075         .set_tso = ethtool_op_set_tso,
2076 };
2077
2078 static int in_range(int val, int lo, int hi)
2079 {
2080         return val < 0 || (val <= hi && val >= lo);
2081 }
2082
2083 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2084 {
2085         struct port_info *pi = netdev_priv(dev);
2086         struct adapter *adapter = pi->adapter;
2087         u32 cmd;
2088         int ret;
2089
2090         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2091                 return -EFAULT;
2092
2093         switch (cmd) {
2094         case CHELSIO_SET_QSET_PARAMS:{
2095                 int i;
2096                 struct qset_params *q;
2097                 struct ch_qset_params t;
2098                 int q1 = pi->first_qset;
2099                 int nqsets = pi->nqsets;
2100
2101                 if (!capable(CAP_NET_ADMIN))
2102                         return -EPERM;
2103                 if (copy_from_user(&t, useraddr, sizeof(t)))
2104                         return -EFAULT;
2105                 if (t.qset_idx >= SGE_QSETS)
2106                         return -EINVAL;
2107                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2108                         !in_range(t.cong_thres, 0, 255) ||
2109                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2110                                 MAX_TXQ_ENTRIES) ||
2111                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2112                                 MAX_TXQ_ENTRIES) ||
2113                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2114                                 MAX_CTRL_TXQ_ENTRIES) ||
2115                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2116                                 MAX_RX_BUFFERS)
2117                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2118                                         MAX_RX_JUMBO_BUFFERS)
2119                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2120                                         MAX_RSPQ_ENTRIES))
2121                         return -EINVAL;
2122
2123                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
2124                         for_each_port(adapter, i) {
2125                                 pi = adap2pinfo(adapter, i);
2126                                 if (t.qset_idx >= pi->first_qset &&
2127                                     t.qset_idx < pi->first_qset + pi->nqsets &&
2128                                     !(pi->rx_offload & T3_RX_CSUM))
2129                                         return -EINVAL;
2130                         }
2131
2132                 if ((adapter->flags & FULL_INIT_DONE) &&
2133                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2134                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2135                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2136                         t.polling >= 0 || t.cong_thres >= 0))
2137                         return -EBUSY;
2138
2139                 /* Allow setting of any available qset when offload enabled */
2140                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2141                         q1 = 0;
2142                         for_each_port(adapter, i) {
2143                                 pi = adap2pinfo(adapter, i);
2144                                 nqsets += pi->first_qset + pi->nqsets;
2145                         }
2146                 }
2147
2148                 if (t.qset_idx < q1)
2149                         return -EINVAL;
2150                 if (t.qset_idx > q1 + nqsets - 1)
2151                         return -EINVAL;
2152
2153                 q = &adapter->params.sge.qset[t.qset_idx];
2154
2155                 if (t.rspq_size >= 0)
2156                         q->rspq_size = t.rspq_size;
2157                 if (t.fl_size[0] >= 0)
2158                         q->fl_size = t.fl_size[0];
2159                 if (t.fl_size[1] >= 0)
2160                         q->jumbo_size = t.fl_size[1];
2161                 if (t.txq_size[0] >= 0)
2162                         q->txq_size[0] = t.txq_size[0];
2163                 if (t.txq_size[1] >= 0)
2164                         q->txq_size[1] = t.txq_size[1];
2165                 if (t.txq_size[2] >= 0)
2166                         q->txq_size[2] = t.txq_size[2];
2167                 if (t.cong_thres >= 0)
2168                         q->cong_thres = t.cong_thres;
2169                 if (t.intr_lat >= 0) {
2170                         struct sge_qset *qs =
2171                                 &adapter->sge.qs[t.qset_idx];
2172
2173                         q->coalesce_usecs = t.intr_lat;
2174                         t3_update_qset_coalesce(qs, q);
2175                 }
2176                 if (t.polling >= 0) {
2177                         if (adapter->flags & USING_MSIX)
2178                                 q->polling = t.polling;
2179                         else {
2180                                 /* No polling with INTx for T3A */
2181                                 if (adapter->params.rev == 0 &&
2182                                         !(adapter->flags & USING_MSI))
2183                                         t.polling = 0;
2184
2185                                 for (i = 0; i < SGE_QSETS; i++) {
2186                                         q = &adapter->params.sge.
2187                                                 qset[i];
2188                                         q->polling = t.polling;
2189                                 }
2190                         }
2191                 }
2192                 if (t.lro >= 0)
2193                         set_qset_lro(dev, t.qset_idx, t.lro);
2194
2195                 break;
2196         }
2197         case CHELSIO_GET_QSET_PARAMS:{
2198                 struct qset_params *q;
2199                 struct ch_qset_params t;
2200                 int q1 = pi->first_qset;
2201                 int nqsets = pi->nqsets;
2202                 int i;
2203
2204                 if (copy_from_user(&t, useraddr, sizeof(t)))
2205                         return -EFAULT;
2206
2207                 /* Display qsets for all ports when offload enabled */
2208                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2209                         q1 = 0;
2210                         for_each_port(adapter, i) {
2211                                 pi = adap2pinfo(adapter, i);
2212                                 nqsets = pi->first_qset + pi->nqsets;
2213                         }
2214                 }
2215
2216                 if (t.qset_idx >= nqsets)
2217                         return -EINVAL;
2218
2219                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
2220                 t.rspq_size = q->rspq_size;
2221                 t.txq_size[0] = q->txq_size[0];
2222                 t.txq_size[1] = q->txq_size[1];
2223                 t.txq_size[2] = q->txq_size[2];
2224                 t.fl_size[0] = q->fl_size;
2225                 t.fl_size[1] = q->jumbo_size;
2226                 t.polling = q->polling;
2227                 t.lro = q->lro;
2228                 t.intr_lat = q->coalesce_usecs;
2229                 t.cong_thres = q->cong_thres;
2230                 t.qnum = q1;
2231
2232                 if (adapter->flags & USING_MSIX)
2233                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2234                 else
2235                         t.vector = adapter->pdev->irq;
2236
2237                 if (copy_to_user(useraddr, &t, sizeof(t)))
2238                         return -EFAULT;
2239                 break;
2240         }
2241         case CHELSIO_SET_QSET_NUM:{
2242                 struct ch_reg edata;
2243                 unsigned int i, first_qset = 0, other_qsets = 0;
2244
2245                 if (!capable(CAP_NET_ADMIN))
2246                         return -EPERM;
2247                 if (adapter->flags & FULL_INIT_DONE)
2248                         return -EBUSY;
2249                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2250                         return -EFAULT;
2251                 if (edata.val < 1 ||
2252                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2253                         return -EINVAL;
2254
2255                 for_each_port(adapter, i)
2256                         if (adapter->port[i] && adapter->port[i] != dev)
2257                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2258
2259                 if (edata.val + other_qsets > SGE_QSETS)
2260                         return -EINVAL;
2261
2262                 pi->nqsets = edata.val;
2263
2264                 for_each_port(adapter, i)
2265                         if (adapter->port[i]) {
2266                                 pi = adap2pinfo(adapter, i);
2267                                 pi->first_qset = first_qset;
2268                                 first_qset += pi->nqsets;
2269                         }
2270                 break;
2271         }
2272         case CHELSIO_GET_QSET_NUM:{
2273                 struct ch_reg edata;
2274
2275                 edata.cmd = CHELSIO_GET_QSET_NUM;
2276                 edata.val = pi->nqsets;
2277                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2278                         return -EFAULT;
2279                 break;
2280         }
2281         case CHELSIO_LOAD_FW:{
2282                 u8 *fw_data;
2283                 struct ch_mem_range t;
2284
2285                 if (!capable(CAP_SYS_RAWIO))
2286                         return -EPERM;
2287                 if (copy_from_user(&t, useraddr, sizeof(t)))
2288                         return -EFAULT;
2289                 /* Check t.len sanity ? */
2290                 fw_data = kmalloc(t.len, GFP_KERNEL);
2291                 if (!fw_data)
2292                         return -ENOMEM;
2293
2294                 if (copy_from_user
2295                         (fw_data, useraddr + sizeof(t), t.len)) {
2296                         kfree(fw_data);
2297                         return -EFAULT;
2298                 }
2299
2300                 ret = t3_load_fw(adapter, fw_data, t.len);
2301                 kfree(fw_data);
2302                 if (ret)
2303                         return ret;
2304                 break;
2305         }
2306         case CHELSIO_SETMTUTAB:{
2307                 struct ch_mtus m;
2308                 int i;
2309
2310                 if (!is_offload(adapter))
2311                         return -EOPNOTSUPP;
2312                 if (!capable(CAP_NET_ADMIN))
2313                         return -EPERM;
2314                 if (offload_running(adapter))
2315                         return -EBUSY;
2316                 if (copy_from_user(&m, useraddr, sizeof(m)))
2317                         return -EFAULT;
2318                 if (m.nmtus != NMTUS)
2319                         return -EINVAL;
2320                 if (m.mtus[0] < 81)     /* accommodate SACK */
2321                         return -EINVAL;
2322
2323                 /* MTUs must be in ascending order */
2324                 for (i = 1; i < NMTUS; ++i)
2325                         if (m.mtus[i] < m.mtus[i - 1])
2326                                 return -EINVAL;
2327
2328                 memcpy(adapter->params.mtus, m.mtus,
2329                         sizeof(adapter->params.mtus));
2330                 break;
2331         }
2332         case CHELSIO_GET_PM:{
2333                 struct tp_params *p = &adapter->params.tp;
2334                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2335
2336                 if (!is_offload(adapter))
2337                         return -EOPNOTSUPP;
2338                 m.tx_pg_sz = p->tx_pg_size;
2339                 m.tx_num_pg = p->tx_num_pgs;
2340                 m.rx_pg_sz = p->rx_pg_size;
2341                 m.rx_num_pg = p->rx_num_pgs;
2342                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2343                 if (copy_to_user(useraddr, &m, sizeof(m)))
2344                         return -EFAULT;
2345                 break;
2346         }
2347         case CHELSIO_SET_PM:{
2348                 struct ch_pm m;
2349                 struct tp_params *p = &adapter->params.tp;
2350
2351                 if (!is_offload(adapter))
2352                         return -EOPNOTSUPP;
2353                 if (!capable(CAP_NET_ADMIN))
2354                         return -EPERM;
2355                 if (adapter->flags & FULL_INIT_DONE)
2356                         return -EBUSY;
2357                 if (copy_from_user(&m, useraddr, sizeof(m)))
2358                         return -EFAULT;
2359                 if (!is_power_of_2(m.rx_pg_sz) ||
2360                         !is_power_of_2(m.tx_pg_sz))
2361                         return -EINVAL; /* not power of 2 */
2362                 if (!(m.rx_pg_sz & 0x14000))
2363                         return -EINVAL; /* not 16KB or 64KB */
2364                 if (!(m.tx_pg_sz & 0x1554000))
2365                         return -EINVAL;
2366                 if (m.tx_num_pg == -1)
2367                         m.tx_num_pg = p->tx_num_pgs;
2368                 if (m.rx_num_pg == -1)
2369                         m.rx_num_pg = p->rx_num_pgs;
2370                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2371                         return -EINVAL;
2372                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2373                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2374                         return -EINVAL;
2375                 p->rx_pg_size = m.rx_pg_sz;
2376                 p->tx_pg_size = m.tx_pg_sz;
2377                 p->rx_num_pgs = m.rx_num_pg;
2378                 p->tx_num_pgs = m.tx_num_pg;
2379                 break;
2380         }
2381         case CHELSIO_GET_MEM:{
2382                 struct ch_mem_range t;
2383                 struct mc7 *mem;
2384                 u64 buf[32];
2385
2386                 if (!is_offload(adapter))
2387                         return -EOPNOTSUPP;
2388                 if (!(adapter->flags & FULL_INIT_DONE))
2389                         return -EIO;    /* need the memory controllers */
2390                 if (copy_from_user(&t, useraddr, sizeof(t)))
2391                         return -EFAULT;
2392                 if ((t.addr & 7) || (t.len & 7))
2393                         return -EINVAL;
2394                 if (t.mem_id == MEM_CM)
2395                         mem = &adapter->cm;
2396                 else if (t.mem_id == MEM_PMRX)
2397                         mem = &adapter->pmrx;
2398                 else if (t.mem_id == MEM_PMTX)
2399                         mem = &adapter->pmtx;
2400                 else
2401                         return -EINVAL;
2402
2403                 /*
2404                  * Version scheme:
2405                  * bits 0..9: chip version
2406                  * bits 10..15: chip revision
2407                  */
2408                 t.version = 3 | (adapter->params.rev << 10);
2409                 if (copy_to_user(useraddr, &t, sizeof(t)))
2410                         return -EFAULT;
2411
2412                 /*
2413                  * Read 256 bytes at a time as len can be large and we don't
2414                  * want to use huge intermediate buffers.
2415                  */
2416                 useraddr += sizeof(t);  /* advance to start of buffer */
2417                 while (t.len) {
2418                         unsigned int chunk =
2419                                 min_t(unsigned int, t.len, sizeof(buf));
2420
2421                         ret =
2422                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2423                                                 buf);
2424                         if (ret)
2425                                 return ret;
2426                         if (copy_to_user(useraddr, buf, chunk))
2427                                 return -EFAULT;
2428                         useraddr += chunk;
2429                         t.addr += chunk;
2430                         t.len -= chunk;
2431                 }
2432                 break;
2433         }
2434         case CHELSIO_SET_TRACE_FILTER:{
2435                 struct ch_trace t;
2436                 const struct trace_params *tp;
2437
2438                 if (!capable(CAP_NET_ADMIN))
2439                         return -EPERM;
2440                 if (!offload_running(adapter))
2441                         return -EAGAIN;
2442                 if (copy_from_user(&t, useraddr, sizeof(t)))
2443                         return -EFAULT;
2444
2445                 tp = (const struct trace_params *)&t.sip;
2446                 if (t.config_tx)
2447                         t3_config_trace_filter(adapter, tp, 0,
2448                                                 t.invert_match,
2449                                                 t.trace_tx);
2450                 if (t.config_rx)
2451                         t3_config_trace_filter(adapter, tp, 1,
2452                                                 t.invert_match,
2453                                                 t.trace_rx);
2454                 break;
2455         }
2456         default:
2457                 return -EOPNOTSUPP;
2458         }
2459         return 0;
2460 }
2461
2462 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2463 {
2464         struct mii_ioctl_data *data = if_mii(req);
2465         struct port_info *pi = netdev_priv(dev);
2466         struct adapter *adapter = pi->adapter;
2467
2468         switch (cmd) {
2469         case SIOCGMIIREG:
2470         case SIOCSMIIREG:
2471                 /* Convert phy_id from older PRTAD/DEVAD format */
2472                 if (is_10G(adapter) &&
2473                     !mdio_phy_id_is_c45(data->phy_id) &&
2474                     (data->phy_id & 0x1f00) &&
2475                     !(data->phy_id & 0xe0e0))
2476                         data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2477                                                        data->phy_id & 0x1f);
2478                 /* FALLTHRU */
2479         case SIOCGMIIPHY:
2480                 return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2481         case SIOCCHIOCTL:
2482                 return cxgb_extension_ioctl(dev, req->ifr_data);
2483         default:
2484                 return -EOPNOTSUPP;
2485         }
2486 }
2487
2488 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2489 {
2490         struct port_info *pi = netdev_priv(dev);
2491         struct adapter *adapter = pi->adapter;
2492         int ret;
2493
2494         if (new_mtu < 81)       /* accommodate SACK */
2495                 return -EINVAL;
2496         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2497                 return ret;
2498         dev->mtu = new_mtu;
2499         init_port_mtus(adapter);
2500         if (adapter->params.rev == 0 && offload_running(adapter))
2501                 t3_load_mtus(adapter, adapter->params.mtus,
2502                              adapter->params.a_wnd, adapter->params.b_wnd,
2503                              adapter->port[0]->mtu);
2504         return 0;
2505 }
2506
2507 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2508 {
2509         struct port_info *pi = netdev_priv(dev);
2510         struct adapter *adapter = pi->adapter;
2511         struct sockaddr *addr = p;
2512
2513         if (!is_valid_ether_addr(addr->sa_data))
2514                 return -EINVAL;
2515
2516         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2517         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2518         if (offload_running(adapter))
2519                 write_smt_entry(adapter, pi->port_id);
2520         return 0;
2521 }
2522
2523 /**
2524  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2525  * @adap: the adapter
2526  * @p: the port
2527  *
2528  * Ensures that current Rx processing on any of the queues associated with
2529  * the given port completes before returning.  We do this by acquiring and
2530  * releasing the locks of the response queues associated with the port.
2531  */
2532 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2533 {
2534         int i;
2535
2536         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2537                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2538
2539                 spin_lock_irq(&q->lock);
2540                 spin_unlock_irq(&q->lock);
2541         }
2542 }
2543
2544 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2545 {
2546         struct port_info *pi = netdev_priv(dev);
2547         struct adapter *adapter = pi->adapter;
2548
2549         pi->vlan_grp = grp;
2550         if (adapter->params.rev > 0)
2551                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2552         else {
2553                 /* single control for all ports */
2554                 unsigned int i, have_vlans = 0;
2555                 for_each_port(adapter, i)
2556                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2557
2558                 t3_set_vlan_accel(adapter, 1, have_vlans);
2559         }
2560         t3_synchronize_rx(adapter, pi);
2561 }
2562
2563 #ifdef CONFIG_NET_POLL_CONTROLLER
2564 static void cxgb_netpoll(struct net_device *dev)
2565 {
2566         struct port_info *pi = netdev_priv(dev);
2567         struct adapter *adapter = pi->adapter;
2568         int qidx;
2569
2570         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2571                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2572                 void *source;
2573
2574                 if (adapter->flags & USING_MSIX)
2575                         source = qs;
2576                 else
2577                         source = adapter;
2578
2579                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2580         }
2581 }
2582 #endif
2583
2584 /*
2585  * Periodic accumulation of MAC statistics.
2586  */
2587 static void mac_stats_update(struct adapter *adapter)
2588 {
2589         int i;
2590
2591         for_each_port(adapter, i) {
2592                 struct net_device *dev = adapter->port[i];
2593                 struct port_info *p = netdev_priv(dev);
2594
2595                 if (netif_running(dev)) {
2596                         spin_lock(&adapter->stats_lock);
2597                         t3_mac_update_stats(&p->mac);
2598                         spin_unlock(&adapter->stats_lock);
2599                 }
2600         }
2601 }
2602
2603 static void check_link_status(struct adapter *adapter)
2604 {
2605         int i;
2606
2607         for_each_port(adapter, i) {
2608                 struct net_device *dev = adapter->port[i];
2609                 struct port_info *p = netdev_priv(dev);
2610                 int link_fault;
2611
2612                 spin_lock_irq(&adapter->work_lock);
2613                 link_fault = p->link_fault;
2614                 spin_unlock_irq(&adapter->work_lock);
2615
2616                 if (link_fault) {
2617                         t3_link_fault(adapter, i);
2618                         continue;
2619                 }
2620
2621                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2622                         t3_xgm_intr_disable(adapter, i);
2623                         t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2624
2625                         t3_link_changed(adapter, i);
2626                         t3_xgm_intr_enable(adapter, i);
2627                 }
2628         }
2629 }
2630
2631 static void check_t3b2_mac(struct adapter *adapter)
2632 {
2633         int i;
2634
2635         if (!rtnl_trylock())    /* synchronize with ifdown */
2636                 return;
2637
2638         for_each_port(adapter, i) {
2639                 struct net_device *dev = adapter->port[i];
2640                 struct port_info *p = netdev_priv(dev);
2641                 int status;
2642
2643                 if (!netif_running(dev))
2644                         continue;
2645
2646                 status = 0;
2647                 if (netif_running(dev) && netif_carrier_ok(dev))
2648                         status = t3b2_mac_watchdog_task(&p->mac);
2649                 if (status == 1)
2650                         p->mac.stats.num_toggled++;
2651                 else if (status == 2) {
2652                         struct cmac *mac = &p->mac;
2653
2654                         t3_mac_set_mtu(mac, dev->mtu);
2655                         t3_mac_set_address(mac, 0, dev->dev_addr);
2656                         cxgb_set_rxmode(dev);
2657                         t3_link_start(&p->phy, mac, &p->link_config);
2658                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2659                         t3_port_intr_enable(adapter, p->port_id);
2660                         p->mac.stats.num_resets++;
2661                 }
2662         }
2663         rtnl_unlock();
2664 }
2665
2666
2667 static void t3_adap_check_task(struct work_struct *work)
2668 {
2669         struct adapter *adapter = container_of(work, struct adapter,
2670                                                adap_check_task.work);
2671         const struct adapter_params *p = &adapter->params;
2672         int port;
2673         unsigned int v, status, reset;
2674
2675         adapter->check_task_cnt++;
2676
2677         check_link_status(adapter);
2678
2679         /* Accumulate MAC stats if needed */
2680         if (!p->linkpoll_period ||
2681             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2682             p->stats_update_period) {
2683                 mac_stats_update(adapter);
2684                 adapter->check_task_cnt = 0;
2685         }
2686
2687         if (p->rev == T3_REV_B2)
2688                 check_t3b2_mac(adapter);
2689
2690         /*
2691          * Scan the XGMAC's to check for various conditions which we want to
2692          * monitor in a periodic polling manner rather than via an interrupt
2693          * condition.  This is used for conditions which would otherwise flood
2694          * the system with interrupts and we only really need to know that the
2695          * conditions are "happening" ...  For each condition we count the
2696          * detection of the condition and reset it for the next polling loop.
2697          */
2698         for_each_port(adapter, port) {
2699                 struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2700                 u32 cause;
2701
2702                 cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2703                 reset = 0;
2704                 if (cause & F_RXFIFO_OVERFLOW) {
2705                         mac->stats.rx_fifo_ovfl++;
2706                         reset |= F_RXFIFO_OVERFLOW;
2707                 }
2708
2709                 t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2710         }
2711
2712         /*
2713          * We do the same as above for FL_EMPTY interrupts.
2714          */
2715         status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2716         reset = 0;
2717
2718         if (status & F_FLEMPTY) {
2719                 struct sge_qset *qs = &adapter->sge.qs[0];
2720                 int i = 0;
2721
2722                 reset |= F_FLEMPTY;
2723
2724                 v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2725                     0xffff;
2726
2727                 while (v) {
2728                         qs->fl[i].empty += (v & 1);
2729                         if (i)
2730                                 qs++;
2731                         i ^= 1;
2732                         v >>= 1;
2733                 }
2734         }
2735
2736         t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2737
2738         /* Schedule the next check update if any port is active. */
2739         spin_lock_irq(&adapter->work_lock);
2740         if (adapter->open_device_map & PORT_MASK)
2741                 schedule_chk_task(adapter);
2742         spin_unlock_irq(&adapter->work_lock);
2743 }
2744
2745 /*
2746  * Processes external (PHY) interrupts in process context.
2747  */
2748 static void ext_intr_task(struct work_struct *work)
2749 {
2750         struct adapter *adapter = container_of(work, struct adapter,
2751                                                ext_intr_handler_task);
2752         int i;
2753
2754         /* Disable link fault interrupts */
2755         for_each_port(adapter, i) {
2756                 struct net_device *dev = adapter->port[i];
2757                 struct port_info *p = netdev_priv(dev);
2758
2759                 t3_xgm_intr_disable(adapter, i);
2760                 t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2761         }
2762
2763         /* Re-enable link fault interrupts */
2764         t3_phy_intr_handler(adapter);
2765
2766         for_each_port(adapter, i)
2767                 t3_xgm_intr_enable(adapter, i);
2768
2769         /* Now reenable external interrupts */
2770         spin_lock_irq(&adapter->work_lock);
2771         if (adapter->slow_intr_mask) {
2772                 adapter->slow_intr_mask |= F_T3DBG;
2773                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2774                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2775                              adapter->slow_intr_mask);
2776         }
2777         spin_unlock_irq(&adapter->work_lock);
2778 }
2779
2780 /*
2781  * Interrupt-context handler for external (PHY) interrupts.
2782  */
2783 void t3_os_ext_intr_handler(struct adapter *adapter)
2784 {
2785         /*
2786          * Schedule a task to handle external interrupts as they may be slow
2787          * and we use a mutex to protect MDIO registers.  We disable PHY
2788          * interrupts in the meantime and let the task reenable them when
2789          * it's done.
2790          */
2791         spin_lock(&adapter->work_lock);
2792         if (adapter->slow_intr_mask) {
2793                 adapter->slow_intr_mask &= ~F_T3DBG;
2794                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2795                              adapter->slow_intr_mask);
2796                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2797         }
2798         spin_unlock(&adapter->work_lock);
2799 }
2800
2801 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2802 {
2803         struct net_device *netdev = adapter->port[port_id];
2804         struct port_info *pi = netdev_priv(netdev);
2805
2806         spin_lock(&adapter->work_lock);
2807         pi->link_fault = 1;
2808         spin_unlock(&adapter->work_lock);
2809 }
2810
2811 static int t3_adapter_error(struct adapter *adapter, int reset)
2812 {
2813         int i, ret = 0;
2814
2815         if (is_offload(adapter) &&
2816             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2817                 cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2818                 offload_close(&adapter->tdev);
2819         }
2820
2821         /* Stop all ports */
2822         for_each_port(adapter, i) {
2823                 struct net_device *netdev = adapter->port[i];
2824
2825                 if (netif_running(netdev))
2826                         cxgb_close(netdev);
2827         }
2828
2829         /* Stop SGE timers */
2830         t3_stop_sge_timers(adapter);
2831
2832         adapter->flags &= ~FULL_INIT_DONE;
2833
2834         if (reset)
2835                 ret = t3_reset_adapter(adapter);
2836
2837         pci_disable_device(adapter->pdev);
2838
2839         return ret;
2840 }
2841
2842 static int t3_reenable_adapter(struct adapter *adapter)
2843 {
2844         if (pci_enable_device(adapter->pdev)) {
2845                 dev_err(&adapter->pdev->dev,
2846                         "Cannot re-enable PCI device after reset.\n");
2847                 goto err;
2848         }
2849         pci_set_master(adapter->pdev);
2850         pci_restore_state(adapter->pdev);
2851
2852         /* Free sge resources */
2853         t3_free_sge_resources(adapter);
2854
2855         if (t3_replay_prep_adapter(adapter))
2856                 goto err;
2857
2858         return 0;
2859 err:
2860         return -1;
2861 }
2862
2863 static void t3_resume_ports(struct adapter *adapter)
2864 {
2865         int i;
2866
2867         /* Restart the ports */
2868         for_each_port(adapter, i) {
2869                 struct net_device *netdev = adapter->port[i];
2870
2871                 if (netif_running(netdev)) {
2872                         if (cxgb_open(netdev)) {
2873                                 dev_err(&adapter->pdev->dev,
2874                                         "can't bring device back up"
2875                                         " after reset\n");
2876                                 continue;
2877                         }
2878                 }
2879         }
2880
2881         if (is_offload(adapter) && !ofld_disable)
2882                 cxgb3_err_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2883 }
2884
2885 /*
2886  * processes a fatal error.
2887  * Bring the ports down, reset the chip, bring the ports back up.
2888  */
2889 static void fatal_error_task(struct work_struct *work)
2890 {
2891         struct adapter *adapter = container_of(work, struct adapter,
2892                                                fatal_error_handler_task);
2893         int err = 0;
2894
2895         rtnl_lock();
2896         err = t3_adapter_error(adapter, 1);
2897         if (!err)
2898                 err = t3_reenable_adapter(adapter);
2899         if (!err)
2900                 t3_resume_ports(adapter);
2901
2902         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2903         rtnl_unlock();
2904 }
2905
2906 void t3_fatal_err(struct adapter *adapter)
2907 {
2908         unsigned int fw_status[4];
2909
2910         if (adapter->flags & FULL_INIT_DONE) {
2911                 t3_sge_stop(adapter);
2912                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2913                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2914                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2915                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2916
2917                 spin_lock(&adapter->work_lock);
2918                 t3_intr_disable(adapter);
2919                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2920                 spin_unlock(&adapter->work_lock);
2921         }
2922         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2923         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2924                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2925                          fw_status[0], fw_status[1],
2926                          fw_status[2], fw_status[3]);
2927 }
2928
2929 /**
2930  * t3_io_error_detected - called when PCI error is detected
2931  * @pdev: Pointer to PCI device
2932  * @state: The current pci connection state
2933  *
2934  * This function is called after a PCI bus error affecting
2935  * this device has been detected.
2936  */
2937 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2938                                              pci_channel_state_t state)
2939 {
2940         struct adapter *adapter = pci_get_drvdata(pdev);
2941         int ret;
2942
2943         if (state == pci_channel_io_perm_failure)
2944                 return PCI_ERS_RESULT_DISCONNECT;
2945
2946         ret = t3_adapter_error(adapter, 0);
2947
2948         /* Request a slot reset. */
2949         return PCI_ERS_RESULT_NEED_RESET;
2950 }
2951
2952 /**
2953  * t3_io_slot_reset - called after the pci bus has been reset.
2954  * @pdev: Pointer to PCI device
2955  *
2956  * Restart the card from scratch, as if from a cold-boot.
2957  */
2958 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2959 {
2960         struct adapter *adapter = pci_get_drvdata(pdev);
2961
2962         if (!t3_reenable_adapter(adapter))
2963                 return PCI_ERS_RESULT_RECOVERED;
2964
2965         return PCI_ERS_RESULT_DISCONNECT;
2966 }
2967
2968 /**
2969  * t3_io_resume - called when traffic can start flowing again.
2970  * @pdev: Pointer to PCI device
2971  *
2972  * This callback is called when the error recovery driver tells us that
2973  * its OK to resume normal operation.
2974  */
2975 static void t3_io_resume(struct pci_dev *pdev)
2976 {
2977         struct adapter *adapter = pci_get_drvdata(pdev);
2978
2979         CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
2980                  t3_read_reg(adapter, A_PCIE_PEX_ERR));
2981
2982         t3_resume_ports(adapter);
2983 }
2984
2985 static struct pci_error_handlers t3_err_handler = {
2986         .error_detected = t3_io_error_detected,
2987         .slot_reset = t3_io_slot_reset,
2988         .resume = t3_io_resume,
2989 };
2990
2991 /*
2992  * Set the number of qsets based on the number of CPUs and the number of ports,
2993  * not to exceed the number of available qsets, assuming there are enough qsets
2994  * per port in HW.
2995  */
2996 static void set_nqsets(struct adapter *adap)
2997 {
2998         int i, j = 0;
2999         int num_cpus = num_online_cpus();
3000         int hwports = adap->params.nports;
3001         int nqsets = adap->msix_nvectors - 1;
3002
3003         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3004                 if (hwports == 2 &&
3005                     (hwports * nqsets > SGE_QSETS ||
3006                      num_cpus >= nqsets / hwports))
3007                         nqsets /= hwports;
3008                 if (nqsets > num_cpus)
3009                         nqsets = num_cpus;
3010                 if (nqsets < 1 || hwports == 4)
3011                         nqsets = 1;
3012         } else
3013                 nqsets = 1;
3014
3015         for_each_port(adap, i) {
3016                 struct port_info *pi = adap2pinfo(adap, i);
3017
3018                 pi->first_qset = j;
3019                 pi->nqsets = nqsets;
3020                 j = pi->first_qset + nqsets;
3021
3022                 dev_info(&adap->pdev->dev,
3023                          "Port %d using %d queue sets.\n", i, nqsets);
3024         }
3025 }
3026
3027 static int __devinit cxgb_enable_msix(struct adapter *adap)
3028 {
3029         struct msix_entry entries[SGE_QSETS + 1];
3030         int vectors;
3031         int i, err;
3032
3033         vectors = ARRAY_SIZE(entries);
3034         for (i = 0; i < vectors; ++i)
3035                 entries[i].entry = i;
3036
3037         while ((err = pci_enable_msix(adap->pdev, entries, vectors)) > 0)
3038                 vectors = err;
3039
3040         if (err < 0)
3041                 pci_disable_msix(adap->pdev);
3042
3043         if (!err && vectors < (adap->params.nports + 1)) {
3044                 pci_disable_msix(adap->pdev);
3045                 err = -1;
3046         }
3047
3048         if (!err) {
3049                 for (i = 0; i < vectors; ++i)
3050                         adap->msix_info[i].vec = entries[i].vector;
3051                 adap->msix_nvectors = vectors;
3052         }
3053
3054         return err;
3055 }
3056
3057 static void __devinit print_port_info(struct adapter *adap,
3058                                       const struct adapter_info *ai)
3059 {
3060         static const char *pci_variant[] = {
3061                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3062         };
3063
3064         int i;
3065         char buf[80];
3066
3067         if (is_pcie(adap))
3068                 snprintf(buf, sizeof(buf), "%s x%d",
3069                          pci_variant[adap->params.pci.variant],
3070                          adap->params.pci.width);
3071         else
3072                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3073                          pci_variant[adap->params.pci.variant],
3074                          adap->params.pci.speed, adap->params.pci.width);
3075
3076         for_each_port(adap, i) {
3077                 struct net_device *dev = adap->port[i];
3078                 const struct port_info *pi = netdev_priv(dev);
3079
3080                 if (!test_bit(i, &adap->registered_device_map))
3081                         continue;
3082                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
3083                        dev->name, ai->desc, pi->phy.desc,
3084                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
3085                        (adap->flags & USING_MSIX) ? " MSI-X" :
3086                        (adap->flags & USING_MSI) ? " MSI" : "");
3087                 if (adap->name == dev->name && adap->params.vpd.mclk)
3088                         printk(KERN_INFO
3089                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3090                                adap->name, t3_mc7_size(&adap->cm) >> 20,
3091                                t3_mc7_size(&adap->pmtx) >> 20,
3092                                t3_mc7_size(&adap->pmrx) >> 20,
3093                                adap->params.vpd.sn);
3094         }
3095 }
3096
3097 static const struct net_device_ops cxgb_netdev_ops = {
3098         .ndo_open               = cxgb_open,
3099         .ndo_stop               = cxgb_close,
3100         .ndo_start_xmit         = t3_eth_xmit,
3101         .ndo_get_stats          = cxgb_get_stats,
3102         .ndo_validate_addr      = eth_validate_addr,
3103         .ndo_set_multicast_list = cxgb_set_rxmode,
3104         .ndo_do_ioctl           = cxgb_ioctl,
3105         .ndo_change_mtu         = cxgb_change_mtu,
3106         .ndo_set_mac_address    = cxgb_set_mac_addr,
3107         .ndo_vlan_rx_register   = vlan_rx_register,
3108 #ifdef CONFIG_NET_POLL_CONTROLLER
3109         .ndo_poll_controller    = cxgb_netpoll,
3110 #endif
3111 };
3112
3113 static int __devinit init_one(struct pci_dev *pdev,
3114                               const struct pci_device_id *ent)
3115 {
3116         static int version_printed;
3117
3118         int i, err, pci_using_dac = 0;
3119         resource_size_t mmio_start, mmio_len;
3120         const struct adapter_info *ai;
3121         struct adapter *adapter = NULL;
3122         struct port_info *pi;
3123
3124         if (!version_printed) {
3125                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3126                 ++version_printed;
3127         }
3128
3129         if (!cxgb3_wq) {
3130                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3131                 if (!cxgb3_wq) {
3132                         printk(KERN_ERR DRV_NAME
3133                                ": cannot initialize work queue\n");
3134                         return -ENOMEM;
3135                 }
3136         }
3137
3138         err = pci_request_regions(pdev, DRV_NAME);
3139         if (err) {
3140                 /* Just info, some other driver may have claimed the device. */
3141                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3142                 return err;
3143         }
3144
3145         err = pci_enable_device(pdev);
3146         if (err) {
3147                 dev_err(&pdev->dev, "cannot enable PCI device\n");
3148                 goto out_release_regions;
3149         }
3150
3151         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3152                 pci_using_dac = 1;
3153                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3154                 if (err) {
3155                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3156                                "coherent allocations\n");
3157                         goto out_disable_device;
3158                 }
3159         } else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3160                 dev_err(&pdev->dev, "no usable DMA configuration\n");
3161                 goto out_disable_device;
3162         }
3163
3164         pci_set_master(pdev);
3165         pci_save_state(pdev);
3166
3167         mmio_start = pci_resource_start(pdev, 0);
3168         mmio_len = pci_resource_len(pdev, 0);
3169         ai = t3_get_adapter_info(ent->driver_data);
3170
3171         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3172         if (!adapter) {
3173                 err = -ENOMEM;
3174                 goto out_disable_device;
3175         }
3176
3177         adapter->nofail_skb =
3178                 alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3179         if (!adapter->nofail_skb) {
3180                 dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3181                 err = -ENOMEM;
3182                 goto out_free_adapter;
3183         }
3184
3185         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3186         if (!adapter->regs) {
3187                 dev_err(&pdev->dev, "cannot map device registers\n");
3188                 err = -ENOMEM;
3189                 goto out_free_adapter;
3190         }
3191
3192         adapter->pdev = pdev;
3193         adapter->name = pci_name(pdev);
3194         adapter->msg_enable = dflt_msg_enable;
3195         adapter->mmio_len = mmio_len;
3196
3197         mutex_init(&adapter->mdio_lock);
3198         spin_lock_init(&adapter->work_lock);
3199         spin_lock_init(&adapter->stats_lock);
3200
3201         INIT_LIST_HEAD(&adapter->adapter_list);
3202         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3203         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3204         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3205
3206         for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3207                 struct net_device *netdev;
3208
3209                 netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3210                 if (!netdev) {
3211                         err = -ENOMEM;
3212                         goto out_free_dev;
3213                 }
3214
3215                 SET_NETDEV_DEV(netdev, &pdev->dev);
3216
3217                 adapter->port[i] = netdev;
3218                 pi = netdev_priv(netdev);
3219                 pi->adapter = adapter;
3220                 pi->rx_offload = T3_RX_CSUM | T3_LRO;
3221                 pi->port_id = i;
3222                 netif_carrier_off(netdev);
3223                 netif_tx_stop_all_queues(netdev);
3224                 netdev->irq = pdev->irq;
3225                 netdev->mem_start = mmio_start;
3226                 netdev->mem_end = mmio_start + mmio_len - 1;
3227                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
3228                 netdev->features |= NETIF_F_GRO;
3229                 if (pci_using_dac)
3230                         netdev->features |= NETIF_F_HIGHDMA;
3231
3232                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3233                 netdev->netdev_ops = &cxgb_netdev_ops;
3234                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3235         }
3236
3237         pci_set_drvdata(pdev, adapter);
3238         if (t3_prep_adapter(adapter, ai, 1) < 0) {
3239                 err = -ENODEV;
3240                 goto out_free_dev;
3241         }
3242
3243         /*
3244          * The card is now ready to go.  If any errors occur during device
3245          * registration we do not fail the whole card but rather proceed only
3246          * with the ports we manage to register successfully.  However we must
3247          * register at least one net device.
3248          */
3249         for_each_port(adapter, i) {
3250                 err = register_netdev(adapter->port[i]);
3251                 if (err)
3252                         dev_warn(&pdev->dev,
3253                                  "cannot register net device %s, skipping\n",
3254                                  adapter->port[i]->name);
3255                 else {
3256                         /*
3257                          * Change the name we use for messages to the name of
3258                          * the first successfully registered interface.
3259                          */
3260                         if (!adapter->registered_device_map)
3261                                 adapter->name = adapter->port[i]->name;
3262
3263                         __set_bit(i, &adapter->registered_device_map);
3264                 }
3265         }
3266         if (!adapter->registered_device_map) {
3267                 dev_err(&pdev->dev, "could not register any net devices\n");
3268                 goto out_free_dev;
3269         }
3270
3271         /* Driver's ready. Reflect it on LEDs */
3272         t3_led_ready(adapter);
3273
3274         if (is_offload(adapter)) {
3275                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3276                 cxgb3_adapter_ofld(adapter);
3277         }
3278
3279         /* See what interrupts we'll be using */
3280         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3281                 adapter->flags |= USING_MSIX;
3282         else if (msi > 0 && pci_enable_msi(pdev) == 0)
3283                 adapter->flags |= USING_MSI;
3284
3285         set_nqsets(adapter);
3286
3287         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3288                                  &cxgb3_attr_group);
3289
3290         print_port_info(adapter, ai);
3291         return 0;
3292
3293 out_free_dev:
3294         iounmap(adapter->regs);
3295         for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3296                 if (adapter->port[i])
3297                         free_netdev(adapter->port[i]);
3298
3299 out_free_adapter:
3300         kfree(adapter);
3301
3302 out_disable_device:
3303         pci_disable_device(pdev);
3304 out_release_regions:
3305         pci_release_regions(pdev);
3306         pci_set_drvdata(pdev, NULL);
3307         return err;
3308 }
3309
3310 static void __devexit remove_one(struct pci_dev *pdev)
3311 {
3312         struct adapter *adapter = pci_get_drvdata(pdev);
3313
3314         if (adapter) {
3315                 int i;
3316
3317                 t3_sge_stop(adapter);
3318                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
3319                                    &cxgb3_attr_group);
3320
3321                 if (is_offload(adapter)) {
3322                         cxgb3_adapter_unofld(adapter);
3323                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3324                                      &adapter->open_device_map))
3325                                 offload_close(&adapter->tdev);
3326                 }
3327
3328                 for_each_port(adapter, i)
3329                     if (test_bit(i, &adapter->registered_device_map))
3330                         unregister_netdev(adapter->port[i]);
3331
3332                 t3_stop_sge_timers(adapter);
3333                 t3_free_sge_resources(adapter);
3334                 cxgb_disable_msi(adapter);
3335
3336                 for_each_port(adapter, i)
3337                         if (adapter->port[i])
3338                                 free_netdev(adapter->port[i]);
3339
3340                 iounmap(adapter->regs);
3341                 if (adapter->nofail_skb)
3342                         kfree_skb(adapter->nofail_skb);
3343                 kfree(adapter);
3344                 pci_release_regions(pdev);
3345                 pci_disable_device(pdev);
3346                 pci_set_drvdata(pdev, NULL);
3347         }
3348 }
3349
3350 static struct pci_driver driver = {
3351         .name = DRV_NAME,
3352         .id_table = cxgb3_pci_tbl,
3353         .probe = init_one,
3354         .remove = __devexit_p(remove_one),
3355         .err_handler = &t3_err_handler,
3356 };
3357
3358 static int __init cxgb3_init_module(void)
3359 {
3360         int ret;
3361
3362         cxgb3_offload_init();
3363
3364         ret = pci_register_driver(&driver);
3365         return ret;
3366 }
3367
3368 static void __exit cxgb3_cleanup_module(void)
3369 {
3370         pci_unregister_driver(&driver);
3371         if (cxgb3_wq)
3372                 destroy_workqueue(cxgb3_wq);
3373 }
3374
3375 module_init(cxgb3_init_module);
3376 module_exit(cxgb3_cleanup_module);