drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 0),     /* PE9000 */
  84         CH_DEVICE(0x21, 1),     /* T302E */
  85         CH_DEVICE(0x22, 2),     /* T310E */
  86         CH_DEVICE(0x23, 3),     /* T320X */
  87         CH_DEVICE(0x24, 1),     /* T302X */
  88         CH_DEVICE(0x25, 3),     /* T320E */
  89         CH_DEVICE(0x26, 2),     /* T310X */
  90         CH_DEVICE(0x30, 2),     /* T3B10 */
  91         CH_DEVICE(0x31, 3),     /* T3B20 */
  92         CH_DEVICE(0x32, 1),     /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 /**
 212  *      t3_os_phymod_changed - handle PHY module changes
 213  *      @phy: the PHY reporting the module change
 214  *      @mod_type: new module type
 215  *
 216  *      This is the OS-dependent handler for PHY module changes.  It is
 217  *      invoked when a PHY module is removed or inserted for any OS-specific
 218  *      processing.
 219  */
 220 void t3_os_phymod_changed(struct adapter *adap, int port_id)
 221 {
 222         static const char *mod_str[] = {
 223                 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
 224         };
 225
 226         const struct net_device *dev = adap->port[port_id];
 227         const struct port_info *pi = netdev_priv(dev);
 228
 229         if (pi->phy.modtype == phy_modtype_none)
 230                 printk(KERN_INFO "%s: PHY module unplugged\n", dev->name);
 231         else
 232                 printk(KERN_INFO "%s: %s PHY module inserted\n", dev->name,
 233                        mod_str[pi->phy.modtype]);
 234 }
 235
 236 static void cxgb_set_rxmode(struct net_device *dev)
 237 {
 238         struct t3_rx_mode rm;
 239         struct port_info *pi = netdev_priv(dev);
 240
 241         init_rx_mode(&rm, dev, dev->mc_list);
 242         t3_mac_set_rx_mode(&pi->mac, &rm);
 243 }
 244
 245 /**
 246  *      link_start - enable a port
 247  *      @dev: the device to enable
 248  *
 249  *      Performs the MAC and PHY actions needed to enable a port.
 250  */
 251 static void link_start(struct net_device *dev)
 252 {
 253         struct t3_rx_mode rm;
 254         struct port_info *pi = netdev_priv(dev);
 255         struct cmac *mac = &pi->mac;
 256
 257         init_rx_mode(&rm, dev, dev->mc_list);
 258         t3_mac_reset(mac);
 259         t3_mac_set_mtu(mac, dev->mtu);
 260         t3_mac_set_address(mac, 0, dev->dev_addr);
 261         t3_mac_set_rx_mode(mac, &rm);
 262         t3_link_start(&pi->phy, mac, &pi->link_config);
 263         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 264 }
 265
 266 static inline void cxgb_disable_msi(struct adapter *adapter)
 267 {
 268         if (adapter->flags & USING_MSIX) {
 269                 pci_disable_msix(adapter->pdev);
 270                 adapter->flags &= ~USING_MSIX;
 271         } else if (adapter->flags & USING_MSI) {
 272                 pci_disable_msi(adapter->pdev);
 273                 adapter->flags &= ~USING_MSI;
 274         }
 275 }
 276
 277 /*
 278  * Interrupt handler for asynchronous events used with MSI-X.
 279  */
 280 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 281 {
 282         t3_slow_intr_handler(cookie);
 283         return IRQ_HANDLED;
 284 }
 285
 286 /*
 287  * Name the MSI-X interrupts.
 288  */
 289 static void name_msix_vecs(struct adapter *adap)
 290 {
 291         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 292
 293         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 294         adap->msix_info[0].desc[n] = 0;
 295
 296         for_each_port(adap, j) {
 297                 struct net_device *d = adap->port[j];
 298                 const struct port_info *pi = netdev_priv(d);
 299
 300                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 301                         snprintf(adap->msix_info[msi_idx].desc, n,
 302                                  "%s-%d", d->name, pi->first_qset + i);
 303                         adap->msix_info[msi_idx].desc[n] = 0;
 304                 }
 305         }
 306 }
 307
 308 static int request_msix_data_irqs(struct adapter *adap)
 309 {
 310         int i, j, err, qidx = 0;
 311
 312         for_each_port(adap, i) {
 313                 int nqsets = adap2pinfo(adap, i)->nqsets;
 314
 315                 for (j = 0; j < nqsets; ++j) {
 316                         err = request_irq(adap->msix_info[qidx + 1].vec,
 317                                           t3_intr_handler(adap,
 318                                                           adap->sge.qs[qidx].
 319                                                           rspq.polling), 0,
 320                                           adap->msix_info[qidx + 1].desc,
 321                                           &adap->sge.qs[qidx]);
 322                         if (err) {
 323                                 while (--qidx >= 0)
 324                                         free_irq(adap->msix_info[qidx + 1].vec,
 325                                                  &adap->sge.qs[qidx]);
 326                                 return err;
 327                         }
 328                         qidx++;
 329                 }
 330         }
 331         return 0;
 332 }
 333
 334 static void free_irq_resources(struct adapter *adapter)
 335 {
 336         if (adapter->flags & USING_MSIX) {
 337                 int i, n = 0;
 338
 339                 free_irq(adapter->msix_info[0].vec, adapter);
 340                 for_each_port(adapter, i)
 341                     n += adap2pinfo(adapter, i)->nqsets;
 342
 343                 for (i = 0; i < n; ++i)
 344                         free_irq(adapter->msix_info[i + 1].vec,
 345                                  &adapter->sge.qs[i]);
 346         } else
 347                 free_irq(adapter->pdev->irq, adapter);
 348 }
 349
 350 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
 351                               unsigned long n)
 352 {
 353         int attempts = 5;
 354
 355         while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
 356                 if (!--attempts)
 357                         return -ETIMEDOUT;
 358                 msleep(10);
 359         }
 360         return 0;
 361 }
 362
 363 static int init_tp_parity(struct adapter *adap)
 364 {
 365         int i;
 366         struct sk_buff *skb;
 367         struct cpl_set_tcb_field *greq;
 368         unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
 369
 370         t3_tp_set_offload_mode(adap, 1);
 371
 372         for (i = 0; i < 16; i++) {
 373                 struct cpl_smt_write_req *req;
 374
 375                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 376                 req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 377                 memset(req, 0, sizeof(*req));
 378                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 379                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
 380                 req->iff = i;
 381                 t3_mgmt_tx(adap, skb);
 382         }
 383
 384         for (i = 0; i < 2048; i++) {
 385                 struct cpl_l2t_write_req *req;
 386
 387                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 388                 req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
 389                 memset(req, 0, sizeof(*req));
 390                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 391                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
 392                 req->params = htonl(V_L2T_W_IDX(i));
 393                 t3_mgmt_tx(adap, skb);
 394         }
 395
 396         for (i = 0; i < 2048; i++) {
 397                 struct cpl_rte_write_req *req;
 398
 399                 skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 400                 req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
 401                 memset(req, 0, sizeof(*req));
 402                 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 403                 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
 404                 req->l2t_idx = htonl(V_L2T_W_IDX(i));
 405                 t3_mgmt_tx(adap, skb);
 406         }
 407
 408         skb = alloc_skb(sizeof(*greq), GFP_KERNEL | __GFP_NOFAIL);
 409         greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
 410         memset(greq, 0, sizeof(*greq));
 411         greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 412         OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
 413         greq->mask = cpu_to_be64(1);
 414         t3_mgmt_tx(adap, skb);
 415
 416         i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
 417         t3_tp_set_offload_mode(adap, 0);
 418         return i;
 419 }
 420
 421 /**
 422  *      setup_rss - configure RSS
 423  *      @adap: the adapter
 424  *
 425  *      Sets up RSS to distribute packets to multiple receive queues.  We
 426  *      configure the RSS CPU lookup table to distribute to the number of HW
 427  *      receive queues, and the response queue lookup table to narrow that
 428  *      down to the response queues actually configured for each port.
 429  *      We always configure the RSS mapping for two ports since the mapping
 430  *      table has plenty of entries.
 431  */
 432 static void setup_rss(struct adapter *adap)
 433 {
 434         int i;
 435         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 436         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 437         u8 cpus[SGE_QSETS + 1];
 438         u16 rspq_map[RSS_TABLE_SIZE];
 439
 440         for (i = 0; i < SGE_QSETS; ++i)
 441                 cpus[i] = i;
 442         cpus[SGE_QSETS] = 0xff; /* terminator */
 443
 444         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 445                 rspq_map[i] = i % nq0;
 446                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 447         }
 448
 449         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 450                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 451                       V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
 452 }
 453
 454 static void init_napi(struct adapter *adap)
 455 {
 456         int i;
 457
 458         for (i = 0; i < SGE_QSETS; i++) {
 459                 struct sge_qset *qs = &adap->sge.qs[i];
 460
 461                 if (qs->adap)
 462                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 463                                        64);
 464         }
 465
 466         /*
 467          * netif_napi_add() can be called only once per napi_struct because it
 468          * adds each new napi_struct to a list.  Be careful not to call it a
 469          * second time, e.g., during EEH recovery, by making a note of it.
 470          */
 471         adap->flags |= NAPI_INIT;
 472 }
 473
 474 /*
 475  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 476  * both netdevices representing interfaces and the dummy ones for the extra
 477  * queues.
 478  */
 479 static void quiesce_rx(struct adapter *adap)
 480 {
 481         int i;
 482
 483         for (i = 0; i < SGE_QSETS; i++)
 484                 if (adap->sge.qs[i].adap)
 485                         napi_disable(&adap->sge.qs[i].napi);
 486 }
 487
 488 static void enable_all_napi(struct adapter *adap)
 489 {
 490         int i;
 491         for (i = 0; i < SGE_QSETS; i++)
 492                 if (adap->sge.qs[i].adap)
 493                         napi_enable(&adap->sge.qs[i].napi);
 494 }
 495
 496 /**
 497  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 498  *      @adap: the adapter
 499  *
 500  *      Determines how many sets of SGE queues to use and initializes them.
 501  *      We support multiple queue sets per port if we have MSI-X, otherwise
 502  *      just one queue set per port.
 503  */
 504 static int setup_sge_qsets(struct adapter *adap)
 505 {
 506         int i, j, err, irq_idx = 0, qset_idx = 0;
 507         unsigned int ntxq = SGE_TXQ_PER_SET;
 508
 509         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 510                 irq_idx = -1;
 511
 512         for_each_port(adap, i) {
 513                 struct net_device *dev = adap->port[i];
 514                 struct port_info *pi = netdev_priv(dev);
 515
 516                 pi->qs = &adap->sge.qs[pi->first_qset];
 517                 for (j = pi->first_qset; j < pi->first_qset + pi->nqsets;
 518                      ++j, ++qset_idx) {
 519                         if (!pi->rx_csum_offload)
 520                                 adap->params.sge.qset[qset_idx].lro = 0;
 521                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 522                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 523                                                              irq_idx,
 524                                 &adap->params.sge.qset[qset_idx], ntxq, dev);
 525                         if (err) {
 526                                 t3_stop_sge_timers(adap);
 527                                 t3_free_sge_resources(adap);
 528                                 return err;
 529                         }
 530                 }
 531         }
 532
 533         return 0;
 534 }
 535
 536 static ssize_t attr_show(struct device *d, char *buf,
 537                          ssize_t(*format) (struct net_device *, char *))
 538 {
 539         ssize_t len;
 540
 541         /* Synchronize with ioctls that may shut down the device */
 542         rtnl_lock();
 543         len = (*format) (to_net_dev(d), buf);
 544         rtnl_unlock();
 545         return len;
 546 }
 547
 548 static ssize_t attr_store(struct device *d,
 549                           const char *buf, size_t len,
 550                           ssize_t(*set) (struct net_device *, unsigned int),
 551                           unsigned int min_val, unsigned int max_val)
 552 {
 553         char *endp;
 554         ssize_t ret;
 555         unsigned int val;
 556
 557         if (!capable(CAP_NET_ADMIN))
 558                 return -EPERM;
 559
 560         val = simple_strtoul(buf, &endp, 0);
 561         if (endp == buf || val < min_val || val > max_val)
 562                 return -EINVAL;
 563
 564         rtnl_lock();
 565         ret = (*set) (to_net_dev(d), val);
 566         if (!ret)
 567                 ret = len;
 568         rtnl_unlock();
 569         return ret;
 570 }
 571
 572 #define CXGB3_SHOW(name, val_expr) \
 573 static ssize_t format_##name(struct net_device *dev, char *buf) \
 574 { \
 575         struct port_info *pi = netdev_priv(dev); \
 576         struct adapter *adap = pi->adapter; \
 577         return sprintf(buf, "%u\n", val_expr); \
 578 } \
 579 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 580                            char *buf) \
 581 { \
 582         return attr_show(d, buf, format_##name); \
 583 }
 584
 585 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 586 {
 587         struct port_info *pi = netdev_priv(dev);
 588         struct adapter *adap = pi->adapter;
 589         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 590
 591         if (adap->flags & FULL_INIT_DONE)
 592                 return -EBUSY;
 593         if (val && adap->params.rev == 0)
 594                 return -EINVAL;
 595         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 596             min_tids)
 597                 return -EINVAL;
 598         adap->params.mc5.nfilters = val;
 599         return 0;
 600 }
 601
 602 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 603                               const char *buf, size_t len)
 604 {
 605         return attr_store(d, buf, len, set_nfilters, 0, ~0);
 606 }
 607
 608 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 609 {
 610         struct port_info *pi = netdev_priv(dev);
 611         struct adapter *adap = pi->adapter;
 612
 613         if (adap->flags & FULL_INIT_DONE)
 614                 return -EBUSY;
 615         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 616             MC5_MIN_TIDS)
 617                 return -EINVAL;
 618         adap->params.mc5.nservers = val;
 619         return 0;
 620 }
 621
 622 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 623                               const char *buf, size_t len)
 624 {
 625         return attr_store(d, buf, len, set_nservers, 0, ~0);
 626 }
 627
 628 #define CXGB3_ATTR_R(name, val_expr) \
 629 CXGB3_SHOW(name, val_expr) \
 630 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 631
 632 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 633 CXGB3_SHOW(name, val_expr) \
 634 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 635
 636 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 637 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 638 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 639
 640 static struct attribute *cxgb3_attrs[] = {
 641         &dev_attr_cam_size.attr,
 642         &dev_attr_nfilters.attr,
 643         &dev_attr_nservers.attr,
 644         NULL
 645 };
 646
 647 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 648
 649 static ssize_t tm_attr_show(struct device *d,
 650                             char *buf, int sched)
 651 {
 652         struct port_info *pi = netdev_priv(to_net_dev(d));
 653         struct adapter *adap = pi->adapter;
 654         unsigned int v, addr, bpt, cpt;
 655         ssize_t len;
 656
 657         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 658         rtnl_lock();
 659         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 660         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 661         if (sched & 1)
 662                 v >>= 16;
 663         bpt = (v >> 8) & 0xff;
 664         cpt = v & 0xff;
 665         if (!cpt)
 666                 len = sprintf(buf, "disabled\n");
 667         else {
 668                 v = (adap->params.vpd.cclk * 1000) / cpt;
 669                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 670         }
 671         rtnl_unlock();
 672         return len;
 673 }
 674
 675 static ssize_t tm_attr_store(struct device *d,
 676                              const char *buf, size_t len, int sched)
 677 {
 678         struct port_info *pi = netdev_priv(to_net_dev(d));
 679         struct adapter *adap = pi->adapter;
 680         unsigned int val;
 681         char *endp;
 682         ssize_t ret;
 683
 684         if (!capable(CAP_NET_ADMIN))
 685                 return -EPERM;
 686
 687         val = simple_strtoul(buf, &endp, 0);
 688         if (endp == buf || val > 10000000)
 689                 return -EINVAL;
 690
 691         rtnl_lock();
 692         ret = t3_config_sched(adap, val, sched);
 693         if (!ret)
 694                 ret = len;
 695         rtnl_unlock();
 696         return ret;
 697 }
 698
 699 #define TM_ATTR(name, sched) \
 700 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 701                            char *buf) \
 702 { \
 703         return tm_attr_show(d, buf, sched); \
 704 } \
 705 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 706                             const char *buf, size_t len) \
 707 { \
 708         return tm_attr_store(d, buf, len, sched); \
 709 } \
 710 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 711
 712 TM_ATTR(sched0, 0);
 713 TM_ATTR(sched1, 1);
 714 TM_ATTR(sched2, 2);
 715 TM_ATTR(sched3, 3);
 716 TM_ATTR(sched4, 4);
 717 TM_ATTR(sched5, 5);
 718 TM_ATTR(sched6, 6);
 719 TM_ATTR(sched7, 7);
 720
 721 static struct attribute *offload_attrs[] = {
 722         &dev_attr_sched0.attr,
 723         &dev_attr_sched1.attr,
 724         &dev_attr_sched2.attr,
 725         &dev_attr_sched3.attr,
 726         &dev_attr_sched4.attr,
 727         &dev_attr_sched5.attr,
 728         &dev_attr_sched6.attr,
 729         &dev_attr_sched7.attr,
 730         NULL
 731 };
 732
 733 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 734
 735 /*
 736  * Sends an sk_buff to an offload queue driver
 737  * after dealing with any active network taps.
 738  */
 739 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 740 {
 741         int ret;
 742
 743         local_bh_disable();
 744         ret = t3_offload_tx(tdev, skb);
 745         local_bh_enable();
 746         return ret;
 747 }
 748
 749 static int write_smt_entry(struct adapter *adapter, int idx)
 750 {
 751         struct cpl_smt_write_req *req;
 752         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 753
 754         if (!skb)
 755                 return -ENOMEM;
 756
 757         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 758         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 759         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 760         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 761         req->iff = idx;
 762         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 763         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 764         skb->priority = 1;
 765         offload_tx(&adapter->tdev, skb);
 766         return 0;
 767 }
 768
 769 static int init_smt(struct adapter *adapter)
 770 {
 771         int i;
 772
 773         for_each_port(adapter, i)
 774             write_smt_entry(adapter, i);
 775         return 0;
 776 }
 777
 778 static void init_port_mtus(struct adapter *adapter)
 779 {
 780         unsigned int mtus = adapter->port[0]->mtu;
 781
 782         if (adapter->port[1])
 783                 mtus |= adapter->port[1]->mtu << 16;
 784         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 785 }
 786
 787 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 788                               int hi, int port)
 789 {
 790         struct sk_buff *skb;
 791         struct mngt_pktsched_wr *req;
 792         int ret;
 793
 794         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 795         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 796         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 797         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 798         req->sched = sched;
 799         req->idx = qidx;
 800         req->min = lo;
 801         req->max = hi;
 802         req->binding = port;
 803         ret = t3_mgmt_tx(adap, skb);
 804
 805         return ret;
 806 }
 807
 808 static int bind_qsets(struct adapter *adap)
 809 {
 810         int i, j, err = 0;
 811
 812         for_each_port(adap, i) {
 813                 const struct port_info *pi = adap2pinfo(adap, i);
 814
 815                 for (j = 0; j < pi->nqsets; ++j) {
 816                         int ret = send_pktsched_cmd(adap, 1,
 817                                                     pi->first_qset + j, -1,
 818                                                     -1, i);
 819                         if (ret)
 820                                 err = ret;
 821                 }
 822         }
 823
 824         return err;
 825 }
 826
 827 #define FW_FNAME "t3fw-%d.%d.%d.bin"
 828 #define TPSRAM_NAME "t3%c_protocol_sram-%d.%d.%d.bin"
 829
 830 static int upgrade_fw(struct adapter *adap)
 831 {
 832         int ret;
 833         char buf[64];
 834         const struct firmware *fw;
 835         struct device *dev = &adap->pdev->dev;
 836
 837         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 838                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 839         ret = request_firmware(&fw, buf, dev);
 840         if (ret < 0) {
 841                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 842                         buf);
 843                 return ret;
 844         }
 845         ret = t3_load_fw(adap, fw->data, fw->size);
 846         release_firmware(fw);
 847
 848         if (ret == 0)
 849                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 850                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 851         else
 852                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 853                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 854
 855         return ret;
 856 }
 857
 858 static inline char t3rev2char(struct adapter *adapter)
 859 {
 860         char rev = 0;
 861
 862         switch(adapter->params.rev) {
 863         case T3_REV_B:
 864         case T3_REV_B2:
 865                 rev = 'b';
 866                 break;
 867         case T3_REV_C:
 868                 rev = 'c';
 869                 break;
 870         }
 871         return rev;
 872 }
 873
 874 static int update_tpsram(struct adapter *adap)
 875 {
 876         const struct firmware *tpsram;
 877         char buf[64];
 878         struct device *dev = &adap->pdev->dev;
 879         int ret;
 880         char rev;
 881
 882         rev = t3rev2char(adap);
 883         if (!rev)
 884                 return 0;
 885
 886         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 887                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 888
 889         ret = request_firmware(&tpsram, buf, dev);
 890         if (ret < 0) {
 891                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 892                         buf);
 893                 return ret;
 894         }
 895
 896         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 897         if (ret)
 898                 goto release_tpsram;
 899
 900         ret = t3_set_proto_sram(adap, tpsram->data);
 901         if (ret == 0)
 902                 dev_info(dev,
 903                          "successful update of protocol engine "
 904                          "to %d.%d.%d\n",
 905                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 906         else
 907                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 908                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 909         if (ret)
 910                 dev_err(dev, "loading protocol SRAM failed\n");
 911
 912 release_tpsram:
 913         release_firmware(tpsram);
 914
 915         return ret;
 916 }
 917
 918 /**
 919  *      cxgb_up - enable the adapter
 920  *      @adapter: adapter being enabled
 921  *
 922  *      Called when the first port is enabled, this function performs the
 923  *      actions necessary to make an adapter operational, such as completing
 924  *      the initialization of HW modules, and enabling interrupts.
 925  *
 926  *      Must be called with the rtnl lock held.
 927  */
 928 static int cxgb_up(struct adapter *adap)
 929 {
 930         int err;
 931         int must_load;
 932
 933         if (!(adap->flags & FULL_INIT_DONE)) {
 934                 err = t3_check_fw_version(adap, &must_load);
 935                 if (err == -EINVAL) {
 936                         err = upgrade_fw(adap);
 937                         if (err && must_load)
 938                                 goto out;
 939                 }
 940
 941                 err = t3_check_tpsram_version(adap, &must_load);
 942                 if (err == -EINVAL) {
 943                         err = update_tpsram(adap);
 944                         if (err && must_load)
 945                                 goto out;
 946                 }
 947
 948                 /*
 949                  * Clear interrupts now to catch errors if t3_init_hw fails.
 950                  * We clear them again later as initialization may trigger
 951                  * conditions that can interrupt.
 952                  */
 953                 t3_intr_clear(adap);
 954
 955                 err = t3_init_hw(adap, 0);
 956                 if (err)
 957                         goto out;
 958
 959                 t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
 960                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 961
 962                 err = setup_sge_qsets(adap);
 963                 if (err)
 964                         goto out;
 965
 966                 setup_rss(adap);
 967                 if (!(adap->flags & NAPI_INIT))
 968                         init_napi(adap);
 969                 adap->flags |= FULL_INIT_DONE;
 970         }
 971
 972         t3_intr_clear(adap);
 973
 974         if (adap->flags & USING_MSIX) {
 975                 name_msix_vecs(adap);
 976                 err = request_irq(adap->msix_info[0].vec,
 977                                   t3_async_intr_handler, 0,
 978                                   adap->msix_info[0].desc, adap);
 979                 if (err)
 980                         goto irq_err;
 981
 982                 err = request_msix_data_irqs(adap);
 983                 if (err) {
 984                         free_irq(adap->msix_info[0].vec, adap);
 985                         goto irq_err;
 986                 }
 987         } else if ((err = request_irq(adap->pdev->irq,
 988                                       t3_intr_handler(adap,
 989                                                       adap->sge.qs[0].rspq.
 990                                                       polling),
 991                                       (adap->flags & USING_MSI) ?
 992                                        0 : IRQF_SHARED,
 993                                       adap->name, adap)))
 994                 goto irq_err;
 995
 996         enable_all_napi(adap);
 997         t3_sge_start(adap);
 998         t3_intr_enable(adap);
 999
1000         if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1001             is_offload(adap) && init_tp_parity(adap) == 0)
1002                 adap->flags |= TP_PARITY_INIT;
1003
1004         if (adap->flags & TP_PARITY_INIT) {
1005                 t3_write_reg(adap, A_TP_INT_CAUSE,
1006                              F_CMCACHEPERR | F_ARPLUTPERR);
1007                 t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1008         }
1009
1010         if (!(adap->flags & QUEUES_BOUND)) {
1011                 err = bind_qsets(adap);
1012                 if (err) {
1013                         CH_ERR(adap, "failed to bind qsets, err %d\n", err);
1014                         t3_intr_disable(adap);
1015                         free_irq_resources(adap);
1016                         goto out;
1017                 }
1018                 adap->flags |= QUEUES_BOUND;
1019         }
1020
1021 out:
1022         return err;
1023 irq_err:
1024         CH_ERR(adap, "request_irq failed, err %d\n", err);
1025         goto out;
1026 }
1027
1028 /*
1029  * Release resources when all the ports and offloading have been stopped.
1030  */
1031 static void cxgb_down(struct adapter *adapter)
1032 {
1033         t3_sge_stop(adapter);
1034         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
1035         t3_intr_disable(adapter);
1036         spin_unlock_irq(&adapter->work_lock);
1037
1038         free_irq_resources(adapter);
1039         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
1040         quiesce_rx(adapter);
1041 }
1042
1043 static void schedule_chk_task(struct adapter *adap)
1044 {
1045         unsigned int timeo;
1046
1047         timeo = adap->params.linkpoll_period ?
1048             (HZ * adap->params.linkpoll_period) / 10 :
1049             adap->params.stats_update_period * HZ;
1050         if (timeo)
1051                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1052 }
1053
1054 static int offload_open(struct net_device *dev)
1055 {
1056         struct port_info *pi = netdev_priv(dev);
1057         struct adapter *adapter = pi->adapter;
1058         struct t3cdev *tdev = dev2t3cdev(dev);
1059         int adap_up = adapter->open_device_map & PORT_MASK;
1060         int err;
1061
1062         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1063                 return 0;
1064
1065         if (!adap_up && (err = cxgb_up(adapter)) < 0)
1066                 goto out;
1067
1068         t3_tp_set_offload_mode(adapter, 1);
1069         tdev->lldev = adapter->port[0];
1070         err = cxgb3_offload_activate(adapter);
1071         if (err)
1072                 goto out;
1073
1074         init_port_mtus(adapter);
1075         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1076                      adapter->params.b_wnd,
1077                      adapter->params.rev == 0 ?
1078                      adapter->port[0]->mtu : 0xffff);
1079         init_smt(adapter);
1080
1081         if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1082                 dev_dbg(&dev->dev, "cannot create sysfs group\n");
1083
1084         /* Call back all registered clients */
1085         cxgb3_add_clients(tdev);
1086
1087 out:
1088         /* restore them in case the offload module has changed them */
1089         if (err) {
1090                 t3_tp_set_offload_mode(adapter, 0);
1091                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1092                 cxgb3_set_dummy_ops(tdev);
1093         }
1094         return err;
1095 }
1096
1097 static int offload_close(struct t3cdev *tdev)
1098 {
1099         struct adapter *adapter = tdev2adap(tdev);
1100
1101         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1102                 return 0;
1103
1104         /* Call back all registered clients */
1105         cxgb3_remove_clients(tdev);
1106
1107         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1108
1109         tdev->lldev = NULL;
1110         cxgb3_set_dummy_ops(tdev);
1111         t3_tp_set_offload_mode(adapter, 0);
1112         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1113
1114         if (!adapter->open_device_map)
1115                 cxgb_down(adapter);
1116
1117         cxgb3_offload_deactivate(adapter);
1118         return 0;
1119 }
1120
1121 static int cxgb_open(struct net_device *dev)
1122 {
1123         struct port_info *pi = netdev_priv(dev);
1124         struct adapter *adapter = pi->adapter;
1125         int other_ports = adapter->open_device_map & PORT_MASK;
1126         int err;
1127
1128         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1129                 return err;
1130
1131         set_bit(pi->port_id, &adapter->open_device_map);
1132         if (is_offload(adapter) && !ofld_disable) {
1133                 err = offload_open(dev);
1134                 if (err)
1135                         printk(KERN_WARNING
1136                                "Could not initialize offload capabilities\n");
1137         }
1138
1139         link_start(dev);
1140         t3_port_intr_enable(adapter, pi->port_id);
1141         netif_start_queue(dev);
1142         if (!other_ports)
1143                 schedule_chk_task(adapter);
1144
1145         return 0;
1146 }
1147
1148 static int cxgb_close(struct net_device *dev)
1149 {
1150         struct port_info *pi = netdev_priv(dev);
1151         struct adapter *adapter = pi->adapter;
1152
1153         t3_port_intr_disable(adapter, pi->port_id);
1154         netif_stop_queue(dev);
1155         pi->phy.ops->power_down(&pi->phy, 1);
1156         netif_carrier_off(dev);
1157         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1158
1159         spin_lock_irq(&adapter->work_lock);     /* sync with update task */
1160         clear_bit(pi->port_id, &adapter->open_device_map);
1161         spin_unlock_irq(&adapter->work_lock);
1162
1163         if (!(adapter->open_device_map & PORT_MASK))
1164                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1165                                                   &adapter->adap_check_task);
1166
1167         if (!adapter->open_device_map)
1168                 cxgb_down(adapter);
1169
1170         return 0;
1171 }
1172
1173 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1174 {
1175         struct port_info *pi = netdev_priv(dev);
1176         struct adapter *adapter = pi->adapter;
1177         struct net_device_stats *ns = &pi->netstats;
1178         const struct mac_stats *pstats;
1179
1180         spin_lock(&adapter->stats_lock);
1181         pstats = t3_mac_update_stats(&pi->mac);
1182         spin_unlock(&adapter->stats_lock);
1183
1184         ns->tx_bytes = pstats->tx_octets;
1185         ns->tx_packets = pstats->tx_frames;
1186         ns->rx_bytes = pstats->rx_octets;
1187         ns->rx_packets = pstats->rx_frames;
1188         ns->multicast = pstats->rx_mcast_frames;
1189
1190         ns->tx_errors = pstats->tx_underrun;
1191         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1192             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1193             pstats->rx_fifo_ovfl;
1194
1195         /* detailed rx_errors */
1196         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1197         ns->rx_over_errors = 0;
1198         ns->rx_crc_errors = pstats->rx_fcs_errs;
1199         ns->rx_frame_errors = pstats->rx_symbol_errs;
1200         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1201         ns->rx_missed_errors = pstats->rx_cong_drops;
1202
1203         /* detailed tx_errors */
1204         ns->tx_aborted_errors = 0;
1205         ns->tx_carrier_errors = 0;
1206         ns->tx_fifo_errors = pstats->tx_underrun;
1207         ns->tx_heartbeat_errors = 0;
1208         ns->tx_window_errors = 0;
1209         return ns;
1210 }
1211
1212 static u32 get_msglevel(struct net_device *dev)
1213 {
1214         struct port_info *pi = netdev_priv(dev);
1215         struct adapter *adapter = pi->adapter;
1216
1217         return adapter->msg_enable;
1218 }
1219
1220 static void set_msglevel(struct net_device *dev, u32 val)
1221 {
1222         struct port_info *pi = netdev_priv(dev);
1223         struct adapter *adapter = pi->adapter;
1224
1225         adapter->msg_enable = val;
1226 }
1227
1228 static char stats_strings[][ETH_GSTRING_LEN] = {
1229         "TxOctetsOK         ",
1230         "TxFramesOK         ",
1231         "TxMulticastFramesOK",
1232         "TxBroadcastFramesOK",
1233         "TxPauseFrames      ",
1234         "TxUnderrun         ",
1235         "TxExtUnderrun      ",
1236
1237         "TxFrames64         ",
1238         "TxFrames65To127    ",
1239         "TxFrames128To255   ",
1240         "TxFrames256To511   ",
1241         "TxFrames512To1023  ",
1242         "TxFrames1024To1518 ",
1243         "TxFrames1519ToMax  ",
1244
1245         "RxOctetsOK         ",
1246         "RxFramesOK         ",
1247         "RxMulticastFramesOK",
1248         "RxBroadcastFramesOK",
1249         "RxPauseFrames      ",
1250         "RxFCSErrors        ",
1251         "RxSymbolErrors     ",
1252         "RxShortErrors      ",
1253         "RxJabberErrors     ",
1254         "RxLengthErrors     ",
1255         "RxFIFOoverflow     ",
1256
1257         "RxFrames64         ",
1258         "RxFrames65To127    ",
1259         "RxFrames128To255   ",
1260         "RxFrames256To511   ",
1261         "RxFrames512To1023  ",
1262         "RxFrames1024To1518 ",
1263         "RxFrames1519ToMax  ",
1264
1265         "PhyFIFOErrors      ",
1266         "TSO                ",
1267         "VLANextractions    ",
1268         "VLANinsertions     ",
1269         "TxCsumOffload      ",
1270         "RxCsumGood         ",
1271         "LroAggregated      ",
1272         "LroFlushed         ",
1273         "LroNoDesc          ",
1274         "RxDrops            ",
1275
1276         "CheckTXEnToggled   ",
1277         "CheckResets        ",
1278
1279 };
1280
1281 static int get_sset_count(struct net_device *dev, int sset)
1282 {
1283         switch (sset) {
1284         case ETH_SS_STATS:
1285                 return ARRAY_SIZE(stats_strings);
1286         default:
1287                 return -EOPNOTSUPP;
1288         }
1289 }
1290
1291 #define T3_REGMAP_SIZE (3 * 1024)
1292
1293 static int get_regs_len(struct net_device *dev)
1294 {
1295         return T3_REGMAP_SIZE;
1296 }
1297
1298 static int get_eeprom_len(struct net_device *dev)
1299 {
1300         return EEPROMSIZE;
1301 }
1302
1303 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1304 {
1305         struct port_info *pi = netdev_priv(dev);
1306         struct adapter *adapter = pi->adapter;
1307         u32 fw_vers = 0;
1308         u32 tp_vers = 0;
1309
1310         spin_lock(&adapter->stats_lock);
1311         t3_get_fw_version(adapter, &fw_vers);
1312         t3_get_tp_version(adapter, &tp_vers);
1313         spin_unlock(&adapter->stats_lock);
1314
1315         strcpy(info->driver, DRV_NAME);
1316         strcpy(info->version, DRV_VERSION);
1317         strcpy(info->bus_info, pci_name(adapter->pdev));
1318         if (!fw_vers)
1319                 strcpy(info->fw_version, "N/A");
1320         else {
1321                 snprintf(info->fw_version, sizeof(info->fw_version),
1322                          "%s %u.%u.%u TP %u.%u.%u",
1323                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1324                          G_FW_VERSION_MAJOR(fw_vers),
1325                          G_FW_VERSION_MINOR(fw_vers),
1326                          G_FW_VERSION_MICRO(fw_vers),
1327                          G_TP_VERSION_MAJOR(tp_vers),
1328                          G_TP_VERSION_MINOR(tp_vers),
1329                          G_TP_VERSION_MICRO(tp_vers));
1330         }
1331 }
1332
1333 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1334 {
1335         if (stringset == ETH_SS_STATS)
1336                 memcpy(data, stats_strings, sizeof(stats_strings));
1337 }
1338
1339 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1340                                             struct port_info *p, int idx)
1341 {
1342         int i;
1343         unsigned long tot = 0;
1344
1345         for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1346                 tot += adapter->sge.qs[i].port_stats[idx];
1347         return tot;
1348 }
1349
1350 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1351                       u64 *data)
1352 {
1353         struct port_info *pi = netdev_priv(dev);
1354         struct adapter *adapter = pi->adapter;
1355         const struct mac_stats *s;
1356
1357         spin_lock(&adapter->stats_lock);
1358         s = t3_mac_update_stats(&pi->mac);
1359         spin_unlock(&adapter->stats_lock);
1360
1361         *data++ = s->tx_octets;
1362         *data++ = s->tx_frames;
1363         *data++ = s->tx_mcast_frames;
1364         *data++ = s->tx_bcast_frames;
1365         *data++ = s->tx_pause;
1366         *data++ = s->tx_underrun;
1367         *data++ = s->tx_fifo_urun;
1368
1369         *data++ = s->tx_frames_64;
1370         *data++ = s->tx_frames_65_127;
1371         *data++ = s->tx_frames_128_255;
1372         *data++ = s->tx_frames_256_511;
1373         *data++ = s->tx_frames_512_1023;
1374         *data++ = s->tx_frames_1024_1518;
1375         *data++ = s->tx_frames_1519_max;
1376
1377         *data++ = s->rx_octets;
1378         *data++ = s->rx_frames;
1379         *data++ = s->rx_mcast_frames;
1380         *data++ = s->rx_bcast_frames;
1381         *data++ = s->rx_pause;
1382         *data++ = s->rx_fcs_errs;
1383         *data++ = s->rx_symbol_errs;
1384         *data++ = s->rx_short;
1385         *data++ = s->rx_jabber;
1386         *data++ = s->rx_too_long;
1387         *data++ = s->rx_fifo_ovfl;
1388
1389         *data++ = s->rx_frames_64;
1390         *data++ = s->rx_frames_65_127;
1391         *data++ = s->rx_frames_128_255;
1392         *data++ = s->rx_frames_256_511;
1393         *data++ = s->rx_frames_512_1023;
1394         *data++ = s->rx_frames_1024_1518;
1395         *data++ = s->rx_frames_1519_max;
1396
1397         *data++ = pi->phy.fifo_errors;
1398
1399         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1400         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1401         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1402         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1403         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1404         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_AGGR);
1405         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_FLUSHED);
1406         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_LRO_NO_DESC);
1407         *data++ = s->rx_cong_drops;
1408
1409         *data++ = s->num_toggled;
1410         *data++ = s->num_resets;
1411 }
1412
1413 static inline void reg_block_dump(struct adapter *ap, void *buf,
1414                                   unsigned int start, unsigned int end)
1415 {
1416         u32 *p = buf + start;
1417
1418         for (; start <= end; start += sizeof(u32))
1419                 *p++ = t3_read_reg(ap, start);
1420 }
1421
1422 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1423                      void *buf)
1424 {
1425         struct port_info *pi = netdev_priv(dev);
1426         struct adapter *ap = pi->adapter;
1427
1428         /*
1429          * Version scheme:
1430          * bits 0..9: chip version
1431          * bits 10..15: chip revision
1432          * bit 31: set for PCIe cards
1433          */
1434         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1435
1436         /*
1437          * We skip the MAC statistics registers because they are clear-on-read.
1438          * Also reading multi-register stats would need to synchronize with the
1439          * periodic mac stats accumulation.  Hard to justify the complexity.
1440          */
1441         memset(buf, 0, T3_REGMAP_SIZE);
1442         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1443         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1444         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1445         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1446         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1447         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1448                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1449         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1450                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1451 }
1452
1453 static int restart_autoneg(struct net_device *dev)
1454 {
1455         struct port_info *p = netdev_priv(dev);
1456
1457         if (!netif_running(dev))
1458                 return -EAGAIN;
1459         if (p->link_config.autoneg != AUTONEG_ENABLE)
1460                 return -EINVAL;
1461         p->phy.ops->autoneg_restart(&p->phy);
1462         return 0;
1463 }
1464
1465 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1466 {
1467         struct port_info *pi = netdev_priv(dev);
1468         struct adapter *adapter = pi->adapter;
1469         int i;
1470
1471         if (data == 0)
1472                 data = 2;
1473
1474         for (i = 0; i < data * 2; i++) {
1475                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1476                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1477                 if (msleep_interruptible(500))
1478                         break;
1479         }
1480         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1481                          F_GPIO0_OUT_VAL);
1482         return 0;
1483 }
1484
1485 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1486 {
1487         struct port_info *p = netdev_priv(dev);
1488
1489         cmd->supported = p->link_config.supported;
1490         cmd->advertising = p->link_config.advertising;
1491
1492         if (netif_carrier_ok(dev)) {
1493                 cmd->speed = p->link_config.speed;
1494                 cmd->duplex = p->link_config.duplex;
1495         } else {
1496                 cmd->speed = -1;
1497                 cmd->duplex = -1;
1498         }
1499
1500         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1501         cmd->phy_address = p->phy.addr;
1502         cmd->transceiver = XCVR_EXTERNAL;
1503         cmd->autoneg = p->link_config.autoneg;
1504         cmd->maxtxpkt = 0;
1505         cmd->maxrxpkt = 0;
1506         return 0;
1507 }
1508
1509 static int speed_duplex_to_caps(int speed, int duplex)
1510 {
1511         int cap = 0;
1512
1513         switch (speed) {
1514         case SPEED_10:
1515                 if (duplex == DUPLEX_FULL)
1516                         cap = SUPPORTED_10baseT_Full;
1517                 else
1518                         cap = SUPPORTED_10baseT_Half;
1519                 break;
1520         case SPEED_100:
1521                 if (duplex == DUPLEX_FULL)
1522                         cap = SUPPORTED_100baseT_Full;
1523                 else
1524                         cap = SUPPORTED_100baseT_Half;
1525                 break;
1526         case SPEED_1000:
1527                 if (duplex == DUPLEX_FULL)
1528                         cap = SUPPORTED_1000baseT_Full;
1529                 else
1530                         cap = SUPPORTED_1000baseT_Half;
1531                 break;
1532         case SPEED_10000:
1533                 if (duplex == DUPLEX_FULL)
1534                         cap = SUPPORTED_10000baseT_Full;
1535         }
1536         return cap;
1537 }
1538
1539 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1540                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1541                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1542                       ADVERTISED_10000baseT_Full)
1543
1544 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1545 {
1546         int cap;
1547         struct port_info *p = netdev_priv(dev);
1548         struct link_config *lc = &p->link_config;
1549
1550         if (!(lc->supported & SUPPORTED_Autoneg)) {
1551                 /*
1552                  * PHY offers a single speed/duplex.  See if that's what's
1553                  * being requested.
1554                  */
1555                 if (cmd->autoneg == AUTONEG_DISABLE) {
1556                         cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1557                         if (lc->supported & cap)
1558                                 return 0;
1559                 }
1560                 return -EINVAL;
1561         }
1562
1563         if (cmd->autoneg == AUTONEG_DISABLE) {
1564                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1565
1566                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1567                         return -EINVAL;
1568                 lc->requested_speed = cmd->speed;
1569                 lc->requested_duplex = cmd->duplex;
1570                 lc->advertising = 0;
1571         } else {
1572                 cmd->advertising &= ADVERTISED_MASK;
1573                 cmd->advertising &= lc->supported;
1574                 if (!cmd->advertising)
1575                         return -EINVAL;
1576                 lc->requested_speed = SPEED_INVALID;
1577                 lc->requested_duplex = DUPLEX_INVALID;
1578                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1579         }
1580         lc->autoneg = cmd->autoneg;
1581         if (netif_running(dev))
1582                 t3_link_start(&p->phy, &p->mac, lc);
1583         return 0;
1584 }
1585
1586 static void get_pauseparam(struct net_device *dev,
1587                            struct ethtool_pauseparam *epause)
1588 {
1589         struct port_info *p = netdev_priv(dev);
1590
1591         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1592         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1593         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1594 }
1595
1596 static int set_pauseparam(struct net_device *dev,
1597                           struct ethtool_pauseparam *epause)
1598 {
1599         struct port_info *p = netdev_priv(dev);
1600         struct link_config *lc = &p->link_config;
1601
1602         if (epause->autoneg == AUTONEG_DISABLE)
1603                 lc->requested_fc = 0;
1604         else if (lc->supported & SUPPORTED_Autoneg)
1605                 lc->requested_fc = PAUSE_AUTONEG;
1606         else
1607                 return -EINVAL;
1608
1609         if (epause->rx_pause)
1610                 lc->requested_fc |= PAUSE_RX;
1611         if (epause->tx_pause)
1612                 lc->requested_fc |= PAUSE_TX;
1613         if (lc->autoneg == AUTONEG_ENABLE) {
1614                 if (netif_running(dev))
1615                         t3_link_start(&p->phy, &p->mac, lc);
1616         } else {
1617                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1618                 if (netif_running(dev))
1619                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1620         }
1621         return 0;
1622 }
1623
1624 static u32 get_rx_csum(struct net_device *dev)
1625 {
1626         struct port_info *p = netdev_priv(dev);
1627
1628         return p->rx_csum_offload;
1629 }
1630
1631 static int set_rx_csum(struct net_device *dev, u32 data)
1632 {
1633         struct port_info *p = netdev_priv(dev);
1634
1635         p->rx_csum_offload = data;
1636         if (!data) {
1637                 struct adapter *adap = p->adapter;
1638                 int i;
1639
1640                 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1641                         adap->params.sge.qset[i].lro = 0;
1642                         adap->sge.qs[i].lro_enabled = 0;
1643                 }
1644         }
1645         return 0;
1646 }
1647
1648 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1649 {
1650         struct port_info *pi = netdev_priv(dev);
1651         struct adapter *adapter = pi->adapter;
1652         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1653
1654         e->rx_max_pending = MAX_RX_BUFFERS;
1655         e->rx_mini_max_pending = 0;
1656         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1657         e->tx_max_pending = MAX_TXQ_ENTRIES;
1658
1659         e->rx_pending = q->fl_size;
1660         e->rx_mini_pending = q->rspq_size;
1661         e->rx_jumbo_pending = q->jumbo_size;
1662         e->tx_pending = q->txq_size[0];
1663 }
1664
1665 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1666 {
1667         struct port_info *pi = netdev_priv(dev);
1668         struct adapter *adapter = pi->adapter;
1669         struct qset_params *q;
1670         int i;
1671
1672         if (e->rx_pending > MAX_RX_BUFFERS ||
1673             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1674             e->tx_pending > MAX_TXQ_ENTRIES ||
1675             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1676             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1677             e->rx_pending < MIN_FL_ENTRIES ||
1678             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1679             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1680                 return -EINVAL;
1681
1682         if (adapter->flags & FULL_INIT_DONE)
1683                 return -EBUSY;
1684
1685         q = &adapter->params.sge.qset[pi->first_qset];
1686         for (i = 0; i < pi->nqsets; ++i, ++q) {
1687                 q->rspq_size = e->rx_mini_pending;
1688                 q->fl_size = e->rx_pending;
1689                 q->jumbo_size = e->rx_jumbo_pending;
1690                 q->txq_size[0] = e->tx_pending;
1691                 q->txq_size[1] = e->tx_pending;
1692                 q->txq_size[2] = e->tx_pending;
1693         }
1694         return 0;
1695 }
1696
1697 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1698 {
1699         struct port_info *pi = netdev_priv(dev);
1700         struct adapter *adapter = pi->adapter;
1701         struct qset_params *qsp = &adapter->params.sge.qset[0];
1702         struct sge_qset *qs = &adapter->sge.qs[0];
1703
1704         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1705                 return -EINVAL;
1706
1707         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1708         t3_update_qset_coalesce(qs, qsp);
1709         return 0;
1710 }
1711
1712 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1713 {
1714         struct port_info *pi = netdev_priv(dev);
1715         struct adapter *adapter = pi->adapter;
1716         struct qset_params *q = adapter->params.sge.qset;
1717
1718         c->rx_coalesce_usecs = q->coalesce_usecs;
1719         return 0;
1720 }
1721
1722 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1723                       u8 * data)
1724 {
1725         struct port_info *pi = netdev_priv(dev);
1726         struct adapter *adapter = pi->adapter;
1727         int i, err = 0;
1728
1729         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1730         if (!buf)
1731                 return -ENOMEM;
1732
1733         e->magic = EEPROM_MAGIC;
1734         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1735                 err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
1736
1737         if (!err)
1738                 memcpy(data, buf + e->offset, e->len);
1739         kfree(buf);
1740         return err;
1741 }
1742
1743 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1744                       u8 * data)
1745 {
1746         struct port_info *pi = netdev_priv(dev);
1747         struct adapter *adapter = pi->adapter;
1748         u32 aligned_offset, aligned_len;
1749         __le32 *p;
1750         u8 *buf;
1751         int err;
1752
1753         if (eeprom->magic != EEPROM_MAGIC)
1754                 return -EINVAL;
1755
1756         aligned_offset = eeprom->offset & ~3;
1757         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1758
1759         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1760                 buf = kmalloc(aligned_len, GFP_KERNEL);
1761                 if (!buf)
1762                         return -ENOMEM;
1763                 err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
1764                 if (!err && aligned_len > 4)
1765                         err = t3_seeprom_read(adapter,
1766                                               aligned_offset + aligned_len - 4,
1767                                               (__le32 *) & buf[aligned_len - 4]);
1768                 if (err)
1769                         goto out;
1770                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1771         } else
1772                 buf = data;
1773
1774         err = t3_seeprom_wp(adapter, 0);
1775         if (err)
1776                 goto out;
1777
1778         for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1779                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1780                 aligned_offset += 4;
1781         }
1782
1783         if (!err)
1784                 err = t3_seeprom_wp(adapter, 1);
1785 out:
1786         if (buf != data)
1787                 kfree(buf);
1788         return err;
1789 }
1790
1791 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1792 {
1793         wol->supported = 0;
1794         wol->wolopts = 0;
1795         memset(&wol->sopass, 0, sizeof(wol->sopass));
1796 }
1797
1798 static const struct ethtool_ops cxgb_ethtool_ops = {
1799         .get_settings = get_settings,
1800         .set_settings = set_settings,
1801         .get_drvinfo = get_drvinfo,
1802         .get_msglevel = get_msglevel,
1803         .set_msglevel = set_msglevel,
1804         .get_ringparam = get_sge_param,
1805         .set_ringparam = set_sge_param,
1806         .get_coalesce = get_coalesce,
1807         .set_coalesce = set_coalesce,
1808         .get_eeprom_len = get_eeprom_len,
1809         .get_eeprom = get_eeprom,
1810         .set_eeprom = set_eeprom,
1811         .get_pauseparam = get_pauseparam,
1812         .set_pauseparam = set_pauseparam,
1813         .get_rx_csum = get_rx_csum,
1814         .set_rx_csum = set_rx_csum,
1815         .set_tx_csum = ethtool_op_set_tx_csum,
1816         .set_sg = ethtool_op_set_sg,
1817         .get_link = ethtool_op_get_link,
1818         .get_strings = get_strings,
1819         .phys_id = cxgb3_phys_id,
1820         .nway_reset = restart_autoneg,
1821         .get_sset_count = get_sset_count,
1822         .get_ethtool_stats = get_stats,
1823         .get_regs_len = get_regs_len,
1824         .get_regs = get_regs,
1825         .get_wol = get_wol,
1826         .set_tso = ethtool_op_set_tso,
1827 };
1828
1829 static int in_range(int val, int lo, int hi)
1830 {
1831         return val < 0 || (val <= hi && val >= lo);
1832 }
1833
1834 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1835 {
1836         struct port_info *pi = netdev_priv(dev);
1837         struct adapter *adapter = pi->adapter;
1838         u32 cmd;
1839         int ret;
1840
1841         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1842                 return -EFAULT;
1843
1844         switch (cmd) {
1845         case CHELSIO_SET_QSET_PARAMS:{
1846                 int i;
1847                 struct qset_params *q;
1848                 struct ch_qset_params t;
1849                 int q1 = pi->first_qset;
1850                 int nqsets = pi->nqsets;
1851
1852                 if (!capable(CAP_NET_ADMIN))
1853                         return -EPERM;
1854                 if (copy_from_user(&t, useraddr, sizeof(t)))
1855                         return -EFAULT;
1856                 if (t.qset_idx >= SGE_QSETS)
1857                         return -EINVAL;
1858                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1859                         !in_range(t.cong_thres, 0, 255) ||
1860                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1861                                 MAX_TXQ_ENTRIES) ||
1862                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1863                                 MAX_TXQ_ENTRIES) ||
1864                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1865                                 MAX_CTRL_TXQ_ENTRIES) ||
1866                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1867                                 MAX_RX_BUFFERS)
1868                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1869                                         MAX_RX_JUMBO_BUFFERS)
1870                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1871                                         MAX_RSPQ_ENTRIES))
1872                         return -EINVAL;
1873
1874                 if ((adapter->flags & FULL_INIT_DONE) && t.lro > 0)
1875                         for_each_port(adapter, i) {
1876                                 pi = adap2pinfo(adapter, i);
1877                                 if (t.qset_idx >= pi->first_qset &&
1878                                     t.qset_idx < pi->first_qset + pi->nqsets &&
1879                                     !pi->rx_csum_offload)
1880                                         return -EINVAL;
1881                         }
1882
1883                 if ((adapter->flags & FULL_INIT_DONE) &&
1884                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1885                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1886                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1887                         t.polling >= 0 || t.cong_thres >= 0))
1888                         return -EBUSY;
1889
1890                 /* Allow setting of any available qset when offload enabled */
1891                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1892                         q1 = 0;
1893                         for_each_port(adapter, i) {
1894                                 pi = adap2pinfo(adapter, i);
1895                                 nqsets += pi->first_qset + pi->nqsets;
1896                         }
1897                 }
1898
1899                 if (t.qset_idx < q1)
1900                         return -EINVAL;
1901                 if (t.qset_idx > q1 + nqsets - 1)
1902                         return -EINVAL;
1903
1904                 q = &adapter->params.sge.qset[t.qset_idx];
1905
1906                 if (t.rspq_size >= 0)
1907                         q->rspq_size = t.rspq_size;
1908                 if (t.fl_size[0] >= 0)
1909                         q->fl_size = t.fl_size[0];
1910                 if (t.fl_size[1] >= 0)
1911                         q->jumbo_size = t.fl_size[1];
1912                 if (t.txq_size[0] >= 0)
1913                         q->txq_size[0] = t.txq_size[0];
1914                 if (t.txq_size[1] >= 0)
1915                         q->txq_size[1] = t.txq_size[1];
1916                 if (t.txq_size[2] >= 0)
1917                         q->txq_size[2] = t.txq_size[2];
1918                 if (t.cong_thres >= 0)
1919                         q->cong_thres = t.cong_thres;
1920                 if (t.intr_lat >= 0) {
1921                         struct sge_qset *qs =
1922                                 &adapter->sge.qs[t.qset_idx];
1923
1924                         q->coalesce_usecs = t.intr_lat;
1925                         t3_update_qset_coalesce(qs, q);
1926                 }
1927                 if (t.polling >= 0) {
1928                         if (adapter->flags & USING_MSIX)
1929                                 q->polling = t.polling;
1930                         else {
1931                                 /* No polling with INTx for T3A */
1932                                 if (adapter->params.rev == 0 &&
1933                                         !(adapter->flags & USING_MSI))
1934                                         t.polling = 0;
1935
1936                                 for (i = 0; i < SGE_QSETS; i++) {
1937                                         q = &adapter->params.sge.
1938                                                 qset[i];
1939                                         q->polling = t.polling;
1940                                 }
1941                         }
1942                 }
1943                 if (t.lro >= 0) {
1944                         struct sge_qset *qs = &adapter->sge.qs[t.qset_idx];
1945                         q->lro = t.lro;
1946                         qs->lro_enabled = t.lro;
1947                 }
1948                 break;
1949         }
1950         case CHELSIO_GET_QSET_PARAMS:{
1951                 struct qset_params *q;
1952                 struct ch_qset_params t;
1953                 int q1 = pi->first_qset;
1954                 int nqsets = pi->nqsets;
1955                 int i;
1956
1957                 if (copy_from_user(&t, useraddr, sizeof(t)))
1958                         return -EFAULT;
1959
1960                 /* Display qsets for all ports when offload enabled */
1961                 if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
1962                         q1 = 0;
1963                         for_each_port(adapter, i) {
1964                                 pi = adap2pinfo(adapter, i);
1965                                 nqsets = pi->first_qset + pi->nqsets;
1966                         }
1967                 }
1968
1969                 if (t.qset_idx >= nqsets)
1970                         return -EINVAL;
1971
1972                 q = &adapter->params.sge.qset[q1 + t.qset_idx];
1973                 t.rspq_size = q->rspq_size;
1974                 t.txq_size[0] = q->txq_size[0];
1975                 t.txq_size[1] = q->txq_size[1];
1976                 t.txq_size[2] = q->txq_size[2];
1977                 t.fl_size[0] = q->fl_size;
1978                 t.fl_size[1] = q->jumbo_size;
1979                 t.polling = q->polling;
1980                 t.lro = q->lro;
1981                 t.intr_lat = q->coalesce_usecs;
1982                 t.cong_thres = q->cong_thres;
1983                 t.qnum = q1;
1984
1985                 if (adapter->flags & USING_MSIX)
1986                         t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
1987                 else
1988                         t.vector = adapter->pdev->irq;
1989
1990                 if (copy_to_user(useraddr, &t, sizeof(t)))
1991                         return -EFAULT;
1992                 break;
1993         }
1994         case CHELSIO_SET_QSET_NUM:{
1995                 struct ch_reg edata;
1996                 unsigned int i, first_qset = 0, other_qsets = 0;
1997
1998                 if (!capable(CAP_NET_ADMIN))
1999                         return -EPERM;
2000                 if (adapter->flags & FULL_INIT_DONE)
2001                         return -EBUSY;
2002                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
2003                         return -EFAULT;
2004                 if (edata.val < 1 ||
2005                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
2006                         return -EINVAL;
2007
2008                 for_each_port(adapter, i)
2009                         if (adapter->port[i] && adapter->port[i] != dev)
2010                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
2011
2012                 if (edata.val + other_qsets > SGE_QSETS)
2013                         return -EINVAL;
2014
2015                 pi->nqsets = edata.val;
2016
2017                 for_each_port(adapter, i)
2018                         if (adapter->port[i]) {
2019                                 pi = adap2pinfo(adapter, i);
2020                                 pi->first_qset = first_qset;
2021                                 first_qset += pi->nqsets;
2022                         }
2023                 break;
2024         }
2025         case CHELSIO_GET_QSET_NUM:{
2026                 struct ch_reg edata;
2027
2028                 edata.cmd = CHELSIO_GET_QSET_NUM;
2029                 edata.val = pi->nqsets;
2030                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
2031                         return -EFAULT;
2032                 break;
2033         }
2034         case CHELSIO_LOAD_FW:{
2035                 u8 *fw_data;
2036                 struct ch_mem_range t;
2037
2038                 if (!capable(CAP_SYS_RAWIO))
2039                         return -EPERM;
2040                 if (copy_from_user(&t, useraddr, sizeof(t)))
2041                         return -EFAULT;
2042                 /* Check t.len sanity ? */
2043                 fw_data = kmalloc(t.len, GFP_KERNEL);
2044                 if (!fw_data)
2045                         return -ENOMEM;
2046
2047                 if (copy_from_user
2048                         (fw_data, useraddr + sizeof(t), t.len)) {
2049                         kfree(fw_data);
2050                         return -EFAULT;
2051                 }
2052
2053                 ret = t3_load_fw(adapter, fw_data, t.len);
2054                 kfree(fw_data);
2055                 if (ret)
2056                         return ret;
2057                 break;
2058         }
2059         case CHELSIO_SETMTUTAB:{
2060                 struct ch_mtus m;
2061                 int i;
2062
2063                 if (!is_offload(adapter))
2064                         return -EOPNOTSUPP;
2065                 if (!capable(CAP_NET_ADMIN))
2066                         return -EPERM;
2067                 if (offload_running(adapter))
2068                         return -EBUSY;
2069                 if (copy_from_user(&m, useraddr, sizeof(m)))
2070                         return -EFAULT;
2071                 if (m.nmtus != NMTUS)
2072                         return -EINVAL;
2073                 if (m.mtus[0] < 81)     /* accommodate SACK */
2074                         return -EINVAL;
2075
2076                 /* MTUs must be in ascending order */
2077                 for (i = 1; i < NMTUS; ++i)
2078                         if (m.mtus[i] < m.mtus[i - 1])
2079                                 return -EINVAL;
2080
2081                 memcpy(adapter->params.mtus, m.mtus,
2082                         sizeof(adapter->params.mtus));
2083                 break;
2084         }
2085         case CHELSIO_GET_PM:{
2086                 struct tp_params *p = &adapter->params.tp;
2087                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2088
2089                 if (!is_offload(adapter))
2090                         return -EOPNOTSUPP;
2091                 m.tx_pg_sz = p->tx_pg_size;
2092                 m.tx_num_pg = p->tx_num_pgs;
2093                 m.rx_pg_sz = p->rx_pg_size;
2094                 m.rx_num_pg = p->rx_num_pgs;
2095                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2096                 if (copy_to_user(useraddr, &m, sizeof(m)))
2097                         return -EFAULT;
2098                 break;
2099         }
2100         case CHELSIO_SET_PM:{
2101                 struct ch_pm m;
2102                 struct tp_params *p = &adapter->params.tp;
2103
2104                 if (!is_offload(adapter))
2105                         return -EOPNOTSUPP;
2106                 if (!capable(CAP_NET_ADMIN))
2107                         return -EPERM;
2108                 if (adapter->flags & FULL_INIT_DONE)
2109                         return -EBUSY;
2110                 if (copy_from_user(&m, useraddr, sizeof(m)))
2111                         return -EFAULT;
2112                 if (!is_power_of_2(m.rx_pg_sz) ||
2113                         !is_power_of_2(m.tx_pg_sz))
2114                         return -EINVAL; /* not power of 2 */
2115                 if (!(m.rx_pg_sz & 0x14000))
2116                         return -EINVAL; /* not 16KB or 64KB */
2117                 if (!(m.tx_pg_sz & 0x1554000))
2118                         return -EINVAL;
2119                 if (m.tx_num_pg == -1)
2120                         m.tx_num_pg = p->tx_num_pgs;
2121                 if (m.rx_num_pg == -1)
2122                         m.rx_num_pg = p->rx_num_pgs;
2123                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2124                         return -EINVAL;
2125                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2126                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2127                         return -EINVAL;
2128                 p->rx_pg_size = m.rx_pg_sz;
2129                 p->tx_pg_size = m.tx_pg_sz;
2130                 p->rx_num_pgs = m.rx_num_pg;
2131                 p->tx_num_pgs = m.tx_num_pg;
2132                 break;
2133         }
2134         case CHELSIO_GET_MEM:{
2135                 struct ch_mem_range t;
2136                 struct mc7 *mem;
2137                 u64 buf[32];
2138
2139                 if (!is_offload(adapter))
2140                         return -EOPNOTSUPP;
2141                 if (!(adapter->flags & FULL_INIT_DONE))
2142                         return -EIO;    /* need the memory controllers */
2143                 if (copy_from_user(&t, useraddr, sizeof(t)))
2144                         return -EFAULT;
2145                 if ((t.addr & 7) || (t.len & 7))
2146                         return -EINVAL;
2147                 if (t.mem_id == MEM_CM)
2148                         mem = &adapter->cm;
2149                 else if (t.mem_id == MEM_PMRX)
2150                         mem = &adapter->pmrx;
2151                 else if (t.mem_id == MEM_PMTX)
2152                         mem = &adapter->pmtx;
2153                 else
2154                         return -EINVAL;
2155
2156                 /*
2157                  * Version scheme:
2158                  * bits 0..9: chip version
2159                  * bits 10..15: chip revision
2160                  */
2161                 t.version = 3 | (adapter->params.rev << 10);
2162                 if (copy_to_user(useraddr, &t, sizeof(t)))
2163                         return -EFAULT;
2164
2165                 /*
2166                  * Read 256 bytes at a time as len can be large and we don't
2167                  * want to use huge intermediate buffers.
2168                  */
2169                 useraddr += sizeof(t);  /* advance to start of buffer */
2170                 while (t.len) {
2171                         unsigned int chunk =
2172                                 min_t(unsigned int, t.len, sizeof(buf));
2173
2174                         ret =
2175                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2176                                                 buf);
2177                         if (ret)
2178                                 return ret;
2179                         if (copy_to_user(useraddr, buf, chunk))
2180                                 return -EFAULT;
2181                         useraddr += chunk;
2182                         t.addr += chunk;
2183                         t.len -= chunk;
2184                 }
2185                 break;
2186         }
2187         case CHELSIO_SET_TRACE_FILTER:{
2188                 struct ch_trace t;
2189                 const struct trace_params *tp;
2190
2191                 if (!capable(CAP_NET_ADMIN))
2192                         return -EPERM;
2193                 if (!offload_running(adapter))
2194                         return -EAGAIN;
2195                 if (copy_from_user(&t, useraddr, sizeof(t)))
2196                         return -EFAULT;
2197
2198                 tp = (const struct trace_params *)&t.sip;
2199                 if (t.config_tx)
2200                         t3_config_trace_filter(adapter, tp, 0,
2201                                                 t.invert_match,
2202                                                 t.trace_tx);
2203                 if (t.config_rx)
2204                         t3_config_trace_filter(adapter, tp, 1,
2205                                                 t.invert_match,
2206                                                 t.trace_rx);
2207                 break;
2208         }
2209         default:
2210                 return -EOPNOTSUPP;
2211         }
2212         return 0;
2213 }
2214
2215 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2216 {
2217         struct mii_ioctl_data *data = if_mii(req);
2218         struct port_info *pi = netdev_priv(dev);
2219         struct adapter *adapter = pi->adapter;
2220         int ret, mmd;
2221
2222         switch (cmd) {
2223         case SIOCGMIIPHY:
2224                 data->phy_id = pi->phy.addr;
2225                 /* FALLTHRU */
2226         case SIOCGMIIREG:{
2227                 u32 val;
2228                 struct cphy *phy = &pi->phy;
2229
2230                 if (!phy->mdio_read)
2231                         return -EOPNOTSUPP;
2232                 if (is_10G(adapter)) {
2233                         mmd = data->phy_id >> 8;
2234                         if (!mmd)
2235                                 mmd = MDIO_DEV_PCS;
2236                         else if (mmd > MDIO_DEV_VEND2)
2237                                 return -EINVAL;
2238
2239                         ret =
2240                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2241                                                 mmd, data->reg_num, &val);
2242                 } else
2243                         ret =
2244                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2245                                                 0, data->reg_num & 0x1f,
2246                                                 &val);
2247                 if (!ret)
2248                         data->val_out = val;
2249                 break;
2250         }
2251         case SIOCSMIIREG:{
2252                 struct cphy *phy = &pi->phy;
2253
2254                 if (!capable(CAP_NET_ADMIN))
2255                         return -EPERM;
2256                 if (!phy->mdio_write)
2257                         return -EOPNOTSUPP;
2258                 if (is_10G(adapter)) {
2259                         mmd = data->phy_id >> 8;
2260                         if (!mmd)
2261                                 mmd = MDIO_DEV_PCS;
2262                         else if (mmd > MDIO_DEV_VEND2)
2263                                 return -EINVAL;
2264
2265                         ret =
2266                                 phy->mdio_write(adapter,
2267                                                 data->phy_id & 0x1f, mmd,
2268                                                 data->reg_num,
2269                                                 data->val_in);
2270                 } else
2271                         ret =
2272                                 phy->mdio_write(adapter,
2273                                                 data->phy_id & 0x1f, 0,
2274                                                 data->reg_num & 0x1f,
2275                                                 data->val_in);
2276                 break;
2277         }
2278         case SIOCCHIOCTL:
2279                 return cxgb_extension_ioctl(dev, req->ifr_data);
2280         default:
2281                 return -EOPNOTSUPP;
2282         }
2283         return ret;
2284 }
2285
2286 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2287 {
2288         struct port_info *pi = netdev_priv(dev);
2289         struct adapter *adapter = pi->adapter;
2290         int ret;
2291
2292         if (new_mtu < 81)       /* accommodate SACK */
2293                 return -EINVAL;
2294         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2295                 return ret;
2296         dev->mtu = new_mtu;
2297         init_port_mtus(adapter);
2298         if (adapter->params.rev == 0 && offload_running(adapter))
2299                 t3_load_mtus(adapter, adapter->params.mtus,
2300                              adapter->params.a_wnd, adapter->params.b_wnd,
2301                              adapter->port[0]->mtu);
2302         return 0;
2303 }
2304
2305 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2306 {
2307         struct port_info *pi = netdev_priv(dev);
2308         struct adapter *adapter = pi->adapter;
2309         struct sockaddr *addr = p;
2310
2311         if (!is_valid_ether_addr(addr->sa_data))
2312                 return -EINVAL;
2313
2314         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2315         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2316         if (offload_running(adapter))
2317                 write_smt_entry(adapter, pi->port_id);
2318         return 0;
2319 }
2320
2321 /**
2322  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2323  * @adap: the adapter
2324  * @p: the port
2325  *
2326  * Ensures that current Rx processing on any of the queues associated with
2327  * the given port completes before returning.  We do this by acquiring and
2328  * releasing the locks of the response queues associated with the port.
2329  */
2330 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2331 {
2332         int i;
2333
2334         for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
2335                 struct sge_rspq *q = &adap->sge.qs[i].rspq;
2336
2337                 spin_lock_irq(&q->lock);
2338                 spin_unlock_irq(&q->lock);
2339         }
2340 }
2341
2342 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2343 {
2344         struct port_info *pi = netdev_priv(dev);
2345         struct adapter *adapter = pi->adapter;
2346
2347         pi->vlan_grp = grp;
2348         if (adapter->params.rev > 0)
2349                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2350         else {
2351                 /* single control for all ports */
2352                 unsigned int i, have_vlans = 0;
2353                 for_each_port(adapter, i)
2354                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2355
2356                 t3_set_vlan_accel(adapter, 1, have_vlans);
2357         }
2358         t3_synchronize_rx(adapter, pi);
2359 }
2360
2361 #ifdef CONFIG_NET_POLL_CONTROLLER
2362 static void cxgb_netpoll(struct net_device *dev)
2363 {
2364         struct port_info *pi = netdev_priv(dev);
2365         struct adapter *adapter = pi->adapter;
2366         int qidx;
2367
2368         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2369                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2370                 void *source;
2371
2372                 if (adapter->flags & USING_MSIX)
2373                         source = qs;
2374                 else
2375                         source = adapter;
2376
2377                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2378         }
2379 }
2380 #endif
2381
2382 /*
2383  * Periodic accumulation of MAC statistics.
2384  */
2385 static void mac_stats_update(struct adapter *adapter)
2386 {
2387         int i;
2388
2389         for_each_port(adapter, i) {
2390                 struct net_device *dev = adapter->port[i];
2391                 struct port_info *p = netdev_priv(dev);
2392
2393                 if (netif_running(dev)) {
2394                         spin_lock(&adapter->stats_lock);
2395                         t3_mac_update_stats(&p->mac);
2396                         spin_unlock(&adapter->stats_lock);
2397                 }
2398         }
2399 }
2400
2401 static void check_link_status(struct adapter *adapter)
2402 {
2403         int i;
2404
2405         for_each_port(adapter, i) {
2406                 struct net_device *dev = adapter->port[i];
2407                 struct port_info *p = netdev_priv(dev);
2408
2409                 if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev))
2410                         t3_link_changed(adapter, i);
2411         }
2412 }
2413
2414 static void check_t3b2_mac(struct adapter *adapter)
2415 {
2416         int i;
2417
2418         if (!rtnl_trylock())    /* synchronize with ifdown */
2419                 return;
2420
2421         for_each_port(adapter, i) {
2422                 struct net_device *dev = adapter->port[i];
2423                 struct port_info *p = netdev_priv(dev);
2424                 int status;
2425
2426                 if (!netif_running(dev))
2427                         continue;
2428
2429                 status = 0;
2430                 if (netif_running(dev) && netif_carrier_ok(dev))
2431                         status = t3b2_mac_watchdog_task(&p->mac);
2432                 if (status == 1)
2433                         p->mac.stats.num_toggled++;
2434                 else if (status == 2) {
2435                         struct cmac *mac = &p->mac;
2436
2437                         t3_mac_set_mtu(mac, dev->mtu);
2438                         t3_mac_set_address(mac, 0, dev->dev_addr);
2439                         cxgb_set_rxmode(dev);
2440                         t3_link_start(&p->phy, mac, &p->link_config);
2441                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2442                         t3_port_intr_enable(adapter, p->port_id);
2443                         p->mac.stats.num_resets++;
2444                 }
2445         }
2446         rtnl_unlock();
2447 }
2448
2449
2450 static void t3_adap_check_task(struct work_struct *work)
2451 {
2452         struct adapter *adapter = container_of(work, struct adapter,
2453                                                adap_check_task.work);
2454         const struct adapter_params *p = &adapter->params;
2455
2456         adapter->check_task_cnt++;
2457
2458         /* Check link status for PHYs without interrupts */
2459         if (p->linkpoll_period)
2460                 check_link_status(adapter);
2461
2462         /* Accumulate MAC stats if needed */
2463         if (!p->linkpoll_period ||
2464             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2465             p->stats_update_period) {
2466                 mac_stats_update(adapter);
2467                 adapter->check_task_cnt = 0;
2468         }
2469
2470         if (p->rev == T3_REV_B2)
2471                 check_t3b2_mac(adapter);
2472
2473         /* Schedule the next check update if any port is active. */
2474         spin_lock_irq(&adapter->work_lock);
2475         if (adapter->open_device_map & PORT_MASK)
2476                 schedule_chk_task(adapter);
2477         spin_unlock_irq(&adapter->work_lock);
2478 }
2479
2480 /*
2481  * Processes external (PHY) interrupts in process context.
2482  */
2483 static void ext_intr_task(struct work_struct *work)
2484 {
2485         struct adapter *adapter = container_of(work, struct adapter,
2486                                                ext_intr_handler_task);
2487
2488         t3_phy_intr_handler(adapter);
2489
2490         /* Now reenable external interrupts */
2491         spin_lock_irq(&adapter->work_lock);
2492         if (adapter->slow_intr_mask) {
2493                 adapter->slow_intr_mask |= F_T3DBG;
2494                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2495                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2496                              adapter->slow_intr_mask);
2497         }
2498         spin_unlock_irq(&adapter->work_lock);
2499 }
2500
2501 /*
2502  * Interrupt-context handler for external (PHY) interrupts.
2503  */
2504 void t3_os_ext_intr_handler(struct adapter *adapter)
2505 {
2506         /*
2507          * Schedule a task to handle external interrupts as they may be slow
2508          * and we use a mutex to protect MDIO registers.  We disable PHY
2509          * interrupts in the meantime and let the task reenable them when
2510          * it's done.
2511          */
2512         spin_lock(&adapter->work_lock);
2513         if (adapter->slow_intr_mask) {
2514                 adapter->slow_intr_mask &= ~F_T3DBG;
2515                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2516                              adapter->slow_intr_mask);
2517                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2518         }
2519         spin_unlock(&adapter->work_lock);
2520 }
2521
2522 static int t3_adapter_error(struct adapter *adapter, int reset)
2523 {
2524         int i, ret = 0;
2525
2526         /* Stop all ports */
2527         for_each_port(adapter, i) {
2528                 struct net_device *netdev = adapter->port[i];
2529
2530                 if (netif_running(netdev))
2531                         cxgb_close(netdev);
2532         }
2533
2534         if (is_offload(adapter) &&
2535             test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
2536                 offload_close(&adapter->tdev);
2537
2538         /* Stop SGE timers */
2539         t3_stop_sge_timers(adapter);
2540
2541         adapter->flags &= ~FULL_INIT_DONE;
2542
2543         if (reset)
2544                 ret = t3_reset_adapter(adapter);
2545
2546         pci_disable_device(adapter->pdev);
2547
2548         return ret;
2549 }
2550
2551 static int t3_reenable_adapter(struct adapter *adapter)
2552 {
2553         if (pci_enable_device(adapter->pdev)) {
2554                 dev_err(&adapter->pdev->dev,
2555                         "Cannot re-enable PCI device after reset.\n");
2556                 goto err;
2557         }
2558         pci_set_master(adapter->pdev);
2559         pci_restore_state(adapter->pdev);
2560
2561         /* Free sge resources */
2562         t3_free_sge_resources(adapter);
2563
2564         if (t3_replay_prep_adapter(adapter))
2565                 goto err;
2566
2567         return 0;
2568 err:
2569         return -1;
2570 }
2571
2572 static void t3_resume_ports(struct adapter *adapter)
2573 {
2574         int i;
2575
2576         /* Restart the ports */
2577         for_each_port(adapter, i) {
2578                 struct net_device *netdev = adapter->port[i];
2579
2580                 if (netif_running(netdev)) {
2581                         if (cxgb_open(netdev)) {
2582                                 dev_err(&adapter->pdev->dev,
2583                                         "can't bring device back up"
2584                                         " after reset\n");
2585                                 continue;
2586                         }
2587                 }
2588         }
2589 }
2590
2591 /*
2592  * processes a fatal error.
2593  * Bring the ports down, reset the chip, bring the ports back up.
2594  */
2595 static void fatal_error_task(struct work_struct *work)
2596 {
2597         struct adapter *adapter = container_of(work, struct adapter,
2598                                                fatal_error_handler_task);
2599         int err = 0;
2600
2601         rtnl_lock();
2602         err = t3_adapter_error(adapter, 1);
2603         if (!err)
2604                 err = t3_reenable_adapter(adapter);
2605         if (!err)
2606                 t3_resume_ports(adapter);
2607
2608         CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2609         rtnl_unlock();
2610 }
2611
2612 void t3_fatal_err(struct adapter *adapter)
2613 {
2614         unsigned int fw_status[4];
2615
2616         if (adapter->flags & FULL_INIT_DONE) {
2617                 t3_sge_stop(adapter);
2618                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2619                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2620                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2621                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2622
2623                 spin_lock(&adapter->work_lock);
2624                 t3_intr_disable(adapter);
2625                 queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2626                 spin_unlock(&adapter->work_lock);
2627         }
2628         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2629         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2630                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2631                          fw_status[0], fw_status[1],
2632                          fw_status[2], fw_status[3]);
2633
2634 }
2635
2636 /**
2637  * t3_io_error_detected - called when PCI error is detected
2638  * @pdev: Pointer to PCI device
2639  * @state: The current pci connection state
2640  *
2641  * This function is called after a PCI bus error affecting
2642  * this device has been detected.
2643  */
2644 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2645                                              pci_channel_state_t state)
2646 {
2647         struct adapter *adapter = pci_get_drvdata(pdev);
2648         int ret;
2649
2650         ret = t3_adapter_error(adapter, 0);
2651
2652         /* Request a slot reset. */
2653         return PCI_ERS_RESULT_NEED_RESET;
2654 }
2655
2656 /**
2657  * t3_io_slot_reset - called after the pci bus has been reset.
2658  * @pdev: Pointer to PCI device
2659  *
2660  * Restart the card from scratch, as if from a cold-boot.
2661  */
2662 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
2663 {
2664         struct adapter *adapter = pci_get_drvdata(pdev);
2665
2666         if (!t3_reenable_adapter(adapter))
2667                 return PCI_ERS_RESULT_RECOVERED;
2668
2669         return PCI_ERS_RESULT_DISCONNECT;
2670 }
2671
2672 /**
2673  * t3_io_resume - called when traffic can start flowing again.
2674  * @pdev: Pointer to PCI device
2675  *
2676  * This callback is called when the error recovery driver tells us that
2677  * its OK to resume normal operation.
2678  */
2679 static void t3_io_resume(struct pci_dev *pdev)
2680 {
2681         struct adapter *adapter = pci_get_drvdata(pdev);
2682
2683         t3_resume_ports(adapter);
2684 }
2685
2686 static struct pci_error_handlers t3_err_handler = {
2687         .error_detected = t3_io_error_detected,
2688         .slot_reset = t3_io_slot_reset,
2689         .resume = t3_io_resume,
2690 };
2691
2692 /*
2693  * Set the number of qsets based on the number of CPUs and the number of ports,
2694  * not to exceed the number of available qsets, assuming there are enough qsets
2695  * per port in HW.
2696  */
2697 static void set_nqsets(struct adapter *adap)
2698 {
2699         int i, j = 0;
2700         int num_cpus = num_online_cpus();
2701         int hwports = adap->params.nports;
2702         int nqsets = SGE_QSETS;
2703
2704         if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
2705                 if (hwports == 2 &&
2706                     (hwports * nqsets > SGE_QSETS ||
2707                      num_cpus >= nqsets / hwports))
2708                         nqsets /= hwports;
2709                 if (nqsets > num_cpus)
2710                         nqsets = num_cpus;
2711                 if (nqsets < 1 || hwports == 4)
2712                         nqsets = 1;
2713         } else
2714                 nqsets = 1;
2715
2716         for_each_port(adap, i) {
2717                 struct port_info *pi = adap2pinfo(adap, i);
2718
2719                 pi->first_qset = j;
2720                 pi->nqsets = nqsets;
2721                 j = pi->first_qset + nqsets;
2722
2723                 dev_info(&adap->pdev->dev,
2724                          "Port %d using %d queue sets.\n", i, nqsets);
2725         }
2726 }
2727
2728 static int __devinit cxgb_enable_msix(struct adapter *adap)
2729 {
2730         struct msix_entry entries[SGE_QSETS + 1];
2731         int i, err;
2732
2733         for (i = 0; i < ARRAY_SIZE(entries); ++i)
2734                 entries[i].entry = i;
2735
2736         err = pci_enable_msix(adap->pdev, entries, ARRAY_SIZE(entries));
2737         if (!err) {
2738                 for (i = 0; i < ARRAY_SIZE(entries); ++i)
2739                         adap->msix_info[i].vec = entries[i].vector;
2740         } else if (err > 0)
2741                 dev_info(&adap->pdev->dev,
2742                        "only %d MSI-X vectors left, not using MSI-X\n", err);
2743         return err;
2744 }
2745
2746 static void __devinit print_port_info(struct adapter *adap,
2747                                       const struct adapter_info *ai)
2748 {
2749         static const char *pci_variant[] = {
2750                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2751         };
2752
2753         int i;
2754         char buf[80];
2755
2756         if (is_pcie(adap))
2757                 snprintf(buf, sizeof(buf), "%s x%d",
2758                          pci_variant[adap->params.pci.variant],
2759                          adap->params.pci.width);
2760         else
2761                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2762                          pci_variant[adap->params.pci.variant],
2763                          adap->params.pci.speed, adap->params.pci.width);
2764
2765         for_each_port(adap, i) {
2766                 struct net_device *dev = adap->port[i];
2767                 const struct port_info *pi = netdev_priv(dev);
2768
2769                 if (!test_bit(i, &adap->registered_device_map))
2770                         continue;
2771                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2772                        dev->name, ai->desc, pi->phy.desc,
2773                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2774                        (adap->flags & USING_MSIX) ? " MSI-X" :
2775                        (adap->flags & USING_MSI) ? " MSI" : "");
2776                 if (adap->name == dev->name && adap->params.vpd.mclk)
2777                         printk(KERN_INFO
2778                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
2779                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2780                                t3_mc7_size(&adap->pmtx) >> 20,
2781                                t3_mc7_size(&adap->pmrx) >> 20,
2782                                adap->params.vpd.sn);
2783         }
2784 }
2785
2786 static int __devinit init_one(struct pci_dev *pdev,
2787                               const struct pci_device_id *ent)
2788 {
2789         static int version_printed;
2790
2791         int i, err, pci_using_dac = 0;
2792         unsigned long mmio_start, mmio_len;
2793         const struct adapter_info *ai;
2794         struct adapter *adapter = NULL;
2795         struct port_info *pi;
2796
2797         if (!version_printed) {
2798                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2799                 ++version_printed;
2800         }
2801
2802         if (!cxgb3_wq) {
2803                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2804                 if (!cxgb3_wq) {
2805                         printk(KERN_ERR DRV_NAME
2806                                ": cannot initialize work queue\n");
2807                         return -ENOMEM;
2808                 }
2809         }
2810
2811         err = pci_request_regions(pdev, DRV_NAME);
2812         if (err) {
2813                 /* Just info, some other driver may have claimed the device. */
2814                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2815                 return err;
2816         }
2817
2818         err = pci_enable_device(pdev);
2819         if (err) {
2820                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2821                 goto out_release_regions;
2822         }
2823
2824         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2825                 pci_using_dac = 1;
2826                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2827                 if (err) {
2828                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2829                                "coherent allocations\n");
2830                         goto out_disable_device;
2831                 }
2832         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2833                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2834                 goto out_disable_device;
2835         }
2836
2837         pci_set_master(pdev);
2838         pci_save_state(pdev);
2839
2840         mmio_start = pci_resource_start(pdev, 0);
2841         mmio_len = pci_resource_len(pdev, 0);
2842         ai = t3_get_adapter_info(ent->driver_data);
2843
2844         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2845         if (!adapter) {
2846                 err = -ENOMEM;
2847                 goto out_disable_device;
2848         }
2849
2850         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2851         if (!adapter->regs) {
2852                 dev_err(&pdev->dev, "cannot map device registers\n");
2853                 err = -ENOMEM;
2854                 goto out_free_adapter;
2855         }
2856
2857         adapter->pdev = pdev;
2858         adapter->name = pci_name(pdev);
2859         adapter->msg_enable = dflt_msg_enable;
2860         adapter->mmio_len = mmio_len;
2861
2862         mutex_init(&adapter->mdio_lock);
2863         spin_lock_init(&adapter->work_lock);
2864         spin_lock_init(&adapter->stats_lock);
2865
2866         INIT_LIST_HEAD(&adapter->adapter_list);
2867         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2868         INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
2869         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2870
2871         for (i = 0; i < ai->nports; ++i) {
2872                 struct net_device *netdev;
2873
2874                 netdev = alloc_etherdev(sizeof(struct port_info));
2875                 if (!netdev) {
2876                         err = -ENOMEM;
2877                         goto out_free_dev;
2878                 }
2879
2880                 SET_NETDEV_DEV(netdev, &pdev->dev);
2881
2882                 adapter->port[i] = netdev;
2883                 pi = netdev_priv(netdev);
2884                 pi->adapter = adapter;
2885                 pi->rx_csum_offload = 1;
2886                 pi->port_id = i;
2887                 netif_carrier_off(netdev);
2888                 netdev->irq = pdev->irq;
2889                 netdev->mem_start = mmio_start;
2890                 netdev->mem_end = mmio_start + mmio_len - 1;
2891                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2892                 netdev->features |= NETIF_F_LLTX;
2893                 if (pci_using_dac)
2894                         netdev->features |= NETIF_F_HIGHDMA;
2895
2896                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2897                 netdev->vlan_rx_register = vlan_rx_register;
2898
2899                 netdev->open = cxgb_open;
2900                 netdev->stop = cxgb_close;
2901                 netdev->hard_start_xmit = t3_eth_xmit;
2902                 netdev->get_stats = cxgb_get_stats;
2903                 netdev->set_multicast_list = cxgb_set_rxmode;
2904                 netdev->do_ioctl = cxgb_ioctl;
2905                 netdev->change_mtu = cxgb_change_mtu;
2906                 netdev->set_mac_address = cxgb_set_mac_addr;
2907 #ifdef CONFIG_NET_POLL_CONTROLLER
2908                 netdev->poll_controller = cxgb_netpoll;
2909 #endif
2910
2911                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2912         }
2913
2914         pci_set_drvdata(pdev, adapter);
2915         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2916                 err = -ENODEV;
2917                 goto out_free_dev;
2918         }
2919
2920         /*
2921          * The card is now ready to go.  If any errors occur during device
2922          * registration we do not fail the whole card but rather proceed only
2923          * with the ports we manage to register successfully.  However we must
2924          * register at least one net device.
2925          */
2926         for_each_port(adapter, i) {
2927                 err = register_netdev(adapter->port[i]);
2928                 if (err)
2929                         dev_warn(&pdev->dev,
2930                                  "cannot register net device %s, skipping\n",
2931                                  adapter->port[i]->name);
2932                 else {
2933                         /*
2934                          * Change the name we use for messages to the name of
2935                          * the first successfully registered interface.
2936                          */
2937                         if (!adapter->registered_device_map)
2938                                 adapter->name = adapter->port[i]->name;
2939
2940                         __set_bit(i, &adapter->registered_device_map);
2941                 }
2942         }
2943         if (!adapter->registered_device_map) {
2944                 dev_err(&pdev->dev, "could not register any net devices\n");
2945                 goto out_free_dev;
2946         }
2947
2948         /* Driver's ready. Reflect it on LEDs */
2949         t3_led_ready(adapter);
2950
2951         if (is_offload(adapter)) {
2952                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
2953                 cxgb3_adapter_ofld(adapter);
2954         }
2955
2956         /* See what interrupts we'll be using */
2957         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
2958                 adapter->flags |= USING_MSIX;
2959         else if (msi > 0 && pci_enable_msi(pdev) == 0)
2960                 adapter->flags |= USING_MSI;
2961
2962         set_nqsets(adapter);
2963
2964         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
2965                                  &cxgb3_attr_group);
2966
2967         print_port_info(adapter, ai);
2968         return 0;
2969
2970 out_free_dev:
2971         iounmap(adapter->regs);
2972         for (i = ai->nports - 1; i >= 0; --i)
2973                 if (adapter->port[i])
2974                         free_netdev(adapter->port[i]);
2975
2976 out_free_adapter:
2977         kfree(adapter);
2978
2979 out_disable_device:
2980         pci_disable_device(pdev);
2981 out_release_regions:
2982         pci_release_regions(pdev);
2983         pci_set_drvdata(pdev, NULL);
2984         return err;
2985 }
2986
2987 static void __devexit remove_one(struct pci_dev *pdev)
2988 {
2989         struct adapter *adapter = pci_get_drvdata(pdev);
2990
2991         if (adapter) {
2992                 int i;
2993
2994                 t3_sge_stop(adapter);
2995                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
2996                                    &cxgb3_attr_group);
2997
2998                 if (is_offload(adapter)) {
2999                         cxgb3_adapter_unofld(adapter);
3000                         if (test_bit(OFFLOAD_DEVMAP_BIT,
3001                                      &adapter->open_device_map))
3002                                 offload_close(&adapter->tdev);
3003                 }
3004
3005                 for_each_port(adapter, i)
3006                     if (test_bit(i, &adapter->registered_device_map))
3007                         unregister_netdev(adapter->port[i]);
3008
3009                 t3_stop_sge_timers(adapter);
3010                 t3_free_sge_resources(adapter);
3011                 cxgb_disable_msi(adapter);
3012
3013                 for_each_port(adapter, i)
3014                         if (adapter->port[i])
3015                                 free_netdev(adapter->port[i]);
3016
3017                 iounmap(adapter->regs);
3018                 kfree(adapter);
3019                 pci_release_regions(pdev);
3020                 pci_disable_device(pdev);
3021                 pci_set_drvdata(pdev, NULL);
3022         }
3023 }
3024
3025 static struct pci_driver driver = {
3026         .name = DRV_NAME,
3027         .id_table = cxgb3_pci_tbl,
3028         .probe = init_one,
3029         .remove = __devexit_p(remove_one),
3030         .err_handler = &t3_err_handler,
3031 };
3032
3033 static int __init cxgb3_init_module(void)
3034 {
3035         int ret;
3036
3037         cxgb3_offload_init();
3038
3039         ret = pci_register_driver(&driver);
3040         return ret;
3041 }
3042
3043 static void __exit cxgb3_cleanup_module(void)
3044 {
3045         pci_unregister_driver(&driver);
3046         if (cxgb3_wq)
3047                 destroy_workqueue(cxgb3_wq);
3048 }
3049
3050 module_init(cxgb3_init_module);
3051 module_exit(cxgb3_cleanup_module);