drivers/net/cxgb3/cxgb3_main.c

   1 /*
   2  * Copyright (c) 2003-2007 Chelsio, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/module.h>
  33 #include <linux/moduleparam.h>
  34 #include <linux/init.h>
  35 #include <linux/pci.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/netdevice.h>
  38 #include <linux/etherdevice.h>
  39 #include <linux/if_vlan.h>
  40 #include <linux/mii.h>
  41 #include <linux/sockios.h>
  42 #include <linux/workqueue.h>
  43 #include <linux/proc_fs.h>
  44 #include <linux/rtnetlink.h>
  45 #include <linux/firmware.h>
  46 #include <linux/log2.h>
  47 #include <asm/uaccess.h>
  48
  49 #include "common.h"
  50 #include "cxgb3_ioctl.h"
  51 #include "regs.h"
  52 #include "cxgb3_offload.h"
  53 #include "version.h"
  54
  55 #include "cxgb3_ctl_defs.h"
  56 #include "t3_cpl.h"
  57 #include "firmware_exports.h"
  58
  59 enum {
  60         MAX_TXQ_ENTRIES = 16384,
  61         MAX_CTRL_TXQ_ENTRIES = 1024,
  62         MAX_RSPQ_ENTRIES = 16384,
  63         MAX_RX_BUFFERS = 16384,
  64         MAX_RX_JUMBO_BUFFERS = 16384,
  65         MIN_TXQ_ENTRIES = 4,
  66         MIN_CTRL_TXQ_ENTRIES = 4,
  67         MIN_RSPQ_ENTRIES = 32,
  68         MIN_FL_ENTRIES = 32
  69 };
  70
  71 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
  72
  73 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
  74                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
  75                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
  76
  77 #define EEPROM_MAGIC 0x38E2F10C
  78
  79 #define CH_DEVICE(devid, ssid, idx) \
  80         { PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, ssid, 0, 0, idx }
  81
  82 static const struct pci_device_id cxgb3_pci_tbl[] = {
  83         CH_DEVICE(0x20, 1, 0),  /* PE9000 */
  84         CH_DEVICE(0x21, 1, 1),  /* T302E */
  85         CH_DEVICE(0x22, 1, 2),  /* T310E */
  86         CH_DEVICE(0x23, 1, 3),  /* T320X */
  87         CH_DEVICE(0x24, 1, 1),  /* T302X */
  88         CH_DEVICE(0x25, 1, 3),  /* T320E */
  89         CH_DEVICE(0x26, 1, 2),  /* T310X */
  90         CH_DEVICE(0x30, 1, 2),  /* T3B10 */
  91         CH_DEVICE(0x31, 1, 3),  /* T3B20 */
  92         CH_DEVICE(0x32, 1, 1),  /* T3B02 */
  93         {0,}
  94 };
  95
  96 MODULE_DESCRIPTION(DRV_DESC);
  97 MODULE_AUTHOR("Chelsio Communications");
  98 MODULE_LICENSE("Dual BSD/GPL");
  99 MODULE_VERSION(DRV_VERSION);
 100 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
 101
 102 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 103
 104 module_param(dflt_msg_enable, int, 0644);
 105 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
 106
 107 /*
 108  * The driver uses the best interrupt scheme available on a platform in the
 109  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
 110  * of these schemes the driver may consider as follows:
 111  *
 112  * msi = 2: choose from among all three options
 113  * msi = 1: only consider MSI and pin interrupts
 114  * msi = 0: force pin interrupts
 115  */
 116 static int msi = 2;
 117
 118 module_param(msi, int, 0644);
 119 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
 120
 121 /*
 122  * The driver enables offload as a default.
 123  * To disable it, use ofld_disable = 1.
 124  */
 125
 126 static int ofld_disable = 0;
 127
 128 module_param(ofld_disable, int, 0644);
 129 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
 130
 131 /*
 132  * We have work elements that we need to cancel when an interface is taken
 133  * down.  Normally the work elements would be executed by keventd but that
 134  * can deadlock because of linkwatch.  If our close method takes the rtnl
 135  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
 136  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
 137  * for our work to complete.  Get our own work queue to solve this.
 138  */
 139 static struct workqueue_struct *cxgb3_wq;
 140
 141 /**
 142  *      link_report - show link status and link speed/duplex
 143  *      @p: the port whose settings are to be reported
 144  *
 145  *      Shows the link status, speed, and duplex of a port.
 146  */
 147 static void link_report(struct net_device *dev)
 148 {
 149         if (!netif_carrier_ok(dev))
 150                 printk(KERN_INFO "%s: link down\n", dev->name);
 151         else {
 152                 const char *s = "10Mbps";
 153                 const struct port_info *p = netdev_priv(dev);
 154
 155                 switch (p->link_config.speed) {
 156                 case SPEED_10000:
 157                         s = "10Gbps";
 158                         break;
 159                 case SPEED_1000:
 160                         s = "1000Mbps";
 161                         break;
 162                 case SPEED_100:
 163                         s = "100Mbps";
 164                         break;
 165                 }
 166
 167                 printk(KERN_INFO "%s: link up, %s, %s-duplex\n", dev->name, s,
 168                        p->link_config.duplex == DUPLEX_FULL ? "full" : "half");
 169         }
 170 }
 171
 172 /**
 173  *      t3_os_link_changed - handle link status changes
 174  *      @adapter: the adapter associated with the link change
 175  *      @port_id: the port index whose limk status has changed
 176  *      @link_stat: the new status of the link
 177  *      @speed: the new speed setting
 178  *      @duplex: the new duplex setting
 179  *      @pause: the new flow-control setting
 180  *
 181  *      This is the OS-dependent handler for link status changes.  The OS
 182  *      neutral handler takes care of most of the processing for these events,
 183  *      then calls this handler for any OS-specific processing.
 184  */
 185 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
 186                         int speed, int duplex, int pause)
 187 {
 188         struct net_device *dev = adapter->port[port_id];
 189         struct port_info *pi = netdev_priv(dev);
 190         struct cmac *mac = &pi->mac;
 191
 192         /* Skip changes from disabled ports. */
 193         if (!netif_running(dev))
 194                 return;
 195
 196         if (link_stat != netif_carrier_ok(dev)) {
 197                 if (link_stat) {
 198                         t3_mac_enable(mac, MAC_DIRECTION_RX);
 199                         netif_carrier_on(dev);
 200                 } else {
 201                         netif_carrier_off(dev);
 202                         pi->phy.ops->power_down(&pi->phy, 1);
 203                         t3_mac_disable(mac, MAC_DIRECTION_RX);
 204                         t3_link_start(&pi->phy, mac, &pi->link_config);
 205                 }
 206
 207                 link_report(dev);
 208         }
 209 }
 210
 211 static void cxgb_set_rxmode(struct net_device *dev)
 212 {
 213         struct t3_rx_mode rm;
 214         struct port_info *pi = netdev_priv(dev);
 215
 216         init_rx_mode(&rm, dev, dev->mc_list);
 217         t3_mac_set_rx_mode(&pi->mac, &rm);
 218 }
 219
 220 /**
 221  *      link_start - enable a port
 222  *      @dev: the device to enable
 223  *
 224  *      Performs the MAC and PHY actions needed to enable a port.
 225  */
 226 static void link_start(struct net_device *dev)
 227 {
 228         struct t3_rx_mode rm;
 229         struct port_info *pi = netdev_priv(dev);
 230         struct cmac *mac = &pi->mac;
 231
 232         init_rx_mode(&rm, dev, dev->mc_list);
 233         t3_mac_reset(mac);
 234         t3_mac_set_mtu(mac, dev->mtu);
 235         t3_mac_set_address(mac, 0, dev->dev_addr);
 236         t3_mac_set_rx_mode(mac, &rm);
 237         t3_link_start(&pi->phy, mac, &pi->link_config);
 238         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
 239 }
 240
 241 static inline void cxgb_disable_msi(struct adapter *adapter)
 242 {
 243         if (adapter->flags & USING_MSIX) {
 244                 pci_disable_msix(adapter->pdev);
 245                 adapter->flags &= ~USING_MSIX;
 246         } else if (adapter->flags & USING_MSI) {
 247                 pci_disable_msi(adapter->pdev);
 248                 adapter->flags &= ~USING_MSI;
 249         }
 250 }
 251
 252 /*
 253  * Interrupt handler for asynchronous events used with MSI-X.
 254  */
 255 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
 256 {
 257         t3_slow_intr_handler(cookie);
 258         return IRQ_HANDLED;
 259 }
 260
 261 /*
 262  * Name the MSI-X interrupts.
 263  */
 264 static void name_msix_vecs(struct adapter *adap)
 265 {
 266         int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
 267
 268         snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
 269         adap->msix_info[0].desc[n] = 0;
 270
 271         for_each_port(adap, j) {
 272                 struct net_device *d = adap->port[j];
 273                 const struct port_info *pi = netdev_priv(d);
 274
 275                 for (i = 0; i < pi->nqsets; i++, msi_idx++) {
 276                         snprintf(adap->msix_info[msi_idx].desc, n,
 277                                  "%s (queue %d)", d->name, i);
 278                         adap->msix_info[msi_idx].desc[n] = 0;
 279                 }
 280         }
 281 }
 282
 283 static int request_msix_data_irqs(struct adapter *adap)
 284 {
 285         int i, j, err, qidx = 0;
 286
 287         for_each_port(adap, i) {
 288                 int nqsets = adap2pinfo(adap, i)->nqsets;
 289
 290                 for (j = 0; j < nqsets; ++j) {
 291                         err = request_irq(adap->msix_info[qidx + 1].vec,
 292                                           t3_intr_handler(adap,
 293                                                           adap->sge.qs[qidx].
 294                                                           rspq.polling), 0,
 295                                           adap->msix_info[qidx + 1].desc,
 296                                           &adap->sge.qs[qidx]);
 297                         if (err) {
 298                                 while (--qidx >= 0)
 299                                         free_irq(adap->msix_info[qidx + 1].vec,
 300                                                  &adap->sge.qs[qidx]);
 301                                 return err;
 302                         }
 303                         qidx++;
 304                 }
 305         }
 306         return 0;
 307 }
 308
 309 /**
 310  *      setup_rss - configure RSS
 311  *      @adap: the adapter
 312  *
 313  *      Sets up RSS to distribute packets to multiple receive queues.  We
 314  *      configure the RSS CPU lookup table to distribute to the number of HW
 315  *      receive queues, and the response queue lookup table to narrow that
 316  *      down to the response queues actually configured for each port.
 317  *      We always configure the RSS mapping for two ports since the mapping
 318  *      table has plenty of entries.
 319  */
 320 static void setup_rss(struct adapter *adap)
 321 {
 322         int i;
 323         unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
 324         unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
 325         u8 cpus[SGE_QSETS + 1];
 326         u16 rspq_map[RSS_TABLE_SIZE];
 327
 328         for (i = 0; i < SGE_QSETS; ++i)
 329                 cpus[i] = i;
 330         cpus[SGE_QSETS] = 0xff; /* terminator */
 331
 332         for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
 333                 rspq_map[i] = i % nq0;
 334                 rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
 335         }
 336
 337         t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
 338                       F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
 339                       V_RRCPLCPUSIZE(6), cpus, rspq_map);
 340 }
 341
 342 static void init_napi(struct adapter *adap)
 343 {
 344         int i;
 345
 346         for (i = 0; i < SGE_QSETS; i++) {
 347                 struct sge_qset *qs = &adap->sge.qs[i];
 348
 349                 if (qs->adap)
 350                         netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
 351                                        64);
 352         }
 353 }
 354
 355 /*
 356  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
 357  * both netdevices representing interfaces and the dummy ones for the extra
 358  * queues.
 359  */
 360 static void quiesce_rx(struct adapter *adap)
 361 {
 362         int i;
 363
 364         for (i = 0; i < SGE_QSETS; i++)
 365                 if (adap->sge.qs[i].adap)
 366                         napi_disable(&adap->sge.qs[i].napi);
 367 }
 368
 369 static void enable_all_napi(struct adapter *adap)
 370 {
 371         int i;
 372         for (i = 0; i < SGE_QSETS; i++)
 373                 if (adap->sge.qs[i].adap)
 374                         napi_enable(&adap->sge.qs[i].napi);
 375 }
 376
 377 /**
 378  *      setup_sge_qsets - configure SGE Tx/Rx/response queues
 379  *      @adap: the adapter
 380  *
 381  *      Determines how many sets of SGE queues to use and initializes them.
 382  *      We support multiple queue sets per port if we have MSI-X, otherwise
 383  *      just one queue set per port.
 384  */
 385 static int setup_sge_qsets(struct adapter *adap)
 386 {
 387         int i, j, err, irq_idx = 0, qset_idx = 0;
 388         unsigned int ntxq = SGE_TXQ_PER_SET;
 389
 390         if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
 391                 irq_idx = -1;
 392
 393         for_each_port(adap, i) {
 394                 struct net_device *dev = adap->port[i];
 395                 struct port_info *pi = netdev_priv(dev);
 396
 397                 pi->qs = &adap->sge.qs[pi->first_qset];
 398                 for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
 399                         err = t3_sge_alloc_qset(adap, qset_idx, 1,
 400                                 (adap->flags & USING_MSIX) ? qset_idx + 1 :
 401                                                              irq_idx,
 402                                 &adap->params.sge.qset[qset_idx], ntxq, dev);
 403                         if (err) {
 404                                 t3_free_sge_resources(adap);
 405                                 return err;
 406                         }
 407                 }
 408         }
 409
 410         return 0;
 411 }
 412
 413 static ssize_t attr_show(struct device *d, struct device_attribute *attr,
 414                          char *buf,
 415                          ssize_t(*format) (struct net_device *, char *))
 416 {
 417         ssize_t len;
 418
 419         /* Synchronize with ioctls that may shut down the device */
 420         rtnl_lock();
 421         len = (*format) (to_net_dev(d), buf);
 422         rtnl_unlock();
 423         return len;
 424 }
 425
 426 static ssize_t attr_store(struct device *d, struct device_attribute *attr,
 427                           const char *buf, size_t len,
 428                           ssize_t(*set) (struct net_device *, unsigned int),
 429                           unsigned int min_val, unsigned int max_val)
 430 {
 431         char *endp;
 432         ssize_t ret;
 433         unsigned int val;
 434
 435         if (!capable(CAP_NET_ADMIN))
 436                 return -EPERM;
 437
 438         val = simple_strtoul(buf, &endp, 0);
 439         if (endp == buf || val < min_val || val > max_val)
 440                 return -EINVAL;
 441
 442         rtnl_lock();
 443         ret = (*set) (to_net_dev(d), val);
 444         if (!ret)
 445                 ret = len;
 446         rtnl_unlock();
 447         return ret;
 448 }
 449
 450 #define CXGB3_SHOW(name, val_expr) \
 451 static ssize_t format_##name(struct net_device *dev, char *buf) \
 452 { \
 453         struct port_info *pi = netdev_priv(dev); \
 454         struct adapter *adap = pi->adapter; \
 455         return sprintf(buf, "%u\n", val_expr); \
 456 } \
 457 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 458                            char *buf) \
 459 { \
 460         return attr_show(d, attr, buf, format_##name); \
 461 }
 462
 463 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
 464 {
 465         struct port_info *pi = netdev_priv(dev);
 466         struct adapter *adap = pi->adapter;
 467         int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
 468
 469         if (adap->flags & FULL_INIT_DONE)
 470                 return -EBUSY;
 471         if (val && adap->params.rev == 0)
 472                 return -EINVAL;
 473         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
 474             min_tids)
 475                 return -EINVAL;
 476         adap->params.mc5.nfilters = val;
 477         return 0;
 478 }
 479
 480 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
 481                               const char *buf, size_t len)
 482 {
 483         return attr_store(d, attr, buf, len, set_nfilters, 0, ~0);
 484 }
 485
 486 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
 487 {
 488         struct port_info *pi = netdev_priv(dev);
 489         struct adapter *adap = pi->adapter;
 490
 491         if (adap->flags & FULL_INIT_DONE)
 492                 return -EBUSY;
 493         if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
 494             MC5_MIN_TIDS)
 495                 return -EINVAL;
 496         adap->params.mc5.nservers = val;
 497         return 0;
 498 }
 499
 500 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
 501                               const char *buf, size_t len)
 502 {
 503         return attr_store(d, attr, buf, len, set_nservers, 0, ~0);
 504 }
 505
 506 #define CXGB3_ATTR_R(name, val_expr) \
 507 CXGB3_SHOW(name, val_expr) \
 508 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 509
 510 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
 511 CXGB3_SHOW(name, val_expr) \
 512 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
 513
 514 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
 515 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
 516 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
 517
 518 static struct attribute *cxgb3_attrs[] = {
 519         &dev_attr_cam_size.attr,
 520         &dev_attr_nfilters.attr,
 521         &dev_attr_nservers.attr,
 522         NULL
 523 };
 524
 525 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
 526
 527 static ssize_t tm_attr_show(struct device *d, struct device_attribute *attr,
 528                             char *buf, int sched)
 529 {
 530         struct port_info *pi = netdev_priv(to_net_dev(d));
 531         struct adapter *adap = pi->adapter;
 532         unsigned int v, addr, bpt, cpt;
 533         ssize_t len;
 534
 535         addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
 536         rtnl_lock();
 537         t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
 538         v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
 539         if (sched & 1)
 540                 v >>= 16;
 541         bpt = (v >> 8) & 0xff;
 542         cpt = v & 0xff;
 543         if (!cpt)
 544                 len = sprintf(buf, "disabled\n");
 545         else {
 546                 v = (adap->params.vpd.cclk * 1000) / cpt;
 547                 len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
 548         }
 549         rtnl_unlock();
 550         return len;
 551 }
 552
 553 static ssize_t tm_attr_store(struct device *d, struct device_attribute *attr,
 554                              const char *buf, size_t len, int sched)
 555 {
 556         struct port_info *pi = netdev_priv(to_net_dev(d));
 557         struct adapter *adap = pi->adapter;
 558         unsigned int val;
 559         char *endp;
 560         ssize_t ret;
 561
 562         if (!capable(CAP_NET_ADMIN))
 563                 return -EPERM;
 564
 565         val = simple_strtoul(buf, &endp, 0);
 566         if (endp == buf || val > 10000000)
 567                 return -EINVAL;
 568
 569         rtnl_lock();
 570         ret = t3_config_sched(adap, val, sched);
 571         if (!ret)
 572                 ret = len;
 573         rtnl_unlock();
 574         return ret;
 575 }
 576
 577 #define TM_ATTR(name, sched) \
 578 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
 579                            char *buf) \
 580 { \
 581         return tm_attr_show(d, attr, buf, sched); \
 582 } \
 583 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
 584                             const char *buf, size_t len) \
 585 { \
 586         return tm_attr_store(d, attr, buf, len, sched); \
 587 } \
 588 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
 589
 590 TM_ATTR(sched0, 0);
 591 TM_ATTR(sched1, 1);
 592 TM_ATTR(sched2, 2);
 593 TM_ATTR(sched3, 3);
 594 TM_ATTR(sched4, 4);
 595 TM_ATTR(sched5, 5);
 596 TM_ATTR(sched6, 6);
 597 TM_ATTR(sched7, 7);
 598
 599 static struct attribute *offload_attrs[] = {
 600         &dev_attr_sched0.attr,
 601         &dev_attr_sched1.attr,
 602         &dev_attr_sched2.attr,
 603         &dev_attr_sched3.attr,
 604         &dev_attr_sched4.attr,
 605         &dev_attr_sched5.attr,
 606         &dev_attr_sched6.attr,
 607         &dev_attr_sched7.attr,
 608         NULL
 609 };
 610
 611 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
 612
 613 /*
 614  * Sends an sk_buff to an offload queue driver
 615  * after dealing with any active network taps.
 616  */
 617 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
 618 {
 619         int ret;
 620
 621         local_bh_disable();
 622         ret = t3_offload_tx(tdev, skb);
 623         local_bh_enable();
 624         return ret;
 625 }
 626
 627 static int write_smt_entry(struct adapter *adapter, int idx)
 628 {
 629         struct cpl_smt_write_req *req;
 630         struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
 631
 632         if (!skb)
 633                 return -ENOMEM;
 634
 635         req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
 636         req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
 637         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
 638         req->mtu_idx = NMTUS - 1;       /* should be 0 but there's a T3 bug */
 639         req->iff = idx;
 640         memset(req->src_mac1, 0, sizeof(req->src_mac1));
 641         memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
 642         skb->priority = 1;
 643         offload_tx(&adapter->tdev, skb);
 644         return 0;
 645 }
 646
 647 static int init_smt(struct adapter *adapter)
 648 {
 649         int i;
 650
 651         for_each_port(adapter, i)
 652             write_smt_entry(adapter, i);
 653         return 0;
 654 }
 655
 656 static void init_port_mtus(struct adapter *adapter)
 657 {
 658         unsigned int mtus = adapter->port[0]->mtu;
 659
 660         if (adapter->port[1])
 661                 mtus |= adapter->port[1]->mtu << 16;
 662         t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
 663 }
 664
 665 static void send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
 666                               int hi, int port)
 667 {
 668         struct sk_buff *skb;
 669         struct mngt_pktsched_wr *req;
 670
 671         skb = alloc_skb(sizeof(*req), GFP_KERNEL | __GFP_NOFAIL);
 672         req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
 673         req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
 674         req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
 675         req->sched = sched;
 676         req->idx = qidx;
 677         req->min = lo;
 678         req->max = hi;
 679         req->binding = port;
 680         t3_mgmt_tx(adap, skb);
 681 }
 682
 683 static void bind_qsets(struct adapter *adap)
 684 {
 685         int i, j;
 686
 687         for_each_port(adap, i) {
 688                 const struct port_info *pi = adap2pinfo(adap, i);
 689
 690                 for (j = 0; j < pi->nqsets; ++j)
 691                         send_pktsched_cmd(adap, 1, pi->first_qset + j, -1,
 692                                           -1, i);
 693         }
 694 }
 695
 696 #define FW_FNAME "t3fw-%d.%d.%d.bin"
 697 #define TPSRAM_NAME "t3%c_protocol_sram-%d.%d.%d.bin"
 698
 699 static int upgrade_fw(struct adapter *adap)
 700 {
 701         int ret;
 702         char buf[64];
 703         const struct firmware *fw;
 704         struct device *dev = &adap->pdev->dev;
 705
 706         snprintf(buf, sizeof(buf), FW_FNAME, FW_VERSION_MAJOR,
 707                  FW_VERSION_MINOR, FW_VERSION_MICRO);
 708         ret = request_firmware(&fw, buf, dev);
 709         if (ret < 0) {
 710                 dev_err(dev, "could not upgrade firmware: unable to load %s\n",
 711                         buf);
 712                 return ret;
 713         }
 714         ret = t3_load_fw(adap, fw->data, fw->size);
 715         release_firmware(fw);
 716
 717         if (ret == 0)
 718                 dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
 719                          FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 720         else
 721                 dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
 722                         FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
 723
 724         return ret;
 725 }
 726
 727 static inline char t3rev2char(struct adapter *adapter)
 728 {
 729         char rev = 0;
 730
 731         switch(adapter->params.rev) {
 732         case T3_REV_B:
 733         case T3_REV_B2:
 734                 rev = 'b';
 735                 break;
 736         case T3_REV_C:
 737                 rev = 'c';
 738                 break;
 739         }
 740         return rev;
 741 }
 742
 743 int update_tpsram(struct adapter *adap)
 744 {
 745         const struct firmware *tpsram;
 746         char buf[64];
 747         struct device *dev = &adap->pdev->dev;
 748         int ret;
 749         char rev;
 750
 751         rev = t3rev2char(adap);
 752         if (!rev)
 753                 return 0;
 754
 755         snprintf(buf, sizeof(buf), TPSRAM_NAME, rev,
 756                  TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 757
 758         ret = request_firmware(&tpsram, buf, dev);
 759         if (ret < 0) {
 760                 dev_err(dev, "could not load TP SRAM: unable to load %s\n",
 761                         buf);
 762                 return ret;
 763         }
 764
 765         ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
 766         if (ret)
 767                 goto release_tpsram;
 768
 769         ret = t3_set_proto_sram(adap, tpsram->data);
 770         if (ret == 0)
 771                 dev_info(dev,
 772                          "successful update of protocol engine "
 773                          "to %d.%d.%d\n",
 774                          TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 775         else
 776                 dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
 777                         TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
 778         if (ret)
 779                 dev_err(dev, "loading protocol SRAM failed\n");
 780
 781 release_tpsram:
 782         release_firmware(tpsram);
 783
 784         return ret;
 785 }
 786
 787 /**
 788  *      cxgb_up - enable the adapter
 789  *      @adapter: adapter being enabled
 790  *
 791  *      Called when the first port is enabled, this function performs the
 792  *      actions necessary to make an adapter operational, such as completing
 793  *      the initialization of HW modules, and enabling interrupts.
 794  *
 795  *      Must be called with the rtnl lock held.
 796  */
 797 static int cxgb_up(struct adapter *adap)
 798 {
 799         int err;
 800         int must_load;
 801
 802         if (!(adap->flags & FULL_INIT_DONE)) {
 803                 err = t3_check_fw_version(adap, &must_load);
 804                 if (err == -EINVAL) {
 805                         err = upgrade_fw(adap);
 806                         if (err && must_load)
 807                                 goto out;
 808                 }
 809
 810                 err = t3_check_tpsram_version(adap, &must_load);
 811                 if (err == -EINVAL) {
 812                         err = update_tpsram(adap);
 813                         if (err && must_load)
 814                                 goto out;
 815                 }
 816
 817                 err = t3_init_hw(adap, 0);
 818                 if (err)
 819                         goto out;
 820
 821                 t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
 822
 823                 err = setup_sge_qsets(adap);
 824                 if (err)
 825                         goto out;
 826
 827                 setup_rss(adap);
 828                 init_napi(adap);
 829                 adap->flags |= FULL_INIT_DONE;
 830         }
 831
 832         t3_intr_clear(adap);
 833
 834         if (adap->flags & USING_MSIX) {
 835                 name_msix_vecs(adap);
 836                 err = request_irq(adap->msix_info[0].vec,
 837                                   t3_async_intr_handler, 0,
 838                                   adap->msix_info[0].desc, adap);
 839                 if (err)
 840                         goto irq_err;
 841
 842                 if (request_msix_data_irqs(adap)) {
 843                         free_irq(adap->msix_info[0].vec, adap);
 844                         goto irq_err;
 845                 }
 846         } else if ((err = request_irq(adap->pdev->irq,
 847                                       t3_intr_handler(adap,
 848                                                       adap->sge.qs[0].rspq.
 849                                                       polling),
 850                                       (adap->flags & USING_MSI) ?
 851                                        0 : IRQF_SHARED,
 852                                       adap->name, adap)))
 853                 goto irq_err;
 854
 855         enable_all_napi(adap);
 856         t3_sge_start(adap);
 857         t3_intr_enable(adap);
 858
 859         if ((adap->flags & (USING_MSIX | QUEUES_BOUND)) == USING_MSIX)
 860                 bind_qsets(adap);
 861         adap->flags |= QUEUES_BOUND;
 862
 863 out:
 864         return err;
 865 irq_err:
 866         CH_ERR(adap, "request_irq failed, err %d\n", err);
 867         goto out;
 868 }
 869
 870 /*
 871  * Release resources when all the ports and offloading have been stopped.
 872  */
 873 static void cxgb_down(struct adapter *adapter)
 874 {
 875         t3_sge_stop(adapter);
 876         spin_lock_irq(&adapter->work_lock);     /* sync with PHY intr task */
 877         t3_intr_disable(adapter);
 878         spin_unlock_irq(&adapter->work_lock);
 879
 880         if (adapter->flags & USING_MSIX) {
 881                 int i, n = 0;
 882
 883                 free_irq(adapter->msix_info[0].vec, adapter);
 884                 for_each_port(adapter, i)
 885                     n += adap2pinfo(adapter, i)->nqsets;
 886
 887                 for (i = 0; i < n; ++i)
 888                         free_irq(adapter->msix_info[i + 1].vec,
 889                                  &adapter->sge.qs[i]);
 890         } else
 891                 free_irq(adapter->pdev->irq, adapter);
 892
 893         flush_workqueue(cxgb3_wq);      /* wait for external IRQ handler */
 894         quiesce_rx(adapter);
 895 }
 896
 897 static void schedule_chk_task(struct adapter *adap)
 898 {
 899         unsigned int timeo;
 900
 901         timeo = adap->params.linkpoll_period ?
 902             (HZ * adap->params.linkpoll_period) / 10 :
 903             adap->params.stats_update_period * HZ;
 904         if (timeo)
 905                 queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
 906 }
 907
 908 static int offload_open(struct net_device *dev)
 909 {
 910         struct port_info *pi = netdev_priv(dev);
 911         struct adapter *adapter = pi->adapter;
 912         struct t3cdev *tdev = dev2t3cdev(dev);
 913         int adap_up = adapter->open_device_map & PORT_MASK;
 914         int err;
 915
 916         if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
 917                 return 0;
 918
 919         if (!adap_up && (err = cxgb_up(adapter)) < 0)
 920                 return err;
 921
 922         t3_tp_set_offload_mode(adapter, 1);
 923         tdev->lldev = adapter->port[0];
 924         err = cxgb3_offload_activate(adapter);
 925         if (err)
 926                 goto out;
 927
 928         init_port_mtus(adapter);
 929         t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
 930                      adapter->params.b_wnd,
 931                      adapter->params.rev == 0 ?
 932                      adapter->port[0]->mtu : 0xffff);
 933         init_smt(adapter);
 934
 935         /* Never mind if the next step fails */
 936         sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group);
 937
 938         /* Call back all registered clients */
 939         cxgb3_add_clients(tdev);
 940
 941 out:
 942         /* restore them in case the offload module has changed them */
 943         if (err) {
 944                 t3_tp_set_offload_mode(adapter, 0);
 945                 clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
 946                 cxgb3_set_dummy_ops(tdev);
 947         }
 948         return err;
 949 }
 950
 951 static int offload_close(struct t3cdev *tdev)
 952 {
 953         struct adapter *adapter = tdev2adap(tdev);
 954
 955         if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
 956                 return 0;
 957
 958         /* Call back all registered clients */
 959         cxgb3_remove_clients(tdev);
 960
 961         sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
 962
 963         tdev->lldev = NULL;
 964         cxgb3_set_dummy_ops(tdev);
 965         t3_tp_set_offload_mode(adapter, 0);
 966         clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
 967
 968         if (!adapter->open_device_map)
 969                 cxgb_down(adapter);
 970
 971         cxgb3_offload_deactivate(adapter);
 972         return 0;
 973 }
 974
 975 static int cxgb_open(struct net_device *dev)
 976 {
 977         struct port_info *pi = netdev_priv(dev);
 978         struct adapter *adapter = pi->adapter;
 979         int other_ports = adapter->open_device_map & PORT_MASK;
 980         int err;
 981
 982         if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0) {
 983                 quiesce_rx(adapter);
 984                 return err;
 985         }
 986
 987         set_bit(pi->port_id, &adapter->open_device_map);
 988         if (is_offload(adapter) && !ofld_disable) {
 989                 err = offload_open(dev);
 990                 if (err)
 991                         printk(KERN_WARNING
 992                                "Could not initialize offload capabilities\n");
 993         }
 994
 995         link_start(dev);
 996         t3_port_intr_enable(adapter, pi->port_id);
 997         netif_start_queue(dev);
 998         if (!other_ports)
 999                 schedule_chk_task(adapter);
1000
1001         return 0;
1002 }
1003
1004 static int cxgb_close(struct net_device *dev)
1005 {
1006         struct port_info *pi = netdev_priv(dev);
1007         struct adapter *adapter = pi->adapter;
1008
1009         t3_port_intr_disable(adapter, pi->port_id);
1010         netif_stop_queue(dev);
1011         pi->phy.ops->power_down(&pi->phy, 1);
1012         netif_carrier_off(dev);
1013         t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1014
1015         spin_lock(&adapter->work_lock); /* sync with update task */
1016         clear_bit(pi->port_id, &adapter->open_device_map);
1017         spin_unlock(&adapter->work_lock);
1018
1019         if (!(adapter->open_device_map & PORT_MASK))
1020                 cancel_rearming_delayed_workqueue(cxgb3_wq,
1021                                                   &adapter->adap_check_task);
1022
1023         if (!adapter->open_device_map)
1024                 cxgb_down(adapter);
1025
1026         return 0;
1027 }
1028
1029 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1030 {
1031         struct port_info *pi = netdev_priv(dev);
1032         struct adapter *adapter = pi->adapter;
1033         struct net_device_stats *ns = &pi->netstats;
1034         const struct mac_stats *pstats;
1035
1036         spin_lock(&adapter->stats_lock);
1037         pstats = t3_mac_update_stats(&pi->mac);
1038         spin_unlock(&adapter->stats_lock);
1039
1040         ns->tx_bytes = pstats->tx_octets;
1041         ns->tx_packets = pstats->tx_frames;
1042         ns->rx_bytes = pstats->rx_octets;
1043         ns->rx_packets = pstats->rx_frames;
1044         ns->multicast = pstats->rx_mcast_frames;
1045
1046         ns->tx_errors = pstats->tx_underrun;
1047         ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1048             pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1049             pstats->rx_fifo_ovfl;
1050
1051         /* detailed rx_errors */
1052         ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1053         ns->rx_over_errors = 0;
1054         ns->rx_crc_errors = pstats->rx_fcs_errs;
1055         ns->rx_frame_errors = pstats->rx_symbol_errs;
1056         ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1057         ns->rx_missed_errors = pstats->rx_cong_drops;
1058
1059         /* detailed tx_errors */
1060         ns->tx_aborted_errors = 0;
1061         ns->tx_carrier_errors = 0;
1062         ns->tx_fifo_errors = pstats->tx_underrun;
1063         ns->tx_heartbeat_errors = 0;
1064         ns->tx_window_errors = 0;
1065         return ns;
1066 }
1067
1068 static u32 get_msglevel(struct net_device *dev)
1069 {
1070         struct port_info *pi = netdev_priv(dev);
1071         struct adapter *adapter = pi->adapter;
1072
1073         return adapter->msg_enable;
1074 }
1075
1076 static void set_msglevel(struct net_device *dev, u32 val)
1077 {
1078         struct port_info *pi = netdev_priv(dev);
1079         struct adapter *adapter = pi->adapter;
1080
1081         adapter->msg_enable = val;
1082 }
1083
1084 static char stats_strings[][ETH_GSTRING_LEN] = {
1085         "TxOctetsOK         ",
1086         "TxFramesOK         ",
1087         "TxMulticastFramesOK",
1088         "TxBroadcastFramesOK",
1089         "TxPauseFrames      ",
1090         "TxUnderrun         ",
1091         "TxExtUnderrun      ",
1092
1093         "TxFrames64         ",
1094         "TxFrames65To127    ",
1095         "TxFrames128To255   ",
1096         "TxFrames256To511   ",
1097         "TxFrames512To1023  ",
1098         "TxFrames1024To1518 ",
1099         "TxFrames1519ToMax  ",
1100
1101         "RxOctetsOK         ",
1102         "RxFramesOK         ",
1103         "RxMulticastFramesOK",
1104         "RxBroadcastFramesOK",
1105         "RxPauseFrames      ",
1106         "RxFCSErrors        ",
1107         "RxSymbolErrors     ",
1108         "RxShortErrors      ",
1109         "RxJabberErrors     ",
1110         "RxLengthErrors     ",
1111         "RxFIFOoverflow     ",
1112
1113         "RxFrames64         ",
1114         "RxFrames65To127    ",
1115         "RxFrames128To255   ",
1116         "RxFrames256To511   ",
1117         "RxFrames512To1023  ",
1118         "RxFrames1024To1518 ",
1119         "RxFrames1519ToMax  ",
1120
1121         "PhyFIFOErrors      ",
1122         "TSO                ",
1123         "VLANextractions    ",
1124         "VLANinsertions     ",
1125         "TxCsumOffload      ",
1126         "RxCsumGood         ",
1127         "RxDrops            ",
1128
1129         "CheckTXEnToggled   ",
1130         "CheckResets        ",
1131
1132 };
1133
1134 static int get_stats_count(struct net_device *dev)
1135 {
1136         return ARRAY_SIZE(stats_strings);
1137 }
1138
1139 #define T3_REGMAP_SIZE (3 * 1024)
1140
1141 static int get_regs_len(struct net_device *dev)
1142 {
1143         return T3_REGMAP_SIZE;
1144 }
1145
1146 static int get_eeprom_len(struct net_device *dev)
1147 {
1148         return EEPROMSIZE;
1149 }
1150
1151 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1152 {
1153         struct port_info *pi = netdev_priv(dev);
1154         struct adapter *adapter = pi->adapter;
1155         u32 fw_vers = 0;
1156         u32 tp_vers = 0;
1157
1158         t3_get_fw_version(adapter, &fw_vers);
1159         t3_get_tp_version(adapter, &tp_vers);
1160
1161         strcpy(info->driver, DRV_NAME);
1162         strcpy(info->version, DRV_VERSION);
1163         strcpy(info->bus_info, pci_name(adapter->pdev));
1164         if (!fw_vers)
1165                 strcpy(info->fw_version, "N/A");
1166         else {
1167                 snprintf(info->fw_version, sizeof(info->fw_version),
1168                          "%s %u.%u.%u TP %u.%u.%u",
1169                          G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1170                          G_FW_VERSION_MAJOR(fw_vers),
1171                          G_FW_VERSION_MINOR(fw_vers),
1172                          G_FW_VERSION_MICRO(fw_vers),
1173                          G_TP_VERSION_MAJOR(tp_vers),
1174                          G_TP_VERSION_MINOR(tp_vers),
1175                          G_TP_VERSION_MICRO(tp_vers));
1176         }
1177 }
1178
1179 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1180 {
1181         if (stringset == ETH_SS_STATS)
1182                 memcpy(data, stats_strings, sizeof(stats_strings));
1183 }
1184
1185 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1186                                             struct port_info *p, int idx)
1187 {
1188         int i;
1189         unsigned long tot = 0;
1190
1191         for (i = 0; i < p->nqsets; ++i)
1192                 tot += adapter->sge.qs[i + p->first_qset].port_stats[idx];
1193         return tot;
1194 }
1195
1196 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1197                       u64 *data)
1198 {
1199         struct port_info *pi = netdev_priv(dev);
1200         struct adapter *adapter = pi->adapter;
1201         const struct mac_stats *s;
1202
1203         spin_lock(&adapter->stats_lock);
1204         s = t3_mac_update_stats(&pi->mac);
1205         spin_unlock(&adapter->stats_lock);
1206
1207         *data++ = s->tx_octets;
1208         *data++ = s->tx_frames;
1209         *data++ = s->tx_mcast_frames;
1210         *data++ = s->tx_bcast_frames;
1211         *data++ = s->tx_pause;
1212         *data++ = s->tx_underrun;
1213         *data++ = s->tx_fifo_urun;
1214
1215         *data++ = s->tx_frames_64;
1216         *data++ = s->tx_frames_65_127;
1217         *data++ = s->tx_frames_128_255;
1218         *data++ = s->tx_frames_256_511;
1219         *data++ = s->tx_frames_512_1023;
1220         *data++ = s->tx_frames_1024_1518;
1221         *data++ = s->tx_frames_1519_max;
1222
1223         *data++ = s->rx_octets;
1224         *data++ = s->rx_frames;
1225         *data++ = s->rx_mcast_frames;
1226         *data++ = s->rx_bcast_frames;
1227         *data++ = s->rx_pause;
1228         *data++ = s->rx_fcs_errs;
1229         *data++ = s->rx_symbol_errs;
1230         *data++ = s->rx_short;
1231         *data++ = s->rx_jabber;
1232         *data++ = s->rx_too_long;
1233         *data++ = s->rx_fifo_ovfl;
1234
1235         *data++ = s->rx_frames_64;
1236         *data++ = s->rx_frames_65_127;
1237         *data++ = s->rx_frames_128_255;
1238         *data++ = s->rx_frames_256_511;
1239         *data++ = s->rx_frames_512_1023;
1240         *data++ = s->rx_frames_1024_1518;
1241         *data++ = s->rx_frames_1519_max;
1242
1243         *data++ = pi->phy.fifo_errors;
1244
1245         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1246         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1247         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1248         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1249         *data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1250         *data++ = s->rx_cong_drops;
1251
1252         *data++ = s->num_toggled;
1253         *data++ = s->num_resets;
1254 }
1255
1256 static inline void reg_block_dump(struct adapter *ap, void *buf,
1257                                   unsigned int start, unsigned int end)
1258 {
1259         u32 *p = buf + start;
1260
1261         for (; start <= end; start += sizeof(u32))
1262                 *p++ = t3_read_reg(ap, start);
1263 }
1264
1265 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1266                      void *buf)
1267 {
1268         struct port_info *pi = netdev_priv(dev);
1269         struct adapter *ap = pi->adapter;
1270
1271         /*
1272          * Version scheme:
1273          * bits 0..9: chip version
1274          * bits 10..15: chip revision
1275          * bit 31: set for PCIe cards
1276          */
1277         regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1278
1279         /*
1280          * We skip the MAC statistics registers because they are clear-on-read.
1281          * Also reading multi-register stats would need to synchronize with the
1282          * periodic mac stats accumulation.  Hard to justify the complexity.
1283          */
1284         memset(buf, 0, T3_REGMAP_SIZE);
1285         reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1286         reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1287         reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1288         reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1289         reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1290         reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1291                        XGM_REG(A_XGM_SERDES_STAT3, 1));
1292         reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1293                        XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1294 }
1295
1296 static int restart_autoneg(struct net_device *dev)
1297 {
1298         struct port_info *p = netdev_priv(dev);
1299
1300         if (!netif_running(dev))
1301                 return -EAGAIN;
1302         if (p->link_config.autoneg != AUTONEG_ENABLE)
1303                 return -EINVAL;
1304         p->phy.ops->autoneg_restart(&p->phy);
1305         return 0;
1306 }
1307
1308 static int cxgb3_phys_id(struct net_device *dev, u32 data)
1309 {
1310         struct port_info *pi = netdev_priv(dev);
1311         struct adapter *adapter = pi->adapter;
1312         int i;
1313
1314         if (data == 0)
1315                 data = 2;
1316
1317         for (i = 0; i < data * 2; i++) {
1318                 t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1319                                  (i & 1) ? F_GPIO0_OUT_VAL : 0);
1320                 if (msleep_interruptible(500))
1321                         break;
1322         }
1323         t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1324                          F_GPIO0_OUT_VAL);
1325         return 0;
1326 }
1327
1328 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1329 {
1330         struct port_info *p = netdev_priv(dev);
1331
1332         cmd->supported = p->link_config.supported;
1333         cmd->advertising = p->link_config.advertising;
1334
1335         if (netif_carrier_ok(dev)) {
1336                 cmd->speed = p->link_config.speed;
1337                 cmd->duplex = p->link_config.duplex;
1338         } else {
1339                 cmd->speed = -1;
1340                 cmd->duplex = -1;
1341         }
1342
1343         cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1344         cmd->phy_address = p->phy.addr;
1345         cmd->transceiver = XCVR_EXTERNAL;
1346         cmd->autoneg = p->link_config.autoneg;
1347         cmd->maxtxpkt = 0;
1348         cmd->maxrxpkt = 0;
1349         return 0;
1350 }
1351
1352 static int speed_duplex_to_caps(int speed, int duplex)
1353 {
1354         int cap = 0;
1355
1356         switch (speed) {
1357         case SPEED_10:
1358                 if (duplex == DUPLEX_FULL)
1359                         cap = SUPPORTED_10baseT_Full;
1360                 else
1361                         cap = SUPPORTED_10baseT_Half;
1362                 break;
1363         case SPEED_100:
1364                 if (duplex == DUPLEX_FULL)
1365                         cap = SUPPORTED_100baseT_Full;
1366                 else
1367                         cap = SUPPORTED_100baseT_Half;
1368                 break;
1369         case SPEED_1000:
1370                 if (duplex == DUPLEX_FULL)
1371                         cap = SUPPORTED_1000baseT_Full;
1372                 else
1373                         cap = SUPPORTED_1000baseT_Half;
1374                 break;
1375         case SPEED_10000:
1376                 if (duplex == DUPLEX_FULL)
1377                         cap = SUPPORTED_10000baseT_Full;
1378         }
1379         return cap;
1380 }
1381
1382 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1383                       ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1384                       ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1385                       ADVERTISED_10000baseT_Full)
1386
1387 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1388 {
1389         struct port_info *p = netdev_priv(dev);
1390         struct link_config *lc = &p->link_config;
1391
1392         if (!(lc->supported & SUPPORTED_Autoneg))
1393                 return -EOPNOTSUPP;     /* can't change speed/duplex */
1394
1395         if (cmd->autoneg == AUTONEG_DISABLE) {
1396                 int cap = speed_duplex_to_caps(cmd->speed, cmd->duplex);
1397
1398                 if (!(lc->supported & cap) || cmd->speed == SPEED_1000)
1399                         return -EINVAL;
1400                 lc->requested_speed = cmd->speed;
1401                 lc->requested_duplex = cmd->duplex;
1402                 lc->advertising = 0;
1403         } else {
1404                 cmd->advertising &= ADVERTISED_MASK;
1405                 cmd->advertising &= lc->supported;
1406                 if (!cmd->advertising)
1407                         return -EINVAL;
1408                 lc->requested_speed = SPEED_INVALID;
1409                 lc->requested_duplex = DUPLEX_INVALID;
1410                 lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1411         }
1412         lc->autoneg = cmd->autoneg;
1413         if (netif_running(dev))
1414                 t3_link_start(&p->phy, &p->mac, lc);
1415         return 0;
1416 }
1417
1418 static void get_pauseparam(struct net_device *dev,
1419                            struct ethtool_pauseparam *epause)
1420 {
1421         struct port_info *p = netdev_priv(dev);
1422
1423         epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1424         epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1425         epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1426 }
1427
1428 static int set_pauseparam(struct net_device *dev,
1429                           struct ethtool_pauseparam *epause)
1430 {
1431         struct port_info *p = netdev_priv(dev);
1432         struct link_config *lc = &p->link_config;
1433
1434         if (epause->autoneg == AUTONEG_DISABLE)
1435                 lc->requested_fc = 0;
1436         else if (lc->supported & SUPPORTED_Autoneg)
1437                 lc->requested_fc = PAUSE_AUTONEG;
1438         else
1439                 return -EINVAL;
1440
1441         if (epause->rx_pause)
1442                 lc->requested_fc |= PAUSE_RX;
1443         if (epause->tx_pause)
1444                 lc->requested_fc |= PAUSE_TX;
1445         if (lc->autoneg == AUTONEG_ENABLE) {
1446                 if (netif_running(dev))
1447                         t3_link_start(&p->phy, &p->mac, lc);
1448         } else {
1449                 lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1450                 if (netif_running(dev))
1451                         t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1452         }
1453         return 0;
1454 }
1455
1456 static u32 get_rx_csum(struct net_device *dev)
1457 {
1458         struct port_info *p = netdev_priv(dev);
1459
1460         return p->rx_csum_offload;
1461 }
1462
1463 static int set_rx_csum(struct net_device *dev, u32 data)
1464 {
1465         struct port_info *p = netdev_priv(dev);
1466
1467         p->rx_csum_offload = data;
1468         return 0;
1469 }
1470
1471 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1472 {
1473         struct port_info *pi = netdev_priv(dev);
1474         struct adapter *adapter = pi->adapter;
1475         const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1476
1477         e->rx_max_pending = MAX_RX_BUFFERS;
1478         e->rx_mini_max_pending = 0;
1479         e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1480         e->tx_max_pending = MAX_TXQ_ENTRIES;
1481
1482         e->rx_pending = q->fl_size;
1483         e->rx_mini_pending = q->rspq_size;
1484         e->rx_jumbo_pending = q->jumbo_size;
1485         e->tx_pending = q->txq_size[0];
1486 }
1487
1488 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1489 {
1490         struct port_info *pi = netdev_priv(dev);
1491         struct adapter *adapter = pi->adapter;
1492         struct qset_params *q;
1493         int i;
1494
1495         if (e->rx_pending > MAX_RX_BUFFERS ||
1496             e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1497             e->tx_pending > MAX_TXQ_ENTRIES ||
1498             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1499             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1500             e->rx_pending < MIN_FL_ENTRIES ||
1501             e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1502             e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1503                 return -EINVAL;
1504
1505         if (adapter->flags & FULL_INIT_DONE)
1506                 return -EBUSY;
1507
1508         q = &adapter->params.sge.qset[pi->first_qset];
1509         for (i = 0; i < pi->nqsets; ++i, ++q) {
1510                 q->rspq_size = e->rx_mini_pending;
1511                 q->fl_size = e->rx_pending;
1512                 q->jumbo_size = e->rx_jumbo_pending;
1513                 q->txq_size[0] = e->tx_pending;
1514                 q->txq_size[1] = e->tx_pending;
1515                 q->txq_size[2] = e->tx_pending;
1516         }
1517         return 0;
1518 }
1519
1520 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1521 {
1522         struct port_info *pi = netdev_priv(dev);
1523         struct adapter *adapter = pi->adapter;
1524         struct qset_params *qsp = &adapter->params.sge.qset[0];
1525         struct sge_qset *qs = &adapter->sge.qs[0];
1526
1527         if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1528                 return -EINVAL;
1529
1530         qsp->coalesce_usecs = c->rx_coalesce_usecs;
1531         t3_update_qset_coalesce(qs, qsp);
1532         return 0;
1533 }
1534
1535 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1536 {
1537         struct port_info *pi = netdev_priv(dev);
1538         struct adapter *adapter = pi->adapter;
1539         struct qset_params *q = adapter->params.sge.qset;
1540
1541         c->rx_coalesce_usecs = q->coalesce_usecs;
1542         return 0;
1543 }
1544
1545 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1546                       u8 * data)
1547 {
1548         struct port_info *pi = netdev_priv(dev);
1549         struct adapter *adapter = pi->adapter;
1550         int i, err = 0;
1551
1552         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1553         if (!buf)
1554                 return -ENOMEM;
1555
1556         e->magic = EEPROM_MAGIC;
1557         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1558                 err = t3_seeprom_read(adapter, i, (u32 *) & buf[i]);
1559
1560         if (!err)
1561                 memcpy(data, buf + e->offset, e->len);
1562         kfree(buf);
1563         return err;
1564 }
1565
1566 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1567                       u8 * data)
1568 {
1569         struct port_info *pi = netdev_priv(dev);
1570         struct adapter *adapter = pi->adapter;
1571         u32 aligned_offset, aligned_len, *p;
1572         u8 *buf;
1573         int err;
1574
1575         if (eeprom->magic != EEPROM_MAGIC)
1576                 return -EINVAL;
1577
1578         aligned_offset = eeprom->offset & ~3;
1579         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1580
1581         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1582                 buf = kmalloc(aligned_len, GFP_KERNEL);
1583                 if (!buf)
1584                         return -ENOMEM;
1585                 err = t3_seeprom_read(adapter, aligned_offset, (u32 *) buf);
1586                 if (!err && aligned_len > 4)
1587                         err = t3_seeprom_read(adapter,
1588                                               aligned_offset + aligned_len - 4,
1589                                               (u32 *) & buf[aligned_len - 4]);
1590                 if (err)
1591                         goto out;
1592                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1593         } else
1594                 buf = data;
1595
1596         err = t3_seeprom_wp(adapter, 0);
1597         if (err)
1598                 goto out;
1599
1600         for (p = (u32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
1601                 err = t3_seeprom_write(adapter, aligned_offset, *p);
1602                 aligned_offset += 4;
1603         }
1604
1605         if (!err)
1606                 err = t3_seeprom_wp(adapter, 1);
1607 out:
1608         if (buf != data)
1609                 kfree(buf);
1610         return err;
1611 }
1612
1613 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1614 {
1615         wol->supported = 0;
1616         wol->wolopts = 0;
1617         memset(&wol->sopass, 0, sizeof(wol->sopass));
1618 }
1619
1620 static const struct ethtool_ops cxgb_ethtool_ops = {
1621         .get_settings = get_settings,
1622         .set_settings = set_settings,
1623         .get_drvinfo = get_drvinfo,
1624         .get_msglevel = get_msglevel,
1625         .set_msglevel = set_msglevel,
1626         .get_ringparam = get_sge_param,
1627         .set_ringparam = set_sge_param,
1628         .get_coalesce = get_coalesce,
1629         .set_coalesce = set_coalesce,
1630         .get_eeprom_len = get_eeprom_len,
1631         .get_eeprom = get_eeprom,
1632         .set_eeprom = set_eeprom,
1633         .get_pauseparam = get_pauseparam,
1634         .set_pauseparam = set_pauseparam,
1635         .get_rx_csum = get_rx_csum,
1636         .set_rx_csum = set_rx_csum,
1637         .set_tx_csum = ethtool_op_set_tx_csum,
1638         .set_sg = ethtool_op_set_sg,
1639         .get_link = ethtool_op_get_link,
1640         .get_strings = get_strings,
1641         .phys_id = cxgb3_phys_id,
1642         .nway_reset = restart_autoneg,
1643         .get_stats_count = get_stats_count,
1644         .get_ethtool_stats = get_stats,
1645         .get_regs_len = get_regs_len,
1646         .get_regs = get_regs,
1647         .get_wol = get_wol,
1648         .set_tso = ethtool_op_set_tso,
1649 };
1650
1651 static int in_range(int val, int lo, int hi)
1652 {
1653         return val < 0 || (val <= hi && val >= lo);
1654 }
1655
1656 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
1657 {
1658         struct port_info *pi = netdev_priv(dev);
1659         struct adapter *adapter = pi->adapter;
1660         u32 cmd;
1661         int ret;
1662
1663         if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
1664                 return -EFAULT;
1665
1666         switch (cmd) {
1667         case CHELSIO_SET_QSET_PARAMS:{
1668                 int i;
1669                 struct qset_params *q;
1670                 struct ch_qset_params t;
1671
1672                 if (!capable(CAP_NET_ADMIN))
1673                         return -EPERM;
1674                 if (copy_from_user(&t, useraddr, sizeof(t)))
1675                         return -EFAULT;
1676                 if (t.qset_idx >= SGE_QSETS)
1677                         return -EINVAL;
1678                 if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
1679                         !in_range(t.cong_thres, 0, 255) ||
1680                         !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
1681                                 MAX_TXQ_ENTRIES) ||
1682                         !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
1683                                 MAX_TXQ_ENTRIES) ||
1684                         !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
1685                                 MAX_CTRL_TXQ_ENTRIES) ||
1686                         !in_range(t.fl_size[0], MIN_FL_ENTRIES,
1687                                 MAX_RX_BUFFERS)
1688                         || !in_range(t.fl_size[1], MIN_FL_ENTRIES,
1689                                         MAX_RX_JUMBO_BUFFERS)
1690                         || !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
1691                                         MAX_RSPQ_ENTRIES))
1692                         return -EINVAL;
1693                 if ((adapter->flags & FULL_INIT_DONE) &&
1694                         (t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
1695                         t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
1696                         t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
1697                         t.polling >= 0 || t.cong_thres >= 0))
1698                         return -EBUSY;
1699
1700                 q = &adapter->params.sge.qset[t.qset_idx];
1701
1702                 if (t.rspq_size >= 0)
1703                         q->rspq_size = t.rspq_size;
1704                 if (t.fl_size[0] >= 0)
1705                         q->fl_size = t.fl_size[0];
1706                 if (t.fl_size[1] >= 0)
1707                         q->jumbo_size = t.fl_size[1];
1708                 if (t.txq_size[0] >= 0)
1709                         q->txq_size[0] = t.txq_size[0];
1710                 if (t.txq_size[1] >= 0)
1711                         q->txq_size[1] = t.txq_size[1];
1712                 if (t.txq_size[2] >= 0)
1713                         q->txq_size[2] = t.txq_size[2];
1714                 if (t.cong_thres >= 0)
1715                         q->cong_thres = t.cong_thres;
1716                 if (t.intr_lat >= 0) {
1717                         struct sge_qset *qs =
1718                                 &adapter->sge.qs[t.qset_idx];
1719
1720                         q->coalesce_usecs = t.intr_lat;
1721                         t3_update_qset_coalesce(qs, q);
1722                 }
1723                 if (t.polling >= 0) {
1724                         if (adapter->flags & USING_MSIX)
1725                                 q->polling = t.polling;
1726                         else {
1727                                 /* No polling with INTx for T3A */
1728                                 if (adapter->params.rev == 0 &&
1729                                         !(adapter->flags & USING_MSI))
1730                                         t.polling = 0;
1731
1732                                 for (i = 0; i < SGE_QSETS; i++) {
1733                                         q = &adapter->params.sge.
1734                                                 qset[i];
1735                                         q->polling = t.polling;
1736                                 }
1737                         }
1738                 }
1739                 break;
1740         }
1741         case CHELSIO_GET_QSET_PARAMS:{
1742                 struct qset_params *q;
1743                 struct ch_qset_params t;
1744
1745                 if (copy_from_user(&t, useraddr, sizeof(t)))
1746                         return -EFAULT;
1747                 if (t.qset_idx >= SGE_QSETS)
1748                         return -EINVAL;
1749
1750                 q = &adapter->params.sge.qset[t.qset_idx];
1751                 t.rspq_size = q->rspq_size;
1752                 t.txq_size[0] = q->txq_size[0];
1753                 t.txq_size[1] = q->txq_size[1];
1754                 t.txq_size[2] = q->txq_size[2];
1755                 t.fl_size[0] = q->fl_size;
1756                 t.fl_size[1] = q->jumbo_size;
1757                 t.polling = q->polling;
1758                 t.intr_lat = q->coalesce_usecs;
1759                 t.cong_thres = q->cong_thres;
1760
1761                 if (copy_to_user(useraddr, &t, sizeof(t)))
1762                         return -EFAULT;
1763                 break;
1764         }
1765         case CHELSIO_SET_QSET_NUM:{
1766                 struct ch_reg edata;
1767                 struct port_info *pi = netdev_priv(dev);
1768                 unsigned int i, first_qset = 0, other_qsets = 0;
1769
1770                 if (!capable(CAP_NET_ADMIN))
1771                         return -EPERM;
1772                 if (adapter->flags & FULL_INIT_DONE)
1773                         return -EBUSY;
1774                 if (copy_from_user(&edata, useraddr, sizeof(edata)))
1775                         return -EFAULT;
1776                 if (edata.val < 1 ||
1777                         (edata.val > 1 && !(adapter->flags & USING_MSIX)))
1778                         return -EINVAL;
1779
1780                 for_each_port(adapter, i)
1781                         if (adapter->port[i] && adapter->port[i] != dev)
1782                                 other_qsets += adap2pinfo(adapter, i)->nqsets;
1783
1784                 if (edata.val + other_qsets > SGE_QSETS)
1785                         return -EINVAL;
1786
1787                 pi->nqsets = edata.val;
1788
1789                 for_each_port(adapter, i)
1790                         if (adapter->port[i]) {
1791                                 pi = adap2pinfo(adapter, i);
1792                                 pi->first_qset = first_qset;
1793                                 first_qset += pi->nqsets;
1794                         }
1795                 break;
1796         }
1797         case CHELSIO_GET_QSET_NUM:{
1798                 struct ch_reg edata;
1799                 struct port_info *pi = netdev_priv(dev);
1800
1801                 edata.cmd = CHELSIO_GET_QSET_NUM;
1802                 edata.val = pi->nqsets;
1803                 if (copy_to_user(useraddr, &edata, sizeof(edata)))
1804                         return -EFAULT;
1805                 break;
1806         }
1807         case CHELSIO_LOAD_FW:{
1808                 u8 *fw_data;
1809                 struct ch_mem_range t;
1810
1811                 if (!capable(CAP_NET_ADMIN))
1812                         return -EPERM;
1813                 if (copy_from_user(&t, useraddr, sizeof(t)))
1814                         return -EFAULT;
1815
1816                 fw_data = kmalloc(t.len, GFP_KERNEL);
1817                 if (!fw_data)
1818                         return -ENOMEM;
1819
1820                 if (copy_from_user
1821                         (fw_data, useraddr + sizeof(t), t.len)) {
1822                         kfree(fw_data);
1823                         return -EFAULT;
1824                 }
1825
1826                 ret = t3_load_fw(adapter, fw_data, t.len);
1827                 kfree(fw_data);
1828                 if (ret)
1829                         return ret;
1830                 break;
1831         }
1832         case CHELSIO_SETMTUTAB:{
1833                 struct ch_mtus m;
1834                 int i;
1835
1836                 if (!is_offload(adapter))
1837                         return -EOPNOTSUPP;
1838                 if (!capable(CAP_NET_ADMIN))
1839                         return -EPERM;
1840                 if (offload_running(adapter))
1841                         return -EBUSY;
1842                 if (copy_from_user(&m, useraddr, sizeof(m)))
1843                         return -EFAULT;
1844                 if (m.nmtus != NMTUS)
1845                         return -EINVAL;
1846                 if (m.mtus[0] < 81)     /* accommodate SACK */
1847                         return -EINVAL;
1848
1849                 /* MTUs must be in ascending order */
1850                 for (i = 1; i < NMTUS; ++i)
1851                         if (m.mtus[i] < m.mtus[i - 1])
1852                                 return -EINVAL;
1853
1854                 memcpy(adapter->params.mtus, m.mtus,
1855                         sizeof(adapter->params.mtus));
1856                 break;
1857         }
1858         case CHELSIO_GET_PM:{
1859                 struct tp_params *p = &adapter->params.tp;
1860                 struct ch_pm m = {.cmd = CHELSIO_GET_PM };
1861
1862                 if (!is_offload(adapter))
1863                         return -EOPNOTSUPP;
1864                 m.tx_pg_sz = p->tx_pg_size;
1865                 m.tx_num_pg = p->tx_num_pgs;
1866                 m.rx_pg_sz = p->rx_pg_size;
1867                 m.rx_num_pg = p->rx_num_pgs;
1868                 m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
1869                 if (copy_to_user(useraddr, &m, sizeof(m)))
1870                         return -EFAULT;
1871                 break;
1872         }
1873         case CHELSIO_SET_PM:{
1874                 struct ch_pm m;
1875                 struct tp_params *p = &adapter->params.tp;
1876
1877                 if (!is_offload(adapter))
1878                         return -EOPNOTSUPP;
1879                 if (!capable(CAP_NET_ADMIN))
1880                         return -EPERM;
1881                 if (adapter->flags & FULL_INIT_DONE)
1882                         return -EBUSY;
1883                 if (copy_from_user(&m, useraddr, sizeof(m)))
1884                         return -EFAULT;
1885                 if (!is_power_of_2(m.rx_pg_sz) ||
1886                         !is_power_of_2(m.tx_pg_sz))
1887                         return -EINVAL; /* not power of 2 */
1888                 if (!(m.rx_pg_sz & 0x14000))
1889                         return -EINVAL; /* not 16KB or 64KB */
1890                 if (!(m.tx_pg_sz & 0x1554000))
1891                         return -EINVAL;
1892                 if (m.tx_num_pg == -1)
1893                         m.tx_num_pg = p->tx_num_pgs;
1894                 if (m.rx_num_pg == -1)
1895                         m.rx_num_pg = p->rx_num_pgs;
1896                 if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
1897                         return -EINVAL;
1898                 if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
1899                         m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
1900                         return -EINVAL;
1901                 p->rx_pg_size = m.rx_pg_sz;
1902                 p->tx_pg_size = m.tx_pg_sz;
1903                 p->rx_num_pgs = m.rx_num_pg;
1904                 p->tx_num_pgs = m.tx_num_pg;
1905                 break;
1906         }
1907         case CHELSIO_GET_MEM:{
1908                 struct ch_mem_range t;
1909                 struct mc7 *mem;
1910                 u64 buf[32];
1911
1912                 if (!is_offload(adapter))
1913                         return -EOPNOTSUPP;
1914                 if (!(adapter->flags & FULL_INIT_DONE))
1915                         return -EIO;    /* need the memory controllers */
1916                 if (copy_from_user(&t, useraddr, sizeof(t)))
1917                         return -EFAULT;
1918                 if ((t.addr & 7) || (t.len & 7))
1919                         return -EINVAL;
1920                 if (t.mem_id == MEM_CM)
1921                         mem = &adapter->cm;
1922                 else if (t.mem_id == MEM_PMRX)
1923                         mem = &adapter->pmrx;
1924                 else if (t.mem_id == MEM_PMTX)
1925                         mem = &adapter->pmtx;
1926                 else
1927                         return -EINVAL;
1928
1929                 /*
1930                  * Version scheme:
1931                  * bits 0..9: chip version
1932                  * bits 10..15: chip revision
1933                  */
1934                 t.version = 3 | (adapter->params.rev << 10);
1935                 if (copy_to_user(useraddr, &t, sizeof(t)))
1936                         return -EFAULT;
1937
1938                 /*
1939                  * Read 256 bytes at a time as len can be large and we don't
1940                  * want to use huge intermediate buffers.
1941                  */
1942                 useraddr += sizeof(t);  /* advance to start of buffer */
1943                 while (t.len) {
1944                         unsigned int chunk =
1945                                 min_t(unsigned int, t.len, sizeof(buf));
1946
1947                         ret =
1948                                 t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
1949                                                 buf);
1950                         if (ret)
1951                                 return ret;
1952                         if (copy_to_user(useraddr, buf, chunk))
1953                                 return -EFAULT;
1954                         useraddr += chunk;
1955                         t.addr += chunk;
1956                         t.len -= chunk;
1957                 }
1958                 break;
1959         }
1960         case CHELSIO_SET_TRACE_FILTER:{
1961                 struct ch_trace t;
1962                 const struct trace_params *tp;
1963
1964                 if (!capable(CAP_NET_ADMIN))
1965                         return -EPERM;
1966                 if (!offload_running(adapter))
1967                         return -EAGAIN;
1968                 if (copy_from_user(&t, useraddr, sizeof(t)))
1969                         return -EFAULT;
1970
1971                 tp = (const struct trace_params *)&t.sip;
1972                 if (t.config_tx)
1973                         t3_config_trace_filter(adapter, tp, 0,
1974                                                 t.invert_match,
1975                                                 t.trace_tx);
1976                 if (t.config_rx)
1977                         t3_config_trace_filter(adapter, tp, 1,
1978                                                 t.invert_match,
1979                                                 t.trace_rx);
1980                 break;
1981         }
1982         default:
1983                 return -EOPNOTSUPP;
1984         }
1985         return 0;
1986 }
1987
1988 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
1989 {
1990         struct mii_ioctl_data *data = if_mii(req);
1991         struct port_info *pi = netdev_priv(dev);
1992         struct adapter *adapter = pi->adapter;
1993         int ret, mmd;
1994
1995         switch (cmd) {
1996         case SIOCGMIIPHY:
1997                 data->phy_id = pi->phy.addr;
1998                 /* FALLTHRU */
1999         case SIOCGMIIREG:{
2000                 u32 val;
2001                 struct cphy *phy = &pi->phy;
2002
2003                 if (!phy->mdio_read)
2004                         return -EOPNOTSUPP;
2005                 if (is_10G(adapter)) {
2006                         mmd = data->phy_id >> 8;
2007                         if (!mmd)
2008                                 mmd = MDIO_DEV_PCS;
2009                         else if (mmd > MDIO_DEV_XGXS)
2010                                 return -EINVAL;
2011
2012                         ret =
2013                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2014                                                 mmd, data->reg_num, &val);
2015                 } else
2016                         ret =
2017                                 phy->mdio_read(adapter, data->phy_id & 0x1f,
2018                                                 0, data->reg_num & 0x1f,
2019                                                 &val);
2020                 if (!ret)
2021                         data->val_out = val;
2022                 break;
2023         }
2024         case SIOCSMIIREG:{
2025                 struct cphy *phy = &pi->phy;
2026
2027                 if (!capable(CAP_NET_ADMIN))
2028                         return -EPERM;
2029                 if (!phy->mdio_write)
2030                         return -EOPNOTSUPP;
2031                 if (is_10G(adapter)) {
2032                         mmd = data->phy_id >> 8;
2033                         if (!mmd)
2034                                 mmd = MDIO_DEV_PCS;
2035                         else if (mmd > MDIO_DEV_XGXS)
2036                                 return -EINVAL;
2037
2038                         ret =
2039                                 phy->mdio_write(adapter,
2040                                                 data->phy_id & 0x1f, mmd,
2041                                                 data->reg_num,
2042                                                 data->val_in);
2043                 } else
2044                         ret =
2045                                 phy->mdio_write(adapter,
2046                                                 data->phy_id & 0x1f, 0,
2047                                                 data->reg_num & 0x1f,
2048                                                 data->val_in);
2049                 break;
2050         }
2051         case SIOCCHIOCTL:
2052                 return cxgb_extension_ioctl(dev, req->ifr_data);
2053         default:
2054                 return -EOPNOTSUPP;
2055         }
2056         return ret;
2057 }
2058
2059 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2060 {
2061         struct port_info *pi = netdev_priv(dev);
2062         struct adapter *adapter = pi->adapter;
2063         int ret;
2064
2065         if (new_mtu < 81)       /* accommodate SACK */
2066                 return -EINVAL;
2067         if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2068                 return ret;
2069         dev->mtu = new_mtu;
2070         init_port_mtus(adapter);
2071         if (adapter->params.rev == 0 && offload_running(adapter))
2072                 t3_load_mtus(adapter, adapter->params.mtus,
2073                              adapter->params.a_wnd, adapter->params.b_wnd,
2074                              adapter->port[0]->mtu);
2075         return 0;
2076 }
2077
2078 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2079 {
2080         struct port_info *pi = netdev_priv(dev);
2081         struct adapter *adapter = pi->adapter;
2082         struct sockaddr *addr = p;
2083
2084         if (!is_valid_ether_addr(addr->sa_data))
2085                 return -EINVAL;
2086
2087         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2088         t3_mac_set_address(&pi->mac, 0, dev->dev_addr);
2089         if (offload_running(adapter))
2090                 write_smt_entry(adapter, pi->port_id);
2091         return 0;
2092 }
2093
2094 /**
2095  * t3_synchronize_rx - wait for current Rx processing on a port to complete
2096  * @adap: the adapter
2097  * @p: the port
2098  *
2099  * Ensures that current Rx processing on any of the queues associated with
2100  * the given port completes before returning.  We do this by acquiring and
2101  * releasing the locks of the response queues associated with the port.
2102  */
2103 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
2104 {
2105         int i;
2106
2107         for (i = 0; i < p->nqsets; i++) {
2108                 struct sge_rspq *q = &adap->sge.qs[i + p->first_qset].rspq;
2109
2110                 spin_lock_irq(&q->lock);
2111                 spin_unlock_irq(&q->lock);
2112         }
2113 }
2114
2115 static void vlan_rx_register(struct net_device *dev, struct vlan_group *grp)
2116 {
2117         struct port_info *pi = netdev_priv(dev);
2118         struct adapter *adapter = pi->adapter;
2119
2120         pi->vlan_grp = grp;
2121         if (adapter->params.rev > 0)
2122                 t3_set_vlan_accel(adapter, 1 << pi->port_id, grp != NULL);
2123         else {
2124                 /* single control for all ports */
2125                 unsigned int i, have_vlans = 0;
2126                 for_each_port(adapter, i)
2127                     have_vlans |= adap2pinfo(adapter, i)->vlan_grp != NULL;
2128
2129                 t3_set_vlan_accel(adapter, 1, have_vlans);
2130         }
2131         t3_synchronize_rx(adapter, pi);
2132 }
2133
2134 #ifdef CONFIG_NET_POLL_CONTROLLER
2135 static void cxgb_netpoll(struct net_device *dev)
2136 {
2137         struct port_info *pi = netdev_priv(dev);
2138         struct adapter *adapter = pi->adapter;
2139         int qidx;
2140
2141         for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2142                 struct sge_qset *qs = &adapter->sge.qs[qidx];
2143                 void *source;
2144
2145                 if (adapter->flags & USING_MSIX)
2146                         source = qs;
2147                 else
2148                         source = adapter;
2149
2150                 t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2151         }
2152 }
2153 #endif
2154
2155 /*
2156  * Periodic accumulation of MAC statistics.
2157  */
2158 static void mac_stats_update(struct adapter *adapter)
2159 {
2160         int i;
2161
2162         for_each_port(adapter, i) {
2163                 struct net_device *dev = adapter->port[i];
2164                 struct port_info *p = netdev_priv(dev);
2165
2166                 if (netif_running(dev)) {
2167                         spin_lock(&adapter->stats_lock);
2168                         t3_mac_update_stats(&p->mac);
2169                         spin_unlock(&adapter->stats_lock);
2170                 }
2171         }
2172 }
2173
2174 static void check_link_status(struct adapter *adapter)
2175 {
2176         int i;
2177
2178         for_each_port(adapter, i) {
2179                 struct net_device *dev = adapter->port[i];
2180                 struct port_info *p = netdev_priv(dev);
2181
2182                 if (!(p->port_type->caps & SUPPORTED_IRQ) && netif_running(dev))
2183                         t3_link_changed(adapter, i);
2184         }
2185 }
2186
2187 static void check_t3b2_mac(struct adapter *adapter)
2188 {
2189         int i;
2190
2191         if (!rtnl_trylock())    /* synchronize with ifdown */
2192                 return;
2193
2194         for_each_port(adapter, i) {
2195                 struct net_device *dev = adapter->port[i];
2196                 struct port_info *p = netdev_priv(dev);
2197                 int status;
2198
2199                 if (!netif_running(dev))
2200                         continue;
2201
2202                 status = 0;
2203                 if (netif_running(dev) && netif_carrier_ok(dev))
2204                         status = t3b2_mac_watchdog_task(&p->mac);
2205                 if (status == 1)
2206                         p->mac.stats.num_toggled++;
2207                 else if (status == 2) {
2208                         struct cmac *mac = &p->mac;
2209
2210                         t3_mac_set_mtu(mac, dev->mtu);
2211                         t3_mac_set_address(mac, 0, dev->dev_addr);
2212                         cxgb_set_rxmode(dev);
2213                         t3_link_start(&p->phy, mac, &p->link_config);
2214                         t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2215                         t3_port_intr_enable(adapter, p->port_id);
2216                         p->mac.stats.num_resets++;
2217                 }
2218         }
2219         rtnl_unlock();
2220 }
2221
2222
2223 static void t3_adap_check_task(struct work_struct *work)
2224 {
2225         struct adapter *adapter = container_of(work, struct adapter,
2226                                                adap_check_task.work);
2227         const struct adapter_params *p = &adapter->params;
2228
2229         adapter->check_task_cnt++;
2230
2231         /* Check link status for PHYs without interrupts */
2232         if (p->linkpoll_period)
2233                 check_link_status(adapter);
2234
2235         /* Accumulate MAC stats if needed */
2236         if (!p->linkpoll_period ||
2237             (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2238             p->stats_update_period) {
2239                 mac_stats_update(adapter);
2240                 adapter->check_task_cnt = 0;
2241         }
2242
2243         if (p->rev == T3_REV_B2)
2244                 check_t3b2_mac(adapter);
2245
2246         /* Schedule the next check update if any port is active. */
2247         spin_lock(&adapter->work_lock);
2248         if (adapter->open_device_map & PORT_MASK)
2249                 schedule_chk_task(adapter);
2250         spin_unlock(&adapter->work_lock);
2251 }
2252
2253 /*
2254  * Processes external (PHY) interrupts in process context.
2255  */
2256 static void ext_intr_task(struct work_struct *work)
2257 {
2258         struct adapter *adapter = container_of(work, struct adapter,
2259                                                ext_intr_handler_task);
2260
2261         t3_phy_intr_handler(adapter);
2262
2263         /* Now reenable external interrupts */
2264         spin_lock_irq(&adapter->work_lock);
2265         if (adapter->slow_intr_mask) {
2266                 adapter->slow_intr_mask |= F_T3DBG;
2267                 t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2268                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2269                              adapter->slow_intr_mask);
2270         }
2271         spin_unlock_irq(&adapter->work_lock);
2272 }
2273
2274 /*
2275  * Interrupt-context handler for external (PHY) interrupts.
2276  */
2277 void t3_os_ext_intr_handler(struct adapter *adapter)
2278 {
2279         /*
2280          * Schedule a task to handle external interrupts as they may be slow
2281          * and we use a mutex to protect MDIO registers.  We disable PHY
2282          * interrupts in the meantime and let the task reenable them when
2283          * it's done.
2284          */
2285         spin_lock(&adapter->work_lock);
2286         if (adapter->slow_intr_mask) {
2287                 adapter->slow_intr_mask &= ~F_T3DBG;
2288                 t3_write_reg(adapter, A_PL_INT_ENABLE0,
2289                              adapter->slow_intr_mask);
2290                 queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2291         }
2292         spin_unlock(&adapter->work_lock);
2293 }
2294
2295 void t3_fatal_err(struct adapter *adapter)
2296 {
2297         unsigned int fw_status[4];
2298
2299         if (adapter->flags & FULL_INIT_DONE) {
2300                 t3_sge_stop(adapter);
2301                 t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2302                 t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2303                 t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2304                 t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2305                 t3_intr_disable(adapter);
2306         }
2307         CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2308         if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2309                 CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2310                          fw_status[0], fw_status[1],
2311                          fw_status[2], fw_status[3]);
2312
2313 }
2314
2315 static int __devinit cxgb_enable_msix(struct adapter *adap)
2316 {
2317         struct msix_entry entries[SGE_QSETS + 1];
2318         int i, err;
2319
2320         for (i = 0; i < ARRAY_SIZE(entries); ++i)
2321                 entries[i].entry = i;
2322
2323         err = pci_enable_msix(adap->pdev, entries, ARRAY_SIZE(entries));
2324         if (!err) {
2325                 for (i = 0; i < ARRAY_SIZE(entries); ++i)
2326                         adap->msix_info[i].vec = entries[i].vector;
2327         } else if (err > 0)
2328                 dev_info(&adap->pdev->dev,
2329                        "only %d MSI-X vectors left, not using MSI-X\n", err);
2330         return err;
2331 }
2332
2333 static void __devinit print_port_info(struct adapter *adap,
2334                                       const struct adapter_info *ai)
2335 {
2336         static const char *pci_variant[] = {
2337                 "PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
2338         };
2339
2340         int i;
2341         char buf[80];
2342
2343         if (is_pcie(adap))
2344                 snprintf(buf, sizeof(buf), "%s x%d",
2345                          pci_variant[adap->params.pci.variant],
2346                          adap->params.pci.width);
2347         else
2348                 snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
2349                          pci_variant[adap->params.pci.variant],
2350                          adap->params.pci.speed, adap->params.pci.width);
2351
2352         for_each_port(adap, i) {
2353                 struct net_device *dev = adap->port[i];
2354                 const struct port_info *pi = netdev_priv(dev);
2355
2356                 if (!test_bit(i, &adap->registered_device_map))
2357                         continue;
2358                 printk(KERN_INFO "%s: %s %s %sNIC (rev %d) %s%s\n",
2359                        dev->name, ai->desc, pi->port_type->desc,
2360                        is_offload(adap) ? "R" : "", adap->params.rev, buf,
2361                        (adap->flags & USING_MSIX) ? " MSI-X" :
2362                        (adap->flags & USING_MSI) ? " MSI" : "");
2363                 if (adap->name == dev->name && adap->params.vpd.mclk)
2364                         printk(KERN_INFO
2365                                "%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
2366                                adap->name, t3_mc7_size(&adap->cm) >> 20,
2367                                t3_mc7_size(&adap->pmtx) >> 20,
2368                                t3_mc7_size(&adap->pmrx) >> 20,
2369                                adap->params.vpd.sn);
2370         }
2371 }
2372
2373 static int __devinit init_one(struct pci_dev *pdev,
2374                               const struct pci_device_id *ent)
2375 {
2376         static int version_printed;
2377
2378         int i, err, pci_using_dac = 0;
2379         unsigned long mmio_start, mmio_len;
2380         const struct adapter_info *ai;
2381         struct adapter *adapter = NULL;
2382         struct port_info *pi;
2383
2384         if (!version_printed) {
2385                 printk(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
2386                 ++version_printed;
2387         }
2388
2389         if (!cxgb3_wq) {
2390                 cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
2391                 if (!cxgb3_wq) {
2392                         printk(KERN_ERR DRV_NAME
2393                                ": cannot initialize work queue\n");
2394                         return -ENOMEM;
2395                 }
2396         }
2397
2398         err = pci_request_regions(pdev, DRV_NAME);
2399         if (err) {
2400                 /* Just info, some other driver may have claimed the device. */
2401                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
2402                 return err;
2403         }
2404
2405         err = pci_enable_device(pdev);
2406         if (err) {
2407                 dev_err(&pdev->dev, "cannot enable PCI device\n");
2408                 goto out_release_regions;
2409         }
2410
2411         if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2412                 pci_using_dac = 1;
2413                 err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2414                 if (err) {
2415                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
2416                                "coherent allocations\n");
2417                         goto out_disable_device;
2418                 }
2419         } else if ((err = pci_set_dma_mask(pdev, DMA_32BIT_MASK)) != 0) {
2420                 dev_err(&pdev->dev, "no usable DMA configuration\n");
2421                 goto out_disable_device;
2422         }
2423
2424         pci_set_master(pdev);
2425
2426         mmio_start = pci_resource_start(pdev, 0);
2427         mmio_len = pci_resource_len(pdev, 0);
2428         ai = t3_get_adapter_info(ent->driver_data);
2429
2430         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2431         if (!adapter) {
2432                 err = -ENOMEM;
2433                 goto out_disable_device;
2434         }
2435
2436         adapter->regs = ioremap_nocache(mmio_start, mmio_len);
2437         if (!adapter->regs) {
2438                 dev_err(&pdev->dev, "cannot map device registers\n");
2439                 err = -ENOMEM;
2440                 goto out_free_adapter;
2441         }
2442
2443         adapter->pdev = pdev;
2444         adapter->name = pci_name(pdev);
2445         adapter->msg_enable = dflt_msg_enable;
2446         adapter->mmio_len = mmio_len;
2447
2448         mutex_init(&adapter->mdio_lock);
2449         spin_lock_init(&adapter->work_lock);
2450         spin_lock_init(&adapter->stats_lock);
2451
2452         INIT_LIST_HEAD(&adapter->adapter_list);
2453         INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
2454         INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
2455
2456         for (i = 0; i < ai->nports; ++i) {
2457                 struct net_device *netdev;
2458
2459                 netdev = alloc_etherdev(sizeof(struct port_info));
2460                 if (!netdev) {
2461                         err = -ENOMEM;
2462                         goto out_free_dev;
2463                 }
2464
2465                 SET_NETDEV_DEV(netdev, &pdev->dev);
2466
2467                 adapter->port[i] = netdev;
2468                 pi = netdev_priv(netdev);
2469                 pi->adapter = adapter;
2470                 pi->rx_csum_offload = 1;
2471                 pi->nqsets = 1;
2472                 pi->first_qset = i;
2473                 pi->activity = 0;
2474                 pi->port_id = i;
2475                 netif_carrier_off(netdev);
2476                 netdev->irq = pdev->irq;
2477                 netdev->mem_start = mmio_start;
2478                 netdev->mem_end = mmio_start + mmio_len - 1;
2479                 netdev->features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO;
2480                 netdev->features |= NETIF_F_LLTX;
2481                 if (pci_using_dac)
2482                         netdev->features |= NETIF_F_HIGHDMA;
2483
2484                 netdev->features |= NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
2485                 netdev->vlan_rx_register = vlan_rx_register;
2486
2487                 netdev->open = cxgb_open;
2488                 netdev->stop = cxgb_close;
2489                 netdev->hard_start_xmit = t3_eth_xmit;
2490                 netdev->get_stats = cxgb_get_stats;
2491                 netdev->set_multicast_list = cxgb_set_rxmode;
2492                 netdev->do_ioctl = cxgb_ioctl;
2493                 netdev->change_mtu = cxgb_change_mtu;
2494                 netdev->set_mac_address = cxgb_set_mac_addr;
2495 #ifdef CONFIG_NET_POLL_CONTROLLER
2496                 netdev->poll_controller = cxgb_netpoll;
2497 #endif
2498
2499                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
2500         }
2501
2502         pci_set_drvdata(pdev, adapter);
2503         if (t3_prep_adapter(adapter, ai, 1) < 0) {
2504                 err = -ENODEV;
2505                 goto out_free_dev;
2506         }
2507
2508         /*
2509          * The card is now ready to go.  If any errors occur during device
2510          * registration we do not fail the whole card but rather proceed only
2511          * with the ports we manage to register successfully.  However we must
2512          * register at least one net device.
2513          */
2514         for_each_port(adapter, i) {
2515                 err = register_netdev(adapter->port[i]);
2516                 if (err)
2517                         dev_warn(&pdev->dev,
2518                                  "cannot register net device %s, skipping\n",
2519                                  adapter->port[i]->name);
2520                 else {
2521                         /*
2522                          * Change the name we use for messages to the name of
2523                          * the first successfully registered interface.
2524                          */
2525                         if (!adapter->registered_device_map)
2526                                 adapter->name = adapter->port[i]->name;
2527
2528                         __set_bit(i, &adapter->registered_device_map);
2529                 }
2530         }
2531         if (!adapter->registered_device_map) {
2532                 dev_err(&pdev->dev, "could not register any net devices\n");
2533                 goto out_free_dev;
2534         }
2535
2536         /* Driver's ready. Reflect it on LEDs */
2537         t3_led_ready(adapter);
2538
2539         if (is_offload(adapter)) {
2540                 __set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
2541                 cxgb3_adapter_ofld(adapter);
2542         }
2543
2544         /* See what interrupts we'll be using */
2545         if (msi > 1 && cxgb_enable_msix(adapter) == 0)
2546                 adapter->flags |= USING_MSIX;
2547         else if (msi > 0 && pci_enable_msi(pdev) == 0)
2548                 adapter->flags |= USING_MSI;
2549
2550         err = sysfs_create_group(&adapter->port[0]->dev.kobj,
2551                                  &cxgb3_attr_group);
2552
2553         print_port_info(adapter, ai);
2554         return 0;
2555
2556 out_free_dev:
2557         iounmap(adapter->regs);
2558         for (i = ai->nports - 1; i >= 0; --i)
2559                 if (adapter->port[i])
2560                         free_netdev(adapter->port[i]);
2561
2562 out_free_adapter:
2563         kfree(adapter);
2564
2565 out_disable_device:
2566         pci_disable_device(pdev);
2567 out_release_regions:
2568         pci_release_regions(pdev);
2569         pci_set_drvdata(pdev, NULL);
2570         return err;
2571 }
2572
2573 static void __devexit remove_one(struct pci_dev *pdev)
2574 {
2575         struct adapter *adapter = pci_get_drvdata(pdev);
2576
2577         if (adapter) {
2578                 int i;
2579
2580                 t3_sge_stop(adapter);
2581                 sysfs_remove_group(&adapter->port[0]->dev.kobj,
2582                                    &cxgb3_attr_group);
2583
2584                 for_each_port(adapter, i)
2585                     if (test_bit(i, &adapter->registered_device_map))
2586                         unregister_netdev(adapter->port[i]);
2587
2588                 if (is_offload(adapter)) {
2589                         cxgb3_adapter_unofld(adapter);
2590                         if (test_bit(OFFLOAD_DEVMAP_BIT,
2591                                      &adapter->open_device_map))
2592                                 offload_close(&adapter->tdev);
2593                 }
2594
2595                 t3_free_sge_resources(adapter);
2596                 cxgb_disable_msi(adapter);
2597
2598                 for_each_port(adapter, i)
2599                         if (adapter->port[i])
2600                                 free_netdev(adapter->port[i]);
2601
2602                 iounmap(adapter->regs);
2603                 kfree(adapter);
2604                 pci_release_regions(pdev);
2605                 pci_disable_device(pdev);
2606                 pci_set_drvdata(pdev, NULL);
2607         }
2608 }
2609
2610 static struct pci_driver driver = {
2611         .name = DRV_NAME,
2612         .id_table = cxgb3_pci_tbl,
2613         .probe = init_one,
2614         .remove = __devexit_p(remove_one),
2615 };
2616
2617 static int __init cxgb3_init_module(void)
2618 {
2619         int ret;
2620
2621         cxgb3_offload_init();
2622
2623         ret = pci_register_driver(&driver);
2624         return ret;
2625 }
2626
2627 static void __exit cxgb3_cleanup_module(void)
2628 {
2629         pci_unregister_driver(&driver);
2630         if (cxgb3_wq)
2631                 destroy_workqueue(cxgb3_wq);
2632 }
2633
2634 module_init(cxgb3_init_module);
2635 module_exit(cxgb3_cleanup_module);