drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <asm/uaccess.h>
  64
  65 #include "cxgb4.h"
  66 #include "t4_regs.h"
  67 #include "t4_msg.h"
  68 #include "t4fw_api.h"
  69 #include "l2t.h"
  70
  71 #define DRV_VERSION "2.0.0-ko"
  72 #define DRV_DESC "Chelsio T4/T5 Network Driver"
  73
  74 /*
  75  * Max interrupt hold-off timer value in us.  Queues fall back to this value
  76  * under extreme memory pressure so it's largish to give the system time to
  77  * recover.
  78  */
  79 #define MAX_SGE_TIMERVAL 200U
  80
  81 enum {
  82         /*
  83          * Physical Function provisioning constants.
  84          */
  85         PFRES_NVI = 4,                  /* # of Virtual Interfaces */
  86         PFRES_NETHCTRL = 128,           /* # of EQs used for ETH or CTRL Qs */
  87         PFRES_NIQFLINT = 128,           /* # of ingress Qs/w Free List(s)/intr
  88                                          */
  89         PFRES_NEQ = 256,                /* # of egress queues */
  90         PFRES_NIQ = 0,                  /* # of ingress queues */
  91         PFRES_TC = 0,                   /* PCI-E traffic class */
  92         PFRES_NEXACTF = 128,            /* # of exact MPS filters */
  93
  94         PFRES_R_CAPS = FW_CMD_CAP_PF,
  95         PFRES_WX_CAPS = FW_CMD_CAP_PF,
  96
  97 #ifdef CONFIG_PCI_IOV
  98         /*
  99          * Virtual Function provisioning constants.  We need two extra Ingress
 100          * Queues with Interrupt capability to serve as the VF's Firmware
 101          * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
 102          * neither will have Free Lists associated with them).  For each
 103          * Ethernet/Control Egress Queue and for each Free List, we need an
 104          * Egress Context.
 105          */
 106         VFRES_NPORTS = 1,               /* # of "ports" per VF */
 107         VFRES_NQSETS = 2,               /* # of "Queue Sets" per VF */
 108
 109         VFRES_NVI = VFRES_NPORTS,       /* # of Virtual Interfaces */
 110         VFRES_NETHCTRL = VFRES_NQSETS,  /* # of EQs used for ETH or CTRL Qs */
 111         VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
 112         VFRES_NEQ = VFRES_NQSETS*2,     /* # of egress queues */
 113         VFRES_NIQ = 0,                  /* # of non-fl/int ingress queues */
 114         VFRES_TC = 0,                   /* PCI-E traffic class */
 115         VFRES_NEXACTF = 16,             /* # of exact MPS filters */
 116
 117         VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
 118         VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
 119 #endif
 120 };
 121
 122 /*
 123  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
 124  * static and likely not to be useful in the long run.  We really need to
 125  * implement some form of persistent configuration which the firmware
 126  * controls.
 127  */
 128 static unsigned int pfvfres_pmask(struct adapter *adapter,
 129                                   unsigned int pf, unsigned int vf)
 130 {
 131         unsigned int portn, portvec;
 132
 133         /*
 134          * Give PF's access to all of the ports.
 135          */
 136         if (vf == 0)
 137                 return FW_PFVF_CMD_PMASK_MASK;
 138
 139         /*
 140          * For VFs, we'll assign them access to the ports based purely on the
 141          * PF.  We assign active ports in order, wrapping around if there are
 142          * fewer active ports than PFs: e.g. active port[pf % nports].
 143          * Unfortunately the adapter's port_info structs haven't been
 144          * initialized yet so we have to compute this.
 145          */
 146         if (adapter->params.nports == 0)
 147                 return 0;
 148
 149         portn = pf % adapter->params.nports;
 150         portvec = adapter->params.portvec;
 151         for (;;) {
 152                 /*
 153                  * Isolate the lowest set bit in the port vector.  If we're at
 154                  * the port number that we want, return that as the pmask.
 155                  * otherwise mask that bit out of the port vector and
 156                  * decrement our port number ...
 157                  */
 158                 unsigned int pmask = portvec ^ (portvec & (portvec-1));
 159                 if (portn == 0)
 160                         return pmask;
 161                 portn--;
 162                 portvec &= ~pmask;
 163         }
 164         /*NOTREACHED*/
 165 }
 166
 167 enum {
 168         MAX_TXQ_ENTRIES      = 16384,
 169         MAX_CTRL_TXQ_ENTRIES = 1024,
 170         MAX_RSPQ_ENTRIES     = 16384,
 171         MAX_RX_BUFFERS       = 16384,
 172         MIN_TXQ_ENTRIES      = 32,
 173         MIN_CTRL_TXQ_ENTRIES = 32,
 174         MIN_RSPQ_ENTRIES     = 128,
 175         MIN_FL_ENTRIES       = 16
 176 };
 177
 178 /* Host shadow copy of ingress filter entry.  This is in host native format
 179  * and doesn't match the ordering or bit order, etc. of the hardware of the
 180  * firmware command.  The use of bit-field structure elements is purely to
 181  * remind ourselves of the field size limitations and save memory in the case
 182  * where the filter table is large.
 183  */
 184 struct filter_entry {
 185         /* Administrative fields for filter.
 186          */
 187         u32 valid:1;            /* filter allocated and valid */
 188         u32 locked:1;           /* filter is administratively locked */
 189
 190         u32 pending:1;          /* filter action is pending firmware reply */
 191         u32 smtidx:8;           /* Source MAC Table index for smac */
 192         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 193
 194         /* The filter itself.  Most of this is a straight copy of information
 195          * provided by the extended ioctl().  Some fields are translated to
 196          * internal forms -- for instance the Ingress Queue ID passed in from
 197          * the ioctl() is translated into the Absolute Ingress Queue ID.
 198          */
 199         struct ch_filter_specification fs;
 200 };
 201
 202 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 203                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 204                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 205
 206 #define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
 207
 208 static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
 209         CH_DEVICE(0xa000, 0),  /* PE10K */
 210         CH_DEVICE(0x4001, -1),
 211         CH_DEVICE(0x4002, -1),
 212         CH_DEVICE(0x4003, -1),
 213         CH_DEVICE(0x4004, -1),
 214         CH_DEVICE(0x4005, -1),
 215         CH_DEVICE(0x4006, -1),
 216         CH_DEVICE(0x4007, -1),
 217         CH_DEVICE(0x4008, -1),
 218         CH_DEVICE(0x4009, -1),
 219         CH_DEVICE(0x400a, -1),
 220         CH_DEVICE(0x4401, 4),
 221         CH_DEVICE(0x4402, 4),
 222         CH_DEVICE(0x4403, 4),
 223         CH_DEVICE(0x4404, 4),
 224         CH_DEVICE(0x4405, 4),
 225         CH_DEVICE(0x4406, 4),
 226         CH_DEVICE(0x4407, 4),
 227         CH_DEVICE(0x4408, 4),
 228         CH_DEVICE(0x4409, 4),
 229         CH_DEVICE(0x440a, 4),
 230         CH_DEVICE(0x440d, 4),
 231         CH_DEVICE(0x440e, 4),
 232         CH_DEVICE(0x5001, 4),
 233         CH_DEVICE(0x5002, 4),
 234         CH_DEVICE(0x5003, 4),
 235         CH_DEVICE(0x5004, 4),
 236         CH_DEVICE(0x5005, 4),
 237         CH_DEVICE(0x5006, 4),
 238         CH_DEVICE(0x5007, 4),
 239         CH_DEVICE(0x5008, 4),
 240         CH_DEVICE(0x5009, 4),
 241         CH_DEVICE(0x500A, 4),
 242         CH_DEVICE(0x500B, 4),
 243         CH_DEVICE(0x500C, 4),
 244         CH_DEVICE(0x500D, 4),
 245         CH_DEVICE(0x500E, 4),
 246         CH_DEVICE(0x500F, 4),
 247         CH_DEVICE(0x5010, 4),
 248         CH_DEVICE(0x5011, 4),
 249         CH_DEVICE(0x5012, 4),
 250         CH_DEVICE(0x5013, 4),
 251         CH_DEVICE(0x5401, 4),
 252         CH_DEVICE(0x5402, 4),
 253         CH_DEVICE(0x5403, 4),
 254         CH_DEVICE(0x5404, 4),
 255         CH_DEVICE(0x5405, 4),
 256         CH_DEVICE(0x5406, 4),
 257         CH_DEVICE(0x5407, 4),
 258         CH_DEVICE(0x5408, 4),
 259         CH_DEVICE(0x5409, 4),
 260         CH_DEVICE(0x540A, 4),
 261         CH_DEVICE(0x540B, 4),
 262         CH_DEVICE(0x540C, 4),
 263         CH_DEVICE(0x540D, 4),
 264         CH_DEVICE(0x540E, 4),
 265         CH_DEVICE(0x540F, 4),
 266         CH_DEVICE(0x5410, 4),
 267         CH_DEVICE(0x5411, 4),
 268         CH_DEVICE(0x5412, 4),
 269         CH_DEVICE(0x5413, 4),
 270         { 0, }
 271 };
 272
 273 #define FW_FNAME "cxgb4/t4fw.bin"
 274 #define FW5_FNAME "cxgb4/t5fw.bin"
 275 #define FW_CFNAME "cxgb4/t4-config.txt"
 276 #define FW5_CFNAME "cxgb4/t5-config.txt"
 277
 278 MODULE_DESCRIPTION(DRV_DESC);
 279 MODULE_AUTHOR("Chelsio Communications");
 280 MODULE_LICENSE("Dual BSD/GPL");
 281 MODULE_VERSION(DRV_VERSION);
 282 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 283 MODULE_FIRMWARE(FW_FNAME);
 284 MODULE_FIRMWARE(FW5_FNAME);
 285
 286 /*
 287  * Normally we're willing to become the firmware's Master PF but will be happy
 288  * if another PF has already become the Master and initialized the adapter.
 289  * Setting "force_init" will cause this driver to forcibly establish itself as
 290  * the Master PF and initialize the adapter.
 291  */
 292 static uint force_init;
 293
 294 module_param(force_init, uint, 0644);
 295 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 296
 297 /*
 298  * Normally if the firmware we connect to has Configuration File support, we
 299  * use that and only fall back to the old Driver-based initialization if the
 300  * Configuration File fails for some reason.  If force_old_init is set, then
 301  * we'll always use the old Driver-based initialization sequence.
 302  */
 303 static uint force_old_init;
 304
 305 module_param(force_old_init, uint, 0644);
 306 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
 307
 308 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 309
 310 module_param(dflt_msg_enable, int, 0644);
 311 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 312
 313 /*
 314  * The driver uses the best interrupt scheme available on a platform in the
 315  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 316  * of these schemes the driver may consider as follows:
 317  *
 318  * msi = 2: choose from among all three options
 319  * msi = 1: only consider MSI and INTx interrupts
 320  * msi = 0: force INTx interrupts
 321  */
 322 static int msi = 2;
 323
 324 module_param(msi, int, 0644);
 325 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 326
 327 /*
 328  * Queue interrupt hold-off timer values.  Queues default to the first of these
 329  * upon creation.
 330  */
 331 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 332
 333 module_param_array(intr_holdoff, uint, NULL, 0644);
 334 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 335                  "0..4 in microseconds");
 336
 337 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 338
 339 module_param_array(intr_cnt, uint, NULL, 0644);
 340 MODULE_PARM_DESC(intr_cnt,
 341                  "thresholds 1..3 for queue interrupt packet counters");
 342
 343 /*
 344  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 345  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 346  * boundaries.  This is a requirement for many architectures which will throw
 347  * a machine check fault if an attempt is made to access one of the 4-byte IP
 348  * header fields on a non-4-byte boundary.  And it's a major performance issue
 349  * even on some architectures which allow it like some implementations of the
 350  * x86 ISA.  However, some architectures don't mind this and for some very
 351  * edge-case performance sensitive applications (like forwarding large volumes
 352  * of small packets), setting this DMA offset to 0 will decrease the number of
 353  * PCI-E Bus transfers enough to measurably affect performance.
 354  */
 355 static int rx_dma_offset = 2;
 356
 357 static bool vf_acls;
 358
 359 #ifdef CONFIG_PCI_IOV
 360 module_param(vf_acls, bool, 0644);
 361 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
 362
 363 /* Configure the number of PCI-E Virtual Function which are to be instantiated
 364  * on SR-IOV Capable Physical Functions.
 365  */
 366 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
 367
 368 module_param_array(num_vf, uint, NULL, 0644);
 369 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 370 #endif
 371
 372 /*
 373  * The filter TCAM has a fixed portion and a variable portion.  The fixed
 374  * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
 375  * ports.  The variable portion is 36 bits which can include things like Exact
 376  * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
 377  * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
 378  * far exceed the 36-bit budget for this "compressed" header portion of the
 379  * filter.  Thus, we have a scarce resource which must be carefully managed.
 380  *
 381  * By default we set this up to mostly match the set of filter matching
 382  * capabilities of T3 but with accommodations for some of T4's more
 383  * interesting features:
 384  *
 385  *   { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
 386  *     [Inner] VLAN (17), Port (3), FCoE (1) }
 387  */
 388 enum {
 389         TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
 390         TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
 391         TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
 392 };
 393
 394 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
 395
 396 module_param(tp_vlan_pri_map, uint, 0644);
 397 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
 398
 399 static struct dentry *cxgb4_debugfs_root;
 400
 401 static LIST_HEAD(adapter_list);
 402 static DEFINE_MUTEX(uld_mutex);
 403 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 404 static const char *uld_str[] = { "RDMA", "iSCSI" };
 405
 406 static void link_report(struct net_device *dev)
 407 {
 408         if (!netif_carrier_ok(dev))
 409                 netdev_info(dev, "link down\n");
 410         else {
 411                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 412
 413                 const char *s = "10Mbps";
 414                 const struct port_info *p = netdev_priv(dev);
 415
 416                 switch (p->link_cfg.speed) {
 417                 case SPEED_10000:
 418                         s = "10Gbps";
 419                         break;
 420                 case SPEED_1000:
 421                         s = "1000Mbps";
 422                         break;
 423                 case SPEED_100:
 424                         s = "100Mbps";
 425                         break;
 426                 }
 427
 428                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 429                             fc[p->link_cfg.fc]);
 430         }
 431 }
 432
 433 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 434 {
 435         struct net_device *dev = adapter->port[port_id];
 436
 437         /* Skip changes from disabled ports. */
 438         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 439                 if (link_stat)
 440                         netif_carrier_on(dev);
 441                 else
 442                         netif_carrier_off(dev);
 443
 444                 link_report(dev);
 445         }
 446 }
 447
 448 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 449 {
 450         static const char *mod_str[] = {
 451                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 452         };
 453
 454         const struct net_device *dev = adap->port[port_id];
 455         const struct port_info *pi = netdev_priv(dev);
 456
 457         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 458                 netdev_info(dev, "port module unplugged\n");
 459         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 460                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 461 }
 462
 463 /*
 464  * Configure the exact and hash address filters to handle a port's multicast
 465  * and secondary unicast MAC addresses.
 466  */
 467 static int set_addr_filters(const struct net_device *dev, bool sleep)
 468 {
 469         u64 mhash = 0;
 470         u64 uhash = 0;
 471         bool free = true;
 472         u16 filt_idx[7];
 473         const u8 *addr[7];
 474         int ret, naddr = 0;
 475         const struct netdev_hw_addr *ha;
 476         int uc_cnt = netdev_uc_count(dev);
 477         int mc_cnt = netdev_mc_count(dev);
 478         const struct port_info *pi = netdev_priv(dev);
 479         unsigned int mb = pi->adapter->fn;
 480
 481         /* first do the secondary unicast addresses */
 482         netdev_for_each_uc_addr(ha, dev) {
 483                 addr[naddr++] = ha->addr;
 484                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 485                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 486                                         naddr, addr, filt_idx, &uhash, sleep);
 487                         if (ret < 0)
 488                                 return ret;
 489
 490                         free = false;
 491                         naddr = 0;
 492                 }
 493         }
 494
 495         /* next set up the multicast addresses */
 496         netdev_for_each_mc_addr(ha, dev) {
 497                 addr[naddr++] = ha->addr;
 498                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 499                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 500                                         naddr, addr, filt_idx, &mhash, sleep);
 501                         if (ret < 0)
 502                                 return ret;
 503
 504                         free = false;
 505                         naddr = 0;
 506                 }
 507         }
 508
 509         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 510                                 uhash | mhash, sleep);
 511 }
 512
 513 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 514 module_param(dbfifo_int_thresh, int, 0644);
 515 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 516
 517 /*
 518  * usecs to sleep while draining the dbfifo
 519  */
 520 static int dbfifo_drain_delay = 1000;
 521 module_param(dbfifo_drain_delay, int, 0644);
 522 MODULE_PARM_DESC(dbfifo_drain_delay,
 523                  "usecs to sleep while draining the dbfifo");
 524
 525 /*
 526  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 527  * If @mtu is -1 it is left unchanged.
 528  */
 529 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 530 {
 531         int ret;
 532         struct port_info *pi = netdev_priv(dev);
 533
 534         ret = set_addr_filters(dev, sleep_ok);
 535         if (ret == 0)
 536                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 537                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 538                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 539                                     sleep_ok);
 540         return ret;
 541 }
 542
 543 static struct workqueue_struct *workq;
 544
 545 /**
 546  *      link_start - enable a port
 547  *      @dev: the port to enable
 548  *
 549  *      Performs the MAC and PHY actions needed to enable a port.
 550  */
 551 static int link_start(struct net_device *dev)
 552 {
 553         int ret;
 554         struct port_info *pi = netdev_priv(dev);
 555         unsigned int mb = pi->adapter->fn;
 556
 557         /*
 558          * We do not set address filters and promiscuity here, the stack does
 559          * that step explicitly.
 560          */
 561         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 562                             !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
 563         if (ret == 0) {
 564                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 565                                     pi->xact_addr_filt, dev->dev_addr, true,
 566                                     true);
 567                 if (ret >= 0) {
 568                         pi->xact_addr_filt = ret;
 569                         ret = 0;
 570                 }
 571         }
 572         if (ret == 0)
 573                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 574                                     &pi->link_cfg);
 575         if (ret == 0)
 576                 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
 577         return ret;
 578 }
 579
 580 /* Clear a filter and release any of its resources that we own.  This also
 581  * clears the filter's "pending" status.
 582  */
 583 static void clear_filter(struct adapter *adap, struct filter_entry *f)
 584 {
 585         /* If the new or old filter have loopback rewriteing rules then we'll
 586          * need to free any existing Layer Two Table (L2T) entries of the old
 587          * filter rule.  The firmware will handle freeing up any Source MAC
 588          * Table (SMT) entries used for rewriting Source MAC Addresses in
 589          * loopback rules.
 590          */
 591         if (f->l2t)
 592                 cxgb4_l2t_release(f->l2t);
 593
 594         /* The zeroing of the filter rule below clears the filter valid,
 595          * pending, locked flags, l2t pointer, etc. so it's all we need for
 596          * this operation.
 597          */
 598         memset(f, 0, sizeof(*f));
 599 }
 600
 601 /* Handle a filter write/deletion reply.
 602  */
 603 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
 604 {
 605         unsigned int idx = GET_TID(rpl);
 606         unsigned int nidx = idx - adap->tids.ftid_base;
 607         unsigned int ret;
 608         struct filter_entry *f;
 609
 610         if (idx >= adap->tids.ftid_base && nidx <
 611            (adap->tids.nftids + adap->tids.nsftids)) {
 612                 idx = nidx;
 613                 ret = GET_TCB_COOKIE(rpl->cookie);
 614                 f = &adap->tids.ftid_tab[idx];
 615
 616                 if (ret == FW_FILTER_WR_FLT_DELETED) {
 617                         /* Clear the filter when we get confirmation from the
 618                          * hardware that the filter has been deleted.
 619                          */
 620                         clear_filter(adap, f);
 621                 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
 622                         dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
 623                                 idx);
 624                         clear_filter(adap, f);
 625                 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
 626                         f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
 627                         f->pending = 0;  /* asynchronous setup completed */
 628                         f->valid = 1;
 629                 } else {
 630                         /* Something went wrong.  Issue a warning about the
 631                          * problem and clear everything out.
 632                          */
 633                         dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
 634                                 idx, ret);
 635                         clear_filter(adap, f);
 636                 }
 637         }
 638 }
 639
 640 /* Response queue handler for the FW event queue.
 641  */
 642 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 643                           const struct pkt_gl *gl)
 644 {
 645         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 646
 647         rsp++;                                          /* skip RSS header */
 648
 649         /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
 650          */
 651         if (unlikely(opcode == CPL_FW4_MSG &&
 652            ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
 653                 rsp++;
 654                 opcode = ((const struct rss_header *)rsp)->opcode;
 655                 rsp++;
 656                 if (opcode != CPL_SGE_EGR_UPDATE) {
 657                         dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
 658                                 , opcode);
 659                         goto out;
 660                 }
 661         }
 662
 663         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 664                 const struct cpl_sge_egr_update *p = (void *)rsp;
 665                 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
 666                 struct sge_txq *txq;
 667
 668                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 669                 txq->restarts++;
 670                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 671                         struct sge_eth_txq *eq;
 672
 673                         eq = container_of(txq, struct sge_eth_txq, q);
 674                         netif_tx_wake_queue(eq->txq);
 675                 } else {
 676                         struct sge_ofld_txq *oq;
 677
 678                         oq = container_of(txq, struct sge_ofld_txq, q);
 679                         tasklet_schedule(&oq->qresume_tsk);
 680                 }
 681         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 682                 const struct cpl_fw6_msg *p = (void *)rsp;
 683
 684                 if (p->type == 0)
 685                         t4_handle_fw_rpl(q->adap, p->data);
 686         } else if (opcode == CPL_L2T_WRITE_RPL) {
 687                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 688
 689                 do_l2t_write_rpl(q->adap, p);
 690         } else if (opcode == CPL_SET_TCB_RPL) {
 691                 const struct cpl_set_tcb_rpl *p = (void *)rsp;
 692
 693                 filter_rpl(q->adap, p);
 694         } else
 695                 dev_err(q->adap->pdev_dev,
 696                         "unexpected CPL %#x on FW event queue\n", opcode);
 697 out:
 698         return 0;
 699 }
 700
 701 /**
 702  *      uldrx_handler - response queue handler for ULD queues
 703  *      @q: the response queue that received the packet
 704  *      @rsp: the response queue descriptor holding the offload message
 705  *      @gl: the gather list of packet fragments
 706  *
 707  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 708  *      the ULD, we just maintain statistics.
 709  */
 710 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 711                          const struct pkt_gl *gl)
 712 {
 713         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 714
 715         /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
 716          */
 717         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
 718             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
 719                 rsp += 2;
 720
 721         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 722                 rxq->stats.nomem++;
 723                 return -1;
 724         }
 725         if (gl == NULL)
 726                 rxq->stats.imm++;
 727         else if (gl == CXGB4_MSG_AN)
 728                 rxq->stats.an++;
 729         else
 730                 rxq->stats.pkts++;
 731         return 0;
 732 }
 733
 734 static void disable_msi(struct adapter *adapter)
 735 {
 736         if (adapter->flags & USING_MSIX) {
 737                 pci_disable_msix(adapter->pdev);
 738                 adapter->flags &= ~USING_MSIX;
 739         } else if (adapter->flags & USING_MSI) {
 740                 pci_disable_msi(adapter->pdev);
 741                 adapter->flags &= ~USING_MSI;
 742         }
 743 }
 744
 745 /*
 746  * Interrupt handler for non-data events used with MSI-X.
 747  */
 748 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 749 {
 750         struct adapter *adap = cookie;
 751
 752         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
 753         if (v & PFSW) {
 754                 adap->swintr = 1;
 755                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
 756         }
 757         t4_slow_intr_handler(adap);
 758         return IRQ_HANDLED;
 759 }
 760
 761 /*
 762  * Name the MSI-X interrupts.
 763  */
 764 static void name_msix_vecs(struct adapter *adap)
 765 {
 766         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 767
 768         /* non-data interrupts */
 769         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 770
 771         /* FW events */
 772         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 773                  adap->port[0]->name);
 774
 775         /* Ethernet queues */
 776         for_each_port(adap, j) {
 777                 struct net_device *d = adap->port[j];
 778                 const struct port_info *pi = netdev_priv(d);
 779
 780                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 781                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 782                                  d->name, i);
 783         }
 784
 785         /* offload queues */
 786         for_each_ofldrxq(&adap->sge, i)
 787                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 788                          adap->port[0]->name, i);
 789
 790         for_each_rdmarxq(&adap->sge, i)
 791                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 792                          adap->port[0]->name, i);
 793 }
 794
 795 static int request_msix_queue_irqs(struct adapter *adap)
 796 {
 797         struct sge *s = &adap->sge;
 798         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi_index = 2;
 799
 800         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 801                           adap->msix_info[1].desc, &s->fw_evtq);
 802         if (err)
 803                 return err;
 804
 805         for_each_ethrxq(s, ethqidx) {
 806                 err = request_irq(adap->msix_info[msi_index].vec,
 807                                   t4_sge_intr_msix, 0,
 808                                   adap->msix_info[msi_index].desc,
 809                                   &s->ethrxq[ethqidx].rspq);
 810                 if (err)
 811                         goto unwind;
 812                 msi_index++;
 813         }
 814         for_each_ofldrxq(s, ofldqidx) {
 815                 err = request_irq(adap->msix_info[msi_index].vec,
 816                                   t4_sge_intr_msix, 0,
 817                                   adap->msix_info[msi_index].desc,
 818                                   &s->ofldrxq[ofldqidx].rspq);
 819                 if (err)
 820                         goto unwind;
 821                 msi_index++;
 822         }
 823         for_each_rdmarxq(s, rdmaqidx) {
 824                 err = request_irq(adap->msix_info[msi_index].vec,
 825                                   t4_sge_intr_msix, 0,
 826                                   adap->msix_info[msi_index].desc,
 827                                   &s->rdmarxq[rdmaqidx].rspq);
 828                 if (err)
 829                         goto unwind;
 830                 msi_index++;
 831         }
 832         return 0;
 833
 834 unwind:
 835         while (--rdmaqidx >= 0)
 836                 free_irq(adap->msix_info[--msi_index].vec,
 837                          &s->rdmarxq[rdmaqidx].rspq);
 838         while (--ofldqidx >= 0)
 839                 free_irq(adap->msix_info[--msi_index].vec,
 840                          &s->ofldrxq[ofldqidx].rspq);
 841         while (--ethqidx >= 0)
 842                 free_irq(adap->msix_info[--msi_index].vec,
 843                          &s->ethrxq[ethqidx].rspq);
 844         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 845         return err;
 846 }
 847
 848 static void free_msix_queue_irqs(struct adapter *adap)
 849 {
 850         int i, msi_index = 2;
 851         struct sge *s = &adap->sge;
 852
 853         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 854         for_each_ethrxq(s, i)
 855                 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
 856         for_each_ofldrxq(s, i)
 857                 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
 858         for_each_rdmarxq(s, i)
 859                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
 860 }
 861
 862 /**
 863  *      write_rss - write the RSS table for a given port
 864  *      @pi: the port
 865  *      @queues: array of queue indices for RSS
 866  *
 867  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 868  *      packets to the Rx queues in @queues.
 869  */
 870 static int write_rss(const struct port_info *pi, const u16 *queues)
 871 {
 872         u16 *rss;
 873         int i, err;
 874         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 875
 876         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 877         if (!rss)
 878                 return -ENOMEM;
 879
 880         /* map the queue indices to queue ids */
 881         for (i = 0; i < pi->rss_size; i++, queues++)
 882                 rss[i] = q[*queues].rspq.abs_id;
 883
 884         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 885                                   pi->rss_size, rss, pi->rss_size);
 886         kfree(rss);
 887         return err;
 888 }
 889
 890 /**
 891  *      setup_rss - configure RSS
 892  *      @adap: the adapter
 893  *
 894  *      Sets up RSS for each port.
 895  */
 896 static int setup_rss(struct adapter *adap)
 897 {
 898         int i, err;
 899
 900         for_each_port(adap, i) {
 901                 const struct port_info *pi = adap2pinfo(adap, i);
 902
 903                 err = write_rss(pi, pi->rss);
 904                 if (err)
 905                         return err;
 906         }
 907         return 0;
 908 }
 909
 910 /*
 911  * Return the channel of the ingress queue with the given qid.
 912  */
 913 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 914 {
 915         qid -= p->ingr_start;
 916         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 917 }
 918
 919 /*
 920  * Wait until all NAPI handlers are descheduled.
 921  */
 922 static void quiesce_rx(struct adapter *adap)
 923 {
 924         int i;
 925
 926         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 927                 struct sge_rspq *q = adap->sge.ingr_map[i];
 928
 929                 if (q && q->handler)
 930                         napi_disable(&q->napi);
 931         }
 932 }
 933
 934 /*
 935  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 936  */
 937 static void enable_rx(struct adapter *adap)
 938 {
 939         int i;
 940
 941         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 942                 struct sge_rspq *q = adap->sge.ingr_map[i];
 943
 944                 if (!q)
 945                         continue;
 946                 if (q->handler)
 947                         napi_enable(&q->napi);
 948                 /* 0-increment GTS to start the timer and enable interrupts */
 949                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
 950                              SEINTARM(q->intr_params) |
 951                              INGRESSQID(q->cntxt_id));
 952         }
 953 }
 954
 955 /**
 956  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 957  *      @adap: the adapter
 958  *
 959  *      Determines how many sets of SGE queues to use and initializes them.
 960  *      We support multiple queue sets per port if we have MSI-X, otherwise
 961  *      just one queue set per port.
 962  */
 963 static int setup_sge_queues(struct adapter *adap)
 964 {
 965         int err, msi_idx, i, j;
 966         struct sge *s = &adap->sge;
 967
 968         bitmap_zero(s->starving_fl, MAX_EGRQ);
 969         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 970
 971         if (adap->flags & USING_MSIX)
 972                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 973         else {
 974                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 975                                        NULL, NULL);
 976                 if (err)
 977                         return err;
 978                 msi_idx = -((int)s->intrq.abs_id + 1);
 979         }
 980
 981         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 982                                msi_idx, NULL, fwevtq_handler);
 983         if (err) {
 984 freeout:        t4_free_sge_resources(adap);
 985                 return err;
 986         }
 987
 988         for_each_port(adap, i) {
 989                 struct net_device *dev = adap->port[i];
 990                 struct port_info *pi = netdev_priv(dev);
 991                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 992                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 993
 994                 for (j = 0; j < pi->nqsets; j++, q++) {
 995                         if (msi_idx > 0)
 996                                 msi_idx++;
 997                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
 998                                                msi_idx, &q->fl,
 999                                                t4_ethrx_handler);
1000                         if (err)
1001                                 goto freeout;
1002                         q->rspq.idx = j;
1003                         memset(&q->stats, 0, sizeof(q->stats));
1004                 }
1005                 for (j = 0; j < pi->nqsets; j++, t++) {
1006                         err = t4_sge_alloc_eth_txq(adap, t, dev,
1007                                         netdev_get_tx_queue(dev, j),
1008                                         s->fw_evtq.cntxt_id);
1009                         if (err)
1010                                 goto freeout;
1011                 }
1012         }
1013
1014         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1015         for_each_ofldrxq(s, i) {
1016                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1017                 struct net_device *dev = adap->port[i / j];
1018
1019                 if (msi_idx > 0)
1020                         msi_idx++;
1021                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1022                                        &q->fl, uldrx_handler);
1023                 if (err)
1024                         goto freeout;
1025                 memset(&q->stats, 0, sizeof(q->stats));
1026                 s->ofld_rxq[i] = q->rspq.abs_id;
1027                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1028                                             s->fw_evtq.cntxt_id);
1029                 if (err)
1030                         goto freeout;
1031         }
1032
1033         for_each_rdmarxq(s, i) {
1034                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1035
1036                 if (msi_idx > 0)
1037                         msi_idx++;
1038                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1039                                        msi_idx, &q->fl, uldrx_handler);
1040                 if (err)
1041                         goto freeout;
1042                 memset(&q->stats, 0, sizeof(q->stats));
1043                 s->rdma_rxq[i] = q->rspq.abs_id;
1044         }
1045
1046         for_each_port(adap, i) {
1047                 /*
1048                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1049                  * have RDMA queues, and that's the right value.
1050                  */
1051                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1052                                             s->fw_evtq.cntxt_id,
1053                                             s->rdmarxq[i].rspq.cntxt_id);
1054                 if (err)
1055                         goto freeout;
1056         }
1057
1058         t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
1059                      RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1060                      QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1061         return 0;
1062 }
1063
1064 /*
1065  * Returns 0 if new FW was successfully loaded, a positive errno if a load was
1066  * started but failed, and a negative errno if flash load couldn't start.
1067  */
1068 static int upgrade_fw(struct adapter *adap)
1069 {
1070         int ret;
1071         u32 vers, exp_major;
1072         const struct fw_hdr *hdr;
1073         const struct firmware *fw;
1074         struct device *dev = adap->pdev_dev;
1075         char *fw_file_name;
1076
1077         switch (CHELSIO_CHIP_VERSION(adap->chip)) {
1078         case CHELSIO_T4:
1079                 fw_file_name = FW_FNAME;
1080                 exp_major = FW_VERSION_MAJOR;
1081                 break;
1082         case CHELSIO_T5:
1083                 fw_file_name = FW5_FNAME;
1084                 exp_major = FW_VERSION_MAJOR_T5;
1085                 break;
1086         default:
1087                 dev_err(dev, "Unsupported chip type, %x\n", adap->chip);
1088                 return -EINVAL;
1089         }
1090
1091         ret = request_firmware(&fw, fw_file_name, dev);
1092         if (ret < 0) {
1093                 dev_err(dev, "unable to load firmware image %s, error %d\n",
1094                         fw_file_name, ret);
1095                 return ret;
1096         }
1097
1098         hdr = (const struct fw_hdr *)fw->data;
1099         vers = ntohl(hdr->fw_ver);
1100         if (FW_HDR_FW_VER_MAJOR_GET(vers) != exp_major) {
1101                 ret = -EINVAL;              /* wrong major version, won't do */
1102                 goto out;
1103         }
1104
1105         /*
1106          * If the flash FW is unusable or we found something newer, load it.
1107          */
1108         if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != exp_major ||
1109             vers > adap->params.fw_vers) {
1110                 dev_info(dev, "upgrading firmware ...\n");
1111                 ret = t4_fw_upgrade(adap, adap->mbox, fw->data, fw->size,
1112                                     /*force=*/false);
1113                 if (!ret)
1114                         dev_info(dev,
1115                                  "firmware upgraded to version %pI4 from %s\n",
1116                                  &hdr->fw_ver, fw_file_name);
1117                 else
1118                         dev_err(dev, "firmware upgrade failed! err=%d\n", -ret);
1119         } else {
1120                 /*
1121                  * Tell our caller that we didn't upgrade the firmware.
1122                  */
1123                 ret = -EINVAL;
1124         }
1125
1126 out:    release_firmware(fw);
1127         return ret;
1128 }
1129
1130 /*
1131  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1132  * The allocated memory is cleared.
1133  */
1134 void *t4_alloc_mem(size_t size)
1135 {
1136         void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1137
1138         if (!p)
1139                 p = vzalloc(size);
1140         return p;
1141 }
1142
1143 /*
1144  * Free memory allocated through alloc_mem().
1145  */
1146 static void t4_free_mem(void *addr)
1147 {
1148         if (is_vmalloc_addr(addr))
1149                 vfree(addr);
1150         else
1151                 kfree(addr);
1152 }
1153
1154 /* Send a Work Request to write the filter at a specified index.  We construct
1155  * a Firmware Filter Work Request to have the work done and put the indicated
1156  * filter into "pending" mode which will prevent any further actions against
1157  * it till we get a reply from the firmware on the completion status of the
1158  * request.
1159  */
1160 static int set_filter_wr(struct adapter *adapter, int fidx)
1161 {
1162         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1163         struct sk_buff *skb;
1164         struct fw_filter_wr *fwr;
1165         unsigned int ftid;
1166
1167         /* If the new filter requires loopback Destination MAC and/or VLAN
1168          * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1169          * the filter.
1170          */
1171         if (f->fs.newdmac || f->fs.newvlan) {
1172                 /* allocate L2T entry for new filter */
1173                 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1174                 if (f->l2t == NULL)
1175                         return -EAGAIN;
1176                 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1177                                         f->fs.eport, f->fs.dmac)) {
1178                         cxgb4_l2t_release(f->l2t);
1179                         f->l2t = NULL;
1180                         return -ENOMEM;
1181                 }
1182         }
1183
1184         ftid = adapter->tids.ftid_base + fidx;
1185
1186         skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1187         fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1188         memset(fwr, 0, sizeof(*fwr));
1189
1190         /* It would be nice to put most of the following in t4_hw.c but most
1191          * of the work is translating the cxgbtool ch_filter_specification
1192          * into the Work Request and the definition of that structure is
1193          * currently in cxgbtool.h which isn't appropriate to pull into the
1194          * common code.  We may eventually try to come up with a more neutral
1195          * filter specification structure but for now it's easiest to simply
1196          * put this fairly direct code in line ...
1197          */
1198         fwr->op_pkd = htonl(FW_WR_OP(FW_FILTER_WR));
1199         fwr->len16_pkd = htonl(FW_WR_LEN16(sizeof(*fwr)/16));
1200         fwr->tid_to_iq =
1201                 htonl(V_FW_FILTER_WR_TID(ftid) |
1202                       V_FW_FILTER_WR_RQTYPE(f->fs.type) |
1203                       V_FW_FILTER_WR_NOREPLY(0) |
1204                       V_FW_FILTER_WR_IQ(f->fs.iq));
1205         fwr->del_filter_to_l2tix =
1206                 htonl(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
1207                       V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
1208                       V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
1209                       V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
1210                       V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
1211                       V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
1212                       V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
1213                       V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
1214                       V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
1215                                              f->fs.newvlan == VLAN_REWRITE) |
1216                       V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
1217                                             f->fs.newvlan == VLAN_REWRITE) |
1218                       V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
1219                       V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
1220                       V_FW_FILTER_WR_PRIO(f->fs.prio) |
1221                       V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
1222         fwr->ethtype = htons(f->fs.val.ethtype);
1223         fwr->ethtypem = htons(f->fs.mask.ethtype);
1224         fwr->frag_to_ovlan_vldm =
1225                 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
1226                  V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
1227                  V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
1228                  V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
1229                  V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
1230                  V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
1231         fwr->smac_sel = 0;
1232         fwr->rx_chan_rx_rpl_iq =
1233                 htons(V_FW_FILTER_WR_RX_CHAN(0) |
1234                       V_FW_FILTER_WR_RX_RPL_IQ(adapter->sge.fw_evtq.abs_id));
1235         fwr->maci_to_matchtypem =
1236                 htonl(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
1237                       V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
1238                       V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
1239                       V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
1240                       V_FW_FILTER_WR_PORT(f->fs.val.iport) |
1241                       V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
1242                       V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
1243                       V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
1244         fwr->ptcl = f->fs.val.proto;
1245         fwr->ptclm = f->fs.mask.proto;
1246         fwr->ttyp = f->fs.val.tos;
1247         fwr->ttypm = f->fs.mask.tos;
1248         fwr->ivlan = htons(f->fs.val.ivlan);
1249         fwr->ivlanm = htons(f->fs.mask.ivlan);
1250         fwr->ovlan = htons(f->fs.val.ovlan);
1251         fwr->ovlanm = htons(f->fs.mask.ovlan);
1252         memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1253         memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1254         memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1255         memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1256         fwr->lp = htons(f->fs.val.lport);
1257         fwr->lpm = htons(f->fs.mask.lport);
1258         fwr->fp = htons(f->fs.val.fport);
1259         fwr->fpm = htons(f->fs.mask.fport);
1260         if (f->fs.newsmac)
1261                 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1262
1263         /* Mark the filter as "pending" and ship off the Filter Work Request.
1264          * When we get the Work Request Reply we'll clear the pending status.
1265          */
1266         f->pending = 1;
1267         set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1268         t4_ofld_send(adapter, skb);
1269         return 0;
1270 }
1271
1272 /* Delete the filter at a specified index.
1273  */
1274 static int del_filter_wr(struct adapter *adapter, int fidx)
1275 {
1276         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1277         struct sk_buff *skb;
1278         struct fw_filter_wr *fwr;
1279         unsigned int len, ftid;
1280
1281         len = sizeof(*fwr);
1282         ftid = adapter->tids.ftid_base + fidx;
1283
1284         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1285         fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1286         t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1287
1288         /* Mark the filter as "pending" and ship off the Filter Work Request.
1289          * When we get the Work Request Reply we'll clear the pending status.
1290          */
1291         f->pending = 1;
1292         t4_mgmt_tx(adapter, skb);
1293         return 0;
1294 }
1295
1296 static inline int is_offload(const struct adapter *adap)
1297 {
1298         return adap->params.offload;
1299 }
1300
1301 /*
1302  * Implementation of ethtool operations.
1303  */
1304
1305 static u32 get_msglevel(struct net_device *dev)
1306 {
1307         return netdev2adap(dev)->msg_enable;
1308 }
1309
1310 static void set_msglevel(struct net_device *dev, u32 val)
1311 {
1312         netdev2adap(dev)->msg_enable = val;
1313 }
1314
1315 static char stats_strings[][ETH_GSTRING_LEN] = {
1316         "TxOctetsOK         ",
1317         "TxFramesOK         ",
1318         "TxBroadcastFrames  ",
1319         "TxMulticastFrames  ",
1320         "TxUnicastFrames    ",
1321         "TxErrorFrames      ",
1322
1323         "TxFrames64         ",
1324         "TxFrames65To127    ",
1325         "TxFrames128To255   ",
1326         "TxFrames256To511   ",
1327         "TxFrames512To1023  ",
1328         "TxFrames1024To1518 ",
1329         "TxFrames1519ToMax  ",
1330
1331         "TxFramesDropped    ",
1332         "TxPauseFrames      ",
1333         "TxPPP0Frames       ",
1334         "TxPPP1Frames       ",
1335         "TxPPP2Frames       ",
1336         "TxPPP3Frames       ",
1337         "TxPPP4Frames       ",
1338         "TxPPP5Frames       ",
1339         "TxPPP6Frames       ",
1340         "TxPPP7Frames       ",
1341
1342         "RxOctetsOK         ",
1343         "RxFramesOK         ",
1344         "RxBroadcastFrames  ",
1345         "RxMulticastFrames  ",
1346         "RxUnicastFrames    ",
1347
1348         "RxFramesTooLong    ",
1349         "RxJabberErrors     ",
1350         "RxFCSErrors        ",
1351         "RxLengthErrors     ",
1352         "RxSymbolErrors     ",
1353         "RxRuntFrames       ",
1354
1355         "RxFrames64         ",
1356         "RxFrames65To127    ",
1357         "RxFrames128To255   ",
1358         "RxFrames256To511   ",
1359         "RxFrames512To1023  ",
1360         "RxFrames1024To1518 ",
1361         "RxFrames1519ToMax  ",
1362
1363         "RxPauseFrames      ",
1364         "RxPPP0Frames       ",
1365         "RxPPP1Frames       ",
1366         "RxPPP2Frames       ",
1367         "RxPPP3Frames       ",
1368         "RxPPP4Frames       ",
1369         "RxPPP5Frames       ",
1370         "RxPPP6Frames       ",
1371         "RxPPP7Frames       ",
1372
1373         "RxBG0FramesDropped ",
1374         "RxBG1FramesDropped ",
1375         "RxBG2FramesDropped ",
1376         "RxBG3FramesDropped ",
1377         "RxBG0FramesTrunc   ",
1378         "RxBG1FramesTrunc   ",
1379         "RxBG2FramesTrunc   ",
1380         "RxBG3FramesTrunc   ",
1381
1382         "TSO                ",
1383         "TxCsumOffload      ",
1384         "RxCsumGood         ",
1385         "VLANextractions    ",
1386         "VLANinsertions     ",
1387         "GROpackets         ",
1388         "GROmerged          ",
1389         "WriteCoalSuccess   ",
1390         "WriteCoalFail      ",
1391 };
1392
1393 static int get_sset_count(struct net_device *dev, int sset)
1394 {
1395         switch (sset) {
1396         case ETH_SS_STATS:
1397                 return ARRAY_SIZE(stats_strings);
1398         default:
1399                 return -EOPNOTSUPP;
1400         }
1401 }
1402
1403 #define T4_REGMAP_SIZE (160 * 1024)
1404 #define T5_REGMAP_SIZE (332 * 1024)
1405
1406 static int get_regs_len(struct net_device *dev)
1407 {
1408         struct adapter *adap = netdev2adap(dev);
1409         if (is_t4(adap->chip))
1410                 return T4_REGMAP_SIZE;
1411         else
1412                 return T5_REGMAP_SIZE;
1413 }
1414
1415 static int get_eeprom_len(struct net_device *dev)
1416 {
1417         return EEPROMSIZE;
1418 }
1419
1420 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1421 {
1422         struct adapter *adapter = netdev2adap(dev);
1423
1424         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1425         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1426         strlcpy(info->bus_info, pci_name(adapter->pdev),
1427                 sizeof(info->bus_info));
1428
1429         if (adapter->params.fw_vers)
1430                 snprintf(info->fw_version, sizeof(info->fw_version),
1431                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1432                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1433                         FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1434                         FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1435                         FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1436                         FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1437                         FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1438                         FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1439                         FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1440 }
1441
1442 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1443 {
1444         if (stringset == ETH_SS_STATS)
1445                 memcpy(data, stats_strings, sizeof(stats_strings));
1446 }
1447
1448 /*
1449  * port stats maintained per queue of the port.  They should be in the same
1450  * order as in stats_strings above.
1451  */
1452 struct queue_port_stats {
1453         u64 tso;
1454         u64 tx_csum;
1455         u64 rx_csum;
1456         u64 vlan_ex;
1457         u64 vlan_ins;
1458         u64 gro_pkts;
1459         u64 gro_merged;
1460 };
1461
1462 static void collect_sge_port_stats(const struct adapter *adap,
1463                 const struct port_info *p, struct queue_port_stats *s)
1464 {
1465         int i;
1466         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1467         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1468
1469         memset(s, 0, sizeof(*s));
1470         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1471                 s->tso += tx->tso;
1472                 s->tx_csum += tx->tx_cso;
1473                 s->rx_csum += rx->stats.rx_cso;
1474                 s->vlan_ex += rx->stats.vlan_ex;
1475                 s->vlan_ins += tx->vlan_ins;
1476                 s->gro_pkts += rx->stats.lro_pkts;
1477                 s->gro_merged += rx->stats.lro_merged;
1478         }
1479 }
1480
1481 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1482                       u64 *data)
1483 {
1484         struct port_info *pi = netdev_priv(dev);
1485         struct adapter *adapter = pi->adapter;
1486         u32 val1, val2;
1487
1488         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1489
1490         data += sizeof(struct port_stats) / sizeof(u64);
1491         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1492         data += sizeof(struct queue_port_stats) / sizeof(u64);
1493         if (!is_t4(adapter->chip)) {
1494                 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1495                 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1496                 val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1497                 *data = val1 - val2;
1498                 data++;
1499                 *data = val2;
1500                 data++;
1501         } else {
1502                 memset(data, 0, 2 * sizeof(u64));
1503                 *data += 2;
1504         }
1505 }
1506
1507 /*
1508  * Return a version number to identify the type of adapter.  The scheme is:
1509  * - bits 0..9: chip version
1510  * - bits 10..15: chip revision
1511  * - bits 16..23: register dump version
1512  */
1513 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1514 {
1515         return CHELSIO_CHIP_VERSION(ap->chip) |
1516                 (CHELSIO_CHIP_RELEASE(ap->chip) << 10) | (1 << 16);
1517 }
1518
1519 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1520                            unsigned int end)
1521 {
1522         u32 *p = buf + start;
1523
1524         for ( ; start <= end; start += sizeof(u32))
1525                 *p++ = t4_read_reg(ap, start);
1526 }
1527
1528 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1529                      void *buf)
1530 {
1531         static const unsigned int t4_reg_ranges[] = {
1532                 0x1008, 0x1108,
1533                 0x1180, 0x11b4,
1534                 0x11fc, 0x123c,
1535                 0x1300, 0x173c,
1536                 0x1800, 0x18fc,
1537                 0x3000, 0x30d8,
1538                 0x30e0, 0x5924,
1539                 0x5960, 0x59d4,
1540                 0x5a00, 0x5af8,
1541                 0x6000, 0x6098,
1542                 0x6100, 0x6150,
1543                 0x6200, 0x6208,
1544                 0x6240, 0x6248,
1545                 0x6280, 0x6338,
1546                 0x6370, 0x638c,
1547                 0x6400, 0x643c,
1548                 0x6500, 0x6524,
1549                 0x6a00, 0x6a38,
1550                 0x6a60, 0x6a78,
1551                 0x6b00, 0x6b84,
1552                 0x6bf0, 0x6c84,
1553                 0x6cf0, 0x6d84,
1554                 0x6df0, 0x6e84,
1555                 0x6ef0, 0x6f84,
1556                 0x6ff0, 0x7084,
1557                 0x70f0, 0x7184,
1558                 0x71f0, 0x7284,
1559                 0x72f0, 0x7384,
1560                 0x73f0, 0x7450,
1561                 0x7500, 0x7530,
1562                 0x7600, 0x761c,
1563                 0x7680, 0x76cc,
1564                 0x7700, 0x7798,
1565                 0x77c0, 0x77fc,
1566                 0x7900, 0x79fc,
1567                 0x7b00, 0x7c38,
1568                 0x7d00, 0x7efc,
1569                 0x8dc0, 0x8e1c,
1570                 0x8e30, 0x8e78,
1571                 0x8ea0, 0x8f6c,
1572                 0x8fc0, 0x9074,
1573                 0x90fc, 0x90fc,
1574                 0x9400, 0x9458,
1575                 0x9600, 0x96bc,
1576                 0x9800, 0x9808,
1577                 0x9820, 0x983c,
1578                 0x9850, 0x9864,
1579                 0x9c00, 0x9c6c,
1580                 0x9c80, 0x9cec,
1581                 0x9d00, 0x9d6c,
1582                 0x9d80, 0x9dec,
1583                 0x9e00, 0x9e6c,
1584                 0x9e80, 0x9eec,
1585                 0x9f00, 0x9f6c,
1586                 0x9f80, 0x9fec,
1587                 0xd004, 0xd03c,
1588                 0xdfc0, 0xdfe0,
1589                 0xe000, 0xea7c,
1590                 0xf000, 0x11190,
1591                 0x19040, 0x1906c,
1592                 0x19078, 0x19080,
1593                 0x1908c, 0x19124,
1594                 0x19150, 0x191b0,
1595                 0x191d0, 0x191e8,
1596                 0x19238, 0x1924c,
1597                 0x193f8, 0x19474,
1598                 0x19490, 0x194f8,
1599                 0x19800, 0x19f30,
1600                 0x1a000, 0x1a06c,
1601                 0x1a0b0, 0x1a120,
1602                 0x1a128, 0x1a138,
1603                 0x1a190, 0x1a1c4,
1604                 0x1a1fc, 0x1a1fc,
1605                 0x1e040, 0x1e04c,
1606                 0x1e284, 0x1e28c,
1607                 0x1e2c0, 0x1e2c0,
1608                 0x1e2e0, 0x1e2e0,
1609                 0x1e300, 0x1e384,
1610                 0x1e3c0, 0x1e3c8,
1611                 0x1e440, 0x1e44c,
1612                 0x1e684, 0x1e68c,
1613                 0x1e6c0, 0x1e6c0,
1614                 0x1e6e0, 0x1e6e0,
1615                 0x1e700, 0x1e784,
1616                 0x1e7c0, 0x1e7c8,
1617                 0x1e840, 0x1e84c,
1618                 0x1ea84, 0x1ea8c,
1619                 0x1eac0, 0x1eac0,
1620                 0x1eae0, 0x1eae0,
1621                 0x1eb00, 0x1eb84,
1622                 0x1ebc0, 0x1ebc8,
1623                 0x1ec40, 0x1ec4c,
1624                 0x1ee84, 0x1ee8c,
1625                 0x1eec0, 0x1eec0,
1626                 0x1eee0, 0x1eee0,
1627                 0x1ef00, 0x1ef84,
1628                 0x1efc0, 0x1efc8,
1629                 0x1f040, 0x1f04c,
1630                 0x1f284, 0x1f28c,
1631                 0x1f2c0, 0x1f2c0,
1632                 0x1f2e0, 0x1f2e0,
1633                 0x1f300, 0x1f384,
1634                 0x1f3c0, 0x1f3c8,
1635                 0x1f440, 0x1f44c,
1636                 0x1f684, 0x1f68c,
1637                 0x1f6c0, 0x1f6c0,
1638                 0x1f6e0, 0x1f6e0,
1639                 0x1f700, 0x1f784,
1640                 0x1f7c0, 0x1f7c8,
1641                 0x1f840, 0x1f84c,
1642                 0x1fa84, 0x1fa8c,
1643                 0x1fac0, 0x1fac0,
1644                 0x1fae0, 0x1fae0,
1645                 0x1fb00, 0x1fb84,
1646                 0x1fbc0, 0x1fbc8,
1647                 0x1fc40, 0x1fc4c,
1648                 0x1fe84, 0x1fe8c,
1649                 0x1fec0, 0x1fec0,
1650                 0x1fee0, 0x1fee0,
1651                 0x1ff00, 0x1ff84,
1652                 0x1ffc0, 0x1ffc8,
1653                 0x20000, 0x2002c,
1654                 0x20100, 0x2013c,
1655                 0x20190, 0x201c8,
1656                 0x20200, 0x20318,
1657                 0x20400, 0x20528,
1658                 0x20540, 0x20614,
1659                 0x21000, 0x21040,
1660                 0x2104c, 0x21060,
1661                 0x210c0, 0x210ec,
1662                 0x21200, 0x21268,
1663                 0x21270, 0x21284,
1664                 0x212fc, 0x21388,
1665                 0x21400, 0x21404,
1666                 0x21500, 0x21518,
1667                 0x2152c, 0x2153c,
1668                 0x21550, 0x21554,
1669                 0x21600, 0x21600,
1670                 0x21608, 0x21628,
1671                 0x21630, 0x2163c,
1672                 0x21700, 0x2171c,
1673                 0x21780, 0x2178c,
1674                 0x21800, 0x21c38,
1675                 0x21c80, 0x21d7c,
1676                 0x21e00, 0x21e04,
1677                 0x22000, 0x2202c,
1678                 0x22100, 0x2213c,
1679                 0x22190, 0x221c8,
1680                 0x22200, 0x22318,
1681                 0x22400, 0x22528,
1682                 0x22540, 0x22614,
1683                 0x23000, 0x23040,
1684                 0x2304c, 0x23060,
1685                 0x230c0, 0x230ec,
1686                 0x23200, 0x23268,
1687                 0x23270, 0x23284,
1688                 0x232fc, 0x23388,
1689                 0x23400, 0x23404,
1690                 0x23500, 0x23518,
1691                 0x2352c, 0x2353c,
1692                 0x23550, 0x23554,
1693                 0x23600, 0x23600,
1694                 0x23608, 0x23628,
1695                 0x23630, 0x2363c,
1696                 0x23700, 0x2371c,
1697                 0x23780, 0x2378c,
1698                 0x23800, 0x23c38,
1699                 0x23c80, 0x23d7c,
1700                 0x23e00, 0x23e04,
1701                 0x24000, 0x2402c,
1702                 0x24100, 0x2413c,
1703                 0x24190, 0x241c8,
1704                 0x24200, 0x24318,
1705                 0x24400, 0x24528,
1706                 0x24540, 0x24614,
1707                 0x25000, 0x25040,
1708                 0x2504c, 0x25060,
1709                 0x250c0, 0x250ec,
1710                 0x25200, 0x25268,
1711                 0x25270, 0x25284,
1712                 0x252fc, 0x25388,
1713                 0x25400, 0x25404,
1714                 0x25500, 0x25518,
1715                 0x2552c, 0x2553c,
1716                 0x25550, 0x25554,
1717                 0x25600, 0x25600,
1718                 0x25608, 0x25628,
1719                 0x25630, 0x2563c,
1720                 0x25700, 0x2571c,
1721                 0x25780, 0x2578c,
1722                 0x25800, 0x25c38,
1723                 0x25c80, 0x25d7c,
1724                 0x25e00, 0x25e04,
1725                 0x26000, 0x2602c,
1726                 0x26100, 0x2613c,
1727                 0x26190, 0x261c8,
1728                 0x26200, 0x26318,
1729                 0x26400, 0x26528,
1730                 0x26540, 0x26614,
1731                 0x27000, 0x27040,
1732                 0x2704c, 0x27060,
1733                 0x270c0, 0x270ec,
1734                 0x27200, 0x27268,
1735                 0x27270, 0x27284,
1736                 0x272fc, 0x27388,
1737                 0x27400, 0x27404,
1738                 0x27500, 0x27518,
1739                 0x2752c, 0x2753c,
1740                 0x27550, 0x27554,
1741                 0x27600, 0x27600,
1742                 0x27608, 0x27628,
1743                 0x27630, 0x2763c,
1744                 0x27700, 0x2771c,
1745                 0x27780, 0x2778c,
1746                 0x27800, 0x27c38,
1747                 0x27c80, 0x27d7c,
1748                 0x27e00, 0x27e04
1749         };
1750
1751         static const unsigned int t5_reg_ranges[] = {
1752                 0x1008, 0x1148,
1753                 0x1180, 0x11b4,
1754                 0x11fc, 0x123c,
1755                 0x1280, 0x173c,
1756                 0x1800, 0x18fc,
1757                 0x3000, 0x3028,
1758                 0x3060, 0x30d8,
1759                 0x30e0, 0x30fc,
1760                 0x3140, 0x357c,
1761                 0x35a8, 0x35cc,
1762                 0x35ec, 0x35ec,
1763                 0x3600, 0x5624,
1764                 0x56cc, 0x575c,
1765                 0x580c, 0x5814,
1766                 0x5890, 0x58bc,
1767                 0x5940, 0x59dc,
1768                 0x59fc, 0x5a18,
1769                 0x5a60, 0x5a9c,
1770                 0x5b9c, 0x5bfc,
1771                 0x6000, 0x6040,
1772                 0x6058, 0x614c,
1773                 0x7700, 0x7798,
1774                 0x77c0, 0x78fc,
1775                 0x7b00, 0x7c54,
1776                 0x7d00, 0x7efc,
1777                 0x8dc0, 0x8de0,
1778                 0x8df8, 0x8e84,
1779                 0x8ea0, 0x8f84,
1780                 0x8fc0, 0x90f8,
1781                 0x9400, 0x9470,
1782                 0x9600, 0x96f4,
1783                 0x9800, 0x9808,
1784                 0x9820, 0x983c,
1785                 0x9850, 0x9864,
1786                 0x9c00, 0x9c6c,
1787                 0x9c80, 0x9cec,
1788                 0x9d00, 0x9d6c,
1789                 0x9d80, 0x9dec,
1790                 0x9e00, 0x9e6c,
1791                 0x9e80, 0x9eec,
1792                 0x9f00, 0x9f6c,
1793                 0x9f80, 0xa020,
1794                 0xd004, 0xd03c,
1795                 0xdfc0, 0xdfe0,
1796                 0xe000, 0x11088,
1797                 0x1109c, 0x1117c,
1798                 0x11190, 0x11204,
1799                 0x19040, 0x1906c,
1800                 0x19078, 0x19080,
1801                 0x1908c, 0x19124,
1802                 0x19150, 0x191b0,
1803                 0x191d0, 0x191e8,
1804                 0x19238, 0x19290,
1805                 0x193f8, 0x19474,
1806                 0x19490, 0x194cc,
1807                 0x194f0, 0x194f8,
1808                 0x19c00, 0x19c60,
1809                 0x19c94, 0x19e10,
1810                 0x19e50, 0x19f34,
1811                 0x19f40, 0x19f50,
1812                 0x19f90, 0x19fe4,
1813                 0x1a000, 0x1a06c,
1814                 0x1a0b0, 0x1a120,
1815                 0x1a128, 0x1a138,
1816                 0x1a190, 0x1a1c4,
1817                 0x1a1fc, 0x1a1fc,
1818                 0x1e008, 0x1e00c,
1819                 0x1e040, 0x1e04c,
1820                 0x1e284, 0x1e290,
1821                 0x1e2c0, 0x1e2c0,
1822                 0x1e2e0, 0x1e2e0,
1823                 0x1e300, 0x1e384,
1824                 0x1e3c0, 0x1e3c8,
1825                 0x1e408, 0x1e40c,
1826                 0x1e440, 0x1e44c,
1827                 0x1e684, 0x1e690,
1828                 0x1e6c0, 0x1e6c0,
1829                 0x1e6e0, 0x1e6e0,
1830                 0x1e700, 0x1e784,
1831                 0x1e7c0, 0x1e7c8,
1832                 0x1e808, 0x1e80c,
1833                 0x1e840, 0x1e84c,
1834                 0x1ea84, 0x1ea90,
1835                 0x1eac0, 0x1eac0,
1836                 0x1eae0, 0x1eae0,
1837                 0x1eb00, 0x1eb84,
1838                 0x1ebc0, 0x1ebc8,
1839                 0x1ec08, 0x1ec0c,
1840                 0x1ec40, 0x1ec4c,
1841                 0x1ee84, 0x1ee90,
1842                 0x1eec0, 0x1eec0,
1843                 0x1eee0, 0x1eee0,
1844                 0x1ef00, 0x1ef84,
1845                 0x1efc0, 0x1efc8,
1846                 0x1f008, 0x1f00c,
1847                 0x1f040, 0x1f04c,
1848                 0x1f284, 0x1f290,
1849                 0x1f2c0, 0x1f2c0,
1850                 0x1f2e0, 0x1f2e0,
1851                 0x1f300, 0x1f384,
1852                 0x1f3c0, 0x1f3c8,
1853                 0x1f408, 0x1f40c,
1854                 0x1f440, 0x1f44c,
1855                 0x1f684, 0x1f690,
1856                 0x1f6c0, 0x1f6c0,
1857                 0x1f6e0, 0x1f6e0,
1858                 0x1f700, 0x1f784,
1859                 0x1f7c0, 0x1f7c8,
1860                 0x1f808, 0x1f80c,
1861                 0x1f840, 0x1f84c,
1862                 0x1fa84, 0x1fa90,
1863                 0x1fac0, 0x1fac0,
1864                 0x1fae0, 0x1fae0,
1865                 0x1fb00, 0x1fb84,
1866                 0x1fbc0, 0x1fbc8,
1867                 0x1fc08, 0x1fc0c,
1868                 0x1fc40, 0x1fc4c,
1869                 0x1fe84, 0x1fe90,
1870                 0x1fec0, 0x1fec0,
1871                 0x1fee0, 0x1fee0,
1872                 0x1ff00, 0x1ff84,
1873                 0x1ffc0, 0x1ffc8,
1874                 0x30000, 0x30030,
1875                 0x30100, 0x30144,
1876                 0x30190, 0x301d0,
1877                 0x30200, 0x30318,
1878                 0x30400, 0x3052c,
1879                 0x30540, 0x3061c,
1880                 0x30800, 0x30834,
1881                 0x308c0, 0x30908,
1882                 0x30910, 0x309ac,
1883                 0x30a00, 0x30a04,
1884                 0x30a0c, 0x30a2c,
1885                 0x30a44, 0x30a50,
1886                 0x30a74, 0x30c24,
1887                 0x30d08, 0x30d14,
1888                 0x30d1c, 0x30d20,
1889                 0x30d3c, 0x30d50,
1890                 0x31200, 0x3120c,
1891                 0x31220, 0x31220,
1892                 0x31240, 0x31240,
1893                 0x31600, 0x31600,
1894                 0x31608, 0x3160c,
1895                 0x31a00, 0x31a1c,
1896                 0x31e04, 0x31e20,
1897                 0x31e38, 0x31e3c,
1898                 0x31e80, 0x31e80,
1899                 0x31e88, 0x31ea8,
1900                 0x31eb0, 0x31eb4,
1901                 0x31ec8, 0x31ed4,
1902                 0x31fb8, 0x32004,
1903                 0x32208, 0x3223c,
1904                 0x32600, 0x32630,
1905                 0x32a00, 0x32abc,
1906                 0x32b00, 0x32b70,
1907                 0x33000, 0x33048,
1908                 0x33060, 0x3309c,
1909                 0x330f0, 0x33148,
1910                 0x33160, 0x3319c,
1911                 0x331f0, 0x332e4,
1912                 0x332f8, 0x333e4,
1913                 0x333f8, 0x33448,
1914                 0x33460, 0x3349c,
1915                 0x334f0, 0x33548,
1916                 0x33560, 0x3359c,
1917                 0x335f0, 0x336e4,
1918                 0x336f8, 0x337e4,
1919                 0x337f8, 0x337fc,
1920                 0x33814, 0x33814,
1921                 0x3382c, 0x3382c,
1922                 0x33880, 0x3388c,
1923                 0x338e8, 0x338ec,
1924                 0x33900, 0x33948,
1925                 0x33960, 0x3399c,
1926                 0x339f0, 0x33ae4,
1927                 0x33af8, 0x33b10,
1928                 0x33b28, 0x33b28,
1929                 0x33b3c, 0x33b50,
1930                 0x33bf0, 0x33c10,
1931                 0x33c28, 0x33c28,
1932                 0x33c3c, 0x33c50,
1933                 0x33cf0, 0x33cfc,
1934                 0x34000, 0x34030,
1935                 0x34100, 0x34144,
1936                 0x34190, 0x341d0,
1937                 0x34200, 0x34318,
1938                 0x34400, 0x3452c,
1939                 0x34540, 0x3461c,
1940                 0x34800, 0x34834,
1941                 0x348c0, 0x34908,
1942                 0x34910, 0x349ac,
1943                 0x34a00, 0x34a04,
1944                 0x34a0c, 0x34a2c,
1945                 0x34a44, 0x34a50,
1946                 0x34a74, 0x34c24,
1947                 0x34d08, 0x34d14,
1948                 0x34d1c, 0x34d20,
1949                 0x34d3c, 0x34d50,
1950                 0x35200, 0x3520c,
1951                 0x35220, 0x35220,
1952                 0x35240, 0x35240,
1953                 0x35600, 0x35600,
1954                 0x35608, 0x3560c,
1955                 0x35a00, 0x35a1c,
1956                 0x35e04, 0x35e20,
1957                 0x35e38, 0x35e3c,
1958                 0x35e80, 0x35e80,
1959                 0x35e88, 0x35ea8,
1960                 0x35eb0, 0x35eb4,
1961                 0x35ec8, 0x35ed4,
1962                 0x35fb8, 0x36004,
1963                 0x36208, 0x3623c,
1964                 0x36600, 0x36630,
1965                 0x36a00, 0x36abc,
1966                 0x36b00, 0x36b70,
1967                 0x37000, 0x37048,
1968                 0x37060, 0x3709c,
1969                 0x370f0, 0x37148,
1970                 0x37160, 0x3719c,
1971                 0x371f0, 0x372e4,
1972                 0x372f8, 0x373e4,
1973                 0x373f8, 0x37448,
1974                 0x37460, 0x3749c,
1975                 0x374f0, 0x37548,
1976                 0x37560, 0x3759c,
1977                 0x375f0, 0x376e4,
1978                 0x376f8, 0x377e4,
1979                 0x377f8, 0x377fc,
1980                 0x37814, 0x37814,
1981                 0x3782c, 0x3782c,
1982                 0x37880, 0x3788c,
1983                 0x378e8, 0x378ec,
1984                 0x37900, 0x37948,
1985                 0x37960, 0x3799c,
1986                 0x379f0, 0x37ae4,
1987                 0x37af8, 0x37b10,
1988                 0x37b28, 0x37b28,
1989                 0x37b3c, 0x37b50,
1990                 0x37bf0, 0x37c10,
1991                 0x37c28, 0x37c28,
1992                 0x37c3c, 0x37c50,
1993                 0x37cf0, 0x37cfc,
1994                 0x38000, 0x38030,
1995                 0x38100, 0x38144,
1996                 0x38190, 0x381d0,
1997                 0x38200, 0x38318,
1998                 0x38400, 0x3852c,
1999                 0x38540, 0x3861c,
2000                 0x38800, 0x38834,
2001                 0x388c0, 0x38908,
2002                 0x38910, 0x389ac,
2003                 0x38a00, 0x38a04,
2004                 0x38a0c, 0x38a2c,
2005                 0x38a44, 0x38a50,
2006                 0x38a74, 0x38c24,
2007                 0x38d08, 0x38d14,
2008                 0x38d1c, 0x38d20,
2009                 0x38d3c, 0x38d50,
2010                 0x39200, 0x3920c,
2011                 0x39220, 0x39220,
2012                 0x39240, 0x39240,
2013                 0x39600, 0x39600,
2014                 0x39608, 0x3960c,
2015                 0x39a00, 0x39a1c,
2016                 0x39e04, 0x39e20,
2017                 0x39e38, 0x39e3c,
2018                 0x39e80, 0x39e80,
2019                 0x39e88, 0x39ea8,
2020                 0x39eb0, 0x39eb4,
2021                 0x39ec8, 0x39ed4,
2022                 0x39fb8, 0x3a004,
2023                 0x3a208, 0x3a23c,
2024                 0x3a600, 0x3a630,
2025                 0x3aa00, 0x3aabc,
2026                 0x3ab00, 0x3ab70,
2027                 0x3b000, 0x3b048,
2028                 0x3b060, 0x3b09c,
2029                 0x3b0f0, 0x3b148,
2030                 0x3b160, 0x3b19c,
2031                 0x3b1f0, 0x3b2e4,
2032                 0x3b2f8, 0x3b3e4,
2033                 0x3b3f8, 0x3b448,
2034                 0x3b460, 0x3b49c,
2035                 0x3b4f0, 0x3b548,
2036                 0x3b560, 0x3b59c,
2037                 0x3b5f0, 0x3b6e4,
2038                 0x3b6f8, 0x3b7e4,
2039                 0x3b7f8, 0x3b7fc,
2040                 0x3b814, 0x3b814,
2041                 0x3b82c, 0x3b82c,
2042                 0x3b880, 0x3b88c,
2043                 0x3b8e8, 0x3b8ec,
2044                 0x3b900, 0x3b948,
2045                 0x3b960, 0x3b99c,
2046                 0x3b9f0, 0x3bae4,
2047                 0x3baf8, 0x3bb10,
2048                 0x3bb28, 0x3bb28,
2049                 0x3bb3c, 0x3bb50,
2050                 0x3bbf0, 0x3bc10,
2051                 0x3bc28, 0x3bc28,
2052                 0x3bc3c, 0x3bc50,
2053                 0x3bcf0, 0x3bcfc,
2054                 0x3c000, 0x3c030,
2055                 0x3c100, 0x3c144,
2056                 0x3c190, 0x3c1d0,
2057                 0x3c200, 0x3c318,
2058                 0x3c400, 0x3c52c,
2059                 0x3c540, 0x3c61c,
2060                 0x3c800, 0x3c834,
2061                 0x3c8c0, 0x3c908,
2062                 0x3c910, 0x3c9ac,
2063                 0x3ca00, 0x3ca04,
2064                 0x3ca0c, 0x3ca2c,
2065                 0x3ca44, 0x3ca50,
2066                 0x3ca74, 0x3cc24,
2067                 0x3cd08, 0x3cd14,
2068                 0x3cd1c, 0x3cd20,
2069                 0x3cd3c, 0x3cd50,
2070                 0x3d200, 0x3d20c,
2071                 0x3d220, 0x3d220,
2072                 0x3d240, 0x3d240,
2073                 0x3d600, 0x3d600,
2074                 0x3d608, 0x3d60c,
2075                 0x3da00, 0x3da1c,
2076                 0x3de04, 0x3de20,
2077                 0x3de38, 0x3de3c,
2078                 0x3de80, 0x3de80,
2079                 0x3de88, 0x3dea8,
2080                 0x3deb0, 0x3deb4,
2081                 0x3dec8, 0x3ded4,
2082                 0x3dfb8, 0x3e004,
2083                 0x3e208, 0x3e23c,
2084                 0x3e600, 0x3e630,
2085                 0x3ea00, 0x3eabc,
2086                 0x3eb00, 0x3eb70,
2087                 0x3f000, 0x3f048,
2088                 0x3f060, 0x3f09c,
2089                 0x3f0f0, 0x3f148,
2090                 0x3f160, 0x3f19c,
2091                 0x3f1f0, 0x3f2e4,
2092                 0x3f2f8, 0x3f3e4,
2093                 0x3f3f8, 0x3f448,
2094                 0x3f460, 0x3f49c,
2095                 0x3f4f0, 0x3f548,
2096                 0x3f560, 0x3f59c,
2097                 0x3f5f0, 0x3f6e4,
2098                 0x3f6f8, 0x3f7e4,
2099                 0x3f7f8, 0x3f7fc,
2100                 0x3f814, 0x3f814,
2101                 0x3f82c, 0x3f82c,
2102                 0x3f880, 0x3f88c,
2103                 0x3f8e8, 0x3f8ec,
2104                 0x3f900, 0x3f948,
2105                 0x3f960, 0x3f99c,
2106                 0x3f9f0, 0x3fae4,
2107                 0x3faf8, 0x3fb10,
2108                 0x3fb28, 0x3fb28,
2109                 0x3fb3c, 0x3fb50,
2110                 0x3fbf0, 0x3fc10,
2111                 0x3fc28, 0x3fc28,
2112                 0x3fc3c, 0x3fc50,
2113                 0x3fcf0, 0x3fcfc,
2114                 0x40000, 0x4000c,
2115                 0x40040, 0x40068,
2116                 0x40080, 0x40144,
2117                 0x40180, 0x4018c,
2118                 0x40200, 0x40298,
2119                 0x402ac, 0x4033c,
2120                 0x403f8, 0x403fc,
2121                 0x41300, 0x413c4,
2122                 0x41400, 0x4141c,
2123                 0x41480, 0x414d0,
2124                 0x44000, 0x44078,
2125                 0x440c0, 0x44278,
2126                 0x442c0, 0x44478,
2127                 0x444c0, 0x44678,
2128                 0x446c0, 0x44878,
2129                 0x448c0, 0x449fc,
2130                 0x45000, 0x45068,
2131                 0x45080, 0x45084,
2132                 0x450a0, 0x450b0,
2133                 0x45200, 0x45268,
2134                 0x45280, 0x45284,
2135                 0x452a0, 0x452b0,
2136                 0x460c0, 0x460e4,
2137                 0x47000, 0x4708c,
2138                 0x47200, 0x47250,
2139                 0x47400, 0x47420,
2140                 0x47600, 0x47618,
2141                 0x47800, 0x47814,
2142                 0x48000, 0x4800c,
2143                 0x48040, 0x48068,
2144                 0x48080, 0x48144,
2145                 0x48180, 0x4818c,
2146                 0x48200, 0x48298,
2147                 0x482ac, 0x4833c,
2148                 0x483f8, 0x483fc,
2149                 0x49300, 0x493c4,
2150                 0x49400, 0x4941c,
2151                 0x49480, 0x494d0,
2152                 0x4c000, 0x4c078,
2153                 0x4c0c0, 0x4c278,
2154                 0x4c2c0, 0x4c478,
2155                 0x4c4c0, 0x4c678,
2156                 0x4c6c0, 0x4c878,
2157                 0x4c8c0, 0x4c9fc,
2158                 0x4d000, 0x4d068,
2159                 0x4d080, 0x4d084,
2160                 0x4d0a0, 0x4d0b0,
2161                 0x4d200, 0x4d268,
2162                 0x4d280, 0x4d284,
2163                 0x4d2a0, 0x4d2b0,
2164                 0x4e0c0, 0x4e0e4,
2165                 0x4f000, 0x4f08c,
2166                 0x4f200, 0x4f250,
2167                 0x4f400, 0x4f420,
2168                 0x4f600, 0x4f618,
2169                 0x4f800, 0x4f814,
2170                 0x50000, 0x500cc,
2171                 0x50400, 0x50400,
2172                 0x50800, 0x508cc,
2173                 0x50c00, 0x50c00,
2174                 0x51000, 0x5101c,
2175                 0x51300, 0x51308,
2176         };
2177
2178         int i;
2179         struct adapter *ap = netdev2adap(dev);
2180         static const unsigned int *reg_ranges;
2181         int arr_size = 0, buf_size = 0;
2182
2183         if (is_t4(ap->chip)) {
2184                 reg_ranges = &t4_reg_ranges[0];
2185                 arr_size = ARRAY_SIZE(t4_reg_ranges);
2186                 buf_size = T4_REGMAP_SIZE;
2187         } else {
2188                 reg_ranges = &t5_reg_ranges[0];
2189                 arr_size = ARRAY_SIZE(t5_reg_ranges);
2190                 buf_size = T5_REGMAP_SIZE;
2191         }
2192
2193         regs->version = mk_adap_vers(ap);
2194
2195         memset(buf, 0, buf_size);
2196         for (i = 0; i < arr_size; i += 2)
2197                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2198 }
2199
2200 static int restart_autoneg(struct net_device *dev)
2201 {
2202         struct port_info *p = netdev_priv(dev);
2203
2204         if (!netif_running(dev))
2205                 return -EAGAIN;
2206         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2207                 return -EINVAL;
2208         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2209         return 0;
2210 }
2211
2212 static int identify_port(struct net_device *dev,
2213                          enum ethtool_phys_id_state state)
2214 {
2215         unsigned int val;
2216         struct adapter *adap = netdev2adap(dev);
2217
2218         if (state == ETHTOOL_ID_ACTIVE)
2219                 val = 0xffff;
2220         else if (state == ETHTOOL_ID_INACTIVE)
2221                 val = 0;
2222         else
2223                 return -EINVAL;
2224
2225         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2226 }
2227
2228 static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
2229 {
2230         unsigned int v = 0;
2231
2232         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2233             type == FW_PORT_TYPE_BT_XAUI) {
2234                 v |= SUPPORTED_TP;
2235                 if (caps & FW_PORT_CAP_SPEED_100M)
2236                         v |= SUPPORTED_100baseT_Full;
2237                 if (caps & FW_PORT_CAP_SPEED_1G)
2238                         v |= SUPPORTED_1000baseT_Full;
2239                 if (caps & FW_PORT_CAP_SPEED_10G)
2240                         v |= SUPPORTED_10000baseT_Full;
2241         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2242                 v |= SUPPORTED_Backplane;
2243                 if (caps & FW_PORT_CAP_SPEED_1G)
2244                         v |= SUPPORTED_1000baseKX_Full;
2245                 if (caps & FW_PORT_CAP_SPEED_10G)
2246                         v |= SUPPORTED_10000baseKX4_Full;
2247         } else if (type == FW_PORT_TYPE_KR)
2248                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2249         else if (type == FW_PORT_TYPE_BP_AP)
2250                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2251                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2252         else if (type == FW_PORT_TYPE_BP4_AP)
2253                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2254                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2255                      SUPPORTED_10000baseKX4_Full;
2256         else if (type == FW_PORT_TYPE_FIBER_XFI ||
2257                  type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
2258                 v |= SUPPORTED_FIBRE;
2259
2260         if (caps & FW_PORT_CAP_ANEG)
2261                 v |= SUPPORTED_Autoneg;
2262         return v;
2263 }
2264
2265 static unsigned int to_fw_linkcaps(unsigned int caps)
2266 {
2267         unsigned int v = 0;
2268
2269         if (caps & ADVERTISED_100baseT_Full)
2270                 v |= FW_PORT_CAP_SPEED_100M;
2271         if (caps & ADVERTISED_1000baseT_Full)
2272                 v |= FW_PORT_CAP_SPEED_1G;
2273         if (caps & ADVERTISED_10000baseT_Full)
2274                 v |= FW_PORT_CAP_SPEED_10G;
2275         return v;
2276 }
2277
2278 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2279 {
2280         const struct port_info *p = netdev_priv(dev);
2281
2282         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2283             p->port_type == FW_PORT_TYPE_BT_XFI ||
2284             p->port_type == FW_PORT_TYPE_BT_XAUI)
2285                 cmd->port = PORT_TP;
2286         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2287                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2288                 cmd->port = PORT_FIBRE;
2289         else if (p->port_type == FW_PORT_TYPE_SFP) {
2290                 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2291                     p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2292                         cmd->port = PORT_DA;
2293                 else
2294                         cmd->port = PORT_FIBRE;
2295         } else
2296                 cmd->port = PORT_OTHER;
2297
2298         if (p->mdio_addr >= 0) {
2299                 cmd->phy_address = p->mdio_addr;
2300                 cmd->transceiver = XCVR_EXTERNAL;
2301                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2302                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2303         } else {
2304                 cmd->phy_address = 0;  /* not really, but no better option */
2305                 cmd->transceiver = XCVR_INTERNAL;
2306                 cmd->mdio_support = 0;
2307         }
2308
2309         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2310         cmd->advertising = from_fw_linkcaps(p->port_type,
2311                                             p->link_cfg.advertising);
2312         ethtool_cmd_speed_set(cmd,
2313                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2314         cmd->duplex = DUPLEX_FULL;
2315         cmd->autoneg = p->link_cfg.autoneg;
2316         cmd->maxtxpkt = 0;
2317         cmd->maxrxpkt = 0;
2318         return 0;
2319 }
2320
2321 static unsigned int speed_to_caps(int speed)
2322 {
2323         if (speed == SPEED_100)
2324                 return FW_PORT_CAP_SPEED_100M;
2325         if (speed == SPEED_1000)
2326                 return FW_PORT_CAP_SPEED_1G;
2327         if (speed == SPEED_10000)
2328                 return FW_PORT_CAP_SPEED_10G;
2329         return 0;
2330 }
2331
2332 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2333 {
2334         unsigned int cap;
2335         struct port_info *p = netdev_priv(dev);
2336         struct link_config *lc = &p->link_cfg;
2337         u32 speed = ethtool_cmd_speed(cmd);
2338
2339         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
2340                 return -EINVAL;
2341
2342         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2343                 /*
2344                  * PHY offers a single speed.  See if that's what's
2345                  * being requested.
2346                  */
2347                 if (cmd->autoneg == AUTONEG_DISABLE &&
2348                     (lc->supported & speed_to_caps(speed)))
2349                         return 0;
2350                 return -EINVAL;
2351         }
2352
2353         if (cmd->autoneg == AUTONEG_DISABLE) {
2354                 cap = speed_to_caps(speed);
2355
2356                 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
2357                     (speed == SPEED_10000))
2358                         return -EINVAL;
2359                 lc->requested_speed = cap;
2360                 lc->advertising = 0;
2361         } else {
2362                 cap = to_fw_linkcaps(cmd->advertising);
2363                 if (!(lc->supported & cap))
2364                         return -EINVAL;
2365                 lc->requested_speed = 0;
2366                 lc->advertising = cap | FW_PORT_CAP_ANEG;
2367         }
2368         lc->autoneg = cmd->autoneg;
2369
2370         if (netif_running(dev))
2371                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2372                                      lc);
2373         return 0;
2374 }
2375
2376 static void get_pauseparam(struct net_device *dev,
2377                            struct ethtool_pauseparam *epause)
2378 {
2379         struct port_info *p = netdev_priv(dev);
2380
2381         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2382         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2383         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2384 }
2385
2386 static int set_pauseparam(struct net_device *dev,
2387                           struct ethtool_pauseparam *epause)
2388 {
2389         struct port_info *p = netdev_priv(dev);
2390         struct link_config *lc = &p->link_cfg;
2391
2392         if (epause->autoneg == AUTONEG_DISABLE)
2393                 lc->requested_fc = 0;
2394         else if (lc->supported & FW_PORT_CAP_ANEG)
2395                 lc->requested_fc = PAUSE_AUTONEG;
2396         else
2397                 return -EINVAL;
2398
2399         if (epause->rx_pause)
2400                 lc->requested_fc |= PAUSE_RX;
2401         if (epause->tx_pause)
2402                 lc->requested_fc |= PAUSE_TX;
2403         if (netif_running(dev))
2404                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2405                                      lc);
2406         return 0;
2407 }
2408
2409 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2410 {
2411         const struct port_info *pi = netdev_priv(dev);
2412         const struct sge *s = &pi->adapter->sge;
2413
2414         e->rx_max_pending = MAX_RX_BUFFERS;
2415         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2416         e->rx_jumbo_max_pending = 0;
2417         e->tx_max_pending = MAX_TXQ_ENTRIES;
2418
2419         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2420         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2421         e->rx_jumbo_pending = 0;
2422         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2423 }
2424
2425 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2426 {
2427         int i;
2428         const struct port_info *pi = netdev_priv(dev);
2429         struct adapter *adapter = pi->adapter;
2430         struct sge *s = &adapter->sge;
2431
2432         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2433             e->tx_pending > MAX_TXQ_ENTRIES ||
2434             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2435             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2436             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2437                 return -EINVAL;
2438
2439         if (adapter->flags & FULL_INIT_DONE)
2440                 return -EBUSY;
2441
2442         for (i = 0; i < pi->nqsets; ++i) {
2443                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2444                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2445                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2446         }
2447         return 0;
2448 }
2449
2450 static int closest_timer(const struct sge *s, int time)
2451 {
2452         int i, delta, match = 0, min_delta = INT_MAX;
2453
2454         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2455                 delta = time - s->timer_val[i];
2456                 if (delta < 0)
2457                         delta = -delta;
2458                 if (delta < min_delta) {
2459                         min_delta = delta;
2460                         match = i;
2461                 }
2462         }
2463         return match;
2464 }
2465
2466 static int closest_thres(const struct sge *s, int thres)
2467 {
2468         int i, delta, match = 0, min_delta = INT_MAX;
2469
2470         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2471                 delta = thres - s->counter_val[i];
2472                 if (delta < 0)
2473                         delta = -delta;
2474                 if (delta < min_delta) {
2475                         min_delta = delta;
2476                         match = i;
2477                 }
2478         }
2479         return match;
2480 }
2481
2482 /*
2483  * Return a queue's interrupt hold-off time in us.  0 means no timer.
2484  */
2485 static unsigned int qtimer_val(const struct adapter *adap,
2486                                const struct sge_rspq *q)
2487 {
2488         unsigned int idx = q->intr_params >> 1;
2489
2490         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2491 }
2492
2493 /**
2494  *      set_rxq_intr_params - set a queue's interrupt holdoff parameters
2495  *      @adap: the adapter
2496  *      @q: the Rx queue
2497  *      @us: the hold-off time in us, or 0 to disable timer
2498  *      @cnt: the hold-off packet count, or 0 to disable counter
2499  *
2500  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
2501  *      one of the two needs to be enabled for the queue to generate interrupts.
2502  */
2503 static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
2504                                unsigned int us, unsigned int cnt)
2505 {
2506         if ((us | cnt) == 0)
2507                 cnt = 1;
2508
2509         if (cnt) {
2510                 int err;
2511                 u32 v, new_idx;
2512
2513                 new_idx = closest_thres(&adap->sge, cnt);
2514                 if (q->desc && q->pktcnt_idx != new_idx) {
2515                         /* the queue has already been created, update it */
2516                         v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
2517                             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2518                             FW_PARAMS_PARAM_YZ(q->cntxt_id);
2519                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2520                                             &new_idx);
2521                         if (err)
2522                                 return err;
2523                 }
2524                 q->pktcnt_idx = new_idx;
2525         }
2526
2527         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2528         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2529         return 0;
2530 }
2531
2532 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2533 {
2534         const struct port_info *pi = netdev_priv(dev);
2535         struct adapter *adap = pi->adapter;
2536         struct sge_rspq *q;
2537         int i;
2538         int r = 0;
2539
2540         for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++) {
2541                 q = &adap->sge.ethrxq[i].rspq;
2542                 r = set_rxq_intr_params(adap, q, c->rx_coalesce_usecs,
2543                         c->rx_max_coalesced_frames);
2544                 if (r) {
2545                         dev_err(&dev->dev, "failed to set coalesce %d\n", r);
2546                         break;
2547                 }
2548         }
2549         return r;
2550 }
2551
2552 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2553 {
2554         const struct port_info *pi = netdev_priv(dev);
2555         const struct adapter *adap = pi->adapter;
2556         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2557
2558         c->rx_coalesce_usecs = qtimer_val(adap, rq);
2559         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2560                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2561         return 0;
2562 }
2563
2564 /**
2565  *      eeprom_ptov - translate a physical EEPROM address to virtual
2566  *      @phys_addr: the physical EEPROM address
2567  *      @fn: the PCI function number
2568  *      @sz: size of function-specific area
2569  *
2570  *      Translate a physical EEPROM address to virtual.  The first 1K is
2571  *      accessed through virtual addresses starting at 31K, the rest is
2572  *      accessed through virtual addresses starting at 0.
2573  *
2574  *      The mapping is as follows:
2575  *      [0..1K) -> [31K..32K)
2576  *      [1K..1K+A) -> [31K-A..31K)
2577  *      [1K+A..ES) -> [0..ES-A-1K)
2578  *
2579  *      where A = @fn * @sz, and ES = EEPROM size.
2580  */
2581 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2582 {
2583         fn *= sz;
2584         if (phys_addr < 1024)
2585                 return phys_addr + (31 << 10);
2586         if (phys_addr < 1024 + fn)
2587                 return 31744 - fn + phys_addr - 1024;
2588         if (phys_addr < EEPROMSIZE)
2589                 return phys_addr - 1024 - fn;
2590         return -EINVAL;
2591 }
2592
2593 /*
2594  * The next two routines implement eeprom read/write from physical addresses.
2595  */
2596 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2597 {
2598         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2599
2600         if (vaddr >= 0)
2601                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2602         return vaddr < 0 ? vaddr : 0;
2603 }
2604
2605 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2606 {
2607         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2608
2609         if (vaddr >= 0)
2610                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2611         return vaddr < 0 ? vaddr : 0;
2612 }
2613
2614 #define EEPROM_MAGIC 0x38E2F10C
2615
2616 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2617                       u8 *data)
2618 {
2619         int i, err = 0;
2620         struct adapter *adapter = netdev2adap(dev);
2621
2622         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2623         if (!buf)
2624                 return -ENOMEM;
2625
2626         e->magic = EEPROM_MAGIC;
2627         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2628                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2629
2630         if (!err)
2631                 memcpy(data, buf + e->offset, e->len);
2632         kfree(buf);
2633         return err;
2634 }
2635
2636 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2637                       u8 *data)
2638 {
2639         u8 *buf;
2640         int err = 0;
2641         u32 aligned_offset, aligned_len, *p;
2642         struct adapter *adapter = netdev2adap(dev);
2643
2644         if (eeprom->magic != EEPROM_MAGIC)
2645                 return -EINVAL;
2646
2647         aligned_offset = eeprom->offset & ~3;
2648         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2649
2650         if (adapter->fn > 0) {
2651                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2652
2653                 if (aligned_offset < start ||
2654                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
2655                         return -EPERM;
2656         }
2657
2658         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2659                 /*
2660                  * RMW possibly needed for first or last words.
2661                  */
2662                 buf = kmalloc(aligned_len, GFP_KERNEL);
2663                 if (!buf)
2664                         return -ENOMEM;
2665                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2666                 if (!err && aligned_len > 4)
2667                         err = eeprom_rd_phys(adapter,
2668                                              aligned_offset + aligned_len - 4,
2669                                              (u32 *)&buf[aligned_len - 4]);
2670                 if (err)
2671                         goto out;
2672                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2673         } else
2674                 buf = data;
2675
2676         err = t4_seeprom_wp(adapter, false);
2677         if (err)
2678                 goto out;
2679
2680         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2681                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2682                 aligned_offset += 4;
2683         }
2684
2685         if (!err)
2686                 err = t4_seeprom_wp(adapter, true);
2687 out:
2688         if (buf != data)
2689                 kfree(buf);
2690         return err;
2691 }
2692
2693 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2694 {
2695         int ret;
2696         const struct firmware *fw;
2697         struct adapter *adap = netdev2adap(netdev);
2698
2699         ef->data[sizeof(ef->data) - 1] = '\0';
2700         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2701         if (ret < 0)
2702                 return ret;
2703
2704         ret = t4_load_fw(adap, fw->data, fw->size);
2705         release_firmware(fw);
2706         if (!ret)
2707                 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
2708         return ret;
2709 }
2710
2711 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2712 #define BCAST_CRC 0xa0ccc1a6
2713
2714 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2715 {
2716         wol->supported = WAKE_BCAST | WAKE_MAGIC;
2717         wol->wolopts = netdev2adap(dev)->wol;
2718         memset(&wol->sopass, 0, sizeof(wol->sopass));
2719 }
2720
2721 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2722 {
2723         int err = 0;
2724         struct port_info *pi = netdev_priv(dev);
2725
2726         if (wol->wolopts & ~WOL_SUPPORTED)
2727                 return -EINVAL;
2728         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2729                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2730         if (wol->wolopts & WAKE_BCAST) {
2731                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2732                                         ~0ULL, 0, false);
2733                 if (!err)
2734                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2735                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
2736         } else
2737                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2738         return err;
2739 }
2740
2741 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2742 {
2743         const struct port_info *pi = netdev_priv(dev);
2744         netdev_features_t changed = dev->features ^ features;
2745         int err;
2746
2747         if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2748                 return 0;
2749
2750         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2751                             -1, -1, -1,
2752                             !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2753         if (unlikely(err))
2754                 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2755         return err;
2756 }
2757
2758 static u32 get_rss_table_size(struct net_device *dev)
2759 {
2760         const struct port_info *pi = netdev_priv(dev);
2761
2762         return pi->rss_size;
2763 }
2764
2765 static int get_rss_table(struct net_device *dev, u32 *p)
2766 {
2767         const struct port_info *pi = netdev_priv(dev);
2768         unsigned int n = pi->rss_size;
2769
2770         while (n--)
2771                 p[n] = pi->rss[n];
2772         return 0;
2773 }
2774
2775 static int set_rss_table(struct net_device *dev, const u32 *p)
2776 {
2777         unsigned int i;
2778         struct port_info *pi = netdev_priv(dev);
2779
2780         for (i = 0; i < pi->rss_size; i++)
2781                 pi->rss[i] = p[i];
2782         if (pi->adapter->flags & FULL_INIT_DONE)
2783                 return write_rss(pi, pi->rss);
2784         return 0;
2785 }
2786
2787 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2788                      u32 *rules)
2789 {
2790         const struct port_info *pi = netdev_priv(dev);
2791
2792         switch (info->cmd) {
2793         case ETHTOOL_GRXFH: {
2794                 unsigned int v = pi->rss_mode;
2795
2796                 info->data = 0;
2797                 switch (info->flow_type) {
2798                 case TCP_V4_FLOW:
2799                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
2800                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2801                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2802                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2803                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2804                         break;
2805                 case UDP_V4_FLOW:
2806                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
2807                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2808                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2809                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2810                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2811                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2812                         break;
2813                 case SCTP_V4_FLOW:
2814                 case AH_ESP_V4_FLOW:
2815                 case IPV4_FLOW:
2816                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
2817                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2818                         break;
2819                 case TCP_V6_FLOW:
2820                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
2821                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2822                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2823                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2824                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2825                         break;
2826                 case UDP_V6_FLOW:
2827                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
2828                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
2829                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2830                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2831                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2832                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2833                         break;
2834                 case SCTP_V6_FLOW:
2835                 case AH_ESP_V6_FLOW:
2836                 case IPV6_FLOW:
2837                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
2838                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2839                         break;
2840                 }
2841                 return 0;
2842         }
2843         case ETHTOOL_GRXRINGS:
2844                 info->data = pi->nqsets;
2845                 return 0;
2846         }
2847         return -EOPNOTSUPP;
2848 }
2849
2850 static const struct ethtool_ops cxgb_ethtool_ops = {
2851         .get_settings      = get_settings,
2852         .set_settings      = set_settings,
2853         .get_drvinfo       = get_drvinfo,
2854         .get_msglevel      = get_msglevel,
2855         .set_msglevel      = set_msglevel,
2856         .get_ringparam     = get_sge_param,
2857         .set_ringparam     = set_sge_param,
2858         .get_coalesce      = get_coalesce,
2859         .set_coalesce      = set_coalesce,
2860         .get_eeprom_len    = get_eeprom_len,
2861         .get_eeprom        = get_eeprom,
2862         .set_eeprom        = set_eeprom,
2863         .get_pauseparam    = get_pauseparam,
2864         .set_pauseparam    = set_pauseparam,
2865         .get_link          = ethtool_op_get_link,
2866         .get_strings       = get_strings,
2867         .set_phys_id       = identify_port,
2868         .nway_reset        = restart_autoneg,
2869         .get_sset_count    = get_sset_count,
2870         .get_ethtool_stats = get_stats,
2871         .get_regs_len      = get_regs_len,
2872         .get_regs          = get_regs,
2873         .get_wol           = get_wol,
2874         .set_wol           = set_wol,
2875         .get_rxnfc         = get_rxnfc,
2876         .get_rxfh_indir_size = get_rss_table_size,
2877         .get_rxfh_indir    = get_rss_table,
2878         .set_rxfh_indir    = set_rss_table,
2879         .flash_device      = set_flash,
2880 };
2881
2882 /*
2883  * debugfs support
2884  */
2885 static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2886                         loff_t *ppos)
2887 {
2888         loff_t pos = *ppos;
2889         loff_t avail = file_inode(file)->i_size;
2890         unsigned int mem = (uintptr_t)file->private_data & 3;
2891         struct adapter *adap = file->private_data - mem;
2892
2893         if (pos < 0)
2894                 return -EINVAL;
2895         if (pos >= avail)
2896                 return 0;
2897         if (count > avail - pos)
2898                 count = avail - pos;
2899
2900         while (count) {
2901                 size_t len;
2902                 int ret, ofst;
2903                 __be32 data[16];
2904
2905                 if ((mem == MEM_MC) || (mem == MEM_MC1))
2906                         ret = t4_mc_read(adap, mem % MEM_MC, pos, data, NULL);
2907                 else
2908                         ret = t4_edc_read(adap, mem, pos, data, NULL);
2909                 if (ret)
2910                         return ret;
2911
2912                 ofst = pos % sizeof(data);
2913                 len = min(count, sizeof(data) - ofst);
2914                 if (copy_to_user(buf, (u8 *)data + ofst, len))
2915                         return -EFAULT;
2916
2917                 buf += len;
2918                 pos += len;
2919                 count -= len;
2920         }
2921         count = pos - *ppos;
2922         *ppos = pos;
2923         return count;
2924 }
2925
2926 static const struct file_operations mem_debugfs_fops = {
2927         .owner   = THIS_MODULE,
2928         .open    = simple_open,
2929         .read    = mem_read,
2930         .llseek  = default_llseek,
2931 };
2932
2933 static void add_debugfs_mem(struct adapter *adap, const char *name,
2934                             unsigned int idx, unsigned int size_mb)
2935 {
2936         struct dentry *de;
2937
2938         de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2939                                  (void *)adap + idx, &mem_debugfs_fops);
2940         if (de && de->d_inode)
2941                 de->d_inode->i_size = size_mb << 20;
2942 }
2943
2944 static int setup_debugfs(struct adapter *adap)
2945 {
2946         int i;
2947         u32 size;
2948
2949         if (IS_ERR_OR_NULL(adap->debugfs_root))
2950                 return -1;
2951
2952         i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2953         if (i & EDRAM0_ENABLE) {
2954                 size = t4_read_reg(adap, MA_EDRAM0_BAR);
2955                 add_debugfs_mem(adap, "edc0", MEM_EDC0, EDRAM_SIZE_GET(size));
2956         }
2957         if (i & EDRAM1_ENABLE) {
2958                 size = t4_read_reg(adap, MA_EDRAM1_BAR);
2959                 add_debugfs_mem(adap, "edc1", MEM_EDC1, EDRAM_SIZE_GET(size));
2960         }
2961         if (is_t4(adap->chip)) {
2962                 size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2963                 if (i & EXT_MEM_ENABLE)
2964                         add_debugfs_mem(adap, "mc", MEM_MC,
2965                                         EXT_MEM_SIZE_GET(size));
2966         } else {
2967                 if (i & EXT_MEM_ENABLE) {
2968                         size = t4_read_reg(adap, MA_EXT_MEMORY_BAR);
2969                         add_debugfs_mem(adap, "mc0", MEM_MC0,
2970                                         EXT_MEM_SIZE_GET(size));
2971                 }
2972                 if (i & EXT_MEM1_ENABLE) {
2973                         size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR);
2974                         add_debugfs_mem(adap, "mc1", MEM_MC1,
2975                                         EXT_MEM_SIZE_GET(size));
2976                 }
2977         }
2978         if (adap->l2t)
2979                 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2980                                     &t4_l2t_fops);
2981         return 0;
2982 }
2983
2984 /*
2985  * upper-layer driver support
2986  */
2987
2988 /*
2989  * Allocate an active-open TID and set it to the supplied value.
2990  */
2991 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2992 {
2993         int atid = -1;
2994
2995         spin_lock_bh(&t->atid_lock);
2996         if (t->afree) {
2997                 union aopen_entry *p = t->afree;
2998
2999                 atid = (p - t->atid_tab) + t->atid_base;
3000                 t->afree = p->next;
3001                 p->data = data;
3002                 t->atids_in_use++;
3003         }
3004         spin_unlock_bh(&t->atid_lock);
3005         return atid;
3006 }
3007 EXPORT_SYMBOL(cxgb4_alloc_atid);
3008
3009 /*
3010  * Release an active-open TID.
3011  */
3012 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3013 {
3014         union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
3015
3016         spin_lock_bh(&t->atid_lock);
3017         p->next = t->afree;
3018         t->afree = p;
3019         t->atids_in_use--;
3020         spin_unlock_bh(&t->atid_lock);
3021 }
3022 EXPORT_SYMBOL(cxgb4_free_atid);
3023
3024 /*
3025  * Allocate a server TID and set it to the supplied value.
3026  */
3027 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3028 {
3029         int stid;
3030
3031         spin_lock_bh(&t->stid_lock);
3032         if (family == PF_INET) {
3033                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3034                 if (stid < t->nstids)
3035                         __set_bit(stid, t->stid_bmap);
3036                 else
3037                         stid = -1;
3038         } else {
3039                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3040                 if (stid < 0)
3041                         stid = -1;
3042         }
3043         if (stid >= 0) {
3044                 t->stid_tab[stid].data = data;
3045                 stid += t->stid_base;
3046                 t->stids_in_use++;
3047         }
3048         spin_unlock_bh(&t->stid_lock);
3049         return stid;
3050 }
3051 EXPORT_SYMBOL(cxgb4_alloc_stid);
3052
3053 /* Allocate a server filter TID and set it to the supplied value.
3054  */
3055 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3056 {
3057         int stid;
3058
3059         spin_lock_bh(&t->stid_lock);
3060         if (family == PF_INET) {
3061                 stid = find_next_zero_bit(t->stid_bmap,
3062                                 t->nstids + t->nsftids, t->nstids);
3063                 if (stid < (t->nstids + t->nsftids))
3064                         __set_bit(stid, t->stid_bmap);
3065                 else
3066                         stid = -1;
3067         } else {
3068                 stid = -1;
3069         }
3070         if (stid >= 0) {
3071                 t->stid_tab[stid].data = data;
3072                 stid += t->stid_base;
3073                 t->stids_in_use++;
3074         }
3075         spin_unlock_bh(&t->stid_lock);
3076         return stid;
3077 }
3078 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3079
3080 /* Release a server TID.
3081  */
3082 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3083 {
3084         stid -= t->stid_base;
3085         spin_lock_bh(&t->stid_lock);
3086         if (family == PF_INET)
3087                 __clear_bit(stid, t->stid_bmap);
3088         else
3089                 bitmap_release_region(t->stid_bmap, stid, 2);
3090         t->stid_tab[stid].data = NULL;
3091         t->stids_in_use--;
3092         spin_unlock_bh(&t->stid_lock);
3093 }
3094 EXPORT_SYMBOL(cxgb4_free_stid);
3095
3096 /*
3097  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
3098  */
3099 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3100                            unsigned int tid)
3101 {
3102         struct cpl_tid_release *req;
3103
3104         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3105         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3106         INIT_TP_WR(req, tid);
3107         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3108 }
3109
3110 /*
3111  * Queue a TID release request and if necessary schedule a work queue to
3112  * process it.
3113  */
3114 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3115                                     unsigned int tid)
3116 {
3117         void **p = &t->tid_tab[tid];
3118         struct adapter *adap = container_of(t, struct adapter, tids);
3119
3120         spin_lock_bh(&adap->tid_release_lock);
3121         *p = adap->tid_release_head;
3122         /* Low 2 bits encode the Tx channel number */
3123         adap->tid_release_head = (void **)((uintptr_t)p | chan);
3124         if (!adap->tid_release_task_busy) {
3125                 adap->tid_release_task_busy = true;
3126                 queue_work(workq, &adap->tid_release_task);
3127         }
3128         spin_unlock_bh(&adap->tid_release_lock);
3129 }
3130
3131 /*
3132  * Process the list of pending TID release requests.
3133  */
3134 static void process_tid_release_list(struct work_struct *work)
3135 {
3136         struct sk_buff *skb;
3137         struct adapter *adap;
3138
3139         adap = container_of(work, struct adapter, tid_release_task);
3140
3141         spin_lock_bh(&adap->tid_release_lock);
3142         while (adap->tid_release_head) {
3143                 void **p = adap->tid_release_head;
3144                 unsigned int chan = (uintptr_t)p & 3;
3145                 p = (void *)p - chan;
3146
3147                 adap->tid_release_head = *p;
3148                 *p = NULL;
3149                 spin_unlock_bh(&adap->tid_release_lock);
3150
3151                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3152                                          GFP_KERNEL)))
3153                         schedule_timeout_uninterruptible(1);
3154
3155                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3156                 t4_ofld_send(adap, skb);
3157                 spin_lock_bh(&adap->tid_release_lock);
3158         }
3159         adap->tid_release_task_busy = false;
3160         spin_unlock_bh(&adap->tid_release_lock);
3161 }
3162
3163 /*
3164  * Release a TID and inform HW.  If we are unable to allocate the release
3165  * message we defer to a work queue.
3166  */
3167 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3168 {
3169         void *old;
3170         struct sk_buff *skb;
3171         struct adapter *adap = container_of(t, struct adapter, tids);
3172
3173         old = t->tid_tab[tid];
3174         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3175         if (likely(skb)) {
3176                 t->tid_tab[tid] = NULL;
3177                 mk_tid_release(skb, chan, tid);
3178                 t4_ofld_send(adap, skb);
3179         } else
3180                 cxgb4_queue_tid_release(t, chan, tid);
3181         if (old)
3182                 atomic_dec(&t->tids_in_use);
3183 }
3184 EXPORT_SYMBOL(cxgb4_remove_tid);
3185
3186 /*
3187  * Allocate and initialize the TID tables.  Returns 0 on success.
3188  */
3189 static int tid_init(struct tid_info *t)
3190 {
3191         size_t size;
3192         unsigned int stid_bmap_size;
3193         unsigned int natids = t->natids;
3194
3195         stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3196         size = t->ntids * sizeof(*t->tid_tab) +
3197                natids * sizeof(*t->atid_tab) +
3198                t->nstids * sizeof(*t->stid_tab) +
3199                t->nsftids * sizeof(*t->stid_tab) +
3200                stid_bmap_size * sizeof(long) +
3201                t->nftids * sizeof(*t->ftid_tab) +
3202                t->nsftids * sizeof(*t->ftid_tab);
3203
3204         t->tid_tab = t4_alloc_mem(size);
3205         if (!t->tid_tab)
3206                 return -ENOMEM;
3207
3208         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3209         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3210         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3211         t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3212         spin_lock_init(&t->stid_lock);
3213         spin_lock_init(&t->atid_lock);
3214
3215         t->stids_in_use = 0;
3216         t->afree = NULL;
3217         t->atids_in_use = 0;
3218         atomic_set(&t->tids_in_use, 0);
3219
3220         /* Setup the free list for atid_tab and clear the stid bitmap. */
3221         if (natids) {
3222                 while (--natids)
3223                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3224                 t->afree = t->atid_tab;
3225         }
3226         bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3227         return 0;
3228 }
3229
3230 /**
3231  *      cxgb4_create_server - create an IP server
3232  *      @dev: the device
3233  *      @stid: the server TID
3234  *      @sip: local IP address to bind server to
3235  *      @sport: the server's TCP port
3236  *      @queue: queue to direct messages from this server to
3237  *
3238  *      Create an IP server for the given port and address.
3239  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3240  */
3241 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3242                         __be32 sip, __be16 sport, __be16 vlan,
3243                         unsigned int queue)
3244 {
3245         unsigned int chan;
3246         struct sk_buff *skb;
3247         struct adapter *adap;
3248         struct cpl_pass_open_req *req;
3249
3250         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3251         if (!skb)
3252                 return -ENOMEM;
3253
3254         adap = netdev2adap(dev);
3255         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3256         INIT_TP_WR(req, 0);
3257         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3258         req->local_port = sport;
3259         req->peer_port = htons(0);
3260         req->local_ip = sip;
3261         req->peer_ip = htonl(0);
3262         chan = rxq_to_chan(&adap->sge, queue);
3263         req->opt0 = cpu_to_be64(TX_CHAN(chan));
3264         req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3265                                 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3266         return t4_mgmt_tx(adap, skb);
3267 }
3268 EXPORT_SYMBOL(cxgb4_create_server);
3269
3270 /**
3271  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3272  *      @mtus: the HW MTU table
3273  *      @mtu: the target MTU
3274  *      @idx: index of selected entry in the MTU table
3275  *
3276  *      Returns the index and the value in the HW MTU table that is closest to
3277  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
3278  *      table, in which case that smallest available value is selected.
3279  */
3280 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3281                             unsigned int *idx)
3282 {
3283         unsigned int i = 0;
3284
3285         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3286                 ++i;
3287         if (idx)
3288                 *idx = i;
3289         return mtus[i];
3290 }
3291 EXPORT_SYMBOL(cxgb4_best_mtu);
3292
3293 /**
3294  *      cxgb4_port_chan - get the HW channel of a port
3295  *      @dev: the net device for the port
3296  *
3297  *      Return the HW Tx channel of the given port.
3298  */
3299 unsigned int cxgb4_port_chan(const struct net_device *dev)
3300 {
3301         return netdev2pinfo(dev)->tx_chan;
3302 }
3303 EXPORT_SYMBOL(cxgb4_port_chan);
3304
3305 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3306 {
3307         struct adapter *adap = netdev2adap(dev);
3308         u32 v1, v2, lp_count, hp_count;
3309
3310         v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3311         v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3312         if (is_t4(adap->chip)) {
3313                 lp_count = G_LP_COUNT(v1);
3314                 hp_count = G_HP_COUNT(v1);
3315         } else {
3316                 lp_count = G_LP_COUNT_T5(v1);
3317                 hp_count = G_HP_COUNT_T5(v2);
3318         }
3319         return lpfifo ? lp_count : hp_count;
3320 }
3321 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3322
3323 /**
3324  *      cxgb4_port_viid - get the VI id of a port
3325  *      @dev: the net device for the port
3326  *
3327  *      Return the VI id of the given port.
3328  */
3329 unsigned int cxgb4_port_viid(const struct net_device *dev)
3330 {
3331         return netdev2pinfo(dev)->viid;
3332 }
3333 EXPORT_SYMBOL(cxgb4_port_viid);
3334
3335 /**
3336  *      cxgb4_port_idx - get the index of a port
3337  *      @dev: the net device for the port
3338  *
3339  *      Return the index of the given port.
3340  */
3341 unsigned int cxgb4_port_idx(const struct net_device *dev)
3342 {
3343         return netdev2pinfo(dev)->port_id;
3344 }
3345 EXPORT_SYMBOL(cxgb4_port_idx);
3346
3347 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3348                          struct tp_tcp_stats *v6)
3349 {
3350         struct adapter *adap = pci_get_drvdata(pdev);
3351
3352         spin_lock(&adap->stats_lock);
3353         t4_tp_get_tcp_stats(adap, v4, v6);
3354         spin_unlock(&adap->stats_lock);
3355 }
3356 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3357
3358 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3359                       const unsigned int *pgsz_order)
3360 {
3361         struct adapter *adap = netdev2adap(dev);
3362
3363         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3364         t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3365                      HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3366                      HPZ3(pgsz_order[3]));
3367 }
3368 EXPORT_SYMBOL(cxgb4_iscsi_init);
3369
3370 int cxgb4_flush_eq_cache(struct net_device *dev)
3371 {
3372         struct adapter *adap = netdev2adap(dev);
3373         int ret;
3374
3375         ret = t4_fwaddrspace_write(adap, adap->mbox,
3376                                    0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3377         return ret;
3378 }
3379 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3380
3381 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3382 {
3383         u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3384         __be64 indices;
3385         int ret;
3386
3387         ret = t4_mem_win_read_len(adap, addr, (__be32 *)&indices, 8);
3388         if (!ret) {
3389                 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3390                 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3391         }
3392         return ret;
3393 }
3394
3395 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3396                         u16 size)
3397 {
3398         struct adapter *adap = netdev2adap(dev);
3399         u16 hw_pidx, hw_cidx;
3400         int ret;
3401
3402         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3403         if (ret)
3404                 goto out;
3405
3406         if (pidx != hw_pidx) {
3407                 u16 delta;
3408
3409                 if (pidx >= hw_pidx)
3410                         delta = pidx - hw_pidx;
3411                 else
3412                         delta = size - hw_pidx + pidx;
3413                 wmb();
3414                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3415                              QID(qid) | PIDX(delta));
3416         }
3417 out:
3418         return ret;
3419 }
3420 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3421
3422 void cxgb4_disable_db_coalescing(struct net_device *dev)
3423 {
3424         struct adapter *adap;
3425
3426         adap = netdev2adap(dev);
3427         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3428                          F_NOCOALESCE);
3429 }
3430 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3431
3432 void cxgb4_enable_db_coalescing(struct net_device *dev)
3433 {
3434         struct adapter *adap;
3435
3436         adap = netdev2adap(dev);
3437         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3438 }
3439 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3440
3441 static struct pci_driver cxgb4_driver;
3442
3443 static void check_neigh_update(struct neighbour *neigh)
3444 {
3445         const struct device *parent;
3446         const struct net_device *netdev = neigh->dev;
3447
3448         if (netdev->priv_flags & IFF_802_1Q_VLAN)
3449                 netdev = vlan_dev_real_dev(netdev);
3450         parent = netdev->dev.parent;
3451         if (parent && parent->driver == &cxgb4_driver.driver)
3452                 t4_l2t_update(dev_get_drvdata(parent), neigh);
3453 }
3454
3455 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3456                        void *data)
3457 {
3458         switch (event) {
3459         case NETEVENT_NEIGH_UPDATE:
3460                 check_neigh_update(data);
3461                 break;
3462         case NETEVENT_REDIRECT:
3463         default:
3464                 break;
3465         }
3466         return 0;
3467 }
3468
3469 static bool netevent_registered;
3470 static struct notifier_block cxgb4_netevent_nb = {
3471         .notifier_call = netevent_cb
3472 };
3473
3474 static void drain_db_fifo(struct adapter *adap, int usecs)
3475 {
3476         u32 v1, v2, lp_count, hp_count;
3477
3478         do {
3479                 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3480                 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3481                 if (is_t4(adap->chip)) {
3482                         lp_count = G_LP_COUNT(v1);
3483                         hp_count = G_HP_COUNT(v1);
3484                 } else {
3485                         lp_count = G_LP_COUNT_T5(v1);
3486                         hp_count = G_HP_COUNT_T5(v2);
3487                 }
3488
3489                 if (lp_count == 0 && hp_count == 0)
3490                         break;
3491                 set_current_state(TASK_UNINTERRUPTIBLE);
3492                 schedule_timeout(usecs_to_jiffies(usecs));
3493         } while (1);
3494 }
3495
3496 static void disable_txq_db(struct sge_txq *q)
3497 {
3498         spin_lock_irq(&q->db_lock);
3499         q->db_disabled = 1;
3500         spin_unlock_irq(&q->db_lock);
3501 }
3502
3503 static void enable_txq_db(struct sge_txq *q)
3504 {
3505         spin_lock_irq(&q->db_lock);
3506         q->db_disabled = 0;
3507         spin_unlock_irq(&q->db_lock);
3508 }
3509
3510 static void disable_dbs(struct adapter *adap)
3511 {
3512         int i;
3513
3514         for_each_ethrxq(&adap->sge, i)
3515                 disable_txq_db(&adap->sge.ethtxq[i].q);
3516         for_each_ofldrxq(&adap->sge, i)
3517                 disable_txq_db(&adap->sge.ofldtxq[i].q);
3518         for_each_port(adap, i)
3519                 disable_txq_db(&adap->sge.ctrlq[i].q);
3520 }
3521
3522 static void enable_dbs(struct adapter *adap)
3523 {
3524         int i;
3525
3526         for_each_ethrxq(&adap->sge, i)
3527                 enable_txq_db(&adap->sge.ethtxq[i].q);
3528         for_each_ofldrxq(&adap->sge, i)
3529                 enable_txq_db(&adap->sge.ofldtxq[i].q);
3530         for_each_port(adap, i)
3531                 enable_txq_db(&adap->sge.ctrlq[i].q);
3532 }
3533
3534 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3535 {
3536         u16 hw_pidx, hw_cidx;
3537         int ret;
3538
3539         spin_lock_bh(&q->db_lock);
3540         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3541         if (ret)
3542                 goto out;
3543         if (q->db_pidx != hw_pidx) {
3544                 u16 delta;
3545
3546                 if (q->db_pidx >= hw_pidx)
3547                         delta = q->db_pidx - hw_pidx;
3548                 else
3549                         delta = q->size - hw_pidx + q->db_pidx;
3550                 wmb();
3551                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3552                              QID(q->cntxt_id) | PIDX(delta));
3553         }
3554 out:
3555         q->db_disabled = 0;
3556         spin_unlock_bh(&q->db_lock);
3557         if (ret)
3558                 CH_WARN(adap, "DB drop recovery failed.\n");
3559 }
3560 static void recover_all_queues(struct adapter *adap)
3561 {
3562         int i;
3563
3564         for_each_ethrxq(&adap->sge, i)
3565                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3566         for_each_ofldrxq(&adap->sge, i)
3567                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3568         for_each_port(adap, i)
3569                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3570 }
3571
3572 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3573 {
3574         mutex_lock(&uld_mutex);
3575         if (adap->uld_handle[CXGB4_ULD_RDMA])
3576                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3577                                 cmd);
3578         mutex_unlock(&uld_mutex);
3579 }
3580
3581 static void process_db_full(struct work_struct *work)
3582 {
3583         struct adapter *adap;
3584
3585         adap = container_of(work, struct adapter, db_full_task);
3586
3587         notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3588         drain_db_fifo(adap, dbfifo_drain_delay);
3589         t4_set_reg_field(adap, SGE_INT_ENABLE3,
3590                          DBFIFO_HP_INT | DBFIFO_LP_INT,
3591                          DBFIFO_HP_INT | DBFIFO_LP_INT);
3592         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3593 }
3594
3595 static void process_db_drop(struct work_struct *work)
3596 {
3597         struct adapter *adap;
3598
3599         adap = container_of(work, struct adapter, db_drop_task);
3600
3601         if (is_t4(adap->chip)) {
3602                 disable_dbs(adap);
3603                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3604                 drain_db_fifo(adap, 1);
3605                 recover_all_queues(adap);
3606                 enable_dbs(adap);
3607         } else {
3608                 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3609                 u16 qid = (dropped_db >> 15) & 0x1ffff;
3610                 u16 pidx_inc = dropped_db & 0x1fff;
3611                 unsigned int s_qpp;
3612                 unsigned short udb_density;
3613                 unsigned long qpshift;
3614                 int page;
3615                 u32 udb;
3616
3617                 dev_warn(adap->pdev_dev,
3618                          "Dropped DB 0x%x qid %d bar2 %d coalesce %d pidx %d\n",
3619                          dropped_db, qid,
3620                          (dropped_db >> 14) & 1,
3621                          (dropped_db >> 13) & 1,
3622                          pidx_inc);
3623
3624                 drain_db_fifo(adap, 1);
3625
3626                 s_qpp = QUEUESPERPAGEPF1 * adap->fn;
3627                 udb_density = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adap,
3628                                 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
3629                 qpshift = PAGE_SHIFT - ilog2(udb_density);
3630                 udb = qid << qpshift;
3631                 udb &= PAGE_MASK;
3632                 page = udb / PAGE_SIZE;
3633                 udb += (qid - (page * udb_density)) * 128;
3634
3635                 writel(PIDX(pidx_inc),  adap->bar2 + udb + 8);
3636
3637                 /* Re-enable BAR2 WC */
3638                 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3639         }
3640
3641         t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
3642 }
3643
3644 void t4_db_full(struct adapter *adap)
3645 {
3646         if (is_t4(adap->chip)) {
3647                 t4_set_reg_field(adap, SGE_INT_ENABLE3,
3648                                  DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
3649                 queue_work(workq, &adap->db_full_task);
3650         }
3651 }
3652
3653 void t4_db_dropped(struct adapter *adap)
3654 {
3655         if (is_t4(adap->chip))
3656                 queue_work(workq, &adap->db_drop_task);
3657 }
3658
3659 static void uld_attach(struct adapter *adap, unsigned int uld)
3660 {
3661         void *handle;
3662         struct cxgb4_lld_info lli;
3663         unsigned short i;
3664
3665         lli.pdev = adap->pdev;
3666         lli.l2t = adap->l2t;
3667         lli.tids = &adap->tids;
3668         lli.ports = adap->port;
3669         lli.vr = &adap->vres;
3670         lli.mtus = adap->params.mtus;
3671         if (uld == CXGB4_ULD_RDMA) {
3672                 lli.rxq_ids = adap->sge.rdma_rxq;
3673                 lli.nrxq = adap->sge.rdmaqs;
3674         } else if (uld == CXGB4_ULD_ISCSI) {
3675                 lli.rxq_ids = adap->sge.ofld_rxq;
3676                 lli.nrxq = adap->sge.ofldqsets;
3677         }
3678         lli.ntxq = adap->sge.ofldqsets;
3679         lli.nchan = adap->params.nports;
3680         lli.nports = adap->params.nports;
3681         lli.wr_cred = adap->params.ofldq_wr_cred;
3682         lli.adapter_type = adap->params.rev;
3683         lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
3684         lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
3685                         t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
3686                         (adap->fn * 4));
3687         lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
3688                         t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
3689                         (adap->fn * 4));
3690         lli.filt_mode = adap->filter_mode;
3691         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3692         for (i = 0; i < NCHAN; i++)
3693                 lli.tx_modq[i] = i;
3694         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
3695         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
3696         lli.fw_vers = adap->params.fw_vers;
3697         lli.dbfifo_int_thresh = dbfifo_int_thresh;
3698         lli.sge_pktshift = adap->sge.pktshift;
3699         lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3700
3701         handle = ulds[uld].add(&lli);
3702         if (IS_ERR(handle)) {
3703                 dev_warn(adap->pdev_dev,
3704                          "could not attach to the %s driver, error %ld\n",
3705                          uld_str[uld], PTR_ERR(handle));
3706                 return;
3707         }
3708
3709         adap->uld_handle[uld] = handle;
3710
3711         if (!netevent_registered) {
3712                 register_netevent_notifier(&cxgb4_netevent_nb);
3713                 netevent_registered = true;
3714         }
3715
3716         if (adap->flags & FULL_INIT_DONE)
3717                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
3718 }
3719
3720 static void attach_ulds(struct adapter *adap)
3721 {
3722         unsigned int i;
3723
3724         mutex_lock(&uld_mutex);
3725         list_add_tail(&adap->list_node, &adapter_list);
3726         for (i = 0; i < CXGB4_ULD_MAX; i++)
3727                 if (ulds[i].add)
3728                         uld_attach(adap, i);
3729         mutex_unlock(&uld_mutex);
3730 }
3731
3732 static void detach_ulds(struct adapter *adap)
3733 {
3734         unsigned int i;
3735
3736         mutex_lock(&uld_mutex);
3737         list_del(&adap->list_node);
3738         for (i = 0; i < CXGB4_ULD_MAX; i++)
3739                 if (adap->uld_handle[i]) {
3740                         ulds[i].state_change(adap->uld_handle[i],
3741                                              CXGB4_STATE_DETACH);
3742                         adap->uld_handle[i] = NULL;
3743                 }
3744         if (netevent_registered && list_empty(&adapter_list)) {
3745                 unregister_netevent_notifier(&cxgb4_netevent_nb);
3746                 netevent_registered = false;
3747         }
3748         mutex_unlock(&uld_mutex);
3749 }
3750
3751 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
3752 {
3753         unsigned int i;
3754
3755         mutex_lock(&uld_mutex);
3756         for (i = 0; i < CXGB4_ULD_MAX; i++)
3757                 if (adap->uld_handle[i])
3758                         ulds[i].state_change(adap->uld_handle[i], new_state);
3759         mutex_unlock(&uld_mutex);
3760 }
3761
3762 /**
3763  *      cxgb4_register_uld - register an upper-layer driver
3764  *      @type: the ULD type
3765  *      @p: the ULD methods
3766  *
3767  *      Registers an upper-layer driver with this driver and notifies the ULD
3768  *      about any presently available devices that support its type.  Returns
3769  *      %-EBUSY if a ULD of the same type is already registered.
3770  */
3771 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
3772 {
3773         int ret = 0;
3774         struct adapter *adap;
3775
3776         if (type >= CXGB4_ULD_MAX)
3777                 return -EINVAL;
3778         mutex_lock(&uld_mutex);
3779         if (ulds[type].add) {
3780                 ret = -EBUSY;
3781                 goto out;
3782         }
3783         ulds[type] = *p;
3784         list_for_each_entry(adap, &adapter_list, list_node)
3785                 uld_attach(adap, type);
3786 out:    mutex_unlock(&uld_mutex);
3787         return ret;
3788 }
3789 EXPORT_SYMBOL(cxgb4_register_uld);
3790
3791 /**
3792  *      cxgb4_unregister_uld - unregister an upper-layer driver
3793  *      @type: the ULD type
3794  *
3795  *      Unregisters an existing upper-layer driver.
3796  */
3797 int cxgb4_unregister_uld(enum cxgb4_uld type)
3798 {
3799         struct adapter *adap;
3800
3801         if (type >= CXGB4_ULD_MAX)
3802                 return -EINVAL;
3803         mutex_lock(&uld_mutex);
3804         list_for_each_entry(adap, &adapter_list, list_node)
3805                 adap->uld_handle[type] = NULL;
3806         ulds[type].add = NULL;
3807         mutex_unlock(&uld_mutex);
3808         return 0;
3809 }
3810 EXPORT_SYMBOL(cxgb4_unregister_uld);
3811
3812 /**
3813  *      cxgb_up - enable the adapter
3814  *      @adap: adapter being enabled
3815  *
3816  *      Called when the first port is enabled, this function performs the
3817  *      actions necessary to make an adapter operational, such as completing
3818  *      the initialization of HW modules, and enabling interrupts.
3819  *
3820  *      Must be called with the rtnl lock held.
3821  */
3822 static int cxgb_up(struct adapter *adap)
3823 {
3824         int err;
3825
3826         err = setup_sge_queues(adap);
3827         if (err)
3828                 goto out;
3829         err = setup_rss(adap);
3830         if (err)
3831                 goto freeq;
3832
3833         if (adap->flags & USING_MSIX) {
3834                 name_msix_vecs(adap);
3835                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
3836                                   adap->msix_info[0].desc, adap);
3837                 if (err)
3838                         goto irq_err;
3839
3840                 err = request_msix_queue_irqs(adap);
3841                 if (err) {
3842                         free_irq(adap->msix_info[0].vec, adap);
3843                         goto irq_err;
3844                 }
3845         } else {
3846                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
3847                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
3848                                   adap->port[0]->name, adap);
3849                 if (err)
3850                         goto irq_err;
3851         }
3852         enable_rx(adap);
3853         t4_sge_start(adap);
3854         t4_intr_enable(adap);
3855         adap->flags |= FULL_INIT_DONE;
3856         notify_ulds(adap, CXGB4_STATE_UP);
3857  out:
3858         return err;
3859  irq_err:
3860         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
3861  freeq:
3862         t4_free_sge_resources(adap);
3863         goto out;
3864 }
3865
3866 static void cxgb_down(struct adapter *adapter)
3867 {
3868         t4_intr_disable(adapter);
3869         cancel_work_sync(&adapter->tid_release_task);
3870         cancel_work_sync(&adapter->db_full_task);
3871         cancel_work_sync(&adapter->db_drop_task);
3872         adapter->tid_release_task_busy = false;
3873         adapter->tid_release_head = NULL;
3874
3875         if (adapter->flags & USING_MSIX) {
3876                 free_msix_queue_irqs(adapter);
3877                 free_irq(adapter->msix_info[0].vec, adapter);
3878         } else
3879                 free_irq(adapter->pdev->irq, adapter);
3880         quiesce_rx(adapter);
3881         t4_sge_stop(adapter);
3882         t4_free_sge_resources(adapter);
3883         adapter->flags &= ~FULL_INIT_DONE;
3884 }
3885
3886 /*
3887  * net_device operations
3888  */
3889 static int cxgb_open(struct net_device *dev)
3890 {
3891         int err;
3892         struct port_info *pi = netdev_priv(dev);
3893         struct adapter *adapter = pi->adapter;
3894
3895         netif_carrier_off(dev);
3896
3897         if (!(adapter->flags & FULL_INIT_DONE)) {
3898                 err = cxgb_up(adapter);
3899                 if (err < 0)
3900                         return err;
3901         }
3902
3903         err = link_start(dev);
3904         if (!err)
3905                 netif_tx_start_all_queues(dev);
3906         return err;
3907 }
3908
3909 static int cxgb_close(struct net_device *dev)
3910 {
3911         struct port_info *pi = netdev_priv(dev);
3912         struct adapter *adapter = pi->adapter;
3913
3914         netif_tx_stop_all_queues(dev);
3915         netif_carrier_off(dev);
3916         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
3917 }
3918
3919 /* Return an error number if the indicated filter isn't writable ...
3920  */
3921 static int writable_filter(struct filter_entry *f)
3922 {
3923         if (f->locked)
3924                 return -EPERM;
3925         if (f->pending)
3926                 return -EBUSY;
3927
3928         return 0;
3929 }
3930
3931 /* Delete the filter at the specified index (if valid).  The checks for all
3932  * the common problems with doing this like the filter being locked, currently
3933  * pending in another operation, etc.
3934  */
3935 static int delete_filter(struct adapter *adapter, unsigned int fidx)
3936 {
3937         struct filter_entry *f;
3938         int ret;
3939
3940         if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
3941                 return -EINVAL;
3942
3943         f = &adapter->tids.ftid_tab[fidx];
3944         ret = writable_filter(f);
3945         if (ret)
3946                 return ret;
3947         if (f->valid)
3948                 return del_filter_wr(adapter, fidx);
3949
3950         return 0;
3951 }
3952
3953 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
3954                 __be32 sip, __be16 sport, __be16 vlan,
3955                 unsigned int queue, unsigned char port, unsigned char mask)
3956 {
3957         int ret;
3958         struct filter_entry *f;
3959         struct adapter *adap;
3960         int i;
3961         u8 *val;
3962
3963         adap = netdev2adap(dev);
3964
3965         /* Adjust stid to correct filter index */
3966         stid -= adap->tids.nstids;
3967         stid += adap->tids.nftids;
3968
3969         /* Check to make sure the filter requested is writable ...
3970          */
3971         f = &adap->tids.ftid_tab[stid];
3972         ret = writable_filter(f);
3973         if (ret)
3974                 return ret;
3975
3976         /* Clear out any old resources being used by the filter before
3977          * we start constructing the new filter.
3978          */
3979         if (f->valid)
3980                 clear_filter(adap, f);
3981
3982         /* Clear out filter specifications */
3983         memset(&f->fs, 0, sizeof(struct ch_filter_specification));
3984         f->fs.val.lport = cpu_to_be16(sport);
3985         f->fs.mask.lport  = ~0;
3986         val = (u8 *)&sip;
3987         if ((val[0] | val[1] | val[2] | val[3]) != 0) {
3988                 for (i = 0; i < 4; i++) {
3989                         f->fs.val.lip[i] = val[i];
3990                         f->fs.mask.lip[i] = ~0;
3991                 }
3992                 if (adap->filter_mode & F_PORT) {
3993                         f->fs.val.iport = port;
3994                         f->fs.mask.iport = mask;
3995                 }
3996         }
3997
3998         f->fs.dirsteer = 1;
3999         f->fs.iq = queue;
4000         /* Mark filter as locked */
4001         f->locked = 1;
4002         f->fs.rpttid = 1;
4003
4004         ret = set_filter_wr(adap, stid);
4005         if (ret) {
4006                 clear_filter(adap, f);
4007                 return ret;
4008         }
4009
4010         return 0;
4011 }
4012 EXPORT_SYMBOL(cxgb4_create_server_filter);
4013
4014 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4015                 unsigned int queue, bool ipv6)
4016 {
4017         int ret;
4018         struct filter_entry *f;
4019         struct adapter *adap;
4020
4021         adap = netdev2adap(dev);
4022
4023         /* Adjust stid to correct filter index */
4024         stid -= adap->tids.nstids;
4025         stid += adap->tids.nftids;
4026
4027         f = &adap->tids.ftid_tab[stid];
4028         /* Unlock the filter */
4029         f->locked = 0;
4030
4031         ret = delete_filter(adap, stid);
4032         if (ret)
4033                 return ret;
4034
4035         return 0;
4036 }
4037 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4038
4039 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4040                                                 struct rtnl_link_stats64 *ns)
4041 {
4042         struct port_stats stats;
4043         struct port_info *p = netdev_priv(dev);
4044         struct adapter *adapter = p->adapter;
4045
4046         spin_lock(&adapter->stats_lock);
4047         t4_get_port_stats(adapter, p->tx_chan, &stats);
4048         spin_unlock(&adapter->stats_lock);
4049
4050         ns->tx_bytes   = stats.tx_octets;
4051         ns->tx_packets = stats.tx_frames;
4052         ns->rx_bytes   = stats.rx_octets;
4053         ns->rx_packets = stats.rx_frames;
4054         ns->multicast  = stats.rx_mcast_frames;
4055
4056         /* detailed rx_errors */
4057         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4058                                stats.rx_runt;
4059         ns->rx_over_errors   = 0;
4060         ns->rx_crc_errors    = stats.rx_fcs_err;
4061         ns->rx_frame_errors  = stats.rx_symbol_err;
4062         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
4063                                stats.rx_ovflow2 + stats.rx_ovflow3 +
4064                                stats.rx_trunc0 + stats.rx_trunc1 +
4065                                stats.rx_trunc2 + stats.rx_trunc3;
4066         ns->rx_missed_errors = 0;
4067
4068         /* detailed tx_errors */
4069         ns->tx_aborted_errors   = 0;
4070         ns->tx_carrier_errors   = 0;
4071         ns->tx_fifo_errors      = 0;
4072         ns->tx_heartbeat_errors = 0;
4073         ns->tx_window_errors    = 0;
4074
4075         ns->tx_errors = stats.tx_error_frames;
4076         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4077                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4078         return ns;
4079 }
4080
4081 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4082 {
4083         unsigned int mbox;
4084         int ret = 0, prtad, devad;
4085         struct port_info *pi = netdev_priv(dev);
4086         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4087
4088         switch (cmd) {
4089         case SIOCGMIIPHY:
4090                 if (pi->mdio_addr < 0)
4091                         return -EOPNOTSUPP;
4092                 data->phy_id = pi->mdio_addr;
4093                 break;
4094         case SIOCGMIIREG:
4095         case SIOCSMIIREG:
4096                 if (mdio_phy_id_is_c45(data->phy_id)) {
4097                         prtad = mdio_phy_id_prtad(data->phy_id);
4098                         devad = mdio_phy_id_devad(data->phy_id);
4099                 } else if (data->phy_id < 32) {
4100                         prtad = data->phy_id;
4101                         devad = 0;
4102                         data->reg_num &= 0x1f;
4103                 } else
4104                         return -EINVAL;
4105
4106                 mbox = pi->adapter->fn;
4107                 if (cmd == SIOCGMIIREG)
4108                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4109                                          data->reg_num, &data->val_out);
4110                 else
4111                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4112                                          data->reg_num, data->val_in);
4113                 break;
4114         default:
4115                 return -EOPNOTSUPP;
4116         }
4117         return ret;
4118 }
4119
4120 static void cxgb_set_rxmode(struct net_device *dev)
4121 {
4122         /* unfortunately we can't return errors to the stack */
4123         set_rxmode(dev, -1, false);
4124 }
4125
4126 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4127 {
4128         int ret;
4129         struct port_info *pi = netdev_priv(dev);
4130
4131         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
4132                 return -EINVAL;
4133         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4134                             -1, -1, -1, true);
4135         if (!ret)
4136                 dev->mtu = new_mtu;
4137         return ret;
4138 }
4139
4140 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4141 {
4142         int ret;
4143         struct sockaddr *addr = p;
4144         struct port_info *pi = netdev_priv(dev);
4145
4146         if (!is_valid_ether_addr(addr->sa_data))
4147                 return -EADDRNOTAVAIL;
4148
4149         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4150                             pi->xact_addr_filt, addr->sa_data, true, true);
4151         if (ret < 0)
4152                 return ret;
4153
4154         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4155         pi->xact_addr_filt = ret;
4156         return 0;
4157 }
4158
4159 #ifdef CONFIG_NET_POLL_CONTROLLER
4160 static void cxgb_netpoll(struct net_device *dev)
4161 {
4162         struct port_info *pi = netdev_priv(dev);
4163         struct adapter *adap = pi->adapter;
4164
4165         if (adap->flags & USING_MSIX) {
4166                 int i;
4167                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4168
4169                 for (i = pi->nqsets; i; i--, rx++)
4170                         t4_sge_intr_msix(0, &rx->rspq);
4171         } else
4172                 t4_intr_handler(adap)(0, adap);
4173 }
4174 #endif
4175
4176 static const struct net_device_ops cxgb4_netdev_ops = {
4177         .ndo_open             = cxgb_open,
4178         .ndo_stop             = cxgb_close,
4179         .ndo_start_xmit       = t4_eth_xmit,
4180         .ndo_get_stats64      = cxgb_get_stats,
4181         .ndo_set_rx_mode      = cxgb_set_rxmode,
4182         .ndo_set_mac_address  = cxgb_set_mac_addr,
4183         .ndo_set_features     = cxgb_set_features,
4184         .ndo_validate_addr    = eth_validate_addr,
4185         .ndo_do_ioctl         = cxgb_ioctl,
4186         .ndo_change_mtu       = cxgb_change_mtu,
4187 #ifdef CONFIG_NET_POLL_CONTROLLER
4188         .ndo_poll_controller  = cxgb_netpoll,
4189 #endif
4190 };
4191
4192 void t4_fatal_err(struct adapter *adap)
4193 {
4194         t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4195         t4_intr_disable(adap);
4196         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4197 }
4198
4199 static void setup_memwin(struct adapter *adap)
4200 {
4201         u32 bar0, mem_win0_base, mem_win1_base, mem_win2_base;
4202
4203         bar0 = pci_resource_start(adap->pdev, 0);  /* truncation intentional */
4204         if (is_t4(adap->chip)) {
4205                 mem_win0_base = bar0 + MEMWIN0_BASE;
4206                 mem_win1_base = bar0 + MEMWIN1_BASE;
4207                 mem_win2_base = bar0 + MEMWIN2_BASE;
4208         } else {
4209                 /* For T5, only relative offset inside the PCIe BAR is passed */
4210                 mem_win0_base = MEMWIN0_BASE;
4211                 mem_win1_base = MEMWIN1_BASE_T5;
4212                 mem_win2_base = MEMWIN2_BASE_T5;
4213         }
4214         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
4215                      mem_win0_base | BIR(0) |
4216                      WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4217         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
4218                      mem_win1_base | BIR(0) |
4219                      WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4220         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
4221                      mem_win2_base | BIR(0) |
4222                      WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
4223 }
4224
4225 static void setup_memwin_rdma(struct adapter *adap)
4226 {
4227         if (adap->vres.ocq.size) {
4228                 unsigned int start, sz_kb;
4229
4230                 start = pci_resource_start(adap->pdev, 2) +
4231                         OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4232                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4233                 t4_write_reg(adap,
4234                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4235                              start | BIR(1) | WINDOW(ilog2(sz_kb)));
4236                 t4_write_reg(adap,
4237                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4238                              adap->vres.ocq.start);
4239                 t4_read_reg(adap,
4240                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4241         }
4242 }
4243
4244 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4245 {
4246         u32 v;
4247         int ret;
4248
4249         /* get device capabilities */
4250         memset(c, 0, sizeof(*c));
4251         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4252                                FW_CMD_REQUEST | FW_CMD_READ);
4253         c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4254         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4255         if (ret < 0)
4256                 return ret;
4257
4258         /* select capabilities we'll be using */
4259         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4260                 if (!vf_acls)
4261                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4262                 else
4263                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4264         } else if (vf_acls) {
4265                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4266                 return ret;
4267         }
4268         c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4269                                FW_CMD_REQUEST | FW_CMD_WRITE);
4270         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4271         if (ret < 0)
4272                 return ret;
4273
4274         ret = t4_config_glbl_rss(adap, adap->fn,
4275                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4276                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4277                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
4278         if (ret < 0)
4279                 return ret;
4280
4281         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4282                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4283         if (ret < 0)
4284                 return ret;
4285
4286         t4_sge_init(adap);
4287
4288         /* tweak some settings */
4289         t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4290         t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4291         t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4292         v = t4_read_reg(adap, TP_PIO_DATA);
4293         t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
4294
4295         /* first 4 Tx modulation queues point to consecutive Tx channels */
4296         adap->params.tp.tx_modq_map = 0xE4;
4297         t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4298                      V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4299
4300         /* associate each Tx modulation queue with consecutive Tx channels */
4301         v = 0x84218421;
4302         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4303                           &v, 1, A_TP_TX_SCHED_HDR);
4304         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4305                           &v, 1, A_TP_TX_SCHED_FIFO);
4306         t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4307                           &v, 1, A_TP_TX_SCHED_PCMD);
4308
4309 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4310         if (is_offload(adap)) {
4311                 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4312                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4313                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4314                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4315                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4316                 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4317                              V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4318                              V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4319                              V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4320                              V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4321         }
4322
4323         /* get basic stuff going */
4324         return t4_early_init(adap, adap->fn);
4325 }
4326
4327 /*
4328  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
4329  */
4330 #define MAX_ATIDS 8192U
4331
4332 /*
4333  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4334  *
4335  * If the firmware we're dealing with has Configuration File support, then
4336  * we use that to perform all configuration
4337  */
4338
4339 /*
4340  * Tweak configuration based on module parameters, etc.  Most of these have
4341  * defaults assigned to them by Firmware Configuration Files (if we're using
4342  * them) but need to be explicitly set if we're using hard-coded
4343  * initialization.  But even in the case of using Firmware Configuration
4344  * Files, we'd like to expose the ability to change these via module
4345  * parameters so these are essentially common tweaks/settings for
4346  * Configuration Files and hard-coded initialization ...
4347  */
4348 static int adap_init0_tweaks(struct adapter *adapter)
4349 {
4350         /*
4351          * Fix up various Host-Dependent Parameters like Page Size, Cache
4352          * Line Size, etc.  The firmware default is for a 4KB Page Size and
4353          * 64B Cache Line Size ...
4354          */
4355         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4356
4357         /*
4358          * Process module parameters which affect early initialization.
4359          */
4360         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4361                 dev_err(&adapter->pdev->dev,
4362                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
4363                         rx_dma_offset);
4364                 rx_dma_offset = 2;
4365         }
4366         t4_set_reg_field(adapter, SGE_CONTROL,
4367                          PKTSHIFT_MASK,
4368                          PKTSHIFT(rx_dma_offset));
4369
4370         /*
4371          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4372          * adds the pseudo header itself.
4373          */
4374         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
4375                                CSUM_HAS_PSEUDO_HDR, 0);
4376
4377         return 0;
4378 }
4379
4380 /*
4381  * Attempt to initialize the adapter via a Firmware Configuration File.
4382  */
4383 static int adap_init0_config(struct adapter *adapter, int reset)
4384 {
4385         struct fw_caps_config_cmd caps_cmd;
4386         const struct firmware *cf;
4387         unsigned long mtype = 0, maddr = 0;
4388         u32 finiver, finicsum, cfcsum;
4389         int ret, using_flash;
4390         char *fw_config_file, fw_config_file_path[256];
4391
4392         /*
4393          * Reset device if necessary.
4394          */
4395         if (reset) {
4396                 ret = t4_fw_reset(adapter, adapter->mbox,
4397                                   PIORSTMODE | PIORST);
4398                 if (ret < 0)
4399                         goto bye;
4400         }
4401
4402         /*
4403          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4404          * then use that.  Otherwise, use the configuration file stored
4405          * in the adapter flash ...
4406          */
4407         switch (CHELSIO_CHIP_VERSION(adapter->chip)) {
4408         case CHELSIO_T4:
4409                 fw_config_file = FW_CFNAME;
4410                 break;
4411         case CHELSIO_T5:
4412                 fw_config_file = FW5_CFNAME;
4413                 break;
4414         default:
4415                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4416                        adapter->pdev->device);
4417                 ret = -EINVAL;
4418                 goto bye;
4419         }
4420
4421         ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4422         if (ret < 0) {
4423                 using_flash = 1;
4424                 mtype = FW_MEMTYPE_CF_FLASH;
4425                 maddr = t4_flash_cfg_addr(adapter);
4426         } else {
4427                 u32 params[7], val[7];
4428
4429                 using_flash = 0;
4430                 if (cf->size >= FLASH_CFG_MAX_SIZE)
4431                         ret = -ENOMEM;
4432                 else {
4433                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4434                              FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4435                         ret = t4_query_params(adapter, adapter->mbox,
4436                                               adapter->fn, 0, 1, params, val);
4437                         if (ret == 0) {
4438                                 /*
4439                                  * For t4_memory_write() below addresses and
4440                                  * sizes have to be in terms of multiples of 4
4441                                  * bytes.  So, if the Configuration File isn't
4442                                  * a multiple of 4 bytes in length we'll have
4443                                  * to write that out separately since we can't
4444                                  * guarantee that the bytes following the
4445                                  * residual byte in the buffer returned by
4446                                  * request_firmware() are zeroed out ...
4447                                  */
4448                                 size_t resid = cf->size & 0x3;
4449                                 size_t size = cf->size & ~0x3;
4450                                 __be32 *data = (__be32 *)cf->data;
4451
4452                                 mtype = FW_PARAMS_PARAM_Y_GET(val[0]);
4453                                 maddr = FW_PARAMS_PARAM_Z_GET(val[0]) << 16;
4454
4455                                 ret = t4_memory_write(adapter, mtype, maddr,
4456                                                       size, data);
4457                                 if (ret == 0 && resid != 0) {
4458                                         union {
4459                                                 __be32 word;
4460                                                 char buf[4];
4461                                         } last;
4462                                         int i;
4463
4464                                         last.word = data[size >> 2];
4465                                         for (i = resid; i < 4; i++)
4466                                                 last.buf[i] = 0;
4467                                         ret = t4_memory_write(adapter, mtype,
4468                                                               maddr + size,
4469                                                               4, &last.word);
4470                                 }
4471                         }
4472                 }
4473
4474                 release_firmware(cf);
4475                 if (ret)
4476                         goto bye;
4477         }
4478
4479         /*
4480          * Issue a Capability Configuration command to the firmware to get it
4481          * to parse the Configuration File.  We don't use t4_fw_config_file()
4482          * because we want the ability to modify various features after we've
4483          * processed the configuration file ...
4484          */
4485         memset(&caps_cmd, 0, sizeof(caps_cmd));
4486         caps_cmd.op_to_write =
4487                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4488                       FW_CMD_REQUEST |
4489                       FW_CMD_READ);
4490         caps_cmd.cfvalid_to_len16 =
4491                 htonl(FW_CAPS_CONFIG_CMD_CFVALID |
4492                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
4493                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) |
4494                       FW_LEN16(caps_cmd));
4495         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4496                          &caps_cmd);
4497         if (ret < 0)
4498                 goto bye;
4499
4500         finiver = ntohl(caps_cmd.finiver);
4501         finicsum = ntohl(caps_cmd.finicsum);
4502         cfcsum = ntohl(caps_cmd.cfcsum);
4503         if (finicsum != cfcsum)
4504                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4505                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4506                          finicsum, cfcsum);
4507
4508         /*
4509          * And now tell the firmware to use the configuration we just loaded.
4510          */
4511         caps_cmd.op_to_write =
4512                 htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4513                       FW_CMD_REQUEST |
4514                       FW_CMD_WRITE);
4515         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4516         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4517                          NULL);
4518         if (ret < 0)
4519                 goto bye;
4520
4521         /*
4522          * Tweak configuration based on system architecture, module
4523          * parameters, etc.
4524          */
4525         ret = adap_init0_tweaks(adapter);
4526         if (ret < 0)
4527                 goto bye;
4528
4529         /*
4530          * And finally tell the firmware to initialize itself using the
4531          * parameters from the Configuration File.
4532          */
4533         ret = t4_fw_initialize(adapter, adapter->mbox);
4534         if (ret < 0)
4535                 goto bye;
4536
4537         sprintf(fw_config_file_path, "/lib/firmware/%s", fw_config_file);
4538         /*
4539          * Return successfully and note that we're operating with parameters
4540          * not supplied by the driver, rather than from hard-wired
4541          * initialization constants burried in the driver.
4542          */
4543         adapter->flags |= USING_SOFT_PARAMS;
4544         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
4545                  "Configuration File %s, version %#x, computed checksum %#x\n",
4546                  (using_flash
4547                   ? "in device FLASH"
4548                   : fw_config_file_path),
4549                  finiver, cfcsum);
4550         return 0;
4551
4552         /*
4553          * Something bad happened.  Return the error ...  (If the "error"
4554          * is that there's no Configuration File on the adapter we don't
4555          * want to issue a warning since this is fairly common.)
4556          */
4557 bye:
4558         if (ret != -ENOENT)
4559                 dev_warn(adapter->pdev_dev, "Configuration file error %d\n",
4560                          -ret);
4561         return ret;
4562 }
4563
4564 /*
4565  * Attempt to initialize the adapter via hard-coded, driver supplied
4566  * parameters ...
4567  */
4568 static int adap_init0_no_config(struct adapter *adapter, int reset)
4569 {
4570         struct sge *s = &adapter->sge;
4571         struct fw_caps_config_cmd caps_cmd;
4572         u32 v;
4573         int i, ret;
4574
4575         /*
4576          * Reset device if necessary
4577          */
4578         if (reset) {
4579                 ret = t4_fw_reset(adapter, adapter->mbox,
4580                                   PIORSTMODE | PIORST);
4581                 if (ret < 0)
4582                         goto bye;
4583         }
4584
4585         /*
4586          * Get device capabilities and select which we'll be using.
4587          */
4588         memset(&caps_cmd, 0, sizeof(caps_cmd));
4589         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4590                                      FW_CMD_REQUEST | FW_CMD_READ);
4591         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4592         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4593                          &caps_cmd);
4594         if (ret < 0)
4595                 goto bye;
4596
4597         if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4598                 if (!vf_acls)
4599                         caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4600                 else
4601                         caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4602         } else if (vf_acls) {
4603                 dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
4604                 goto bye;
4605         }
4606         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
4607                               FW_CMD_REQUEST | FW_CMD_WRITE);
4608         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4609                          NULL);
4610         if (ret < 0)
4611                 goto bye;
4612
4613         /*
4614          * Tweak configuration based on system architecture, module
4615          * parameters, etc.
4616          */
4617         ret = adap_init0_tweaks(adapter);
4618         if (ret < 0)
4619                 goto bye;
4620
4621         /*
4622          * Select RSS Global Mode we want to use.  We use "Basic Virtual"
4623          * mode which maps each Virtual Interface to its own section of
4624          * the RSS Table and we turn on all map and hash enables ...
4625          */
4626         adapter->flags |= RSS_TNLALLLOOKUP;
4627         ret = t4_config_glbl_rss(adapter, adapter->mbox,
4628                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4629                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
4630                                  FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
4631                                  ((adapter->flags & RSS_TNLALLLOOKUP) ?
4632                                         FW_RSS_GLB_CONFIG_CMD_TNLALLLKP : 0));
4633         if (ret < 0)
4634                 goto bye;
4635
4636         /*
4637          * Set up our own fundamental resource provisioning ...
4638          */
4639         ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
4640                           PFRES_NEQ, PFRES_NETHCTRL,
4641                           PFRES_NIQFLINT, PFRES_NIQ,
4642                           PFRES_TC, PFRES_NVI,
4643                           FW_PFVF_CMD_CMASK_MASK,
4644                           pfvfres_pmask(adapter, adapter->fn, 0),
4645                           PFRES_NEXACTF,
4646                           PFRES_R_CAPS, PFRES_WX_CAPS);
4647         if (ret < 0)
4648                 goto bye;
4649
4650         /*
4651          * Perform low level SGE initialization.  We need to do this before we
4652          * send the firmware the INITIALIZE command because that will cause
4653          * any other PF Drivers which are waiting for the Master
4654          * Initialization to proceed forward.
4655          */
4656         for (i = 0; i < SGE_NTIMERS - 1; i++)
4657                 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
4658         s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
4659         s->counter_val[0] = 1;
4660         for (i = 1; i < SGE_NCOUNTERS; i++)
4661                 s->counter_val[i] = min(intr_cnt[i - 1],
4662                                         THRESHOLD_0_GET(THRESHOLD_0_MASK));
4663         t4_sge_init(adapter);
4664
4665 #ifdef CONFIG_PCI_IOV
4666         /*
4667          * Provision resource limits for Virtual Functions.  We currently
4668          * grant them all the same static resource limits except for the Port
4669          * Access Rights Mask which we're assigning based on the PF.  All of
4670          * the static provisioning stuff for both the PF and VF really needs
4671          * to be managed in a persistent manner for each device which the
4672          * firmware controls.
4673          */
4674         {
4675                 int pf, vf;
4676
4677                 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
4678                         if (num_vf[pf] <= 0)
4679                                 continue;
4680
4681                         /* VF numbering starts at 1! */
4682                         for (vf = 1; vf <= num_vf[pf]; vf++) {
4683                                 ret = t4_cfg_pfvf(adapter, adapter->mbox,
4684                                                   pf, vf,
4685                                                   VFRES_NEQ, VFRES_NETHCTRL,
4686                                                   VFRES_NIQFLINT, VFRES_NIQ,
4687                                                   VFRES_TC, VFRES_NVI,
4688                                                   FW_PFVF_CMD_CMASK_MASK,
4689                                                   pfvfres_pmask(
4690                                                   adapter, pf, vf),
4691                                                   VFRES_NEXACTF,
4692                                                   VFRES_R_CAPS, VFRES_WX_CAPS);
4693                                 if (ret < 0)
4694                                         dev_warn(adapter->pdev_dev,
4695                                                  "failed to "\
4696                                                  "provision pf/vf=%d/%d; "
4697                                                  "err=%d\n", pf, vf, ret);
4698                         }
4699                 }
4700         }
4701 #endif
4702
4703         /*
4704          * Set up the default filter mode.  Later we'll want to implement this
4705          * via a firmware command, etc. ...  This needs to be done before the
4706          * firmare initialization command ...  If the selected set of fields
4707          * isn't equal to the default value, we'll need to make sure that the
4708          * field selections will fit in the 36-bit budget.
4709          */
4710         if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
4711                 int j, bits = 0;
4712
4713                 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
4714                         switch (tp_vlan_pri_map & (1 << j)) {
4715                         case 0:
4716                                 /* compressed filter field not enabled */
4717                                 break;
4718                         case FCOE_MASK:
4719                                 bits +=  1;
4720                                 break;
4721                         case PORT_MASK:
4722                                 bits +=  3;
4723                                 break;
4724                         case VNIC_ID_MASK:
4725                                 bits += 17;
4726                                 break;
4727                         case VLAN_MASK:
4728                                 bits += 17;
4729                                 break;
4730                         case TOS_MASK:
4731                                 bits +=  8;
4732                                 break;
4733                         case PROTOCOL_MASK:
4734                                 bits +=  8;
4735                                 break;
4736                         case ETHERTYPE_MASK:
4737                                 bits += 16;
4738                                 break;
4739                         case MACMATCH_MASK:
4740                                 bits +=  9;
4741                                 break;
4742                         case MPSHITTYPE_MASK:
4743                                 bits +=  3;
4744                                 break;
4745                         case FRAGMENTATION_MASK:
4746                                 bits +=  1;
4747                                 break;
4748                         }
4749
4750                 if (bits > 36) {
4751                         dev_err(adapter->pdev_dev,
4752                                 "tp_vlan_pri_map=%#x needs %d bits > 36;"\
4753                                 " using %#x\n", tp_vlan_pri_map, bits,
4754                                 TP_VLAN_PRI_MAP_DEFAULT);
4755                         tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
4756                 }
4757         }
4758         v = tp_vlan_pri_map;
4759         t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
4760                           &v, 1, TP_VLAN_PRI_MAP);
4761
4762         /*
4763          * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
4764          * to support any of the compressed filter fields above.  Newer
4765          * versions of the firmware do this automatically but it doesn't hurt
4766          * to set it here.  Meanwhile, we do _not_ need to set Lookup Every
4767          * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
4768          * since the firmware automatically turns this on and off when we have
4769          * a non-zero number of filters active (since it does have a
4770          * performance impact).
4771          */
4772         if (tp_vlan_pri_map)
4773                 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
4774                                  FIVETUPLELOOKUP_MASK,
4775                                  FIVETUPLELOOKUP_MASK);
4776
4777         /*
4778          * Tweak some settings.
4779          */
4780         t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
4781                      RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
4782                      PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
4783                      KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
4784
4785         /*
4786          * Get basic stuff going by issuing the Firmware Initialize command.
4787          * Note that this _must_ be after all PFVF commands ...
4788          */
4789         ret = t4_fw_initialize(adapter, adapter->mbox);
4790         if (ret < 0)
4791                 goto bye;
4792
4793         /*
4794          * Return successfully!
4795          */
4796         dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
4797                  "driver parameters\n");
4798         return 0;
4799
4800         /*
4801          * Something bad happened.  Return the error ...
4802          */
4803 bye:
4804         return ret;
4805 }
4806
4807 /*
4808  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4809  */
4810 static int adap_init0(struct adapter *adap)
4811 {
4812         int ret;
4813         u32 v, port_vec;
4814         enum dev_state state;
4815         u32 params[7], val[7];
4816         struct fw_caps_config_cmd caps_cmd;
4817         int reset = 1, j;
4818
4819         /*
4820          * Contact FW, advertising Master capability (and potentially forcing
4821          * ourselves as the Master PF if our module parameter force_init is
4822          * set).
4823          */
4824         ret = t4_fw_hello(adap, adap->mbox, adap->fn,
4825                           force_init ? MASTER_MUST : MASTER_MAY,
4826                           &state);
4827         if (ret < 0) {
4828                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
4829                         ret);
4830                 return ret;
4831         }
4832         if (ret == adap->mbox)
4833                 adap->flags |= MASTER_PF;
4834         if (force_init && state == DEV_STATE_INIT)
4835                 state = DEV_STATE_UNINIT;
4836
4837         /*
4838          * If we're the Master PF Driver and the device is uninitialized,
4839          * then let's consider upgrading the firmware ...  (We always want
4840          * to check the firmware version number in order to A. get it for
4841          * later reporting and B. to warn if the currently loaded firmware
4842          * is excessively mismatched relative to the driver.)
4843          */
4844         ret = t4_check_fw_version(adap);
4845
4846         /* The error code -EFAULT is returned by t4_check_fw_version() if
4847          * firmware on adapter < supported firmware. If firmware on adapter
4848          * is too old (not supported by driver) and we're the MASTER_PF set
4849          * adapter state to DEV_STATE_UNINIT to force firmware upgrade
4850          * and reinitialization.
4851          */
4852         if ((adap->flags & MASTER_PF) && ret == -EFAULT)
4853                 state = DEV_STATE_UNINIT;
4854         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
4855                 if (ret == -EINVAL || ret == -EFAULT || ret > 0) {
4856                         if (upgrade_fw(adap) >= 0) {
4857                                 /*
4858                                  * Note that the chip was reset as part of the
4859                                  * firmware upgrade so we don't reset it again
4860                                  * below and grab the new firmware version.
4861                                  */
4862                                 reset = 0;
4863                                 ret = t4_check_fw_version(adap);
4864                         } else
4865                                 if (ret == -EFAULT) {
4866                                         /*
4867                                          * Firmware is old but still might
4868                                          * work if we force reinitialization
4869                                          * of the adapter. Ignoring FW upgrade
4870                                          * failure.
4871                                          */
4872                                         dev_warn(adap->pdev_dev,
4873                                                  "Ignoring firmware upgrade "
4874                                                  "failure, and forcing driver "
4875                                                  "to reinitialize the "
4876                                                  "adapter.\n");
4877                                         ret = 0;
4878                                 }
4879                 }
4880                 if (ret < 0)
4881                         return ret;
4882         }
4883
4884         /*
4885          * Grab VPD parameters.  This should be done after we establish a
4886          * connection to the firmware since some of the VPD parameters
4887          * (notably the Core Clock frequency) are retrieved via requests to
4888          * the firmware.  On the other hand, we need these fairly early on
4889          * so we do this right after getting ahold of the firmware.
4890          */
4891         ret = get_vpd_params(adap, &adap->params.vpd);
4892         if (ret < 0)
4893                 goto bye;
4894
4895         /*
4896          * Find out what ports are available to us.  Note that we need to do
4897          * this before calling adap_init0_no_config() since it needs nports
4898          * and portvec ...
4899          */
4900         v =
4901             FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4902             FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_PORTVEC);
4903         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
4904         if (ret < 0)
4905                 goto bye;
4906
4907         adap->params.nports = hweight32(port_vec);
4908         adap->params.portvec = port_vec;
4909
4910         /*
4911          * If the firmware is initialized already (and we're not forcing a
4912          * master initialization), note that we're living with existing
4913          * adapter parameters.  Otherwise, it's time to try initializing the
4914          * adapter ...
4915          */
4916         if (state == DEV_STATE_INIT) {
4917                 dev_info(adap->pdev_dev, "Coming up as %s: "\
4918                          "Adapter already initialized\n",
4919                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
4920                 adap->flags |= USING_SOFT_PARAMS;
4921         } else {
4922                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
4923                          "Initializing adapter\n");
4924
4925                 /*
4926                  * If the firmware doesn't support Configuration
4927                  * Files warn user and exit,
4928                  */
4929                 if (ret < 0)
4930                         dev_warn(adap->pdev_dev, "Firmware doesn't support "
4931                                  "configuration file.\n");
4932                 if (force_old_init)
4933                         ret = adap_init0_no_config(adap, reset);
4934                 else {
4935                         /*
4936                          * Find out whether we're dealing with a version of
4937                          * the firmware which has configuration file support.
4938                          */
4939                         params[0] = (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
4940                                      FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CF));
4941                         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
4942                                               params, val);
4943
4944                         /*
4945                          * If the firmware doesn't support Configuration
4946                          * Files, use the old Driver-based, hard-wired
4947                          * initialization.  Otherwise, try using the
4948                          * Configuration File support and fall back to the
4949                          * Driver-based initialization if there's no
4950                          * Configuration File found.
4951                          */
4952                         if (ret < 0)
4953                                 ret = adap_init0_no_config(adap, reset);
4954                         else {
4955                                 /*
4956                                  * The firmware provides us with a memory
4957                                  * buffer where we can load a Configuration
4958                                  * File from the host if we want to override
4959                                  * the Configuration File in flash.
4960                                  */
4961
4962                                 ret = adap_init0_config(adap, reset);
4963                                 if (ret == -ENOENT) {
4964                                         dev_info(adap->pdev_dev,
4965                                             "No Configuration File present "
4966                                             "on adapter.  Using hard-wired "
4967                                             "configuration parameters.\n");
4968                                         ret = adap_init0_no_config(adap, reset);
4969                                 }
4970                         }
4971                 }
4972                 if (ret < 0) {
4973                         dev_err(adap->pdev_dev,
4974                                 "could not initialize adapter, error %d\n",
4975                                 -ret);
4976                         goto bye;
4977                 }
4978         }
4979
4980         /*
4981          * If we're living with non-hard-coded parameters (either from a
4982          * Firmware Configuration File or values programmed by a different PF
4983          * Driver), give the SGE code a chance to pull in anything that it
4984          * needs ...  Note that this must be called after we retrieve our VPD
4985          * parameters in order to know how to convert core ticks to seconds.
4986          */
4987         if (adap->flags & USING_SOFT_PARAMS) {
4988                 ret = t4_sge_init(adap);
4989                 if (ret < 0)
4990                         goto bye;
4991         }
4992
4993         if (is_bypass_device(adap->pdev->device))
4994                 adap->params.bypass = 1;
4995
4996         /*
4997          * Grab some of our basic fundamental operating parameters.
4998          */
4999 #define FW_PARAM_DEV(param) \
5000         (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
5001         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
5002
5003 #define FW_PARAM_PFVF(param) \
5004         FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
5005         FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)|  \
5006         FW_PARAMS_PARAM_Y(0) | \
5007         FW_PARAMS_PARAM_Z(0)
5008
5009         params[0] = FW_PARAM_PFVF(EQ_START);
5010         params[1] = FW_PARAM_PFVF(L2T_START);
5011         params[2] = FW_PARAM_PFVF(L2T_END);
5012         params[3] = FW_PARAM_PFVF(FILTER_START);
5013         params[4] = FW_PARAM_PFVF(FILTER_END);
5014         params[5] = FW_PARAM_PFVF(IQFLINT_START);
5015         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5016         if (ret < 0)
5017                 goto bye;
5018         adap->sge.egr_start = val[0];
5019         adap->l2t_start = val[1];
5020         adap->l2t_end = val[2];
5021         adap->tids.ftid_base = val[3];
5022         adap->tids.nftids = val[4] - val[3] + 1;
5023         adap->sge.ingr_start = val[5];
5024
5025         /* query params related to active filter region */
5026         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5027         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5028         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5029         /* If Active filter size is set we enable establishing
5030          * offload connection through firmware work request
5031          */
5032         if ((val[0] != val[1]) && (ret >= 0)) {
5033                 adap->flags |= FW_OFLD_CONN;
5034                 adap->tids.aftid_base = val[0];
5035                 adap->tids.aftid_end = val[1];
5036         }
5037
5038         /* If we're running on newer firmware, let it know that we're
5039          * prepared to deal with encapsulated CPL messages.  Older
5040          * firmware won't understand this and we'll just get
5041          * unencapsulated messages ...
5042          */
5043         params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5044         val[0] = 1;
5045         (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5046
5047         /*
5048          * Get device capabilities so we can determine what resources we need
5049          * to manage.
5050          */
5051         memset(&caps_cmd, 0, sizeof(caps_cmd));
5052         caps_cmd.op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
5053                                      FW_CMD_REQUEST | FW_CMD_READ);
5054         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5055         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5056                          &caps_cmd);
5057         if (ret < 0)
5058                 goto bye;
5059
5060         if (caps_cmd.ofldcaps) {
5061                 /* query offload-related parameters */
5062                 params[0] = FW_PARAM_DEV(NTID);
5063                 params[1] = FW_PARAM_PFVF(SERVER_START);
5064                 params[2] = FW_PARAM_PFVF(SERVER_END);
5065                 params[3] = FW_PARAM_PFVF(TDDP_START);
5066                 params[4] = FW_PARAM_PFVF(TDDP_END);
5067                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5068                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5069                                       params, val);
5070                 if (ret < 0)
5071                         goto bye;
5072                 adap->tids.ntids = val[0];
5073                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5074                 adap->tids.stid_base = val[1];
5075                 adap->tids.nstids = val[2] - val[1] + 1;
5076                 /*
5077                  * Setup server filter region. Divide the availble filter
5078                  * region into two parts. Regular filters get 1/3rd and server
5079                  * filters get 2/3rd part. This is only enabled if workarond
5080                  * path is enabled.
5081                  * 1. For regular filters.
5082                  * 2. Server filter: This are special filters which are used
5083                  * to redirect SYN packets to offload queue.
5084                  */
5085                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5086                         adap->tids.sftid_base = adap->tids.ftid_base +
5087                                         DIV_ROUND_UP(adap->tids.nftids, 3);
5088                         adap->tids.nsftids = adap->tids.nftids -
5089                                          DIV_ROUND_UP(adap->tids.nftids, 3);
5090                         adap->tids.nftids = adap->tids.sftid_base -
5091                                                 adap->tids.ftid_base;
5092                 }
5093                 adap->vres.ddp.start = val[3];
5094                 adap->vres.ddp.size = val[4] - val[3] + 1;
5095                 adap->params.ofldq_wr_cred = val[5];
5096
5097                 adap->params.offload = 1;
5098         }
5099         if (caps_cmd.rdmacaps) {
5100                 params[0] = FW_PARAM_PFVF(STAG_START);
5101                 params[1] = FW_PARAM_PFVF(STAG_END);
5102                 params[2] = FW_PARAM_PFVF(RQ_START);
5103                 params[3] = FW_PARAM_PFVF(RQ_END);
5104                 params[4] = FW_PARAM_PFVF(PBL_START);
5105                 params[5] = FW_PARAM_PFVF(PBL_END);
5106                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5107                                       params, val);
5108                 if (ret < 0)
5109                         goto bye;
5110                 adap->vres.stag.start = val[0];
5111                 adap->vres.stag.size = val[1] - val[0] + 1;
5112                 adap->vres.rq.start = val[2];
5113                 adap->vres.rq.size = val[3] - val[2] + 1;
5114                 adap->vres.pbl.start = val[4];
5115                 adap->vres.pbl.size = val[5] - val[4] + 1;
5116
5117                 params[0] = FW_PARAM_PFVF(SQRQ_START);
5118                 params[1] = FW_PARAM_PFVF(SQRQ_END);
5119                 params[2] = FW_PARAM_PFVF(CQ_START);
5120                 params[3] = FW_PARAM_PFVF(CQ_END);
5121                 params[4] = FW_PARAM_PFVF(OCQ_START);
5122                 params[5] = FW_PARAM_PFVF(OCQ_END);
5123                 ret = t4_query_params(adap, 0, 0, 0, 6, params, val);
5124                 if (ret < 0)
5125                         goto bye;
5126                 adap->vres.qp.start = val[0];
5127                 adap->vres.qp.size = val[1] - val[0] + 1;
5128                 adap->vres.cq.start = val[2];
5129                 adap->vres.cq.size = val[3] - val[2] + 1;
5130                 adap->vres.ocq.start = val[4];
5131                 adap->vres.ocq.size = val[5] - val[4] + 1;
5132         }
5133         if (caps_cmd.iscsicaps) {
5134                 params[0] = FW_PARAM_PFVF(ISCSI_START);
5135                 params[1] = FW_PARAM_PFVF(ISCSI_END);
5136                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5137                                       params, val);
5138                 if (ret < 0)
5139                         goto bye;
5140                 adap->vres.iscsi.start = val[0];
5141                 adap->vres.iscsi.size = val[1] - val[0] + 1;
5142         }
5143 #undef FW_PARAM_PFVF
5144 #undef FW_PARAM_DEV
5145
5146         /*
5147          * These are finalized by FW initialization, load their values now.
5148          */
5149         v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
5150         adap->params.tp.tre = TIMERRESOLUTION_GET(v);
5151         adap->params.tp.dack_re = DELAYEDACKRESOLUTION_GET(v);
5152         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5153         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5154                      adap->params.b_wnd);
5155
5156         /* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
5157         for (j = 0; j < NCHAN; j++)
5158                 adap->params.tp.tx_modq[j] = j;
5159
5160         t4_read_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
5161                          &adap->filter_mode, 1,
5162                          TP_VLAN_PRI_MAP);
5163
5164         adap->flags |= FW_OK;
5165         return 0;
5166
5167         /*
5168          * Something bad happened.  If a command timed out or failed with EIO
5169          * FW does not operate within its spec or something catastrophic
5170          * happened to HW/FW, stop issuing commands.
5171          */
5172 bye:
5173         if (ret != -ETIMEDOUT && ret != -EIO)
5174                 t4_fw_bye(adap, adap->mbox);
5175         return ret;
5176 }
5177
5178 /* EEH callbacks */
5179
5180 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5181                                          pci_channel_state_t state)
5182 {
5183         int i;
5184         struct adapter *adap = pci_get_drvdata(pdev);
5185
5186         if (!adap)
5187                 goto out;
5188
5189         rtnl_lock();
5190         adap->flags &= ~FW_OK;
5191         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5192         for_each_port(adap, i) {
5193                 struct net_device *dev = adap->port[i];
5194
5195                 netif_device_detach(dev);
5196                 netif_carrier_off(dev);
5197         }
5198         if (adap->flags & FULL_INIT_DONE)
5199                 cxgb_down(adap);
5200         rtnl_unlock();
5201         pci_disable_device(pdev);
5202 out:    return state == pci_channel_io_perm_failure ?
5203                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5204 }
5205
5206 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5207 {
5208         int i, ret;
5209         struct fw_caps_config_cmd c;
5210         struct adapter *adap = pci_get_drvdata(pdev);
5211
5212         if (!adap) {
5213                 pci_restore_state(pdev);
5214                 pci_save_state(pdev);
5215                 return PCI_ERS_RESULT_RECOVERED;
5216         }
5217
5218         if (pci_enable_device(pdev)) {
5219                 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
5220                 return PCI_ERS_RESULT_DISCONNECT;
5221         }
5222
5223         pci_set_master(pdev);
5224         pci_restore_state(pdev);
5225         pci_save_state(pdev);
5226         pci_cleanup_aer_uncorrect_error_status(pdev);
5227
5228         if (t4_wait_dev_ready(adap) < 0)
5229                 return PCI_ERS_RESULT_DISCONNECT;
5230         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5231                 return PCI_ERS_RESULT_DISCONNECT;
5232         adap->flags |= FW_OK;
5233         if (adap_init1(adap, &c))
5234                 return PCI_ERS_RESULT_DISCONNECT;
5235
5236         for_each_port(adap, i) {
5237                 struct port_info *p = adap2pinfo(adap, i);
5238
5239                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5240                                   NULL, NULL);
5241                 if (ret < 0)
5242                         return PCI_ERS_RESULT_DISCONNECT;
5243                 p->viid = ret;
5244                 p->xact_addr_filt = -1;
5245         }
5246
5247         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5248                      adap->params.b_wnd);
5249         setup_memwin(adap);
5250         if (cxgb_up(adap))
5251                 return PCI_ERS_RESULT_DISCONNECT;
5252         return PCI_ERS_RESULT_RECOVERED;
5253 }
5254
5255 static void eeh_resume(struct pci_dev *pdev)
5256 {
5257         int i;
5258         struct adapter *adap = pci_get_drvdata(pdev);
5259
5260         if (!adap)
5261                 return;
5262
5263         rtnl_lock();
5264         for_each_port(adap, i) {
5265                 struct net_device *dev = adap->port[i];
5266
5267                 if (netif_running(dev)) {
5268                         link_start(dev);
5269                         cxgb_set_rxmode(dev);
5270                 }
5271                 netif_device_attach(dev);
5272         }
5273         rtnl_unlock();
5274 }
5275
5276 static const struct pci_error_handlers cxgb4_eeh = {
5277         .error_detected = eeh_err_detected,
5278         .slot_reset     = eeh_slot_reset,
5279         .resume         = eeh_resume,
5280 };
5281
5282 static inline bool is_10g_port(const struct link_config *lc)
5283 {
5284         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
5285 }
5286
5287 static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
5288                              unsigned int size, unsigned int iqe_size)
5289 {
5290         q->intr_params = QINTR_TIMER_IDX(timer_idx) |
5291                          (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
5292         q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
5293         q->iqe_len = iqe_size;
5294         q->size = size;
5295 }
5296
5297 /*
5298  * Perform default configuration of DMA queues depending on the number and type
5299  * of ports we found and the number of available CPUs.  Most settings can be
5300  * modified by the admin prior to actual use.
5301  */
5302 static void cfg_queues(struct adapter *adap)
5303 {
5304         struct sge *s = &adap->sge;
5305         int i, q10g = 0, n10g = 0, qidx = 0;
5306
5307         for_each_port(adap, i)
5308                 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
5309
5310         /*
5311          * We default to 1 queue per non-10G port and up to # of cores queues
5312          * per 10G port.
5313          */
5314         if (n10g)
5315                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5316         if (q10g > netif_get_num_default_rss_queues())
5317                 q10g = netif_get_num_default_rss_queues();
5318
5319         for_each_port(adap, i) {
5320                 struct port_info *pi = adap2pinfo(adap, i);
5321
5322                 pi->first_qset = qidx;
5323                 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
5324                 qidx += pi->nqsets;
5325         }
5326
5327         s->ethqsets = qidx;
5328         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
5329
5330         if (is_offload(adap)) {
5331                 /*
5332                  * For offload we use 1 queue/channel if all ports are up to 1G,
5333                  * otherwise we divide all available queues amongst the channels
5334                  * capped by the number of available cores.
5335                  */
5336                 if (n10g) {
5337                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5338                                   num_online_cpus());
5339                         s->ofldqsets = roundup(i, adap->params.nports);
5340                 } else
5341                         s->ofldqsets = adap->params.nports;
5342                 /* For RDMA one Rx queue per channel suffices */
5343                 s->rdmaqs = adap->params.nports;
5344         }
5345
5346         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5347                 struct sge_eth_rxq *r = &s->ethrxq[i];
5348
5349                 init_rspq(&r->rspq, 0, 0, 1024, 64);
5350                 r->fl.size = 72;
5351         }
5352
5353         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5354                 s->ethtxq[i].q.size = 1024;
5355
5356         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5357                 s->ctrlq[i].q.size = 512;
5358
5359         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5360                 s->ofldtxq[i].q.size = 1024;
5361
5362         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5363                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5364
5365                 init_rspq(&r->rspq, 0, 0, 1024, 64);
5366                 r->rspq.uld = CXGB4_ULD_ISCSI;
5367                 r->fl.size = 72;
5368         }
5369
5370         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5371                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5372
5373                 init_rspq(&r->rspq, 0, 0, 511, 64);
5374                 r->rspq.uld = CXGB4_ULD_RDMA;
5375                 r->fl.size = 72;
5376         }
5377
5378         init_rspq(&s->fw_evtq, 6, 0, 512, 64);
5379         init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
5380 }
5381
5382 /*
5383  * Reduce the number of Ethernet queues across all ports to at most n.
5384  * n provides at least one queue per port.
5385  */
5386 static void reduce_ethqs(struct adapter *adap, int n)
5387 {
5388         int i;
5389         struct port_info *pi;
5390
5391         while (n < adap->sge.ethqsets)
5392                 for_each_port(adap, i) {
5393                         pi = adap2pinfo(adap, i);
5394                         if (pi->nqsets > 1) {
5395                                 pi->nqsets--;
5396                                 adap->sge.ethqsets--;
5397                                 if (adap->sge.ethqsets <= n)
5398                                         break;
5399                         }
5400                 }
5401
5402         n = 0;
5403         for_each_port(adap, i) {
5404                 pi = adap2pinfo(adap, i);
5405                 pi->first_qset = n;
5406                 n += pi->nqsets;
5407         }
5408 }
5409
5410 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5411 #define EXTRA_VECS 2
5412
5413 static int enable_msix(struct adapter *adap)
5414 {
5415         int ofld_need = 0;
5416         int i, err, want, need;
5417         struct sge *s = &adap->sge;
5418         unsigned int nchan = adap->params.nports;
5419         struct msix_entry entries[MAX_INGQ + 1];
5420
5421         for (i = 0; i < ARRAY_SIZE(entries); ++i)
5422                 entries[i].entry = i;
5423
5424         want = s->max_ethqsets + EXTRA_VECS;
5425         if (is_offload(adap)) {
5426                 want += s->rdmaqs + s->ofldqsets;
5427                 /* need nchan for each possible ULD */
5428                 ofld_need = 2 * nchan;
5429         }
5430         need = adap->params.nports + EXTRA_VECS + ofld_need;
5431
5432         while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
5433                 want = err;
5434
5435         if (!err) {
5436                 /*
5437                  * Distribute available vectors to the various queue groups.
5438                  * Every group gets its minimum requirement and NIC gets top
5439                  * priority for leftovers.
5440                  */
5441                 i = want - EXTRA_VECS - ofld_need;
5442                 if (i < s->max_ethqsets) {
5443                         s->max_ethqsets = i;
5444                         if (i < s->ethqsets)
5445                                 reduce_ethqs(adap, i);
5446                 }
5447                 if (is_offload(adap)) {
5448                         i = want - EXTRA_VECS - s->max_ethqsets;
5449                         i -= ofld_need - nchan;
5450                         s->ofldqsets = (i / nchan) * nchan;  /* round down */
5451                 }
5452                 for (i = 0; i < want; ++i)
5453                         adap->msix_info[i].vec = entries[i].vector;
5454         } else if (err > 0)
5455                 dev_info(adap->pdev_dev,
5456                          "only %d MSI-X vectors left, not using MSI-X\n", err);
5457         return err;
5458 }
5459
5460 #undef EXTRA_VECS
5461
5462 static int init_rss(struct adapter *adap)
5463 {
5464         unsigned int i, j;
5465
5466         for_each_port(adap, i) {
5467                 struct port_info *pi = adap2pinfo(adap, i);
5468
5469                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5470                 if (!pi->rss)
5471                         return -ENOMEM;
5472                 for (j = 0; j < pi->rss_size; j++)
5473                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5474         }
5475         return 0;
5476 }
5477
5478 static void print_port_info(const struct net_device *dev)
5479 {
5480         static const char *base[] = {
5481                 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
5482                 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
5483         };
5484
5485         char buf[80];
5486         char *bufp = buf;
5487         const char *spd = "";
5488         const struct port_info *pi = netdev_priv(dev);
5489         const struct adapter *adap = pi->adapter;
5490
5491         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5492                 spd = " 2.5 GT/s";
5493         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5494                 spd = " 5 GT/s";
5495
5496         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5497                 bufp += sprintf(bufp, "100/");
5498         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5499                 bufp += sprintf(bufp, "1000/");
5500         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5501                 bufp += sprintf(bufp, "10G/");
5502         if (bufp != buf)
5503                 --bufp;
5504         sprintf(bufp, "BASE-%s", base[pi->port_type]);
5505
5506         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5507                     adap->params.vpd.id,
5508                     CHELSIO_CHIP_RELEASE(adap->params.rev), buf,
5509                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5510                     (adap->flags & USING_MSIX) ? " MSI-X" :
5511                     (adap->flags & USING_MSI) ? " MSI" : "");
5512         netdev_info(dev, "S/N: %s, E/C: %s\n",
5513                     adap->params.vpd.sn, adap->params.vpd.ec);
5514 }
5515
5516 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5517 {
5518         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5519 }
5520
5521 /*
5522  * Free the following resources:
5523  * - memory used for tables
5524  * - MSI/MSI-X
5525  * - net devices
5526  * - resources FW is holding for us
5527  */
5528 static void free_some_resources(struct adapter *adapter)
5529 {
5530         unsigned int i;
5531
5532         t4_free_mem(adapter->l2t);
5533         t4_free_mem(adapter->tids.tid_tab);
5534         disable_msi(adapter);
5535
5536         for_each_port(adapter, i)
5537                 if (adapter->port[i]) {
5538                         kfree(adap2pinfo(adapter, i)->rss);
5539                         free_netdev(adapter->port[i]);
5540                 }
5541         if (adapter->flags & FW_OK)
5542                 t4_fw_bye(adapter, adapter->fn);
5543 }
5544
5545 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5546 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5547                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5548 #define SEGMENT_SIZE 128
5549
5550 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5551 {
5552         int func, i, err, s_qpp, qpp, num_seg;
5553         struct port_info *pi;
5554         bool highdma = false;
5555         struct adapter *adapter = NULL;
5556
5557         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5558
5559         err = pci_request_regions(pdev, KBUILD_MODNAME);
5560         if (err) {
5561                 /* Just info, some other driver may have claimed the device. */
5562                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5563                 return err;
5564         }
5565
5566         /* We control everything through one PF */
5567         func = PCI_FUNC(pdev->devfn);
5568         if (func != ent->driver_data) {
5569                 pci_save_state(pdev);        /* to restore SR-IOV later */
5570                 goto sriov;
5571         }
5572
5573         err = pci_enable_device(pdev);
5574         if (err) {
5575                 dev_err(&pdev->dev, "cannot enable PCI device\n");
5576                 goto out_release_regions;
5577         }
5578
5579         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
5580                 highdma = true;
5581                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5582                 if (err) {
5583                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5584                                 "coherent allocations\n");
5585                         goto out_disable_device;
5586                 }
5587         } else {
5588                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5589                 if (err) {
5590                         dev_err(&pdev->dev, "no usable DMA configuration\n");
5591                         goto out_disable_device;
5592                 }
5593         }
5594
5595         pci_enable_pcie_error_reporting(pdev);
5596         enable_pcie_relaxed_ordering(pdev);
5597         pci_set_master(pdev);
5598         pci_save_state(pdev);
5599
5600         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5601         if (!adapter) {
5602                 err = -ENOMEM;
5603                 goto out_disable_device;
5604         }
5605
5606         adapter->regs = pci_ioremap_bar(pdev, 0);
5607         if (!adapter->regs) {
5608                 dev_err(&pdev->dev, "cannot map device registers\n");
5609                 err = -ENOMEM;
5610                 goto out_free_adapter;
5611         }
5612
5613         adapter->pdev = pdev;
5614         adapter->pdev_dev = &pdev->dev;
5615         adapter->mbox = func;
5616         adapter->fn = func;
5617         adapter->msg_enable = dflt_msg_enable;
5618         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
5619
5620         spin_lock_init(&adapter->stats_lock);
5621         spin_lock_init(&adapter->tid_release_lock);
5622
5623         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
5624         INIT_WORK(&adapter->db_full_task, process_db_full);
5625         INIT_WORK(&adapter->db_drop_task, process_db_drop);
5626
5627         err = t4_prep_adapter(adapter);
5628         if (err)
5629                 goto out_unmap_bar0;
5630
5631         if (!is_t4(adapter->chip)) {
5632                 s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
5633                 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
5634                       SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
5635                 num_seg = PAGE_SIZE / SEGMENT_SIZE;
5636
5637                 /* Each segment size is 128B. Write coalescing is enabled only
5638                  * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
5639                  * queue is less no of segments that can be accommodated in
5640                  * a page size.
5641                  */
5642                 if (qpp > num_seg) {
5643                         dev_err(&pdev->dev,
5644                                 "Incorrect number of egress queues per page\n");
5645                         err = -EINVAL;
5646                         goto out_unmap_bar0;
5647                 }
5648                 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
5649                 pci_resource_len(pdev, 2));
5650                 if (!adapter->bar2) {
5651                         dev_err(&pdev->dev, "cannot map device bar2 region\n");
5652                         err = -ENOMEM;
5653                         goto out_unmap_bar0;
5654                 }
5655         }
5656
5657         setup_memwin(adapter);
5658         err = adap_init0(adapter);
5659         setup_memwin_rdma(adapter);
5660         if (err)
5661                 goto out_unmap_bar;
5662
5663         for_each_port(adapter, i) {
5664                 struct net_device *netdev;
5665
5666                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
5667                                            MAX_ETH_QSETS);
5668                 if (!netdev) {
5669                         err = -ENOMEM;
5670                         goto out_free_dev;
5671                 }
5672
5673                 SET_NETDEV_DEV(netdev, &pdev->dev);
5674
5675                 adapter->port[i] = netdev;
5676                 pi = netdev_priv(netdev);
5677                 pi->adapter = adapter;
5678                 pi->xact_addr_filt = -1;
5679                 pi->port_id = i;
5680                 netdev->irq = pdev->irq;
5681
5682                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
5683                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
5684                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
5685                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
5686                 if (highdma)
5687                         netdev->hw_features |= NETIF_F_HIGHDMA;
5688                 netdev->features |= netdev->hw_features;
5689                 netdev->vlan_features = netdev->features & VLAN_FEAT;
5690
5691                 netdev->priv_flags |= IFF_UNICAST_FLT;
5692
5693                 netdev->netdev_ops = &cxgb4_netdev_ops;
5694                 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
5695         }
5696
5697         pci_set_drvdata(pdev, adapter);
5698
5699         if (adapter->flags & FW_OK) {
5700                 err = t4_port_init(adapter, func, func, 0);
5701                 if (err)
5702                         goto out_free_dev;
5703         }
5704
5705         /*
5706          * Configure queues and allocate tables now, they can be needed as
5707          * soon as the first register_netdev completes.
5708          */
5709         cfg_queues(adapter);
5710
5711         adapter->l2t = t4_init_l2t();
5712         if (!adapter->l2t) {
5713                 /* We tolerate a lack of L2T, giving up some functionality */
5714                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
5715                 adapter->params.offload = 0;
5716         }
5717
5718         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
5719                 dev_warn(&pdev->dev, "could not allocate TID table, "
5720                          "continuing\n");
5721                 adapter->params.offload = 0;
5722         }
5723
5724         /* See what interrupts we'll be using */
5725         if (msi > 1 && enable_msix(adapter) == 0)
5726                 adapter->flags |= USING_MSIX;
5727         else if (msi > 0 && pci_enable_msi(pdev) == 0)
5728                 adapter->flags |= USING_MSI;
5729
5730         err = init_rss(adapter);
5731         if (err)
5732                 goto out_free_dev;
5733
5734         /*
5735          * The card is now ready to go.  If any errors occur during device
5736          * registration we do not fail the whole card but rather proceed only
5737          * with the ports we manage to register successfully.  However we must
5738          * register at least one net device.
5739          */
5740         for_each_port(adapter, i) {
5741                 pi = adap2pinfo(adapter, i);
5742                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
5743                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
5744
5745                 err = register_netdev(adapter->port[i]);
5746                 if (err)
5747                         break;
5748                 adapter->chan_map[pi->tx_chan] = i;
5749                 print_port_info(adapter->port[i]);
5750         }
5751         if (i == 0) {
5752                 dev_err(&pdev->dev, "could not register any net devices\n");
5753                 goto out_free_dev;
5754         }
5755         if (err) {
5756                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
5757                 err = 0;
5758         }
5759
5760         if (cxgb4_debugfs_root) {
5761                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
5762                                                            cxgb4_debugfs_root);
5763                 setup_debugfs(adapter);
5764         }
5765
5766         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
5767         pdev->needs_freset = 1;
5768
5769         if (is_offload(adapter))
5770                 attach_ulds(adapter);
5771
5772 sriov:
5773 #ifdef CONFIG_PCI_IOV
5774         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
5775                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
5776                         dev_info(&pdev->dev,
5777                                  "instantiated %u virtual functions\n",
5778                                  num_vf[func]);
5779 #endif
5780         return 0;
5781
5782  out_free_dev:
5783         free_some_resources(adapter);
5784  out_unmap_bar:
5785         if (!is_t4(adapter->chip))
5786                 iounmap(adapter->bar2);
5787  out_unmap_bar0:
5788         iounmap(adapter->regs);
5789  out_free_adapter:
5790         kfree(adapter);
5791  out_disable_device:
5792         pci_disable_pcie_error_reporting(pdev);
5793         pci_disable_device(pdev);
5794  out_release_regions:
5795         pci_release_regions(pdev);
5796         pci_set_drvdata(pdev, NULL);
5797         return err;
5798 }
5799
5800 static void remove_one(struct pci_dev *pdev)
5801 {
5802         struct adapter *adapter = pci_get_drvdata(pdev);
5803
5804 #ifdef CONFIG_PCI_IOV
5805         pci_disable_sriov(pdev);
5806
5807 #endif
5808
5809         if (adapter) {
5810                 int i;
5811
5812                 if (is_offload(adapter))
5813                         detach_ulds(adapter);
5814
5815                 for_each_port(adapter, i)
5816                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
5817                                 unregister_netdev(adapter->port[i]);
5818
5819                 if (adapter->debugfs_root)
5820                         debugfs_remove_recursive(adapter->debugfs_root);
5821
5822                 /* If we allocated filters, free up state associated with any
5823                  * valid filters ...
5824                  */
5825                 if (adapter->tids.ftid_tab) {
5826                         struct filter_entry *f = &adapter->tids.ftid_tab[0];
5827                         for (i = 0; i < (adapter->tids.nftids +
5828                                         adapter->tids.nsftids); i++, f++)
5829                                 if (f->valid)
5830                                         clear_filter(adapter, f);
5831                 }
5832
5833                 if (adapter->flags & FULL_INIT_DONE)
5834                         cxgb_down(adapter);
5835
5836                 free_some_resources(adapter);
5837                 iounmap(adapter->regs);
5838                 if (!is_t4(adapter->chip))
5839                         iounmap(adapter->bar2);
5840                 kfree(adapter);
5841                 pci_disable_pcie_error_reporting(pdev);
5842                 pci_disable_device(pdev);
5843                 pci_release_regions(pdev);
5844                 pci_set_drvdata(pdev, NULL);
5845         } else
5846                 pci_release_regions(pdev);
5847 }
5848
5849 static struct pci_driver cxgb4_driver = {
5850         .name     = KBUILD_MODNAME,
5851         .id_table = cxgb4_pci_tbl,
5852         .probe    = init_one,
5853         .remove   = remove_one,
5854         .err_handler = &cxgb4_eeh,
5855 };
5856
5857 static int __init cxgb4_init_module(void)
5858 {
5859         int ret;
5860
5861         workq = create_singlethread_workqueue("cxgb4");
5862         if (!workq)
5863                 return -ENOMEM;
5864
5865         /* Debugfs support is optional, just warn if this fails */
5866         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
5867         if (!cxgb4_debugfs_root)
5868                 pr_warn("could not create debugfs entry, continuing\n");
5869
5870         ret = pci_register_driver(&cxgb4_driver);
5871         if (ret < 0)
5872                 debugfs_remove(cxgb4_debugfs_root);
5873         return ret;
5874 }
5875
5876 static void __exit cxgb4_cleanup_module(void)
5877 {
5878         pci_unregister_driver(&cxgb4_driver);
5879         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
5880         flush_workqueue(workq);
5881         destroy_workqueue(workq);
5882 }
5883
5884 module_init(cxgb4_init_module);
5885 module_exit(cxgb4_cleanup_module);