drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c

   1 /*
   2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
   3  *
   4  * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  */
  34
  35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  36
  37 #include <linux/bitmap.h>
  38 #include <linux/crc32.h>
  39 #include <linux/ctype.h>
  40 #include <linux/debugfs.h>
  41 #include <linux/err.h>
  42 #include <linux/etherdevice.h>
  43 #include <linux/firmware.h>
  44 #include <linux/if.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/init.h>
  47 #include <linux/log2.h>
  48 #include <linux/mdio.h>
  49 #include <linux/module.h>
  50 #include <linux/moduleparam.h>
  51 #include <linux/mutex.h>
  52 #include <linux/netdevice.h>
  53 #include <linux/pci.h>
  54 #include <linux/aer.h>
  55 #include <linux/rtnetlink.h>
  56 #include <linux/sched.h>
  57 #include <linux/seq_file.h>
  58 #include <linux/sockios.h>
  59 #include <linux/vmalloc.h>
  60 #include <linux/workqueue.h>
  61 #include <net/neighbour.h>
  62 #include <net/netevent.h>
  63 #include <net/addrconf.h>
  64 #include <net/bonding.h>
  65 #include <net/addrconf.h>
  66 #include <asm/uaccess.h>
  67
  68 #include "cxgb4.h"
  69 #include "t4_regs.h"
  70 #include "t4_values.h"
  71 #include "t4_msg.h"
  72 #include "t4fw_api.h"
  73 #include "t4fw_version.h"
  74 #include "cxgb4_dcb.h"
  75 #include "cxgb4_debugfs.h"
  76 #include "clip_tbl.h"
  77 #include "l2t.h"
  78
  79 #ifdef DRV_VERSION
  80 #undef DRV_VERSION
  81 #endif
  82 #define DRV_VERSION "2.0.0-ko"
  83 #define DRV_DESC "Chelsio T4/T5 Network Driver"
  84
  85 enum {
  86         MAX_TXQ_ENTRIES      = 16384,
  87         MAX_CTRL_TXQ_ENTRIES = 1024,
  88         MAX_RSPQ_ENTRIES     = 16384,
  89         MAX_RX_BUFFERS       = 16384,
  90         MIN_TXQ_ENTRIES      = 32,
  91         MIN_CTRL_TXQ_ENTRIES = 32,
  92         MIN_RSPQ_ENTRIES     = 128,
  93         MIN_FL_ENTRIES       = 16
  94 };
  95
  96 /* Host shadow copy of ingress filter entry.  This is in host native format
  97  * and doesn't match the ordering or bit order, etc. of the hardware of the
  98  * firmware command.  The use of bit-field structure elements is purely to
  99  * remind ourselves of the field size limitations and save memory in the case
 100  * where the filter table is large.
 101  */
 102 struct filter_entry {
 103         /* Administrative fields for filter.
 104          */
 105         u32 valid:1;            /* filter allocated and valid */
 106         u32 locked:1;           /* filter is administratively locked */
 107
 108         u32 pending:1;          /* filter action is pending firmware reply */
 109         u32 smtidx:8;           /* Source MAC Table index for smac */
 110         struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
 111
 112         /* The filter itself.  Most of this is a straight copy of information
 113          * provided by the extended ioctl().  Some fields are translated to
 114          * internal forms -- for instance the Ingress Queue ID passed in from
 115          * the ioctl() is translated into the Absolute Ingress Queue ID.
 116          */
 117         struct ch_filter_specification fs;
 118 };
 119
 120 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
 121                          NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
 122                          NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
 123
 124 /* Macros needed to support the PCI Device ID Table ...
 125  */
 126 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
 127         static struct pci_device_id cxgb4_pci_tbl[] = {
 128 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
 129
 130 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
 131  * called for both.
 132  */
 133 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
 134
 135 #define CH_PCI_ID_TABLE_ENTRY(devid) \
 136                 {PCI_VDEVICE(CHELSIO, (devid)), 4}
 137
 138 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
 139                 { 0, } \
 140         }
 141
 142 #include "t4_pci_id_tbl.h"
 143
 144 #define FW4_FNAME "cxgb4/t4fw.bin"
 145 #define FW5_FNAME "cxgb4/t5fw.bin"
 146 #define FW4_CFNAME "cxgb4/t4-config.txt"
 147 #define FW5_CFNAME "cxgb4/t5-config.txt"
 148
 149 MODULE_DESCRIPTION(DRV_DESC);
 150 MODULE_AUTHOR("Chelsio Communications");
 151 MODULE_LICENSE("Dual BSD/GPL");
 152 MODULE_VERSION(DRV_VERSION);
 153 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
 154 MODULE_FIRMWARE(FW4_FNAME);
 155 MODULE_FIRMWARE(FW5_FNAME);
 156
 157 /*
 158  * Normally we're willing to become the firmware's Master PF but will be happy
 159  * if another PF has already become the Master and initialized the adapter.
 160  * Setting "force_init" will cause this driver to forcibly establish itself as
 161  * the Master PF and initialize the adapter.
 162  */
 163 static uint force_init;
 164
 165 module_param(force_init, uint, 0644);
 166 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
 167
 168 /*
 169  * Normally if the firmware we connect to has Configuration File support, we
 170  * use that and only fall back to the old Driver-based initialization if the
 171  * Configuration File fails for some reason.  If force_old_init is set, then
 172  * we'll always use the old Driver-based initialization sequence.
 173  */
 174 static uint force_old_init;
 175
 176 module_param(force_old_init, uint, 0644);
 177 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated"
 178                  " parameter");
 179
 180 static int dflt_msg_enable = DFLT_MSG_ENABLE;
 181
 182 module_param(dflt_msg_enable, int, 0644);
 183 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
 184
 185 /*
 186  * The driver uses the best interrupt scheme available on a platform in the
 187  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
 188  * of these schemes the driver may consider as follows:
 189  *
 190  * msi = 2: choose from among all three options
 191  * msi = 1: only consider MSI and INTx interrupts
 192  * msi = 0: force INTx interrupts
 193  */
 194 static int msi = 2;
 195
 196 module_param(msi, int, 0644);
 197 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
 198
 199 /*
 200  * Queue interrupt hold-off timer values.  Queues default to the first of these
 201  * upon creation.
 202  */
 203 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
 204
 205 module_param_array(intr_holdoff, uint, NULL, 0644);
 206 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
 207                  "0..4 in microseconds, deprecated parameter");
 208
 209 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
 210
 211 module_param_array(intr_cnt, uint, NULL, 0644);
 212 MODULE_PARM_DESC(intr_cnt,
 213                  "thresholds 1..3 for queue interrupt packet counters, "
 214                  "deprecated parameter");
 215
 216 /*
 217  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
 218  * offset by 2 bytes in order to have the IP headers line up on 4-byte
 219  * boundaries.  This is a requirement for many architectures which will throw
 220  * a machine check fault if an attempt is made to access one of the 4-byte IP
 221  * header fields on a non-4-byte boundary.  And it's a major performance issue
 222  * even on some architectures which allow it like some implementations of the
 223  * x86 ISA.  However, some architectures don't mind this and for some very
 224  * edge-case performance sensitive applications (like forwarding large volumes
 225  * of small packets), setting this DMA offset to 0 will decrease the number of
 226  * PCI-E Bus transfers enough to measurably affect performance.
 227  */
 228 static int rx_dma_offset = 2;
 229
 230 static bool vf_acls;
 231
 232 #ifdef CONFIG_PCI_IOV
 233 module_param(vf_acls, bool, 0644);
 234 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, "
 235                  "deprecated parameter");
 236
 237 /* Configure the number of PCI-E Virtual Function which are to be instantiated
 238  * on SR-IOV Capable Physical Functions.
 239  */
 240 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
 241
 242 module_param_array(num_vf, uint, NULL, 0644);
 243 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
 244 #endif
 245
 246 /* TX Queue select used to determine what algorithm to use for selecting TX
 247  * queue. Select between the kernel provided function (select_queue=0) or user
 248  * cxgb_select_queue function (select_queue=1)
 249  *
 250  * Default: select_queue=0
 251  */
 252 static int select_queue;
 253 module_param(select_queue, int, 0644);
 254 MODULE_PARM_DESC(select_queue,
 255                  "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
 256
 257 static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC;
 258
 259 module_param(tp_vlan_pri_map, uint, 0644);
 260 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, "
 261                  "deprecated parameter");
 262
 263 static struct dentry *cxgb4_debugfs_root;
 264
 265 static LIST_HEAD(adapter_list);
 266 static DEFINE_MUTEX(uld_mutex);
 267 /* Adapter list to be accessed from atomic context */
 268 static LIST_HEAD(adap_rcu_list);
 269 static DEFINE_SPINLOCK(adap_rcu_lock);
 270 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
 271 static const char *uld_str[] = { "RDMA", "iSCSI" };
 272
 273 static void link_report(struct net_device *dev)
 274 {
 275         if (!netif_carrier_ok(dev))
 276                 netdev_info(dev, "link down\n");
 277         else {
 278                 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
 279
 280                 const char *s = "10Mbps";
 281                 const struct port_info *p = netdev_priv(dev);
 282
 283                 switch (p->link_cfg.speed) {
 284                 case 10000:
 285                         s = "10Gbps";
 286                         break;
 287                 case 1000:
 288                         s = "1000Mbps";
 289                         break;
 290                 case 100:
 291                         s = "100Mbps";
 292                         break;
 293                 case 40000:
 294                         s = "40Gbps";
 295                         break;
 296                 }
 297
 298                 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
 299                             fc[p->link_cfg.fc]);
 300         }
 301 }
 302
 303 #ifdef CONFIG_CHELSIO_T4_DCB
 304 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */
 305 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
 306 {
 307         struct port_info *pi = netdev_priv(dev);
 308         struct adapter *adap = pi->adapter;
 309         struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
 310         int i;
 311
 312         /* We use a simple mapping of Port TX Queue Index to DCB
 313          * Priority when we're enabling DCB.
 314          */
 315         for (i = 0; i < pi->nqsets; i++, txq++) {
 316                 u32 name, value;
 317                 int err;
 318
 319                 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
 320                         FW_PARAMS_PARAM_X_V(
 321                                 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
 322                         FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
 323                 value = enable ? i : 0xffffffff;
 324
 325                 /* Since we can be called while atomic (from "interrupt
 326                  * level") we need to issue the Set Parameters Commannd
 327                  * without sleeping (timeout < 0).
 328                  */
 329                 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
 330                                             &name, &value);
 331
 332                 if (err)
 333                         dev_err(adap->pdev_dev,
 334                                 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
 335                                 enable ? "set" : "unset", pi->port_id, i, -err);
 336                 else
 337                         txq->dcb_prio = value;
 338         }
 339 }
 340 #endif /* CONFIG_CHELSIO_T4_DCB */
 341
 342 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
 343 {
 344         struct net_device *dev = adapter->port[port_id];
 345
 346         /* Skip changes from disabled ports. */
 347         if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
 348                 if (link_stat)
 349                         netif_carrier_on(dev);
 350                 else {
 351 #ifdef CONFIG_CHELSIO_T4_DCB
 352                         cxgb4_dcb_state_init(dev);
 353                         dcb_tx_queue_prio_enable(dev, false);
 354 #endif /* CONFIG_CHELSIO_T4_DCB */
 355                         netif_carrier_off(dev);
 356                 }
 357
 358                 link_report(dev);
 359         }
 360 }
 361
 362 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
 363 {
 364         static const char *mod_str[] = {
 365                 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
 366         };
 367
 368         const struct net_device *dev = adap->port[port_id];
 369         const struct port_info *pi = netdev_priv(dev);
 370
 371         if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
 372                 netdev_info(dev, "port module unplugged\n");
 373         else if (pi->mod_type < ARRAY_SIZE(mod_str))
 374                 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
 375 }
 376
 377 /*
 378  * Configure the exact and hash address filters to handle a port's multicast
 379  * and secondary unicast MAC addresses.
 380  */
 381 static int set_addr_filters(const struct net_device *dev, bool sleep)
 382 {
 383         u64 mhash = 0;
 384         u64 uhash = 0;
 385         bool free = true;
 386         u16 filt_idx[7];
 387         const u8 *addr[7];
 388         int ret, naddr = 0;
 389         const struct netdev_hw_addr *ha;
 390         int uc_cnt = netdev_uc_count(dev);
 391         int mc_cnt = netdev_mc_count(dev);
 392         const struct port_info *pi = netdev_priv(dev);
 393         unsigned int mb = pi->adapter->fn;
 394
 395         /* first do the secondary unicast addresses */
 396         netdev_for_each_uc_addr(ha, dev) {
 397                 addr[naddr++] = ha->addr;
 398                 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 399                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 400                                         naddr, addr, filt_idx, &uhash, sleep);
 401                         if (ret < 0)
 402                                 return ret;
 403
 404                         free = false;
 405                         naddr = 0;
 406                 }
 407         }
 408
 409         /* next set up the multicast addresses */
 410         netdev_for_each_mc_addr(ha, dev) {
 411                 addr[naddr++] = ha->addr;
 412                 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
 413                         ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
 414                                         naddr, addr, filt_idx, &mhash, sleep);
 415                         if (ret < 0)
 416                                 return ret;
 417
 418                         free = false;
 419                         naddr = 0;
 420                 }
 421         }
 422
 423         return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
 424                                 uhash | mhash, sleep);
 425 }
 426
 427 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
 428 module_param(dbfifo_int_thresh, int, 0644);
 429 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
 430
 431 /*
 432  * usecs to sleep while draining the dbfifo
 433  */
 434 static int dbfifo_drain_delay = 1000;
 435 module_param(dbfifo_drain_delay, int, 0644);
 436 MODULE_PARM_DESC(dbfifo_drain_delay,
 437                  "usecs to sleep while draining the dbfifo");
 438
 439 /*
 440  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
 441  * If @mtu is -1 it is left unchanged.
 442  */
 443 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
 444 {
 445         int ret;
 446         struct port_info *pi = netdev_priv(dev);
 447
 448         ret = set_addr_filters(dev, sleep_ok);
 449         if (ret == 0)
 450                 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
 451                                     (dev->flags & IFF_PROMISC) ? 1 : 0,
 452                                     (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
 453                                     sleep_ok);
 454         return ret;
 455 }
 456
 457 /**
 458  *      link_start - enable a port
 459  *      @dev: the port to enable
 460  *
 461  *      Performs the MAC and PHY actions needed to enable a port.
 462  */
 463 static int link_start(struct net_device *dev)
 464 {
 465         int ret;
 466         struct port_info *pi = netdev_priv(dev);
 467         unsigned int mb = pi->adapter->fn;
 468
 469         /*
 470          * We do not set address filters and promiscuity here, the stack does
 471          * that step explicitly.
 472          */
 473         ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
 474                             !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
 475         if (ret == 0) {
 476                 ret = t4_change_mac(pi->adapter, mb, pi->viid,
 477                                     pi->xact_addr_filt, dev->dev_addr, true,
 478                                     true);
 479                 if (ret >= 0) {
 480                         pi->xact_addr_filt = ret;
 481                         ret = 0;
 482                 }
 483         }
 484         if (ret == 0)
 485                 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
 486                                     &pi->link_cfg);
 487         if (ret == 0) {
 488                 local_bh_disable();
 489                 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
 490                                           true, CXGB4_DCB_ENABLED);
 491                 local_bh_enable();
 492         }
 493
 494         return ret;
 495 }
 496
 497 int cxgb4_dcb_enabled(const struct net_device *dev)
 498 {
 499 #ifdef CONFIG_CHELSIO_T4_DCB
 500         struct port_info *pi = netdev_priv(dev);
 501
 502         if (!pi->dcb.enabled)
 503                 return 0;
 504
 505         return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
 506                 (pi->dcb.state == CXGB4_DCB_STATE_HOST));
 507 #else
 508         return 0;
 509 #endif
 510 }
 511 EXPORT_SYMBOL(cxgb4_dcb_enabled);
 512
 513 #ifdef CONFIG_CHELSIO_T4_DCB
 514 /* Handle a Data Center Bridging update message from the firmware. */
 515 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
 516 {
 517         int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
 518         struct net_device *dev = adap->port[port];
 519         int old_dcb_enabled = cxgb4_dcb_enabled(dev);
 520         int new_dcb_enabled;
 521
 522         cxgb4_dcb_handle_fw_update(adap, pcmd);
 523         new_dcb_enabled = cxgb4_dcb_enabled(dev);
 524
 525         /* If the DCB has become enabled or disabled on the port then we're
 526          * going to need to set up/tear down DCB Priority parameters for the
 527          * TX Queues associated with the port.
 528          */
 529         if (new_dcb_enabled != old_dcb_enabled)
 530                 dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
 531 }
 532 #endif /* CONFIG_CHELSIO_T4_DCB */
 533
 534 /* Clear a filter and release any of its resources that we own.  This also
 535  * clears the filter's "pending" status.
 536  */
 537 static void clear_filter(struct adapter *adap, struct filter_entry *f)
 538 {
 539         /* If the new or old filter have loopback rewriteing rules then we'll
 540          * need to free any existing Layer Two Table (L2T) entries of the old
 541          * filter rule.  The firmware will handle freeing up any Source MAC
 542          * Table (SMT) entries used for rewriting Source MAC Addresses in
 543          * loopback rules.
 544          */
 545         if (f->l2t)
 546                 cxgb4_l2t_release(f->l2t);
 547
 548         /* The zeroing of the filter rule below clears the filter valid,
 549          * pending, locked flags, l2t pointer, etc. so it's all we need for
 550          * this operation.
 551          */
 552         memset(f, 0, sizeof(*f));
 553 }
 554
 555 /* Handle a filter write/deletion reply.
 556  */
 557 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
 558 {
 559         unsigned int idx = GET_TID(rpl);
 560         unsigned int nidx = idx - adap->tids.ftid_base;
 561         unsigned int ret;
 562         struct filter_entry *f;
 563
 564         if (idx >= adap->tids.ftid_base && nidx <
 565            (adap->tids.nftids + adap->tids.nsftids)) {
 566                 idx = nidx;
 567                 ret = TCB_COOKIE_G(rpl->cookie);
 568                 f = &adap->tids.ftid_tab[idx];
 569
 570                 if (ret == FW_FILTER_WR_FLT_DELETED) {
 571                         /* Clear the filter when we get confirmation from the
 572                          * hardware that the filter has been deleted.
 573                          */
 574                         clear_filter(adap, f);
 575                 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
 576                         dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
 577                                 idx);
 578                         clear_filter(adap, f);
 579                 } else if (ret == FW_FILTER_WR_FLT_ADDED) {
 580                         f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
 581                         f->pending = 0;  /* asynchronous setup completed */
 582                         f->valid = 1;
 583                 } else {
 584                         /* Something went wrong.  Issue a warning about the
 585                          * problem and clear everything out.
 586                          */
 587                         dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
 588                                 idx, ret);
 589                         clear_filter(adap, f);
 590                 }
 591         }
 592 }
 593
 594 /* Response queue handler for the FW event queue.
 595  */
 596 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
 597                           const struct pkt_gl *gl)
 598 {
 599         u8 opcode = ((const struct rss_header *)rsp)->opcode;
 600
 601         rsp++;                                          /* skip RSS header */
 602
 603         /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
 604          */
 605         if (unlikely(opcode == CPL_FW4_MSG &&
 606            ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
 607                 rsp++;
 608                 opcode = ((const struct rss_header *)rsp)->opcode;
 609                 rsp++;
 610                 if (opcode != CPL_SGE_EGR_UPDATE) {
 611                         dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
 612                                 , opcode);
 613                         goto out;
 614                 }
 615         }
 616
 617         if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
 618                 const struct cpl_sge_egr_update *p = (void *)rsp;
 619                 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid));
 620                 struct sge_txq *txq;
 621
 622                 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
 623                 txq->restarts++;
 624                 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
 625                         struct sge_eth_txq *eq;
 626
 627                         eq = container_of(txq, struct sge_eth_txq, q);
 628                         netif_tx_wake_queue(eq->txq);
 629                 } else {
 630                         struct sge_ofld_txq *oq;
 631
 632                         oq = container_of(txq, struct sge_ofld_txq, q);
 633                         tasklet_schedule(&oq->qresume_tsk);
 634                 }
 635         } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
 636                 const struct cpl_fw6_msg *p = (void *)rsp;
 637
 638 #ifdef CONFIG_CHELSIO_T4_DCB
 639                 const struct fw_port_cmd *pcmd = (const void *)p->data;
 640                 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
 641                 unsigned int action =
 642                         FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
 643
 644                 if (cmd == FW_PORT_CMD &&
 645                     action == FW_PORT_ACTION_GET_PORT_INFO) {
 646                         int port = FW_PORT_CMD_PORTID_G(
 647                                         be32_to_cpu(pcmd->op_to_portid));
 648                         struct net_device *dev = q->adap->port[port];
 649                         int state_input = ((pcmd->u.info.dcbxdis_pkd &
 650                                             FW_PORT_CMD_DCBXDIS_F)
 651                                            ? CXGB4_DCB_INPUT_FW_DISABLED
 652                                            : CXGB4_DCB_INPUT_FW_ENABLED);
 653
 654                         cxgb4_dcb_state_fsm(dev, state_input);
 655                 }
 656
 657                 if (cmd == FW_PORT_CMD &&
 658                     action == FW_PORT_ACTION_L2_DCB_CFG)
 659                         dcb_rpl(q->adap, pcmd);
 660                 else
 661 #endif
 662                         if (p->type == 0)
 663                                 t4_handle_fw_rpl(q->adap, p->data);
 664         } else if (opcode == CPL_L2T_WRITE_RPL) {
 665                 const struct cpl_l2t_write_rpl *p = (void *)rsp;
 666
 667                 do_l2t_write_rpl(q->adap, p);
 668         } else if (opcode == CPL_SET_TCB_RPL) {
 669                 const struct cpl_set_tcb_rpl *p = (void *)rsp;
 670
 671                 filter_rpl(q->adap, p);
 672         } else
 673                 dev_err(q->adap->pdev_dev,
 674                         "unexpected CPL %#x on FW event queue\n", opcode);
 675 out:
 676         return 0;
 677 }
 678
 679 /**
 680  *      uldrx_handler - response queue handler for ULD queues
 681  *      @q: the response queue that received the packet
 682  *      @rsp: the response queue descriptor holding the offload message
 683  *      @gl: the gather list of packet fragments
 684  *
 685  *      Deliver an ingress offload packet to a ULD.  All processing is done by
 686  *      the ULD, we just maintain statistics.
 687  */
 688 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
 689                          const struct pkt_gl *gl)
 690 {
 691         struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
 692
 693         /* FW can send CPLs encapsulated in a CPL_FW4_MSG.
 694          */
 695         if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
 696             ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
 697                 rsp += 2;
 698
 699         if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
 700                 rxq->stats.nomem++;
 701                 return -1;
 702         }
 703         if (gl == NULL)
 704                 rxq->stats.imm++;
 705         else if (gl == CXGB4_MSG_AN)
 706                 rxq->stats.an++;
 707         else
 708                 rxq->stats.pkts++;
 709         return 0;
 710 }
 711
 712 static void disable_msi(struct adapter *adapter)
 713 {
 714         if (adapter->flags & USING_MSIX) {
 715                 pci_disable_msix(adapter->pdev);
 716                 adapter->flags &= ~USING_MSIX;
 717         } else if (adapter->flags & USING_MSI) {
 718                 pci_disable_msi(adapter->pdev);
 719                 adapter->flags &= ~USING_MSI;
 720         }
 721 }
 722
 723 /*
 724  * Interrupt handler for non-data events used with MSI-X.
 725  */
 726 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
 727 {
 728         struct adapter *adap = cookie;
 729         u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A));
 730
 731         if (v & PFSW_F) {
 732                 adap->swintr = 1;
 733                 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v);
 734         }
 735         t4_slow_intr_handler(adap);
 736         return IRQ_HANDLED;
 737 }
 738
 739 /*
 740  * Name the MSI-X interrupts.
 741  */
 742 static void name_msix_vecs(struct adapter *adap)
 743 {
 744         int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
 745
 746         /* non-data interrupts */
 747         snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
 748
 749         /* FW events */
 750         snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
 751                  adap->port[0]->name);
 752
 753         /* Ethernet queues */
 754         for_each_port(adap, j) {
 755                 struct net_device *d = adap->port[j];
 756                 const struct port_info *pi = netdev_priv(d);
 757
 758                 for (i = 0; i < pi->nqsets; i++, msi_idx++)
 759                         snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
 760                                  d->name, i);
 761         }
 762
 763         /* offload queues */
 764         for_each_ofldrxq(&adap->sge, i)
 765                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
 766                          adap->port[0]->name, i);
 767
 768         for_each_rdmarxq(&adap->sge, i)
 769                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
 770                          adap->port[0]->name, i);
 771
 772         for_each_rdmaciq(&adap->sge, i)
 773                 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
 774                          adap->port[0]->name, i);
 775 }
 776
 777 static int request_msix_queue_irqs(struct adapter *adap)
 778 {
 779         struct sge *s = &adap->sge;
 780         int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
 781         int msi_index = 2;
 782
 783         err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
 784                           adap->msix_info[1].desc, &s->fw_evtq);
 785         if (err)
 786                 return err;
 787
 788         for_each_ethrxq(s, ethqidx) {
 789                 err = request_irq(adap->msix_info[msi_index].vec,
 790                                   t4_sge_intr_msix, 0,
 791                                   adap->msix_info[msi_index].desc,
 792                                   &s->ethrxq[ethqidx].rspq);
 793                 if (err)
 794                         goto unwind;
 795                 msi_index++;
 796         }
 797         for_each_ofldrxq(s, ofldqidx) {
 798                 err = request_irq(adap->msix_info[msi_index].vec,
 799                                   t4_sge_intr_msix, 0,
 800                                   adap->msix_info[msi_index].desc,
 801                                   &s->ofldrxq[ofldqidx].rspq);
 802                 if (err)
 803                         goto unwind;
 804                 msi_index++;
 805         }
 806         for_each_rdmarxq(s, rdmaqidx) {
 807                 err = request_irq(adap->msix_info[msi_index].vec,
 808                                   t4_sge_intr_msix, 0,
 809                                   adap->msix_info[msi_index].desc,
 810                                   &s->rdmarxq[rdmaqidx].rspq);
 811                 if (err)
 812                         goto unwind;
 813                 msi_index++;
 814         }
 815         for_each_rdmaciq(s, rdmaciqqidx) {
 816                 err = request_irq(adap->msix_info[msi_index].vec,
 817                                   t4_sge_intr_msix, 0,
 818                                   adap->msix_info[msi_index].desc,
 819                                   &s->rdmaciq[rdmaciqqidx].rspq);
 820                 if (err)
 821                         goto unwind;
 822                 msi_index++;
 823         }
 824         return 0;
 825
 826 unwind:
 827         while (--rdmaciqqidx >= 0)
 828                 free_irq(adap->msix_info[--msi_index].vec,
 829                          &s->rdmaciq[rdmaciqqidx].rspq);
 830         while (--rdmaqidx >= 0)
 831                 free_irq(adap->msix_info[--msi_index].vec,
 832                          &s->rdmarxq[rdmaqidx].rspq);
 833         while (--ofldqidx >= 0)
 834                 free_irq(adap->msix_info[--msi_index].vec,
 835                          &s->ofldrxq[ofldqidx].rspq);
 836         while (--ethqidx >= 0)
 837                 free_irq(adap->msix_info[--msi_index].vec,
 838                          &s->ethrxq[ethqidx].rspq);
 839         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 840         return err;
 841 }
 842
 843 static void free_msix_queue_irqs(struct adapter *adap)
 844 {
 845         int i, msi_index = 2;
 846         struct sge *s = &adap->sge;
 847
 848         free_irq(adap->msix_info[1].vec, &s->fw_evtq);
 849         for_each_ethrxq(s, i)
 850                 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
 851         for_each_ofldrxq(s, i)
 852                 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
 853         for_each_rdmarxq(s, i)
 854                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
 855         for_each_rdmaciq(s, i)
 856                 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
 857 }
 858
 859 /**
 860  *      write_rss - write the RSS table for a given port
 861  *      @pi: the port
 862  *      @queues: array of queue indices for RSS
 863  *
 864  *      Sets up the portion of the HW RSS table for the port's VI to distribute
 865  *      packets to the Rx queues in @queues.
 866  */
 867 static int write_rss(const struct port_info *pi, const u16 *queues)
 868 {
 869         u16 *rss;
 870         int i, err;
 871         const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
 872
 873         rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
 874         if (!rss)
 875                 return -ENOMEM;
 876
 877         /* map the queue indices to queue ids */
 878         for (i = 0; i < pi->rss_size; i++, queues++)
 879                 rss[i] = q[*queues].rspq.abs_id;
 880
 881         err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
 882                                   pi->rss_size, rss, pi->rss_size);
 883         kfree(rss);
 884         return err;
 885 }
 886
 887 /**
 888  *      setup_rss - configure RSS
 889  *      @adap: the adapter
 890  *
 891  *      Sets up RSS for each port.
 892  */
 893 static int setup_rss(struct adapter *adap)
 894 {
 895         int i, err;
 896
 897         for_each_port(adap, i) {
 898                 const struct port_info *pi = adap2pinfo(adap, i);
 899
 900                 err = write_rss(pi, pi->rss);
 901                 if (err)
 902                         return err;
 903         }
 904         return 0;
 905 }
 906
 907 /*
 908  * Return the channel of the ingress queue with the given qid.
 909  */
 910 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
 911 {
 912         qid -= p->ingr_start;
 913         return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
 914 }
 915
 916 /*
 917  * Wait until all NAPI handlers are descheduled.
 918  */
 919 static void quiesce_rx(struct adapter *adap)
 920 {
 921         int i;
 922
 923         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 924                 struct sge_rspq *q = adap->sge.ingr_map[i];
 925
 926                 if (q && q->handler) {
 927                         napi_disable(&q->napi);
 928                         local_bh_disable();
 929                         while (!cxgb_poll_lock_napi(q))
 930                                 mdelay(1);
 931                         local_bh_enable();
 932                 }
 933
 934         }
 935 }
 936
 937 /*
 938  * Enable NAPI scheduling and interrupt generation for all Rx queues.
 939  */
 940 static void enable_rx(struct adapter *adap)
 941 {
 942         int i;
 943
 944         for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
 945                 struct sge_rspq *q = adap->sge.ingr_map[i];
 946
 947                 if (!q)
 948                         continue;
 949                 if (q->handler) {
 950                         cxgb_busy_poll_init_lock(q);
 951                         napi_enable(&q->napi);
 952                 }
 953                 /* 0-increment GTS to start the timer and enable interrupts */
 954                 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A),
 955                              SEINTARM_V(q->intr_params) |
 956                              INGRESSQID_V(q->cntxt_id));
 957         }
 958 }
 959
 960 /**
 961  *      setup_sge_queues - configure SGE Tx/Rx/response queues
 962  *      @adap: the adapter
 963  *
 964  *      Determines how many sets of SGE queues to use and initializes them.
 965  *      We support multiple queue sets per port if we have MSI-X, otherwise
 966  *      just one queue set per port.
 967  */
 968 static int setup_sge_queues(struct adapter *adap)
 969 {
 970         int err, msi_idx, i, j;
 971         struct sge *s = &adap->sge;
 972
 973         bitmap_zero(s->starving_fl, MAX_EGRQ);
 974         bitmap_zero(s->txq_maperr, MAX_EGRQ);
 975
 976         if (adap->flags & USING_MSIX)
 977                 msi_idx = 1;         /* vector 0 is for non-queue interrupts */
 978         else {
 979                 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
 980                                        NULL, NULL);
 981                 if (err)
 982                         return err;
 983                 msi_idx = -((int)s->intrq.abs_id + 1);
 984         }
 985
 986         err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
 987                                msi_idx, NULL, fwevtq_handler);
 988         if (err) {
 989 freeout:        t4_free_sge_resources(adap);
 990                 return err;
 991         }
 992
 993         for_each_port(adap, i) {
 994                 struct net_device *dev = adap->port[i];
 995                 struct port_info *pi = netdev_priv(dev);
 996                 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
 997                 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
 998
 999                 for (j = 0; j < pi->nqsets; j++, q++) {
1000                         if (msi_idx > 0)
1001                                 msi_idx++;
1002                         err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1003                                                msi_idx, &q->fl,
1004                                                t4_ethrx_handler);
1005                         if (err)
1006                                 goto freeout;
1007                         q->rspq.idx = j;
1008                         memset(&q->stats, 0, sizeof(q->stats));
1009                 }
1010                 for (j = 0; j < pi->nqsets; j++, t++) {
1011                         err = t4_sge_alloc_eth_txq(adap, t, dev,
1012                                         netdev_get_tx_queue(dev, j),
1013                                         s->fw_evtq.cntxt_id);
1014                         if (err)
1015                                 goto freeout;
1016                 }
1017         }
1018
1019         j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1020         for_each_ofldrxq(s, i) {
1021                 struct sge_ofld_rxq *q = &s->ofldrxq[i];
1022                 struct net_device *dev = adap->port[i / j];
1023
1024                 if (msi_idx > 0)
1025                         msi_idx++;
1026                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1027                                        q->fl.size ? &q->fl : NULL,
1028                                        uldrx_handler);
1029                 if (err)
1030                         goto freeout;
1031                 memset(&q->stats, 0, sizeof(q->stats));
1032                 s->ofld_rxq[i] = q->rspq.abs_id;
1033                 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1034                                             s->fw_evtq.cntxt_id);
1035                 if (err)
1036                         goto freeout;
1037         }
1038
1039         for_each_rdmarxq(s, i) {
1040                 struct sge_ofld_rxq *q = &s->rdmarxq[i];
1041
1042                 if (msi_idx > 0)
1043                         msi_idx++;
1044                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1045                                        msi_idx, q->fl.size ? &q->fl : NULL,
1046                                        uldrx_handler);
1047                 if (err)
1048                         goto freeout;
1049                 memset(&q->stats, 0, sizeof(q->stats));
1050                 s->rdma_rxq[i] = q->rspq.abs_id;
1051         }
1052
1053         for_each_rdmaciq(s, i) {
1054                 struct sge_ofld_rxq *q = &s->rdmaciq[i];
1055
1056                 if (msi_idx > 0)
1057                         msi_idx++;
1058                 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1059                                        msi_idx, q->fl.size ? &q->fl : NULL,
1060                                        uldrx_handler);
1061                 if (err)
1062                         goto freeout;
1063                 memset(&q->stats, 0, sizeof(q->stats));
1064                 s->rdma_ciq[i] = q->rspq.abs_id;
1065         }
1066
1067         for_each_port(adap, i) {
1068                 /*
1069                  * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1070                  * have RDMA queues, and that's the right value.
1071                  */
1072                 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1073                                             s->fw_evtq.cntxt_id,
1074                                             s->rdmarxq[i].rspq.cntxt_id);
1075                 if (err)
1076                         goto freeout;
1077         }
1078
1079         t4_write_reg(adap, is_t4(adap->params.chip) ?
1080                                 MPS_TRC_RSS_CONTROL_A :
1081                                 MPS_T5_TRC_RSS_CONTROL_A,
1082                      RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) |
1083                      QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id));
1084         return 0;
1085 }
1086
1087 /*
1088  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1089  * The allocated memory is cleared.
1090  */
1091 void *t4_alloc_mem(size_t size)
1092 {
1093         void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1094
1095         if (!p)
1096                 p = vzalloc(size);
1097         return p;
1098 }
1099
1100 /*
1101  * Free memory allocated through alloc_mem().
1102  */
1103 void t4_free_mem(void *addr)
1104 {
1105         if (is_vmalloc_addr(addr))
1106                 vfree(addr);
1107         else
1108                 kfree(addr);
1109 }
1110
1111 /* Send a Work Request to write the filter at a specified index.  We construct
1112  * a Firmware Filter Work Request to have the work done and put the indicated
1113  * filter into "pending" mode which will prevent any further actions against
1114  * it till we get a reply from the firmware on the completion status of the
1115  * request.
1116  */
1117 static int set_filter_wr(struct adapter *adapter, int fidx)
1118 {
1119         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1120         struct sk_buff *skb;
1121         struct fw_filter_wr *fwr;
1122         unsigned int ftid;
1123
1124         /* If the new filter requires loopback Destination MAC and/or VLAN
1125          * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1126          * the filter.
1127          */
1128         if (f->fs.newdmac || f->fs.newvlan) {
1129                 /* allocate L2T entry for new filter */
1130                 f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1131                 if (f->l2t == NULL)
1132                         return -EAGAIN;
1133                 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1134                                         f->fs.eport, f->fs.dmac)) {
1135                         cxgb4_l2t_release(f->l2t);
1136                         f->l2t = NULL;
1137                         return -ENOMEM;
1138                 }
1139         }
1140
1141         ftid = adapter->tids.ftid_base + fidx;
1142
1143         skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1144         fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1145         memset(fwr, 0, sizeof(*fwr));
1146
1147         /* It would be nice to put most of the following in t4_hw.c but most
1148          * of the work is translating the cxgbtool ch_filter_specification
1149          * into the Work Request and the definition of that structure is
1150          * currently in cxgbtool.h which isn't appropriate to pull into the
1151          * common code.  We may eventually try to come up with a more neutral
1152          * filter specification structure but for now it's easiest to simply
1153          * put this fairly direct code in line ...
1154          */
1155         fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1156         fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
1157         fwr->tid_to_iq =
1158                 htonl(FW_FILTER_WR_TID_V(ftid) |
1159                       FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1160                       FW_FILTER_WR_NOREPLY_V(0) |
1161                       FW_FILTER_WR_IQ_V(f->fs.iq));
1162         fwr->del_filter_to_l2tix =
1163                 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1164                       FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1165                       FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1166                       FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1167                       FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1168                       FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1169                       FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1170                       FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1171                       FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
1172                                              f->fs.newvlan == VLAN_REWRITE) |
1173                       FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
1174                                             f->fs.newvlan == VLAN_REWRITE) |
1175                       FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1176                       FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1177                       FW_FILTER_WR_PRIO_V(f->fs.prio) |
1178                       FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
1179         fwr->ethtype = htons(f->fs.val.ethtype);
1180         fwr->ethtypem = htons(f->fs.mask.ethtype);
1181         fwr->frag_to_ovlan_vldm =
1182                 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1183                  FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1184                  FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1185                  FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1186                  FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1187                  FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
1188         fwr->smac_sel = 0;
1189         fwr->rx_chan_rx_rpl_iq =
1190                 htons(FW_FILTER_WR_RX_CHAN_V(0) |
1191                       FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
1192         fwr->maci_to_matchtypem =
1193                 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1194                       FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1195                       FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1196                       FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1197                       FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1198                       FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1199                       FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1200                       FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
1201         fwr->ptcl = f->fs.val.proto;
1202         fwr->ptclm = f->fs.mask.proto;
1203         fwr->ttyp = f->fs.val.tos;
1204         fwr->ttypm = f->fs.mask.tos;
1205         fwr->ivlan = htons(f->fs.val.ivlan);
1206         fwr->ivlanm = htons(f->fs.mask.ivlan);
1207         fwr->ovlan = htons(f->fs.val.ovlan);
1208         fwr->ovlanm = htons(f->fs.mask.ovlan);
1209         memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1210         memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1211         memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1212         memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1213         fwr->lp = htons(f->fs.val.lport);
1214         fwr->lpm = htons(f->fs.mask.lport);
1215         fwr->fp = htons(f->fs.val.fport);
1216         fwr->fpm = htons(f->fs.mask.fport);
1217         if (f->fs.newsmac)
1218                 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1219
1220         /* Mark the filter as "pending" and ship off the Filter Work Request.
1221          * When we get the Work Request Reply we'll clear the pending status.
1222          */
1223         f->pending = 1;
1224         set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1225         t4_ofld_send(adapter, skb);
1226         return 0;
1227 }
1228
1229 /* Delete the filter at a specified index.
1230  */
1231 static int del_filter_wr(struct adapter *adapter, int fidx)
1232 {
1233         struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1234         struct sk_buff *skb;
1235         struct fw_filter_wr *fwr;
1236         unsigned int len, ftid;
1237
1238         len = sizeof(*fwr);
1239         ftid = adapter->tids.ftid_base + fidx;
1240
1241         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1242         fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1243         t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1244
1245         /* Mark the filter as "pending" and ship off the Filter Work Request.
1246          * When we get the Work Request Reply we'll clear the pending status.
1247          */
1248         f->pending = 1;
1249         t4_mgmt_tx(adapter, skb);
1250         return 0;
1251 }
1252
1253 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1254                              void *accel_priv, select_queue_fallback_t fallback)
1255 {
1256         int txq;
1257
1258 #ifdef CONFIG_CHELSIO_T4_DCB
1259         /* If a Data Center Bridging has been successfully negotiated on this
1260          * link then we'll use the skb's priority to map it to a TX Queue.
1261          * The skb's priority is determined via the VLAN Tag Priority Code
1262          * Point field.
1263          */
1264         if (cxgb4_dcb_enabled(dev)) {
1265                 u16 vlan_tci;
1266                 int err;
1267
1268                 err = vlan_get_tag(skb, &vlan_tci);
1269                 if (unlikely(err)) {
1270                         if (net_ratelimit())
1271                                 netdev_warn(dev,
1272                                             "TX Packet without VLAN Tag on DCB Link\n");
1273                         txq = 0;
1274                 } else {
1275                         txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1276                 }
1277                 return txq;
1278         }
1279 #endif /* CONFIG_CHELSIO_T4_DCB */
1280
1281         if (select_queue) {
1282                 txq = (skb_rx_queue_recorded(skb)
1283                         ? skb_get_rx_queue(skb)
1284                         : smp_processor_id());
1285
1286                 while (unlikely(txq >= dev->real_num_tx_queues))
1287                         txq -= dev->real_num_tx_queues;
1288
1289                 return txq;
1290         }
1291
1292         return fallback(dev, skb) % dev->real_num_tx_queues;
1293 }
1294
1295 static inline int is_offload(const struct adapter *adap)
1296 {
1297         return adap->params.offload;
1298 }
1299
1300 /*
1301  * Implementation of ethtool operations.
1302  */
1303
1304 static u32 get_msglevel(struct net_device *dev)
1305 {
1306         return netdev2adap(dev)->msg_enable;
1307 }
1308
1309 static void set_msglevel(struct net_device *dev, u32 val)
1310 {
1311         netdev2adap(dev)->msg_enable = val;
1312 }
1313
1314 static char stats_strings[][ETH_GSTRING_LEN] = {
1315         "TxOctetsOK         ",
1316         "TxFramesOK         ",
1317         "TxBroadcastFrames  ",
1318         "TxMulticastFrames  ",
1319         "TxUnicastFrames    ",
1320         "TxErrorFrames      ",
1321
1322         "TxFrames64         ",
1323         "TxFrames65To127    ",
1324         "TxFrames128To255   ",
1325         "TxFrames256To511   ",
1326         "TxFrames512To1023  ",
1327         "TxFrames1024To1518 ",
1328         "TxFrames1519ToMax  ",
1329
1330         "TxFramesDropped    ",
1331         "TxPauseFrames      ",
1332         "TxPPP0Frames       ",
1333         "TxPPP1Frames       ",
1334         "TxPPP2Frames       ",
1335         "TxPPP3Frames       ",
1336         "TxPPP4Frames       ",
1337         "TxPPP5Frames       ",
1338         "TxPPP6Frames       ",
1339         "TxPPP7Frames       ",
1340
1341         "RxOctetsOK         ",
1342         "RxFramesOK         ",
1343         "RxBroadcastFrames  ",
1344         "RxMulticastFrames  ",
1345         "RxUnicastFrames    ",
1346
1347         "RxFramesTooLong    ",
1348         "RxJabberErrors     ",
1349         "RxFCSErrors        ",
1350         "RxLengthErrors     ",
1351         "RxSymbolErrors     ",
1352         "RxRuntFrames       ",
1353
1354         "RxFrames64         ",
1355         "RxFrames65To127    ",
1356         "RxFrames128To255   ",
1357         "RxFrames256To511   ",
1358         "RxFrames512To1023  ",
1359         "RxFrames1024To1518 ",
1360         "RxFrames1519ToMax  ",
1361
1362         "RxPauseFrames      ",
1363         "RxPPP0Frames       ",
1364         "RxPPP1Frames       ",
1365         "RxPPP2Frames       ",
1366         "RxPPP3Frames       ",
1367         "RxPPP4Frames       ",
1368         "RxPPP5Frames       ",
1369         "RxPPP6Frames       ",
1370         "RxPPP7Frames       ",
1371
1372         "RxBG0FramesDropped ",
1373         "RxBG1FramesDropped ",
1374         "RxBG2FramesDropped ",
1375         "RxBG3FramesDropped ",
1376         "RxBG0FramesTrunc   ",
1377         "RxBG1FramesTrunc   ",
1378         "RxBG2FramesTrunc   ",
1379         "RxBG3FramesTrunc   ",
1380
1381         "TSO                ",
1382         "TxCsumOffload      ",
1383         "RxCsumGood         ",
1384         "VLANextractions    ",
1385         "VLANinsertions     ",
1386         "GROpackets         ",
1387         "GROmerged          ",
1388         "WriteCoalSuccess   ",
1389         "WriteCoalFail      ",
1390 };
1391
1392 static int get_sset_count(struct net_device *dev, int sset)
1393 {
1394         switch (sset) {
1395         case ETH_SS_STATS:
1396                 return ARRAY_SIZE(stats_strings);
1397         default:
1398                 return -EOPNOTSUPP;
1399         }
1400 }
1401
1402 #define T4_REGMAP_SIZE (160 * 1024)
1403 #define T5_REGMAP_SIZE (332 * 1024)
1404
1405 static int get_regs_len(struct net_device *dev)
1406 {
1407         struct adapter *adap = netdev2adap(dev);
1408         if (is_t4(adap->params.chip))
1409                 return T4_REGMAP_SIZE;
1410         else
1411                 return T5_REGMAP_SIZE;
1412 }
1413
1414 static int get_eeprom_len(struct net_device *dev)
1415 {
1416         return EEPROMSIZE;
1417 }
1418
1419 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1420 {
1421         struct adapter *adapter = netdev2adap(dev);
1422         u32 exprom_vers;
1423
1424         strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1425         strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1426         strlcpy(info->bus_info, pci_name(adapter->pdev),
1427                 sizeof(info->bus_info));
1428
1429         if (adapter->params.fw_vers)
1430                 snprintf(info->fw_version, sizeof(info->fw_version),
1431                         "%u.%u.%u.%u, TP %u.%u.%u.%u",
1432                         FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1433                         FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1434                         FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1435                         FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1436                         FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1437                         FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1438                         FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1439                         FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
1440
1441         if (!t4_get_exprom_version(adapter, &exprom_vers))
1442                 snprintf(info->erom_version, sizeof(info->erom_version),
1443                          "%u.%u.%u.%u",
1444                          FW_HDR_FW_VER_MAJOR_G(exprom_vers),
1445                          FW_HDR_FW_VER_MINOR_G(exprom_vers),
1446                          FW_HDR_FW_VER_MICRO_G(exprom_vers),
1447                          FW_HDR_FW_VER_BUILD_G(exprom_vers));
1448 }
1449
1450 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1451 {
1452         if (stringset == ETH_SS_STATS)
1453                 memcpy(data, stats_strings, sizeof(stats_strings));
1454 }
1455
1456 /*
1457  * port stats maintained per queue of the port.  They should be in the same
1458  * order as in stats_strings above.
1459  */
1460 struct queue_port_stats {
1461         u64 tso;
1462         u64 tx_csum;
1463         u64 rx_csum;
1464         u64 vlan_ex;
1465         u64 vlan_ins;
1466         u64 gro_pkts;
1467         u64 gro_merged;
1468 };
1469
1470 static void collect_sge_port_stats(const struct adapter *adap,
1471                 const struct port_info *p, struct queue_port_stats *s)
1472 {
1473         int i;
1474         const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1475         const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1476
1477         memset(s, 0, sizeof(*s));
1478         for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1479                 s->tso += tx->tso;
1480                 s->tx_csum += tx->tx_cso;
1481                 s->rx_csum += rx->stats.rx_cso;
1482                 s->vlan_ex += rx->stats.vlan_ex;
1483                 s->vlan_ins += tx->vlan_ins;
1484                 s->gro_pkts += rx->stats.lro_pkts;
1485                 s->gro_merged += rx->stats.lro_merged;
1486         }
1487 }
1488
1489 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1490                       u64 *data)
1491 {
1492         struct port_info *pi = netdev_priv(dev);
1493         struct adapter *adapter = pi->adapter;
1494         u32 val1, val2;
1495
1496         t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1497
1498         data += sizeof(struct port_stats) / sizeof(u64);
1499         collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1500         data += sizeof(struct queue_port_stats) / sizeof(u64);
1501         if (!is_t4(adapter->params.chip)) {
1502                 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7));
1503                 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A);
1504                 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A);
1505                 *data = val1 - val2;
1506                 data++;
1507                 *data = val2;
1508                 data++;
1509         } else {
1510                 memset(data, 0, 2 * sizeof(u64));
1511                 *data += 2;
1512         }
1513 }
1514
1515 /*
1516  * Return a version number to identify the type of adapter.  The scheme is:
1517  * - bits 0..9: chip version
1518  * - bits 10..15: chip revision
1519  * - bits 16..23: register dump version
1520  */
1521 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1522 {
1523         return CHELSIO_CHIP_VERSION(ap->params.chip) |
1524                 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1525 }
1526
1527 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1528                            unsigned int end)
1529 {
1530         u32 *p = buf + start;
1531
1532         for ( ; start <= end; start += sizeof(u32))
1533                 *p++ = t4_read_reg(ap, start);
1534 }
1535
1536 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1537                      void *buf)
1538 {
1539         static const unsigned int t4_reg_ranges[] = {
1540                 0x1008, 0x1108,
1541                 0x1180, 0x11b4,
1542                 0x11fc, 0x123c,
1543                 0x1300, 0x173c,
1544                 0x1800, 0x18fc,
1545                 0x3000, 0x30d8,
1546                 0x30e0, 0x5924,
1547                 0x5960, 0x59d4,
1548                 0x5a00, 0x5af8,
1549                 0x6000, 0x6098,
1550                 0x6100, 0x6150,
1551                 0x6200, 0x6208,
1552                 0x6240, 0x6248,
1553                 0x6280, 0x6338,
1554                 0x6370, 0x638c,
1555                 0x6400, 0x643c,
1556                 0x6500, 0x6524,
1557                 0x6a00, 0x6a38,
1558                 0x6a60, 0x6a78,
1559                 0x6b00, 0x6b84,
1560                 0x6bf0, 0x6c84,
1561                 0x6cf0, 0x6d84,
1562                 0x6df0, 0x6e84,
1563                 0x6ef0, 0x6f84,
1564                 0x6ff0, 0x7084,
1565                 0x70f0, 0x7184,
1566                 0x71f0, 0x7284,
1567                 0x72f0, 0x7384,
1568                 0x73f0, 0x7450,
1569                 0x7500, 0x7530,
1570                 0x7600, 0x761c,
1571                 0x7680, 0x76cc,
1572                 0x7700, 0x7798,
1573                 0x77c0, 0x77fc,
1574                 0x7900, 0x79fc,
1575                 0x7b00, 0x7c38,
1576                 0x7d00, 0x7efc,
1577                 0x8dc0, 0x8e1c,
1578                 0x8e30, 0x8e78,
1579                 0x8ea0, 0x8f6c,
1580                 0x8fc0, 0x9074,
1581                 0x90fc, 0x90fc,
1582                 0x9400, 0x9458,
1583                 0x9600, 0x96bc,
1584                 0x9800, 0x9808,
1585                 0x9820, 0x983c,
1586                 0x9850, 0x9864,
1587                 0x9c00, 0x9c6c,
1588                 0x9c80, 0x9cec,
1589                 0x9d00, 0x9d6c,
1590                 0x9d80, 0x9dec,
1591                 0x9e00, 0x9e6c,
1592                 0x9e80, 0x9eec,
1593                 0x9f00, 0x9f6c,
1594                 0x9f80, 0x9fec,
1595                 0xd004, 0xd03c,
1596                 0xdfc0, 0xdfe0,
1597                 0xe000, 0xea7c,
1598                 0xf000, 0x11110,
1599                 0x11118, 0x11190,
1600                 0x19040, 0x1906c,
1601                 0x19078, 0x19080,
1602                 0x1908c, 0x19124,
1603                 0x19150, 0x191b0,
1604                 0x191d0, 0x191e8,
1605                 0x19238, 0x1924c,
1606                 0x193f8, 0x19474,
1607                 0x19490, 0x194f8,
1608                 0x19800, 0x19f30,
1609                 0x1a000, 0x1a06c,
1610                 0x1a0b0, 0x1a120,
1611                 0x1a128, 0x1a138,
1612                 0x1a190, 0x1a1c4,
1613                 0x1a1fc, 0x1a1fc,
1614                 0x1e040, 0x1e04c,
1615                 0x1e284, 0x1e28c,
1616                 0x1e2c0, 0x1e2c0,
1617                 0x1e2e0, 0x1e2e0,
1618                 0x1e300, 0x1e384,
1619                 0x1e3c0, 0x1e3c8,
1620                 0x1e440, 0x1e44c,
1621                 0x1e684, 0x1e68c,
1622                 0x1e6c0, 0x1e6c0,
1623                 0x1e6e0, 0x1e6e0,
1624                 0x1e700, 0x1e784,
1625                 0x1e7c0, 0x1e7c8,
1626                 0x1e840, 0x1e84c,
1627                 0x1ea84, 0x1ea8c,
1628                 0x1eac0, 0x1eac0,
1629                 0x1eae0, 0x1eae0,
1630                 0x1eb00, 0x1eb84,
1631                 0x1ebc0, 0x1ebc8,
1632                 0x1ec40, 0x1ec4c,
1633                 0x1ee84, 0x1ee8c,
1634                 0x1eec0, 0x1eec0,
1635                 0x1eee0, 0x1eee0,
1636                 0x1ef00, 0x1ef84,
1637                 0x1efc0, 0x1efc8,
1638                 0x1f040, 0x1f04c,
1639                 0x1f284, 0x1f28c,
1640                 0x1f2c0, 0x1f2c0,
1641                 0x1f2e0, 0x1f2e0,
1642                 0x1f300, 0x1f384,
1643                 0x1f3c0, 0x1f3c8,
1644                 0x1f440, 0x1f44c,
1645                 0x1f684, 0x1f68c,
1646                 0x1f6c0, 0x1f6c0,
1647                 0x1f6e0, 0x1f6e0,
1648                 0x1f700, 0x1f784,
1649                 0x1f7c0, 0x1f7c8,
1650                 0x1f840, 0x1f84c,
1651                 0x1fa84, 0x1fa8c,
1652                 0x1fac0, 0x1fac0,
1653                 0x1fae0, 0x1fae0,
1654                 0x1fb00, 0x1fb84,
1655                 0x1fbc0, 0x1fbc8,
1656                 0x1fc40, 0x1fc4c,
1657                 0x1fe84, 0x1fe8c,
1658                 0x1fec0, 0x1fec0,
1659                 0x1fee0, 0x1fee0,
1660                 0x1ff00, 0x1ff84,
1661                 0x1ffc0, 0x1ffc8,
1662                 0x20000, 0x2002c,
1663                 0x20100, 0x2013c,
1664                 0x20190, 0x201c8,
1665                 0x20200, 0x20318,
1666                 0x20400, 0x20528,
1667                 0x20540, 0x20614,
1668                 0x21000, 0x21040,
1669                 0x2104c, 0x21060,
1670                 0x210c0, 0x210ec,
1671                 0x21200, 0x21268,
1672                 0x21270, 0x21284,
1673                 0x212fc, 0x21388,
1674                 0x21400, 0x21404,
1675                 0x21500, 0x21518,
1676                 0x2152c, 0x2153c,
1677                 0x21550, 0x21554,
1678                 0x21600, 0x21600,
1679                 0x21608, 0x21628,
1680                 0x21630, 0x2163c,
1681                 0x21700, 0x2171c,
1682                 0x21780, 0x2178c,
1683                 0x21800, 0x21c38,
1684                 0x21c80, 0x21d7c,
1685                 0x21e00, 0x21e04,
1686                 0x22000, 0x2202c,
1687                 0x22100, 0x2213c,
1688                 0x22190, 0x221c8,
1689                 0x22200, 0x22318,
1690                 0x22400, 0x22528,
1691                 0x22540, 0x22614,
1692                 0x23000, 0x23040,
1693                 0x2304c, 0x23060,
1694                 0x230c0, 0x230ec,
1695                 0x23200, 0x23268,
1696                 0x23270, 0x23284,
1697                 0x232fc, 0x23388,
1698                 0x23400, 0x23404,
1699                 0x23500, 0x23518,
1700                 0x2352c, 0x2353c,
1701                 0x23550, 0x23554,
1702                 0x23600, 0x23600,
1703                 0x23608, 0x23628,
1704                 0x23630, 0x2363c,
1705                 0x23700, 0x2371c,
1706                 0x23780, 0x2378c,
1707                 0x23800, 0x23c38,
1708                 0x23c80, 0x23d7c,
1709                 0x23e00, 0x23e04,
1710                 0x24000, 0x2402c,
1711                 0x24100, 0x2413c,
1712                 0x24190, 0x241c8,
1713                 0x24200, 0x24318,
1714                 0x24400, 0x24528,
1715                 0x24540, 0x24614,
1716                 0x25000, 0x25040,
1717                 0x2504c, 0x25060,
1718                 0x250c0, 0x250ec,
1719                 0x25200, 0x25268,
1720                 0x25270, 0x25284,
1721                 0x252fc, 0x25388,
1722                 0x25400, 0x25404,
1723                 0x25500, 0x25518,
1724                 0x2552c, 0x2553c,
1725                 0x25550, 0x25554,
1726                 0x25600, 0x25600,
1727                 0x25608, 0x25628,
1728                 0x25630, 0x2563c,
1729                 0x25700, 0x2571c,
1730                 0x25780, 0x2578c,
1731                 0x25800, 0x25c38,
1732                 0x25c80, 0x25d7c,
1733                 0x25e00, 0x25e04,
1734                 0x26000, 0x2602c,
1735                 0x26100, 0x2613c,
1736                 0x26190, 0x261c8,
1737                 0x26200, 0x26318,
1738                 0x26400, 0x26528,
1739                 0x26540, 0x26614,
1740                 0x27000, 0x27040,
1741                 0x2704c, 0x27060,
1742                 0x270c0, 0x270ec,
1743                 0x27200, 0x27268,
1744                 0x27270, 0x27284,
1745                 0x272fc, 0x27388,
1746                 0x27400, 0x27404,
1747                 0x27500, 0x27518,
1748                 0x2752c, 0x2753c,
1749                 0x27550, 0x27554,
1750                 0x27600, 0x27600,
1751                 0x27608, 0x27628,
1752                 0x27630, 0x2763c,
1753                 0x27700, 0x2771c,
1754                 0x27780, 0x2778c,
1755                 0x27800, 0x27c38,
1756                 0x27c80, 0x27d7c,
1757                 0x27e00, 0x27e04
1758         };
1759
1760         static const unsigned int t5_reg_ranges[] = {
1761                 0x1008, 0x1148,
1762                 0x1180, 0x11b4,
1763                 0x11fc, 0x123c,
1764                 0x1280, 0x173c,
1765                 0x1800, 0x18fc,
1766                 0x3000, 0x3028,
1767                 0x3060, 0x30d8,
1768                 0x30e0, 0x30fc,
1769                 0x3140, 0x357c,
1770                 0x35a8, 0x35cc,
1771                 0x35ec, 0x35ec,
1772                 0x3600, 0x5624,
1773                 0x56cc, 0x575c,
1774                 0x580c, 0x5814,
1775                 0x5890, 0x58bc,
1776                 0x5940, 0x59dc,
1777                 0x59fc, 0x5a18,
1778                 0x5a60, 0x5a9c,
1779                 0x5b9c, 0x5bfc,
1780                 0x6000, 0x6040,
1781                 0x6058, 0x614c,
1782                 0x7700, 0x7798,
1783                 0x77c0, 0x78fc,
1784                 0x7b00, 0x7c54,
1785                 0x7d00, 0x7efc,
1786                 0x8dc0, 0x8de0,
1787                 0x8df8, 0x8e84,
1788                 0x8ea0, 0x8f84,
1789                 0x8fc0, 0x90f8,
1790                 0x9400, 0x9470,
1791                 0x9600, 0x96f4,
1792                 0x9800, 0x9808,
1793                 0x9820, 0x983c,
1794                 0x9850, 0x9864,
1795                 0x9c00, 0x9c6c,
1796                 0x9c80, 0x9cec,
1797                 0x9d00, 0x9d6c,
1798                 0x9d80, 0x9dec,
1799                 0x9e00, 0x9e6c,
1800                 0x9e80, 0x9eec,
1801                 0x9f00, 0x9f6c,
1802                 0x9f80, 0xa020,
1803                 0xd004, 0xd03c,
1804                 0xdfc0, 0xdfe0,
1805                 0xe000, 0x11088,
1806                 0x1109c, 0x11110,
1807                 0x11118, 0x1117c,
1808                 0x11190, 0x11204,
1809                 0x19040, 0x1906c,
1810                 0x19078, 0x19080,
1811                 0x1908c, 0x19124,
1812                 0x19150, 0x191b0,
1813                 0x191d0, 0x191e8,
1814                 0x19238, 0x19290,
1815                 0x193f8, 0x19474,
1816                 0x19490, 0x194cc,
1817                 0x194f0, 0x194f8,
1818                 0x19c00, 0x19c60,
1819                 0x19c94, 0x19e10,
1820                 0x19e50, 0x19f34,
1821                 0x19f40, 0x19f50,
1822                 0x19f90, 0x19fe4,
1823                 0x1a000, 0x1a06c,
1824                 0x1a0b0, 0x1a120,
1825                 0x1a128, 0x1a138,
1826                 0x1a190, 0x1a1c4,
1827                 0x1a1fc, 0x1a1fc,
1828                 0x1e008, 0x1e00c,
1829                 0x1e040, 0x1e04c,
1830                 0x1e284, 0x1e290,
1831                 0x1e2c0, 0x1e2c0,
1832                 0x1e2e0, 0x1e2e0,
1833                 0x1e300, 0x1e384,
1834                 0x1e3c0, 0x1e3c8,
1835                 0x1e408, 0x1e40c,
1836                 0x1e440, 0x1e44c,
1837                 0x1e684, 0x1e690,
1838                 0x1e6c0, 0x1e6c0,
1839                 0x1e6e0, 0x1e6e0,
1840                 0x1e700, 0x1e784,
1841                 0x1e7c0, 0x1e7c8,
1842                 0x1e808, 0x1e80c,
1843                 0x1e840, 0x1e84c,
1844                 0x1ea84, 0x1ea90,
1845                 0x1eac0, 0x1eac0,
1846                 0x1eae0, 0x1eae0,
1847                 0x1eb00, 0x1eb84,
1848                 0x1ebc0, 0x1ebc8,
1849                 0x1ec08, 0x1ec0c,
1850                 0x1ec40, 0x1ec4c,
1851                 0x1ee84, 0x1ee90,
1852                 0x1eec0, 0x1eec0,
1853                 0x1eee0, 0x1eee0,
1854                 0x1ef00, 0x1ef84,
1855                 0x1efc0, 0x1efc8,
1856                 0x1f008, 0x1f00c,
1857                 0x1f040, 0x1f04c,
1858                 0x1f284, 0x1f290,
1859                 0x1f2c0, 0x1f2c0,
1860                 0x1f2e0, 0x1f2e0,
1861                 0x1f300, 0x1f384,
1862                 0x1f3c0, 0x1f3c8,
1863                 0x1f408, 0x1f40c,
1864                 0x1f440, 0x1f44c,
1865                 0x1f684, 0x1f690,
1866                 0x1f6c0, 0x1f6c0,
1867                 0x1f6e0, 0x1f6e0,
1868                 0x1f700, 0x1f784,
1869                 0x1f7c0, 0x1f7c8,
1870                 0x1f808, 0x1f80c,
1871                 0x1f840, 0x1f84c,
1872                 0x1fa84, 0x1fa90,
1873                 0x1fac0, 0x1fac0,
1874                 0x1fae0, 0x1fae0,
1875                 0x1fb00, 0x1fb84,
1876                 0x1fbc0, 0x1fbc8,
1877                 0x1fc08, 0x1fc0c,
1878                 0x1fc40, 0x1fc4c,
1879                 0x1fe84, 0x1fe90,
1880                 0x1fec0, 0x1fec0,
1881                 0x1fee0, 0x1fee0,
1882                 0x1ff00, 0x1ff84,
1883                 0x1ffc0, 0x1ffc8,
1884                 0x30000, 0x30030,
1885                 0x30100, 0x30144,
1886                 0x30190, 0x301d0,
1887                 0x30200, 0x30318,
1888                 0x30400, 0x3052c,
1889                 0x30540, 0x3061c,
1890                 0x30800, 0x30834,
1891                 0x308c0, 0x30908,
1892                 0x30910, 0x309ac,
1893                 0x30a00, 0x30a04,
1894                 0x30a0c, 0x30a2c,
1895                 0x30a44, 0x30a50,
1896                 0x30a74, 0x30c24,
1897                 0x30d08, 0x30d14,
1898                 0x30d1c, 0x30d20,
1899                 0x30d3c, 0x30d50,
1900                 0x31200, 0x3120c,
1901                 0x31220, 0x31220,
1902                 0x31240, 0x31240,
1903                 0x31600, 0x31600,
1904                 0x31608, 0x3160c,
1905                 0x31a00, 0x31a1c,
1906                 0x31e04, 0x31e20,
1907                 0x31e38, 0x31e3c,
1908                 0x31e80, 0x31e80,
1909                 0x31e88, 0x31ea8,
1910                 0x31eb0, 0x31eb4,
1911                 0x31ec8, 0x31ed4,
1912                 0x31fb8, 0x32004,
1913                 0x32208, 0x3223c,
1914                 0x32600, 0x32630,
1915                 0x32a00, 0x32abc,
1916                 0x32b00, 0x32b70,
1917                 0x33000, 0x33048,
1918                 0x33060, 0x3309c,
1919                 0x330f0, 0x33148,
1920                 0x33160, 0x3319c,
1921                 0x331f0, 0x332e4,
1922                 0x332f8, 0x333e4,
1923                 0x333f8, 0x33448,
1924                 0x33460, 0x3349c,
1925                 0x334f0, 0x33548,
1926                 0x33560, 0x3359c,
1927                 0x335f0, 0x336e4,
1928                 0x336f8, 0x337e4,
1929                 0x337f8, 0x337fc,
1930                 0x33814, 0x33814,
1931                 0x3382c, 0x3382c,
1932                 0x33880, 0x3388c,
1933                 0x338e8, 0x338ec,
1934                 0x33900, 0x33948,
1935                 0x33960, 0x3399c,
1936                 0x339f0, 0x33ae4,
1937                 0x33af8, 0x33b10,
1938                 0x33b28, 0x33b28,
1939                 0x33b3c, 0x33b50,
1940                 0x33bf0, 0x33c10,
1941                 0x33c28, 0x33c28,
1942                 0x33c3c, 0x33c50,
1943                 0x33cf0, 0x33cfc,
1944                 0x34000, 0x34030,
1945                 0x34100, 0x34144,
1946                 0x34190, 0x341d0,
1947                 0x34200, 0x34318,
1948                 0x34400, 0x3452c,
1949                 0x34540, 0x3461c,
1950                 0x34800, 0x34834,
1951                 0x348c0, 0x34908,
1952                 0x34910, 0x349ac,
1953                 0x34a00, 0x34a04,
1954                 0x34a0c, 0x34a2c,
1955                 0x34a44, 0x34a50,
1956                 0x34a74, 0x34c24,
1957                 0x34d08, 0x34d14,
1958                 0x34d1c, 0x34d20,
1959                 0x34d3c, 0x34d50,
1960                 0x35200, 0x3520c,
1961                 0x35220, 0x35220,
1962                 0x35240, 0x35240,
1963                 0x35600, 0x35600,
1964                 0x35608, 0x3560c,
1965                 0x35a00, 0x35a1c,
1966                 0x35e04, 0x35e20,
1967                 0x35e38, 0x35e3c,
1968                 0x35e80, 0x35e80,
1969                 0x35e88, 0x35ea8,
1970                 0x35eb0, 0x35eb4,
1971                 0x35ec8, 0x35ed4,
1972                 0x35fb8, 0x36004,
1973                 0x36208, 0x3623c,
1974                 0x36600, 0x36630,
1975                 0x36a00, 0x36abc,
1976                 0x36b00, 0x36b70,
1977                 0x37000, 0x37048,
1978                 0x37060, 0x3709c,
1979                 0x370f0, 0x37148,
1980                 0x37160, 0x3719c,
1981                 0x371f0, 0x372e4,
1982                 0x372f8, 0x373e4,
1983                 0x373f8, 0x37448,
1984                 0x37460, 0x3749c,
1985                 0x374f0, 0x37548,
1986                 0x37560, 0x3759c,
1987                 0x375f0, 0x376e4,
1988                 0x376f8, 0x377e4,
1989                 0x377f8, 0x377fc,
1990                 0x37814, 0x37814,
1991                 0x3782c, 0x3782c,
1992                 0x37880, 0x3788c,
1993                 0x378e8, 0x378ec,
1994                 0x37900, 0x37948,
1995                 0x37960, 0x3799c,
1996                 0x379f0, 0x37ae4,
1997                 0x37af8, 0x37b10,
1998                 0x37b28, 0x37b28,
1999                 0x37b3c, 0x37b50,
2000                 0x37bf0, 0x37c10,
2001                 0x37c28, 0x37c28,
2002                 0x37c3c, 0x37c50,
2003                 0x37cf0, 0x37cfc,
2004                 0x38000, 0x38030,
2005                 0x38100, 0x38144,
2006                 0x38190, 0x381d0,
2007                 0x38200, 0x38318,
2008                 0x38400, 0x3852c,
2009                 0x38540, 0x3861c,
2010                 0x38800, 0x38834,
2011                 0x388c0, 0x38908,
2012                 0x38910, 0x389ac,
2013                 0x38a00, 0x38a04,
2014                 0x38a0c, 0x38a2c,
2015                 0x38a44, 0x38a50,
2016                 0x38a74, 0x38c24,
2017                 0x38d08, 0x38d14,
2018                 0x38d1c, 0x38d20,
2019                 0x38d3c, 0x38d50,
2020                 0x39200, 0x3920c,
2021                 0x39220, 0x39220,
2022                 0x39240, 0x39240,
2023                 0x39600, 0x39600,
2024                 0x39608, 0x3960c,
2025                 0x39a00, 0x39a1c,
2026                 0x39e04, 0x39e20,
2027                 0x39e38, 0x39e3c,
2028                 0x39e80, 0x39e80,
2029                 0x39e88, 0x39ea8,
2030                 0x39eb0, 0x39eb4,
2031                 0x39ec8, 0x39ed4,
2032                 0x39fb8, 0x3a004,
2033                 0x3a208, 0x3a23c,
2034                 0x3a600, 0x3a630,
2035                 0x3aa00, 0x3aabc,
2036                 0x3ab00, 0x3ab70,
2037                 0x3b000, 0x3b048,
2038                 0x3b060, 0x3b09c,
2039                 0x3b0f0, 0x3b148,
2040                 0x3b160, 0x3b19c,
2041                 0x3b1f0, 0x3b2e4,
2042                 0x3b2f8, 0x3b3e4,
2043                 0x3b3f8, 0x3b448,
2044                 0x3b460, 0x3b49c,
2045                 0x3b4f0, 0x3b548,
2046                 0x3b560, 0x3b59c,
2047                 0x3b5f0, 0x3b6e4,
2048                 0x3b6f8, 0x3b7e4,
2049                 0x3b7f8, 0x3b7fc,
2050                 0x3b814, 0x3b814,
2051                 0x3b82c, 0x3b82c,
2052                 0x3b880, 0x3b88c,
2053                 0x3b8e8, 0x3b8ec,
2054                 0x3b900, 0x3b948,
2055                 0x3b960, 0x3b99c,
2056                 0x3b9f0, 0x3bae4,
2057                 0x3baf8, 0x3bb10,
2058                 0x3bb28, 0x3bb28,
2059                 0x3bb3c, 0x3bb50,
2060                 0x3bbf0, 0x3bc10,
2061                 0x3bc28, 0x3bc28,
2062                 0x3bc3c, 0x3bc50,
2063                 0x3bcf0, 0x3bcfc,
2064                 0x3c000, 0x3c030,
2065                 0x3c100, 0x3c144,
2066                 0x3c190, 0x3c1d0,
2067                 0x3c200, 0x3c318,
2068                 0x3c400, 0x3c52c,
2069                 0x3c540, 0x3c61c,
2070                 0x3c800, 0x3c834,
2071                 0x3c8c0, 0x3c908,
2072                 0x3c910, 0x3c9ac,
2073                 0x3ca00, 0x3ca04,
2074                 0x3ca0c, 0x3ca2c,
2075                 0x3ca44, 0x3ca50,
2076                 0x3ca74, 0x3cc24,
2077                 0x3cd08, 0x3cd14,
2078                 0x3cd1c, 0x3cd20,
2079                 0x3cd3c, 0x3cd50,
2080                 0x3d200, 0x3d20c,
2081                 0x3d220, 0x3d220,
2082                 0x3d240, 0x3d240,
2083                 0x3d600, 0x3d600,
2084                 0x3d608, 0x3d60c,
2085                 0x3da00, 0x3da1c,
2086                 0x3de04, 0x3de20,
2087                 0x3de38, 0x3de3c,
2088                 0x3de80, 0x3de80,
2089                 0x3de88, 0x3dea8,
2090                 0x3deb0, 0x3deb4,
2091                 0x3dec8, 0x3ded4,
2092                 0x3dfb8, 0x3e004,
2093                 0x3e208, 0x3e23c,
2094                 0x3e600, 0x3e630,
2095                 0x3ea00, 0x3eabc,
2096                 0x3eb00, 0x3eb70,
2097                 0x3f000, 0x3f048,
2098                 0x3f060, 0x3f09c,
2099                 0x3f0f0, 0x3f148,
2100                 0x3f160, 0x3f19c,
2101                 0x3f1f0, 0x3f2e4,
2102                 0x3f2f8, 0x3f3e4,
2103                 0x3f3f8, 0x3f448,
2104                 0x3f460, 0x3f49c,
2105                 0x3f4f0, 0x3f548,
2106                 0x3f560, 0x3f59c,
2107                 0x3f5f0, 0x3f6e4,
2108                 0x3f6f8, 0x3f7e4,
2109                 0x3f7f8, 0x3f7fc,
2110                 0x3f814, 0x3f814,
2111                 0x3f82c, 0x3f82c,
2112                 0x3f880, 0x3f88c,
2113                 0x3f8e8, 0x3f8ec,
2114                 0x3f900, 0x3f948,
2115                 0x3f960, 0x3f99c,
2116                 0x3f9f0, 0x3fae4,
2117                 0x3faf8, 0x3fb10,
2118                 0x3fb28, 0x3fb28,
2119                 0x3fb3c, 0x3fb50,
2120                 0x3fbf0, 0x3fc10,
2121                 0x3fc28, 0x3fc28,
2122                 0x3fc3c, 0x3fc50,
2123                 0x3fcf0, 0x3fcfc,
2124                 0x40000, 0x4000c,
2125                 0x40040, 0x40068,
2126                 0x40080, 0x40144,
2127                 0x40180, 0x4018c,
2128                 0x40200, 0x40298,
2129                 0x402ac, 0x4033c,
2130                 0x403f8, 0x403fc,
2131                 0x41304, 0x413c4,
2132                 0x41400, 0x4141c,
2133                 0x41480, 0x414d0,
2134                 0x44000, 0x44078,
2135                 0x440c0, 0x44278,
2136                 0x442c0, 0x44478,
2137                 0x444c0, 0x44678,
2138                 0x446c0, 0x44878,
2139                 0x448c0, 0x449fc,
2140                 0x45000, 0x45068,
2141                 0x45080, 0x45084,
2142                 0x450a0, 0x450b0,
2143                 0x45200, 0x45268,
2144                 0x45280, 0x45284,
2145                 0x452a0, 0x452b0,
2146                 0x460c0, 0x460e4,
2147                 0x47000, 0x4708c,
2148                 0x47200, 0x47250,
2149                 0x47400, 0x47420,
2150                 0x47600, 0x47618,
2151                 0x47800, 0x47814,
2152                 0x48000, 0x4800c,
2153                 0x48040, 0x48068,
2154                 0x48080, 0x48144,
2155                 0x48180, 0x4818c,
2156                 0x48200, 0x48298,
2157                 0x482ac, 0x4833c,
2158                 0x483f8, 0x483fc,
2159                 0x49304, 0x493c4,
2160                 0x49400, 0x4941c,
2161                 0x49480, 0x494d0,
2162                 0x4c000, 0x4c078,
2163                 0x4c0c0, 0x4c278,
2164                 0x4c2c0, 0x4c478,
2165                 0x4c4c0, 0x4c678,
2166                 0x4c6c0, 0x4c878,
2167                 0x4c8c0, 0x4c9fc,
2168                 0x4d000, 0x4d068,
2169                 0x4d080, 0x4d084,
2170                 0x4d0a0, 0x4d0b0,
2171                 0x4d200, 0x4d268,
2172                 0x4d280, 0x4d284,
2173                 0x4d2a0, 0x4d2b0,
2174                 0x4e0c0, 0x4e0e4,
2175                 0x4f000, 0x4f08c,
2176                 0x4f200, 0x4f250,
2177                 0x4f400, 0x4f420,
2178                 0x4f600, 0x4f618,
2179                 0x4f800, 0x4f814,
2180                 0x50000, 0x500cc,
2181                 0x50400, 0x50400,
2182                 0x50800, 0x508cc,
2183                 0x50c00, 0x50c00,
2184                 0x51000, 0x5101c,
2185                 0x51300, 0x51308,
2186         };
2187
2188         int i;
2189         struct adapter *ap = netdev2adap(dev);
2190         static const unsigned int *reg_ranges;
2191         int arr_size = 0, buf_size = 0;
2192
2193         if (is_t4(ap->params.chip)) {
2194                 reg_ranges = &t4_reg_ranges[0];
2195                 arr_size = ARRAY_SIZE(t4_reg_ranges);
2196                 buf_size = T4_REGMAP_SIZE;
2197         } else {
2198                 reg_ranges = &t5_reg_ranges[0];
2199                 arr_size = ARRAY_SIZE(t5_reg_ranges);
2200                 buf_size = T5_REGMAP_SIZE;
2201         }
2202
2203         regs->version = mk_adap_vers(ap);
2204
2205         memset(buf, 0, buf_size);
2206         for (i = 0; i < arr_size; i += 2)
2207                 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2208 }
2209
2210 static int restart_autoneg(struct net_device *dev)
2211 {
2212         struct port_info *p = netdev_priv(dev);
2213
2214         if (!netif_running(dev))
2215                 return -EAGAIN;
2216         if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2217                 return -EINVAL;
2218         t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2219         return 0;
2220 }
2221
2222 static int identify_port(struct net_device *dev,
2223                          enum ethtool_phys_id_state state)
2224 {
2225         unsigned int val;
2226         struct adapter *adap = netdev2adap(dev);
2227
2228         if (state == ETHTOOL_ID_ACTIVE)
2229                 val = 0xffff;
2230         else if (state == ETHTOOL_ID_INACTIVE)
2231                 val = 0;
2232         else
2233                 return -EINVAL;
2234
2235         return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2236 }
2237
2238 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
2239 {
2240         unsigned int v = 0;
2241
2242         if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2243             type == FW_PORT_TYPE_BT_XAUI) {
2244                 v |= SUPPORTED_TP;
2245                 if (caps & FW_PORT_CAP_SPEED_100M)
2246                         v |= SUPPORTED_100baseT_Full;
2247                 if (caps & FW_PORT_CAP_SPEED_1G)
2248                         v |= SUPPORTED_1000baseT_Full;
2249                 if (caps & FW_PORT_CAP_SPEED_10G)
2250                         v |= SUPPORTED_10000baseT_Full;
2251         } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2252                 v |= SUPPORTED_Backplane;
2253                 if (caps & FW_PORT_CAP_SPEED_1G)
2254                         v |= SUPPORTED_1000baseKX_Full;
2255                 if (caps & FW_PORT_CAP_SPEED_10G)
2256                         v |= SUPPORTED_10000baseKX4_Full;
2257         } else if (type == FW_PORT_TYPE_KR)
2258                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2259         else if (type == FW_PORT_TYPE_BP_AP)
2260                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2261                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2262         else if (type == FW_PORT_TYPE_BP4_AP)
2263                 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2264                      SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2265                      SUPPORTED_10000baseKX4_Full;
2266         else if (type == FW_PORT_TYPE_FIBER_XFI ||
2267                  type == FW_PORT_TYPE_FIBER_XAUI ||
2268                  type == FW_PORT_TYPE_SFP ||
2269                  type == FW_PORT_TYPE_QSFP_10G ||
2270                  type == FW_PORT_TYPE_QSA) {
2271                 v |= SUPPORTED_FIBRE;
2272                 if (caps & FW_PORT_CAP_SPEED_1G)
2273                         v |= SUPPORTED_1000baseT_Full;
2274                 if (caps & FW_PORT_CAP_SPEED_10G)
2275                         v |= SUPPORTED_10000baseT_Full;
2276         } else if (type == FW_PORT_TYPE_BP40_BA ||
2277                    type == FW_PORT_TYPE_QSFP) {
2278                 v |= SUPPORTED_40000baseSR4_Full;
2279                 v |= SUPPORTED_FIBRE;
2280         }
2281
2282         if (caps & FW_PORT_CAP_ANEG)
2283                 v |= SUPPORTED_Autoneg;
2284         return v;
2285 }
2286
2287 static unsigned int to_fw_linkcaps(unsigned int caps)
2288 {
2289         unsigned int v = 0;
2290
2291         if (caps & ADVERTISED_100baseT_Full)
2292                 v |= FW_PORT_CAP_SPEED_100M;
2293         if (caps & ADVERTISED_1000baseT_Full)
2294                 v |= FW_PORT_CAP_SPEED_1G;
2295         if (caps & ADVERTISED_10000baseT_Full)
2296                 v |= FW_PORT_CAP_SPEED_10G;
2297         if (caps & ADVERTISED_40000baseSR4_Full)
2298                 v |= FW_PORT_CAP_SPEED_40G;
2299         return v;
2300 }
2301
2302 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2303 {
2304         const struct port_info *p = netdev_priv(dev);
2305
2306         if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2307             p->port_type == FW_PORT_TYPE_BT_XFI ||
2308             p->port_type == FW_PORT_TYPE_BT_XAUI)
2309                 cmd->port = PORT_TP;
2310         else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2311                  p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2312                 cmd->port = PORT_FIBRE;
2313         else if (p->port_type == FW_PORT_TYPE_SFP ||
2314                  p->port_type == FW_PORT_TYPE_QSFP_10G ||
2315                  p->port_type == FW_PORT_TYPE_QSA ||
2316                  p->port_type == FW_PORT_TYPE_QSFP) {
2317                 if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2318                     p->mod_type == FW_PORT_MOD_TYPE_SR ||
2319                     p->mod_type == FW_PORT_MOD_TYPE_ER ||
2320                     p->mod_type == FW_PORT_MOD_TYPE_LRM)
2321                         cmd->port = PORT_FIBRE;
2322                 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2323                          p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2324                         cmd->port = PORT_DA;
2325                 else
2326                         cmd->port = PORT_OTHER;
2327         } else
2328                 cmd->port = PORT_OTHER;
2329
2330         if (p->mdio_addr >= 0) {
2331                 cmd->phy_address = p->mdio_addr;
2332                 cmd->transceiver = XCVR_EXTERNAL;
2333                 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2334                         MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2335         } else {
2336                 cmd->phy_address = 0;  /* not really, but no better option */
2337                 cmd->transceiver = XCVR_INTERNAL;
2338                 cmd->mdio_support = 0;
2339         }
2340
2341         cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2342         cmd->advertising = from_fw_linkcaps(p->port_type,
2343                                             p->link_cfg.advertising);
2344         ethtool_cmd_speed_set(cmd,
2345                               netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2346         cmd->duplex = DUPLEX_FULL;
2347         cmd->autoneg = p->link_cfg.autoneg;
2348         cmd->maxtxpkt = 0;
2349         cmd->maxrxpkt = 0;
2350         return 0;
2351 }
2352
2353 static unsigned int speed_to_caps(int speed)
2354 {
2355         if (speed == 100)
2356                 return FW_PORT_CAP_SPEED_100M;
2357         if (speed == 1000)
2358                 return FW_PORT_CAP_SPEED_1G;
2359         if (speed == 10000)
2360                 return FW_PORT_CAP_SPEED_10G;
2361         if (speed == 40000)
2362                 return FW_PORT_CAP_SPEED_40G;
2363         return 0;
2364 }
2365
2366 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2367 {
2368         unsigned int cap;
2369         struct port_info *p = netdev_priv(dev);
2370         struct link_config *lc = &p->link_cfg;
2371         u32 speed = ethtool_cmd_speed(cmd);
2372
2373         if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
2374                 return -EINVAL;
2375
2376         if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2377                 /*
2378                  * PHY offers a single speed.  See if that's what's
2379                  * being requested.
2380                  */
2381                 if (cmd->autoneg == AUTONEG_DISABLE &&
2382                     (lc->supported & speed_to_caps(speed)))
2383                         return 0;
2384                 return -EINVAL;
2385         }
2386
2387         if (cmd->autoneg == AUTONEG_DISABLE) {
2388                 cap = speed_to_caps(speed);
2389
2390                 if (!(lc->supported & cap) ||
2391                     (speed == 1000) ||
2392                     (speed == 10000) ||
2393                     (speed == 40000))
2394                         return -EINVAL;
2395                 lc->requested_speed = cap;
2396                 lc->advertising = 0;
2397         } else {
2398                 cap = to_fw_linkcaps(cmd->advertising);
2399                 if (!(lc->supported & cap))
2400                         return -EINVAL;
2401                 lc->requested_speed = 0;
2402                 lc->advertising = cap | FW_PORT_CAP_ANEG;
2403         }
2404         lc->autoneg = cmd->autoneg;
2405
2406         if (netif_running(dev))
2407                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2408                                      lc);
2409         return 0;
2410 }
2411
2412 static void get_pauseparam(struct net_device *dev,
2413                            struct ethtool_pauseparam *epause)
2414 {
2415         struct port_info *p = netdev_priv(dev);
2416
2417         epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2418         epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2419         epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2420 }
2421
2422 static int set_pauseparam(struct net_device *dev,
2423                           struct ethtool_pauseparam *epause)
2424 {
2425         struct port_info *p = netdev_priv(dev);
2426         struct link_config *lc = &p->link_cfg;
2427
2428         if (epause->autoneg == AUTONEG_DISABLE)
2429                 lc->requested_fc = 0;
2430         else if (lc->supported & FW_PORT_CAP_ANEG)
2431                 lc->requested_fc = PAUSE_AUTONEG;
2432         else
2433                 return -EINVAL;
2434
2435         if (epause->rx_pause)
2436                 lc->requested_fc |= PAUSE_RX;
2437         if (epause->tx_pause)
2438                 lc->requested_fc |= PAUSE_TX;
2439         if (netif_running(dev))
2440                 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2441                                      lc);
2442         return 0;
2443 }
2444
2445 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2446 {
2447         const struct port_info *pi = netdev_priv(dev);
2448         const struct sge *s = &pi->adapter->sge;
2449
2450         e->rx_max_pending = MAX_RX_BUFFERS;
2451         e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2452         e->rx_jumbo_max_pending = 0;
2453         e->tx_max_pending = MAX_TXQ_ENTRIES;
2454
2455         e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2456         e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2457         e->rx_jumbo_pending = 0;
2458         e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2459 }
2460
2461 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2462 {
2463         int i;
2464         const struct port_info *pi = netdev_priv(dev);
2465         struct adapter *adapter = pi->adapter;
2466         struct sge *s = &adapter->sge;
2467
2468         if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2469             e->tx_pending > MAX_TXQ_ENTRIES ||
2470             e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2471             e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2472             e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2473                 return -EINVAL;
2474
2475         if (adapter->flags & FULL_INIT_DONE)
2476                 return -EBUSY;
2477
2478         for (i = 0; i < pi->nqsets; ++i) {
2479                 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2480                 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2481                 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2482         }
2483         return 0;
2484 }
2485
2486 static int closest_timer(const struct sge *s, int time)
2487 {
2488         int i, delta, match = 0, min_delta = INT_MAX;
2489
2490         for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2491                 delta = time - s->timer_val[i];
2492                 if (delta < 0)
2493                         delta = -delta;
2494                 if (delta < min_delta) {
2495                         min_delta = delta;
2496                         match = i;
2497                 }
2498         }
2499         return match;
2500 }
2501
2502 static int closest_thres(const struct sge *s, int thres)
2503 {
2504         int i, delta, match = 0, min_delta = INT_MAX;
2505
2506         for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2507                 delta = thres - s->counter_val[i];
2508                 if (delta < 0)
2509                         delta = -delta;
2510                 if (delta < min_delta) {
2511                         min_delta = delta;
2512                         match = i;
2513                 }
2514         }
2515         return match;
2516 }
2517
2518 /*
2519  * Return a queue's interrupt hold-off time in us.  0 means no timer.
2520  */
2521 unsigned int qtimer_val(const struct adapter *adap,
2522                         const struct sge_rspq *q)
2523 {
2524         unsigned int idx = q->intr_params >> 1;
2525
2526         return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2527 }
2528
2529 /**
2530  *      set_rspq_intr_params - set a queue's interrupt holdoff parameters
2531  *      @q: the Rx queue
2532  *      @us: the hold-off time in us, or 0 to disable timer
2533  *      @cnt: the hold-off packet count, or 0 to disable counter
2534  *
2535  *      Sets an Rx queue's interrupt hold-off time and packet count.  At least
2536  *      one of the two needs to be enabled for the queue to generate interrupts.
2537  */
2538 static int set_rspq_intr_params(struct sge_rspq *q,
2539                                 unsigned int us, unsigned int cnt)
2540 {
2541         struct adapter *adap = q->adap;
2542
2543         if ((us | cnt) == 0)
2544                 cnt = 1;
2545
2546         if (cnt) {
2547                 int err;
2548                 u32 v, new_idx;
2549
2550                 new_idx = closest_thres(&adap->sge, cnt);
2551                 if (q->desc && q->pktcnt_idx != new_idx) {
2552                         /* the queue has already been created, update it */
2553                         v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2554                             FW_PARAMS_PARAM_X_V(
2555                                         FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2556                             FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
2557                         err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2558                                             &new_idx);
2559                         if (err)
2560                                 return err;
2561                 }
2562                 q->pktcnt_idx = new_idx;
2563         }
2564
2565         us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2566         q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2567         return 0;
2568 }
2569
2570 /**
2571  * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2572  * @dev: the network device
2573  * @us: the hold-off time in us, or 0 to disable timer
2574  * @cnt: the hold-off packet count, or 0 to disable counter
2575  *
2576  * Set the RX interrupt hold-off parameters for a network device.
2577  */
2578 static int set_rx_intr_params(struct net_device *dev,
2579                               unsigned int us, unsigned int cnt)
2580 {
2581         int i, err;
2582         struct port_info *pi = netdev_priv(dev);
2583         struct adapter *adap = pi->adapter;
2584         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2585
2586         for (i = 0; i < pi->nqsets; i++, q++) {
2587                 err = set_rspq_intr_params(&q->rspq, us, cnt);
2588                 if (err)
2589                         return err;
2590         }
2591         return 0;
2592 }
2593
2594 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2595 {
2596         int i;
2597         struct port_info *pi = netdev_priv(dev);
2598         struct adapter *adap = pi->adapter;
2599         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2600
2601         for (i = 0; i < pi->nqsets; i++, q++)
2602                 q->rspq.adaptive_rx = adaptive_rx;
2603
2604         return 0;
2605 }
2606
2607 static int get_adaptive_rx_setting(struct net_device *dev)
2608 {
2609         struct port_info *pi = netdev_priv(dev);
2610         struct adapter *adap = pi->adapter;
2611         struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2612
2613         return q->rspq.adaptive_rx;
2614 }
2615
2616 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2617 {
2618         set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
2619         return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2620                                   c->rx_max_coalesced_frames);
2621 }
2622
2623 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2624 {
2625         const struct port_info *pi = netdev_priv(dev);
2626         const struct adapter *adap = pi->adapter;
2627         const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2628
2629         c->rx_coalesce_usecs = qtimer_val(adap, rq);
2630         c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2631                 adap->sge.counter_val[rq->pktcnt_idx] : 0;
2632         c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
2633         return 0;
2634 }
2635
2636 /**
2637  *      eeprom_ptov - translate a physical EEPROM address to virtual
2638  *      @phys_addr: the physical EEPROM address
2639  *      @fn: the PCI function number
2640  *      @sz: size of function-specific area
2641  *
2642  *      Translate a physical EEPROM address to virtual.  The first 1K is
2643  *      accessed through virtual addresses starting at 31K, the rest is
2644  *      accessed through virtual addresses starting at 0.
2645  *
2646  *      The mapping is as follows:
2647  *      [0..1K) -> [31K..32K)
2648  *      [1K..1K+A) -> [31K-A..31K)
2649  *      [1K+A..ES) -> [0..ES-A-1K)
2650  *
2651  *      where A = @fn * @sz, and ES = EEPROM size.
2652  */
2653 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2654 {
2655         fn *= sz;
2656         if (phys_addr < 1024)
2657                 return phys_addr + (31 << 10);
2658         if (phys_addr < 1024 + fn)
2659                 return 31744 - fn + phys_addr - 1024;
2660         if (phys_addr < EEPROMSIZE)
2661                 return phys_addr - 1024 - fn;
2662         return -EINVAL;
2663 }
2664
2665 /*
2666  * The next two routines implement eeprom read/write from physical addresses.
2667  */
2668 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2669 {
2670         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2671
2672         if (vaddr >= 0)
2673                 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2674         return vaddr < 0 ? vaddr : 0;
2675 }
2676
2677 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2678 {
2679         int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2680
2681         if (vaddr >= 0)
2682                 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2683         return vaddr < 0 ? vaddr : 0;
2684 }
2685
2686 #define EEPROM_MAGIC 0x38E2F10C
2687
2688 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2689                       u8 *data)
2690 {
2691         int i, err = 0;
2692         struct adapter *adapter = netdev2adap(dev);
2693
2694         u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2695         if (!buf)
2696                 return -ENOMEM;
2697
2698         e->magic = EEPROM_MAGIC;
2699         for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2700                 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2701
2702         if (!err)
2703                 memcpy(data, buf + e->offset, e->len);
2704         kfree(buf);
2705         return err;
2706 }
2707
2708 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2709                       u8 *data)
2710 {
2711         u8 *buf;
2712         int err = 0;
2713         u32 aligned_offset, aligned_len, *p;
2714         struct adapter *adapter = netdev2adap(dev);
2715
2716         if (eeprom->magic != EEPROM_MAGIC)
2717                 return -EINVAL;
2718
2719         aligned_offset = eeprom->offset & ~3;
2720         aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2721
2722         if (adapter->fn > 0) {
2723                 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2724
2725                 if (aligned_offset < start ||
2726                     aligned_offset + aligned_len > start + EEPROMPFSIZE)
2727                         return -EPERM;
2728         }
2729
2730         if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2731                 /*
2732                  * RMW possibly needed for first or last words.
2733                  */
2734                 buf = kmalloc(aligned_len, GFP_KERNEL);
2735                 if (!buf)
2736                         return -ENOMEM;
2737                 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2738                 if (!err && aligned_len > 4)
2739                         err = eeprom_rd_phys(adapter,
2740                                              aligned_offset + aligned_len - 4,
2741                                              (u32 *)&buf[aligned_len - 4]);
2742                 if (err)
2743                         goto out;
2744                 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2745         } else
2746                 buf = data;
2747
2748         err = t4_seeprom_wp(adapter, false);
2749         if (err)
2750                 goto out;
2751
2752         for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2753                 err = eeprom_wr_phys(adapter, aligned_offset, *p);
2754                 aligned_offset += 4;
2755         }
2756
2757         if (!err)
2758                 err = t4_seeprom_wp(adapter, true);
2759 out:
2760         if (buf != data)
2761                 kfree(buf);
2762         return err;
2763 }
2764
2765 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2766 {
2767         int ret;
2768         const struct firmware *fw;
2769         struct adapter *adap = netdev2adap(netdev);
2770         unsigned int mbox = PCIE_FW_MASTER_M + 1;
2771
2772         ef->data[sizeof(ef->data) - 1] = '\0';
2773         ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2774         if (ret < 0)
2775                 return ret;
2776
2777         /* If the adapter has been fully initialized then we'll go ahead and
2778          * try to get the firmware's cooperation in upgrading to the new
2779          * firmware image otherwise we'll try to do the entire job from the
2780          * host ... and we always "force" the operation in this path.
2781          */
2782         if (adap->flags & FULL_INIT_DONE)
2783                 mbox = adap->mbox;
2784
2785         ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
2786         release_firmware(fw);
2787         if (!ret)
2788                 dev_info(adap->pdev_dev, "loaded firmware %s,"
2789                          " reload cxgb4 driver\n", ef->data);
2790         return ret;
2791 }
2792
2793 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2794 #define BCAST_CRC 0xa0ccc1a6
2795
2796 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2797 {
2798         wol->supported = WAKE_BCAST | WAKE_MAGIC;
2799         wol->wolopts = netdev2adap(dev)->wol;
2800         memset(&wol->sopass, 0, sizeof(wol->sopass));
2801 }
2802
2803 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2804 {
2805         int err = 0;
2806         struct port_info *pi = netdev_priv(dev);
2807
2808         if (wol->wolopts & ~WOL_SUPPORTED)
2809                 return -EINVAL;
2810         t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2811                             (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2812         if (wol->wolopts & WAKE_BCAST) {
2813                 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2814                                         ~0ULL, 0, false);
2815                 if (!err)
2816                         err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2817                                                 ~6ULL, ~0ULL, BCAST_CRC, true);
2818         } else
2819                 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2820         return err;
2821 }
2822
2823 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2824 {
2825         const struct port_info *pi = netdev_priv(dev);
2826         netdev_features_t changed = dev->features ^ features;
2827         int err;
2828
2829         if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2830                 return 0;
2831
2832         err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2833                             -1, -1, -1,
2834                             !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2835         if (unlikely(err))
2836                 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2837         return err;
2838 }
2839
2840 static u32 get_rss_table_size(struct net_device *dev)
2841 {
2842         const struct port_info *pi = netdev_priv(dev);
2843
2844         return pi->rss_size;
2845 }
2846
2847 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
2848 {
2849         const struct port_info *pi = netdev_priv(dev);
2850         unsigned int n = pi->rss_size;
2851
2852         if (hfunc)
2853                 *hfunc = ETH_RSS_HASH_TOP;
2854         if (!p)
2855                 return 0;
2856         while (n--)
2857                 p[n] = pi->rss[n];
2858         return 0;
2859 }
2860
2861 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2862                          const u8 hfunc)
2863 {
2864         unsigned int i;
2865         struct port_info *pi = netdev_priv(dev);
2866
2867         /* We require at least one supported parameter to be changed and no
2868          * change in any of the unsupported parameters
2869          */
2870         if (key ||
2871             (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2872                 return -EOPNOTSUPP;
2873         if (!p)
2874                 return 0;
2875
2876         for (i = 0; i < pi->rss_size; i++)
2877                 pi->rss[i] = p[i];
2878         if (pi->adapter->flags & FULL_INIT_DONE)
2879                 return write_rss(pi, pi->rss);
2880         return 0;
2881 }
2882
2883 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2884                      u32 *rules)
2885 {
2886         const struct port_info *pi = netdev_priv(dev);
2887
2888         switch (info->cmd) {
2889         case ETHTOOL_GRXFH: {
2890                 unsigned int v = pi->rss_mode;
2891
2892                 info->data = 0;
2893                 switch (info->flow_type) {
2894                 case TCP_V4_FLOW:
2895                         if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
2896                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2897                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2898                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2899                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2900                         break;
2901                 case UDP_V4_FLOW:
2902                         if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2903                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2904                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2905                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2906                         else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2907                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2908                         break;
2909                 case SCTP_V4_FLOW:
2910                 case AH_ESP_V4_FLOW:
2911                 case IPV4_FLOW:
2912                         if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2913                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2914                         break;
2915                 case TCP_V6_FLOW:
2916                         if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
2917                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2918                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2919                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2920                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2921                         break;
2922                 case UDP_V6_FLOW:
2923                         if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
2924                             (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2925                                 info->data = RXH_IP_SRC | RXH_IP_DST |
2926                                              RXH_L4_B_0_1 | RXH_L4_B_2_3;
2927                         else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2928                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2929                         break;
2930                 case SCTP_V6_FLOW:
2931                 case AH_ESP_V6_FLOW:
2932                 case IPV6_FLOW:
2933                         if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
2934                                 info->data = RXH_IP_SRC | RXH_IP_DST;
2935                         break;
2936                 }
2937                 return 0;
2938         }
2939         case ETHTOOL_GRXRINGS:
2940                 info->data = pi->nqsets;
2941                 return 0;
2942         }
2943         return -EOPNOTSUPP;
2944 }
2945
2946 static const struct ethtool_ops cxgb_ethtool_ops = {
2947         .get_settings      = get_settings,
2948         .set_settings      = set_settings,
2949         .get_drvinfo       = get_drvinfo,
2950         .get_msglevel      = get_msglevel,
2951         .set_msglevel      = set_msglevel,
2952         .get_ringparam     = get_sge_param,
2953         .set_ringparam     = set_sge_param,
2954         .get_coalesce      = get_coalesce,
2955         .set_coalesce      = set_coalesce,
2956         .get_eeprom_len    = get_eeprom_len,
2957         .get_eeprom        = get_eeprom,
2958         .set_eeprom        = set_eeprom,
2959         .get_pauseparam    = get_pauseparam,
2960         .set_pauseparam    = set_pauseparam,
2961         .get_link          = ethtool_op_get_link,
2962         .get_strings       = get_strings,
2963         .set_phys_id       = identify_port,
2964         .nway_reset        = restart_autoneg,
2965         .get_sset_count    = get_sset_count,
2966         .get_ethtool_stats = get_stats,
2967         .get_regs_len      = get_regs_len,
2968         .get_regs          = get_regs,
2969         .get_wol           = get_wol,
2970         .set_wol           = set_wol,
2971         .get_rxnfc         = get_rxnfc,
2972         .get_rxfh_indir_size = get_rss_table_size,
2973         .get_rxfh          = get_rss_table,
2974         .set_rxfh          = set_rss_table,
2975         .flash_device      = set_flash,
2976 };
2977
2978 static int setup_debugfs(struct adapter *adap)
2979 {
2980         if (IS_ERR_OR_NULL(adap->debugfs_root))
2981                 return -1;
2982
2983 #ifdef CONFIG_DEBUG_FS
2984         t4_setup_debugfs(adap);
2985 #endif
2986         return 0;
2987 }
2988
2989 /*
2990  * upper-layer driver support
2991  */
2992
2993 /*
2994  * Allocate an active-open TID and set it to the supplied value.
2995  */
2996 int cxgb4_alloc_atid(struct tid_info *t, void *data)
2997 {
2998         int atid = -1;
2999
3000         spin_lock_bh(&t->atid_lock);
3001         if (t->afree) {
3002                 union aopen_entry *p = t->afree;
3003
3004                 atid = (p - t->atid_tab) + t->atid_base;
3005                 t->afree = p->next;
3006                 p->data = data;
3007                 t->atids_in_use++;
3008         }
3009         spin_unlock_bh(&t->atid_lock);
3010         return atid;
3011 }
3012 EXPORT_SYMBOL(cxgb4_alloc_atid);
3013
3014 /*
3015  * Release an active-open TID.
3016  */
3017 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3018 {
3019         union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
3020
3021         spin_lock_bh(&t->atid_lock);
3022         p->next = t->afree;
3023         t->afree = p;
3024         t->atids_in_use--;
3025         spin_unlock_bh(&t->atid_lock);
3026 }
3027 EXPORT_SYMBOL(cxgb4_free_atid);
3028
3029 /*
3030  * Allocate a server TID and set it to the supplied value.
3031  */
3032 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3033 {
3034         int stid;
3035
3036         spin_lock_bh(&t->stid_lock);
3037         if (family == PF_INET) {
3038                 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3039                 if (stid < t->nstids)
3040                         __set_bit(stid, t->stid_bmap);
3041                 else
3042                         stid = -1;
3043         } else {
3044                 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3045                 if (stid < 0)
3046                         stid = -1;
3047         }
3048         if (stid >= 0) {
3049                 t->stid_tab[stid].data = data;
3050                 stid += t->stid_base;
3051                 /* IPv6 requires max of 520 bits or 16 cells in TCAM
3052                  * This is equivalent to 4 TIDs. With CLIP enabled it
3053                  * needs 2 TIDs.
3054                  */
3055                 if (family == PF_INET)
3056                         t->stids_in_use++;
3057                 else
3058                         t->stids_in_use += 4;
3059         }
3060         spin_unlock_bh(&t->stid_lock);
3061         return stid;
3062 }
3063 EXPORT_SYMBOL(cxgb4_alloc_stid);
3064
3065 /* Allocate a server filter TID and set it to the supplied value.
3066  */
3067 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3068 {
3069         int stid;
3070
3071         spin_lock_bh(&t->stid_lock);
3072         if (family == PF_INET) {
3073                 stid = find_next_zero_bit(t->stid_bmap,
3074                                 t->nstids + t->nsftids, t->nstids);
3075                 if (stid < (t->nstids + t->nsftids))
3076                         __set_bit(stid, t->stid_bmap);
3077                 else
3078                         stid = -1;
3079         } else {
3080                 stid = -1;
3081         }
3082         if (stid >= 0) {
3083                 t->stid_tab[stid].data = data;
3084                 stid -= t->nstids;
3085                 stid += t->sftid_base;
3086                 t->stids_in_use++;
3087         }
3088         spin_unlock_bh(&t->stid_lock);
3089         return stid;
3090 }
3091 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3092
3093 /* Release a server TID.
3094  */
3095 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3096 {
3097         /* Is it a server filter TID? */
3098         if (t->nsftids && (stid >= t->sftid_base)) {
3099                 stid -= t->sftid_base;
3100                 stid += t->nstids;
3101         } else {
3102                 stid -= t->stid_base;
3103         }
3104
3105         spin_lock_bh(&t->stid_lock);
3106         if (family == PF_INET)
3107                 __clear_bit(stid, t->stid_bmap);
3108         else
3109                 bitmap_release_region(t->stid_bmap, stid, 2);
3110         t->stid_tab[stid].data = NULL;
3111         if (family == PF_INET)
3112                 t->stids_in_use--;
3113         else
3114                 t->stids_in_use -= 4;
3115         spin_unlock_bh(&t->stid_lock);
3116 }
3117 EXPORT_SYMBOL(cxgb4_free_stid);
3118
3119 /*
3120  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
3121  */
3122 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3123                            unsigned int tid)
3124 {
3125         struct cpl_tid_release *req;
3126
3127         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3128         req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3129         INIT_TP_WR(req, tid);
3130         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3131 }
3132
3133 /*
3134  * Queue a TID release request and if necessary schedule a work queue to
3135  * process it.
3136  */
3137 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3138                                     unsigned int tid)
3139 {
3140         void **p = &t->tid_tab[tid];
3141         struct adapter *adap = container_of(t, struct adapter, tids);
3142
3143         spin_lock_bh(&adap->tid_release_lock);
3144         *p = adap->tid_release_head;
3145         /* Low 2 bits encode the Tx channel number */
3146         adap->tid_release_head = (void **)((uintptr_t)p | chan);
3147         if (!adap->tid_release_task_busy) {
3148                 adap->tid_release_task_busy = true;
3149                 queue_work(adap->workq, &adap->tid_release_task);
3150         }
3151         spin_unlock_bh(&adap->tid_release_lock);
3152 }
3153
3154 /*
3155  * Process the list of pending TID release requests.
3156  */
3157 static void process_tid_release_list(struct work_struct *work)
3158 {
3159         struct sk_buff *skb;
3160         struct adapter *adap;
3161
3162         adap = container_of(work, struct adapter, tid_release_task);
3163
3164         spin_lock_bh(&adap->tid_release_lock);
3165         while (adap->tid_release_head) {
3166                 void **p = adap->tid_release_head;
3167                 unsigned int chan = (uintptr_t)p & 3;
3168                 p = (void *)p - chan;
3169
3170                 adap->tid_release_head = *p;
3171                 *p = NULL;
3172                 spin_unlock_bh(&adap->tid_release_lock);
3173
3174                 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3175                                          GFP_KERNEL)))
3176                         schedule_timeout_uninterruptible(1);
3177
3178                 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3179                 t4_ofld_send(adap, skb);
3180                 spin_lock_bh(&adap->tid_release_lock);
3181         }
3182         adap->tid_release_task_busy = false;
3183         spin_unlock_bh(&adap->tid_release_lock);
3184 }
3185
3186 /*
3187  * Release a TID and inform HW.  If we are unable to allocate the release
3188  * message we defer to a work queue.
3189  */
3190 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3191 {
3192         void *old;
3193         struct sk_buff *skb;
3194         struct adapter *adap = container_of(t, struct adapter, tids);
3195
3196         old = t->tid_tab[tid];
3197         skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3198         if (likely(skb)) {
3199                 t->tid_tab[tid] = NULL;
3200                 mk_tid_release(skb, chan, tid);
3201                 t4_ofld_send(adap, skb);
3202         } else
3203                 cxgb4_queue_tid_release(t, chan, tid);
3204         if (old)
3205                 atomic_dec(&t->tids_in_use);
3206 }
3207 EXPORT_SYMBOL(cxgb4_remove_tid);
3208
3209 /*
3210  * Allocate and initialize the TID tables.  Returns 0 on success.
3211  */
3212 static int tid_init(struct tid_info *t)
3213 {
3214         size_t size;
3215         unsigned int stid_bmap_size;
3216         unsigned int natids = t->natids;
3217         struct adapter *adap = container_of(t, struct adapter, tids);
3218
3219         stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3220         size = t->ntids * sizeof(*t->tid_tab) +
3221                natids * sizeof(*t->atid_tab) +
3222                t->nstids * sizeof(*t->stid_tab) +
3223                t->nsftids * sizeof(*t->stid_tab) +
3224                stid_bmap_size * sizeof(long) +
3225                t->nftids * sizeof(*t->ftid_tab) +
3226                t->nsftids * sizeof(*t->ftid_tab);
3227
3228         t->tid_tab = t4_alloc_mem(size);
3229         if (!t->tid_tab)
3230                 return -ENOMEM;
3231
3232         t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3233         t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3234         t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3235         t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3236         spin_lock_init(&t->stid_lock);
3237         spin_lock_init(&t->atid_lock);
3238
3239         t->stids_in_use = 0;
3240         t->afree = NULL;
3241         t->atids_in_use = 0;
3242         atomic_set(&t->tids_in_use, 0);
3243
3244         /* Setup the free list for atid_tab and clear the stid bitmap. */
3245         if (natids) {
3246                 while (--natids)
3247                         t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3248                 t->afree = t->atid_tab;
3249         }
3250         bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3251         /* Reserve stid 0 for T4/T5 adapters */
3252         if (!t->stid_base &&
3253             (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3254                 __set_bit(0, t->stid_bmap);
3255
3256         return 0;
3257 }
3258
3259 /**
3260  *      cxgb4_create_server - create an IP server
3261  *      @dev: the device
3262  *      @stid: the server TID
3263  *      @sip: local IP address to bind server to
3264  *      @sport: the server's TCP port
3265  *      @queue: queue to direct messages from this server to
3266  *
3267  *      Create an IP server for the given port and address.
3268  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3269  */
3270 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3271                         __be32 sip, __be16 sport, __be16 vlan,
3272                         unsigned int queue)
3273 {
3274         unsigned int chan;
3275         struct sk_buff *skb;
3276         struct adapter *adap;
3277         struct cpl_pass_open_req *req;
3278         int ret;
3279
3280         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3281         if (!skb)
3282                 return -ENOMEM;
3283
3284         adap = netdev2adap(dev);
3285         req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3286         INIT_TP_WR(req, 0);
3287         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3288         req->local_port = sport;
3289         req->peer_port = htons(0);
3290         req->local_ip = sip;
3291         req->peer_ip = htonl(0);
3292         chan = rxq_to_chan(&adap->sge, queue);
3293         req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3294         req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3295                                 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3296         ret = t4_mgmt_tx(adap, skb);
3297         return net_xmit_eval(ret);
3298 }
3299 EXPORT_SYMBOL(cxgb4_create_server);
3300
3301 /*      cxgb4_create_server6 - create an IPv6 server
3302  *      @dev: the device
3303  *      @stid: the server TID
3304  *      @sip: local IPv6 address to bind server to
3305  *      @sport: the server's TCP port
3306  *      @queue: queue to direct messages from this server to
3307  *
3308  *      Create an IPv6 server for the given port and address.
3309  *      Returns <0 on error and one of the %NET_XMIT_* values on success.
3310  */
3311 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3312                          const struct in6_addr *sip, __be16 sport,
3313                          unsigned int queue)
3314 {
3315         unsigned int chan;
3316         struct sk_buff *skb;
3317         struct adapter *adap;
3318         struct cpl_pass_open_req6 *req;
3319         int ret;
3320
3321         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3322         if (!skb)
3323                 return -ENOMEM;
3324
3325         adap = netdev2adap(dev);
3326         req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3327         INIT_TP_WR(req, 0);
3328         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3329         req->local_port = sport;
3330         req->peer_port = htons(0);
3331         req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3332         req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3333         req->peer_ip_hi = cpu_to_be64(0);
3334         req->peer_ip_lo = cpu_to_be64(0);
3335         chan = rxq_to_chan(&adap->sge, queue);
3336         req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3337         req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) |
3338                                 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue));
3339         ret = t4_mgmt_tx(adap, skb);
3340         return net_xmit_eval(ret);
3341 }
3342 EXPORT_SYMBOL(cxgb4_create_server6);
3343
3344 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3345                         unsigned int queue, bool ipv6)
3346 {
3347         struct sk_buff *skb;
3348         struct adapter *adap;
3349         struct cpl_close_listsvr_req *req;
3350         int ret;
3351
3352         adap = netdev2adap(dev);
3353
3354         skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3355         if (!skb)
3356                 return -ENOMEM;
3357
3358         req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3359         INIT_TP_WR(req, 0);
3360         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3361         req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) :
3362                                 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue));
3363         ret = t4_mgmt_tx(adap, skb);
3364         return net_xmit_eval(ret);
3365 }
3366 EXPORT_SYMBOL(cxgb4_remove_server);
3367
3368 /**
3369  *      cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3370  *      @mtus: the HW MTU table
3371  *      @mtu: the target MTU
3372  *      @idx: index of selected entry in the MTU table
3373  *
3374  *      Returns the index and the value in the HW MTU table that is closest to
3375  *      but does not exceed @mtu, unless @mtu is smaller than any value in the
3376  *      table, in which case that smallest available value is selected.
3377  */
3378 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3379                             unsigned int *idx)
3380 {
3381         unsigned int i = 0;
3382
3383         while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3384                 ++i;
3385         if (idx)
3386                 *idx = i;
3387         return mtus[i];
3388 }
3389 EXPORT_SYMBOL(cxgb4_best_mtu);
3390
3391 /**
3392  *     cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3393  *     @mtus: the HW MTU table
3394  *     @header_size: Header Size
3395  *     @data_size_max: maximum Data Segment Size
3396  *     @data_size_align: desired Data Segment Size Alignment (2^N)
3397  *     @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3398  *
3399  *     Similar to cxgb4_best_mtu() but instead of searching the Hardware
3400  *     MTU Table based solely on a Maximum MTU parameter, we break that
3401  *     parameter up into a Header Size and Maximum Data Segment Size, and
3402  *     provide a desired Data Segment Size Alignment.  If we find an MTU in
3403  *     the Hardware MTU Table which will result in a Data Segment Size with
3404  *     the requested alignment _and_ that MTU isn't "too far" from the
3405  *     closest MTU, then we'll return that rather than the closest MTU.
3406  */
3407 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3408                                     unsigned short header_size,
3409                                     unsigned short data_size_max,
3410                                     unsigned short data_size_align,
3411                                     unsigned int *mtu_idxp)
3412 {
3413         unsigned short max_mtu = header_size + data_size_max;
3414         unsigned short data_size_align_mask = data_size_align - 1;
3415         int mtu_idx, aligned_mtu_idx;
3416
3417         /* Scan the MTU Table till we find an MTU which is larger than our
3418          * Maximum MTU or we reach the end of the table.  Along the way,
3419          * record the last MTU found, if any, which will result in a Data
3420          * Segment Length matching the requested alignment.
3421          */
3422         for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3423                 unsigned short data_size = mtus[mtu_idx] - header_size;
3424
3425                 /* If this MTU minus the Header Size would result in a
3426                  * Data Segment Size of the desired alignment, remember it.
3427                  */
3428                 if ((data_size & data_size_align_mask) == 0)
3429                         aligned_mtu_idx = mtu_idx;
3430
3431                 /* If we're not at the end of the Hardware MTU Table and the
3432                  * next element is larger than our Maximum MTU, drop out of
3433                  * the loop.
3434                  */
3435                 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3436                         break;
3437         }
3438
3439         /* If we fell out of the loop because we ran to the end of the table,
3440          * then we just have to use the last [largest] entry.
3441          */
3442         if (mtu_idx == NMTUS)
3443                 mtu_idx--;
3444
3445         /* If we found an MTU which resulted in the requested Data Segment
3446          * Length alignment and that's "not far" from the largest MTU which is
3447          * less than or equal to the maximum MTU, then use that.
3448          */
3449         if (aligned_mtu_idx >= 0 &&
3450             mtu_idx - aligned_mtu_idx <= 1)
3451                 mtu_idx = aligned_mtu_idx;
3452
3453         /* If the caller has passed in an MTU Index pointer, pass the
3454          * MTU Index back.  Return the MTU value.
3455          */
3456         if (mtu_idxp)
3457                 *mtu_idxp = mtu_idx;
3458         return mtus[mtu_idx];
3459 }
3460 EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3461
3462 /**
3463  *      cxgb4_port_chan - get the HW channel of a port
3464  *      @dev: the net device for the port
3465  *
3466  *      Return the HW Tx channel of the given port.
3467  */
3468 unsigned int cxgb4_port_chan(const struct net_device *dev)
3469 {
3470         return netdev2pinfo(dev)->tx_chan;
3471 }
3472 EXPORT_SYMBOL(cxgb4_port_chan);
3473
3474 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3475 {
3476         struct adapter *adap = netdev2adap(dev);
3477         u32 v1, v2, lp_count, hp_count;
3478
3479         v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3480         v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3481         if (is_t4(adap->params.chip)) {
3482                 lp_count = LP_COUNT_G(v1);
3483                 hp_count = HP_COUNT_G(v1);
3484         } else {
3485                 lp_count = LP_COUNT_T5_G(v1);
3486                 hp_count = HP_COUNT_T5_G(v2);
3487         }
3488         return lpfifo ? lp_count : hp_count;
3489 }
3490 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3491
3492 /**
3493  *      cxgb4_port_viid - get the VI id of a port
3494  *      @dev: the net device for the port
3495  *
3496  *      Return the VI id of the given port.
3497  */
3498 unsigned int cxgb4_port_viid(const struct net_device *dev)
3499 {
3500         return netdev2pinfo(dev)->viid;
3501 }
3502 EXPORT_SYMBOL(cxgb4_port_viid);
3503
3504 /**
3505  *      cxgb4_port_idx - get the index of a port
3506  *      @dev: the net device for the port
3507  *
3508  *      Return the index of the given port.
3509  */
3510 unsigned int cxgb4_port_idx(const struct net_device *dev)
3511 {
3512         return netdev2pinfo(dev)->port_id;
3513 }
3514 EXPORT_SYMBOL(cxgb4_port_idx);
3515
3516 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3517                          struct tp_tcp_stats *v6)
3518 {
3519         struct adapter *adap = pci_get_drvdata(pdev);
3520
3521         spin_lock(&adap->stats_lock);
3522         t4_tp_get_tcp_stats(adap, v4, v6);
3523         spin_unlock(&adap->stats_lock);
3524 }
3525 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3526
3527 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3528                       const unsigned int *pgsz_order)
3529 {
3530         struct adapter *adap = netdev2adap(dev);
3531
3532         t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask);
3533         t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) |
3534                      HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) |
3535                      HPZ3_V(pgsz_order[3]));
3536 }
3537 EXPORT_SYMBOL(cxgb4_iscsi_init);
3538
3539 int cxgb4_flush_eq_cache(struct net_device *dev)
3540 {
3541         struct adapter *adap = netdev2adap(dev);
3542         int ret;
3543
3544         ret = t4_fwaddrspace_write(adap, adap->mbox,
3545                                    0xe1000000 + SGE_CTXT_CMD_A, 0x20000000);
3546         return ret;
3547 }
3548 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3549
3550 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3551 {
3552         u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8;
3553         __be64 indices;
3554         int ret;
3555
3556         spin_lock(&adap->win0_lock);
3557         ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3558                            sizeof(indices), (__be32 *)&indices,
3559                            T4_MEMORY_READ);
3560         spin_unlock(&adap->win0_lock);
3561         if (!ret) {
3562                 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3563                 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3564         }
3565         return ret;
3566 }
3567
3568 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3569                         u16 size)
3570 {
3571         struct adapter *adap = netdev2adap(dev);
3572         u16 hw_pidx, hw_cidx;
3573         int ret;
3574
3575         ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3576         if (ret)
3577                 goto out;
3578
3579         if (pidx != hw_pidx) {
3580                 u16 delta;
3581                 u32 val;
3582
3583                 if (pidx >= hw_pidx)
3584                         delta = pidx - hw_pidx;
3585                 else
3586                         delta = size - hw_pidx + pidx;
3587
3588                 if (is_t4(adap->params.chip))
3589                         val = PIDX_V(delta);
3590                 else
3591                         val = PIDX_T5_V(delta);
3592                 wmb();
3593                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3594                              QID_V(qid) | val);
3595         }
3596 out:
3597         return ret;
3598 }
3599 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3600
3601 void cxgb4_disable_db_coalescing(struct net_device *dev)
3602 {
3603         struct adapter *adap;
3604
3605         adap = netdev2adap(dev);
3606         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F,
3607                          NOCOALESCE_F);
3608 }
3609 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3610
3611 void cxgb4_enable_db_coalescing(struct net_device *dev)
3612 {
3613         struct adapter *adap;
3614
3615         adap = netdev2adap(dev);
3616         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0);
3617 }
3618 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3619
3620 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3621 {
3622         struct adapter *adap;
3623         u32 offset, memtype, memaddr;
3624         u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
3625         u32 edc0_end, edc1_end, mc0_end, mc1_end;
3626         int ret;
3627
3628         adap = netdev2adap(dev);
3629
3630         offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3631
3632         /* Figure out where the offset lands in the Memory Type/Address scheme.
3633          * This code assumes that the memory is laid out starting at offset 0
3634          * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3635          * and EDC1.  Some cards will have neither MC0 nor MC1, most cards have
3636          * MC0, and some have both MC0 and MC1.
3637          */
3638         size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3639         edc0_size = EDRAM0_SIZE_G(size) << 20;
3640         size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3641         edc1_size = EDRAM1_SIZE_G(size) << 20;
3642         size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3643         mc0_size = EXT_MEM0_SIZE_G(size) << 20;
3644
3645         edc0_end = edc0_size;
3646         edc1_end = edc0_end + edc1_size;
3647         mc0_end = edc1_end + mc0_size;
3648
3649         if (offset < edc0_end) {
3650                 memtype = MEM_EDC0;
3651                 memaddr = offset;
3652         } else if (offset < edc1_end) {
3653                 memtype = MEM_EDC1;
3654                 memaddr = offset - edc0_end;
3655         } else {
3656                 if (offset < mc0_end) {
3657                         memtype = MEM_MC0;
3658                         memaddr = offset - edc1_end;
3659                 } else if (is_t4(adap->params.chip)) {
3660                         /* T4 only has a single memory channel */
3661                         goto err;
3662                 } else {
3663                         size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3664                         mc1_size = EXT_MEM1_SIZE_G(size) << 20;
3665                         mc1_end = mc0_end + mc1_size;
3666                         if (offset < mc1_end) {
3667                                 memtype = MEM_MC1;
3668                                 memaddr = offset - mc0_end;
3669                         } else {
3670                                 /* offset beyond the end of any memory */
3671                                 goto err;
3672                         }
3673                 }
3674         }
3675
3676         spin_lock(&adap->win0_lock);
3677         ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3678         spin_unlock(&adap->win0_lock);
3679         return ret;
3680
3681 err:
3682         dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3683                 stag, offset);
3684         return -EINVAL;
3685 }
3686 EXPORT_SYMBOL(cxgb4_read_tpte);
3687
3688 u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3689 {
3690         u32 hi, lo;
3691         struct adapter *adap;
3692
3693         adap = netdev2adap(dev);
3694         lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A);
3695         hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A));
3696
3697         return ((u64)hi << 32) | (u64)lo;
3698 }
3699 EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3700
3701 int cxgb4_bar2_sge_qregs(struct net_device *dev,
3702                          unsigned int qid,
3703                          enum cxgb4_bar2_qtype qtype,
3704                          u64 *pbar2_qoffset,
3705                          unsigned int *pbar2_qid)
3706 {
3707         return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
3708                                  qid,
3709                                  (qtype == CXGB4_BAR2_QTYPE_EGRESS
3710                                   ? T4_BAR2_QTYPE_EGRESS
3711                                   : T4_BAR2_QTYPE_INGRESS),
3712                                  pbar2_qoffset,
3713                                  pbar2_qid);
3714 }
3715 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3716
3717 static struct pci_driver cxgb4_driver;
3718
3719 static void check_neigh_update(struct neighbour *neigh)
3720 {
3721         const struct device *parent;
3722         const struct net_device *netdev = neigh->dev;
3723
3724         if (netdev->priv_flags & IFF_802_1Q_VLAN)
3725                 netdev = vlan_dev_real_dev(netdev);
3726         parent = netdev->dev.parent;
3727         if (parent && parent->driver == &cxgb4_driver.driver)
3728                 t4_l2t_update(dev_get_drvdata(parent), neigh);
3729 }
3730
3731 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3732                        void *data)
3733 {
3734         switch (event) {
3735         case NETEVENT_NEIGH_UPDATE:
3736                 check_neigh_update(data);
3737                 break;
3738         case NETEVENT_REDIRECT:
3739         default:
3740                 break;
3741         }
3742         return 0;
3743 }
3744
3745 static bool netevent_registered;
3746 static struct notifier_block cxgb4_netevent_nb = {
3747         .notifier_call = netevent_cb
3748 };
3749
3750 static void drain_db_fifo(struct adapter *adap, int usecs)
3751 {
3752         u32 v1, v2, lp_count, hp_count;
3753
3754         do {
3755                 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A);
3756                 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A);
3757                 if (is_t4(adap->params.chip)) {
3758                         lp_count = LP_COUNT_G(v1);
3759                         hp_count = HP_COUNT_G(v1);
3760                 } else {
3761                         lp_count = LP_COUNT_T5_G(v1);
3762                         hp_count = HP_COUNT_T5_G(v2);
3763                 }
3764
3765                 if (lp_count == 0 && hp_count == 0)
3766                         break;
3767                 set_current_state(TASK_UNINTERRUPTIBLE);
3768                 schedule_timeout(usecs_to_jiffies(usecs));
3769         } while (1);
3770 }
3771
3772 static void disable_txq_db(struct sge_txq *q)
3773 {
3774         unsigned long flags;
3775
3776         spin_lock_irqsave(&q->db_lock, flags);
3777         q->db_disabled = 1;
3778         spin_unlock_irqrestore(&q->db_lock, flags);
3779 }
3780
3781 static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3782 {
3783         spin_lock_irq(&q->db_lock);
3784         if (q->db_pidx_inc) {
3785                 /* Make sure that all writes to the TX descriptors
3786                  * are committed before we tell HW about them.
3787                  */
3788                 wmb();
3789                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3790                              QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc));
3791                 q->db_pidx_inc = 0;
3792         }
3793         q->db_disabled = 0;
3794         spin_unlock_irq(&q->db_lock);
3795 }
3796
3797 static void disable_dbs(struct adapter *adap)
3798 {
3799         int i;
3800
3801         for_each_ethrxq(&adap->sge, i)
3802                 disable_txq_db(&adap->sge.ethtxq[i].q);
3803         for_each_ofldrxq(&adap->sge, i)
3804                 disable_txq_db(&adap->sge.ofldtxq[i].q);
3805         for_each_port(adap, i)
3806                 disable_txq_db(&adap->sge.ctrlq[i].q);
3807 }
3808
3809 static void enable_dbs(struct adapter *adap)
3810 {
3811         int i;
3812
3813         for_each_ethrxq(&adap->sge, i)
3814                 enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3815         for_each_ofldrxq(&adap->sge, i)
3816                 enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3817         for_each_port(adap, i)
3818                 enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3819 }
3820
3821 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3822 {
3823         if (adap->uld_handle[CXGB4_ULD_RDMA])
3824                 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3825                                 cmd);
3826 }
3827
3828 static void process_db_full(struct work_struct *work)
3829 {
3830         struct adapter *adap;
3831
3832         adap = container_of(work, struct adapter, db_full_task);
3833
3834         drain_db_fifo(adap, dbfifo_drain_delay);
3835         enable_dbs(adap);
3836         notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3837         t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3838                          DBFIFO_HP_INT_F | DBFIFO_LP_INT_F,
3839                          DBFIFO_HP_INT_F | DBFIFO_LP_INT_F);
3840 }
3841
3842 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3843 {
3844         u16 hw_pidx, hw_cidx;
3845         int ret;
3846
3847         spin_lock_irq(&q->db_lock);
3848         ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3849         if (ret)
3850                 goto out;
3851         if (q->db_pidx != hw_pidx) {
3852                 u16 delta;
3853                 u32 val;
3854
3855                 if (q->db_pidx >= hw_pidx)
3856                         delta = q->db_pidx - hw_pidx;
3857                 else
3858                         delta = q->size - hw_pidx + q->db_pidx;
3859
3860                 if (is_t4(adap->params.chip))
3861                         val = PIDX_V(delta);
3862                 else
3863                         val = PIDX_T5_V(delta);
3864                 wmb();
3865                 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A),
3866                              QID_V(q->cntxt_id) | val);
3867         }
3868 out:
3869         q->db_disabled = 0;
3870         q->db_pidx_inc = 0;
3871         spin_unlock_irq(&q->db_lock);
3872         if (ret)
3873                 CH_WARN(adap, "DB drop recovery failed.\n");
3874 }
3875 static void recover_all_queues(struct adapter *adap)
3876 {
3877         int i;
3878
3879         for_each_ethrxq(&adap->sge, i)
3880                 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3881         for_each_ofldrxq(&adap->sge, i)
3882                 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3883         for_each_port(adap, i)
3884                 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3885 }
3886
3887 static void process_db_drop(struct work_struct *work)
3888 {
3889         struct adapter *adap;
3890
3891         adap = container_of(work, struct adapter, db_drop_task);
3892
3893         if (is_t4(adap->params.chip)) {
3894                 drain_db_fifo(adap, dbfifo_drain_delay);
3895                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
3896                 drain_db_fifo(adap, dbfifo_drain_delay);
3897                 recover_all_queues(adap);
3898                 drain_db_fifo(adap, dbfifo_drain_delay);
3899                 enable_dbs(adap);
3900                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3901         } else {
3902                 u32 dropped_db = t4_read_reg(adap, 0x010ac);
3903                 u16 qid = (dropped_db >> 15) & 0x1ffff;
3904                 u16 pidx_inc = dropped_db & 0x1fff;
3905                 u64 bar2_qoffset;
3906                 unsigned int bar2_qid;
3907                 int ret;
3908
3909                 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
3910                                         &bar2_qoffset, &bar2_qid);
3911                 if (ret)
3912                         dev_err(adap->pdev_dev, "doorbell drop recovery: "
3913                                 "qid=%d, pidx_inc=%d\n", qid, pidx_inc);
3914                 else
3915                         writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid),
3916                                adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
3917
3918                 /* Re-enable BAR2 WC */
3919                 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
3920         }
3921
3922         t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0);
3923 }
3924
3925 void t4_db_full(struct adapter *adap)
3926 {
3927         if (is_t4(adap->params.chip)) {
3928                 disable_dbs(adap);
3929                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3930                 t4_set_reg_field(adap, SGE_INT_ENABLE3_A,
3931                                  DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0);
3932                 queue_work(adap->workq, &adap->db_full_task);
3933         }
3934 }
3935
3936 void t4_db_dropped(struct adapter *adap)
3937 {
3938         if (is_t4(adap->params.chip)) {
3939                 disable_dbs(adap);
3940                 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
3941         }
3942         queue_work(adap->workq, &adap->db_drop_task);
3943 }
3944
3945 static void uld_attach(struct adapter *adap, unsigned int uld)
3946 {
3947         void *handle;
3948         struct cxgb4_lld_info lli;
3949         unsigned short i;
3950
3951         lli.pdev = adap->pdev;
3952         lli.pf = adap->fn;
3953         lli.l2t = adap->l2t;
3954         lli.tids = &adap->tids;
3955         lli.ports = adap->port;
3956         lli.vr = &adap->vres;
3957         lli.mtus = adap->params.mtus;
3958         if (uld == CXGB4_ULD_RDMA) {
3959                 lli.rxq_ids = adap->sge.rdma_rxq;
3960                 lli.ciq_ids = adap->sge.rdma_ciq;
3961                 lli.nrxq = adap->sge.rdmaqs;
3962                 lli.nciq = adap->sge.rdmaciqs;
3963         } else if (uld == CXGB4_ULD_ISCSI) {
3964                 lli.rxq_ids = adap->sge.ofld_rxq;
3965                 lli.nrxq = adap->sge.ofldqsets;
3966         }
3967         lli.ntxq = adap->sge.ofldqsets;
3968         lli.nchan = adap->params.nports;
3969         lli.nports = adap->params.nports;
3970         lli.wr_cred = adap->params.ofldq_wr_cred;
3971         lli.adapter_type = adap->params.chip;
3972         lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A));
3973         lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
3974         lli.udb_density = 1 << adap->params.sge.eq_qpp;
3975         lli.ucq_density = 1 << adap->params.sge.iq_qpp;
3976         lli.filt_mode = adap->params.tp.vlan_pri_map;
3977         /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
3978         for (i = 0; i < NCHAN; i++)
3979                 lli.tx_modq[i] = i;
3980         lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A);
3981         lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A);
3982         lli.fw_vers = adap->params.fw_vers;
3983         lli.dbfifo_int_thresh = dbfifo_int_thresh;
3984         lli.sge_ingpadboundary = adap->sge.fl_align;
3985         lli.sge_egrstatuspagesize = adap->sge.stat_len;
3986         lli.sge_pktshift = adap->sge.pktshift;
3987         lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
3988         lli.max_ordird_qp = adap->params.max_ordird_qp;
3989         lli.max_ird_adapter = adap->params.max_ird_adapter;
3990         lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
3991
3992         handle = ulds[uld].add(&lli);
3993         if (IS_ERR(handle)) {
3994                 dev_warn(adap->pdev_dev,
3995                          "could not attach to the %s driver, error %ld\n",
3996                          uld_str[uld], PTR_ERR(handle));
3997                 return;
3998         }
3999
4000         adap->uld_handle[uld] = handle;
4001
4002         if (!netevent_registered) {
4003                 register_netevent_notifier(&cxgb4_netevent_nb);
4004                 netevent_registered = true;
4005         }
4006
4007         if (adap->flags & FULL_INIT_DONE)
4008                 ulds[uld].state_change(handle, CXGB4_STATE_UP);
4009 }
4010
4011 static void attach_ulds(struct adapter *adap)
4012 {
4013         unsigned int i;
4014
4015         spin_lock(&adap_rcu_lock);
4016         list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4017         spin_unlock(&adap_rcu_lock);
4018
4019         mutex_lock(&uld_mutex);
4020         list_add_tail(&adap->list_node, &adapter_list);
4021         for (i = 0; i < CXGB4_ULD_MAX; i++)
4022                 if (ulds[i].add)
4023                         uld_attach(adap, i);
4024         mutex_unlock(&uld_mutex);
4025 }
4026
4027 static void detach_ulds(struct adapter *adap)
4028 {
4029         unsigned int i;
4030
4031         mutex_lock(&uld_mutex);
4032         list_del(&adap->list_node);
4033         for (i = 0; i < CXGB4_ULD_MAX; i++)
4034                 if (adap->uld_handle[i]) {
4035                         ulds[i].state_change(adap->uld_handle[i],
4036                                              CXGB4_STATE_DETACH);
4037                         adap->uld_handle[i] = NULL;
4038                 }
4039         if (netevent_registered && list_empty(&adapter_list)) {
4040                 unregister_netevent_notifier(&cxgb4_netevent_nb);
4041                 netevent_registered = false;
4042         }
4043         mutex_unlock(&uld_mutex);
4044
4045         spin_lock(&adap_rcu_lock);
4046         list_del_rcu(&adap->rcu_node);
4047         spin_unlock(&adap_rcu_lock);
4048 }
4049
4050 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4051 {
4052         unsigned int i;
4053
4054         mutex_lock(&uld_mutex);
4055         for (i = 0; i < CXGB4_ULD_MAX; i++)
4056                 if (adap->uld_handle[i])
4057                         ulds[i].state_change(adap->uld_handle[i], new_state);
4058         mutex_unlock(&uld_mutex);
4059 }
4060
4061 /**
4062  *      cxgb4_register_uld - register an upper-layer driver
4063  *      @type: the ULD type
4064  *      @p: the ULD methods
4065  *
4066  *      Registers an upper-layer driver with this driver and notifies the ULD
4067  *      about any presently available devices that support its type.  Returns
4068  *      %-EBUSY if a ULD of the same type is already registered.
4069  */
4070 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4071 {
4072         int ret = 0;
4073         struct adapter *adap;
4074
4075         if (type >= CXGB4_ULD_MAX)
4076                 return -EINVAL;
4077         mutex_lock(&uld_mutex);
4078         if (ulds[type].add) {
4079                 ret = -EBUSY;
4080                 goto out;
4081         }
4082         ulds[type] = *p;
4083         list_for_each_entry(adap, &adapter_list, list_node)
4084                 uld_attach(adap, type);
4085 out:    mutex_unlock(&uld_mutex);
4086         return ret;
4087 }
4088 EXPORT_SYMBOL(cxgb4_register_uld);
4089
4090 /**
4091  *      cxgb4_unregister_uld - unregister an upper-layer driver
4092  *      @type: the ULD type
4093  *
4094  *      Unregisters an existing upper-layer driver.
4095  */
4096 int cxgb4_unregister_uld(enum cxgb4_uld type)
4097 {
4098         struct adapter *adap;
4099
4100         if (type >= CXGB4_ULD_MAX)
4101                 return -EINVAL;
4102         mutex_lock(&uld_mutex);
4103         list_for_each_entry(adap, &adapter_list, list_node)
4104                 adap->uld_handle[type] = NULL;
4105         ulds[type].add = NULL;
4106         mutex_unlock(&uld_mutex);
4107         return 0;
4108 }
4109 EXPORT_SYMBOL(cxgb4_unregister_uld);
4110
4111 #if IS_ENABLED(CONFIG_IPV6)
4112 static int cxgb4_inet6addr_handler(struct notifier_block *this,
4113                                    unsigned long event, void *data)
4114 {
4115         struct inet6_ifaddr *ifa = data;
4116         struct net_device *event_dev = ifa->idev->dev;
4117         const struct device *parent = NULL;
4118 #if IS_ENABLED(CONFIG_BONDING)
4119         struct adapter *adap;
4120 #endif
4121         if (event_dev->priv_flags & IFF_802_1Q_VLAN)
4122                 event_dev = vlan_dev_real_dev(event_dev);
4123 #if IS_ENABLED(CONFIG_BONDING)
4124         if (event_dev->flags & IFF_MASTER) {
4125                 list_for_each_entry(adap, &adapter_list, list_node) {
4126                         switch (event) {
4127                         case NETDEV_UP:
4128                                 cxgb4_clip_get(adap->port[0],
4129                                                (const u32 *)ifa, 1);
4130                                 break;
4131                         case NETDEV_DOWN:
4132                                 cxgb4_clip_release(adap->port[0],
4133                                                    (const u32 *)ifa, 1);
4134                                 break;
4135                         default:
4136                                 break;
4137                         }
4138                 }
4139                 return NOTIFY_OK;
4140         }
4141 #endif
4142
4143         if (event_dev)
4144                 parent = event_dev->dev.parent;
4145
4146         if (parent && parent->driver == &cxgb4_driver.driver) {
4147                 switch (event) {
4148                 case NETDEV_UP:
4149                         cxgb4_clip_get(event_dev, (const u32 *)ifa, 1);
4150                         break;
4151                 case NETDEV_DOWN:
4152                         cxgb4_clip_release(event_dev, (const u32 *)ifa, 1);
4153                         break;
4154                 default:
4155                         break;
4156                 }
4157         }
4158         return NOTIFY_OK;
4159 }
4160
4161 static bool inet6addr_registered;
4162 static struct notifier_block cxgb4_inet6addr_notifier = {
4163         .notifier_call = cxgb4_inet6addr_handler
4164 };
4165
4166 static void update_clip(const struct adapter *adap)
4167 {
4168         int i;
4169         struct net_device *dev;
4170         int ret;
4171
4172         rcu_read_lock();
4173
4174         for (i = 0; i < MAX_NPORTS; i++) {
4175                 dev = adap->port[i];
4176                 ret = 0;
4177
4178                 if (dev)
4179                         ret = cxgb4_update_root_dev_clip(dev);
4180
4181                 if (ret < 0)
4182                         break;
4183         }
4184         rcu_read_unlock();
4185 }
4186 #endif /* IS_ENABLED(CONFIG_IPV6) */
4187
4188 /**
4189  *      cxgb_up - enable the adapter
4190  *      @adap: adapter being enabled
4191  *
4192  *      Called when the first port is enabled, this function performs the
4193  *      actions necessary to make an adapter operational, such as completing
4194  *      the initialization of HW modules, and enabling interrupts.
4195  *
4196  *      Must be called with the rtnl lock held.
4197  */
4198 static int cxgb_up(struct adapter *adap)
4199 {
4200         int err;
4201
4202         err = setup_sge_queues(adap);
4203         if (err)
4204                 goto out;
4205         err = setup_rss(adap);
4206         if (err)
4207                 goto freeq;
4208
4209         if (adap->flags & USING_MSIX) {
4210                 name_msix_vecs(adap);
4211                 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4212                                   adap->msix_info[0].desc, adap);
4213                 if (err)
4214                         goto irq_err;
4215
4216                 err = request_msix_queue_irqs(adap);
4217                 if (err) {
4218                         free_irq(adap->msix_info[0].vec, adap);
4219                         goto irq_err;
4220                 }
4221         } else {
4222                 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4223                                   (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4224                                   adap->port[0]->name, adap);
4225                 if (err)
4226                         goto irq_err;
4227         }
4228         enable_rx(adap);
4229         t4_sge_start(adap);
4230         t4_intr_enable(adap);
4231         adap->flags |= FULL_INIT_DONE;
4232         notify_ulds(adap, CXGB4_STATE_UP);
4233 #if IS_ENABLED(CONFIG_IPV6)
4234         update_clip(adap);
4235 #endif
4236  out:
4237         return err;
4238  irq_err:
4239         dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4240  freeq:
4241         t4_free_sge_resources(adap);
4242         goto out;
4243 }
4244
4245 static void cxgb_down(struct adapter *adapter)
4246 {
4247         t4_intr_disable(adapter);
4248         cancel_work_sync(&adapter->tid_release_task);
4249         cancel_work_sync(&adapter->db_full_task);
4250         cancel_work_sync(&adapter->db_drop_task);
4251         adapter->tid_release_task_busy = false;
4252         adapter->tid_release_head = NULL;
4253
4254         if (adapter->flags & USING_MSIX) {
4255                 free_msix_queue_irqs(adapter);
4256                 free_irq(adapter->msix_info[0].vec, adapter);
4257         } else
4258                 free_irq(adapter->pdev->irq, adapter);
4259         quiesce_rx(adapter);
4260         t4_sge_stop(adapter);
4261         t4_free_sge_resources(adapter);
4262         adapter->flags &= ~FULL_INIT_DONE;
4263 }
4264
4265 /*
4266  * net_device operations
4267  */
4268 static int cxgb_open(struct net_device *dev)
4269 {
4270         int err;
4271         struct port_info *pi = netdev_priv(dev);
4272         struct adapter *adapter = pi->adapter;
4273
4274         netif_carrier_off(dev);
4275
4276         if (!(adapter->flags & FULL_INIT_DONE)) {
4277                 err = cxgb_up(adapter);
4278                 if (err < 0)
4279                         return err;
4280         }
4281
4282         err = link_start(dev);
4283         if (!err)
4284                 netif_tx_start_all_queues(dev);
4285         return err;
4286 }
4287
4288 static int cxgb_close(struct net_device *dev)
4289 {
4290         struct port_info *pi = netdev_priv(dev);
4291         struct adapter *adapter = pi->adapter;
4292
4293         netif_tx_stop_all_queues(dev);
4294         netif_carrier_off(dev);
4295         return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4296 }
4297
4298 /* Return an error number if the indicated filter isn't writable ...
4299  */
4300 static int writable_filter(struct filter_entry *f)
4301 {
4302         if (f->locked)
4303                 return -EPERM;
4304         if (f->pending)
4305                 return -EBUSY;
4306
4307         return 0;
4308 }
4309
4310 /* Delete the filter at the specified index (if valid).  The checks for all
4311  * the common problems with doing this like the filter being locked, currently
4312  * pending in another operation, etc.
4313  */
4314 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4315 {
4316         struct filter_entry *f;
4317         int ret;
4318
4319         if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4320                 return -EINVAL;
4321
4322         f = &adapter->tids.ftid_tab[fidx];
4323         ret = writable_filter(f);
4324         if (ret)
4325                 return ret;
4326         if (f->valid)
4327                 return del_filter_wr(adapter, fidx);
4328
4329         return 0;
4330 }
4331
4332 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4333                 __be32 sip, __be16 sport, __be16 vlan,
4334                 unsigned int queue, unsigned char port, unsigned char mask)
4335 {
4336         int ret;
4337         struct filter_entry *f;
4338         struct adapter *adap;
4339         int i;
4340         u8 *val;
4341
4342         adap = netdev2adap(dev);
4343
4344         /* Adjust stid to correct filter index */
4345         stid -= adap->tids.sftid_base;
4346         stid += adap->tids.nftids;
4347
4348         /* Check to make sure the filter requested is writable ...
4349          */
4350         f = &adap->tids.ftid_tab[stid];
4351         ret = writable_filter(f);
4352         if (ret)
4353                 return ret;
4354
4355         /* Clear out any old resources being used by the filter before
4356          * we start constructing the new filter.
4357          */
4358         if (f->valid)
4359                 clear_filter(adap, f);
4360
4361         /* Clear out filter specifications */
4362         memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4363         f->fs.val.lport = cpu_to_be16(sport);
4364         f->fs.mask.lport  = ~0;
4365         val = (u8 *)&sip;
4366         if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4367                 for (i = 0; i < 4; i++) {
4368                         f->fs.val.lip[i] = val[i];
4369                         f->fs.mask.lip[i] = ~0;
4370                 }
4371                 if (adap->params.tp.vlan_pri_map & PORT_F) {
4372                         f->fs.val.iport = port;
4373                         f->fs.mask.iport = mask;
4374                 }
4375         }
4376
4377         if (adap->params.tp.vlan_pri_map & PROTOCOL_F) {
4378                 f->fs.val.proto = IPPROTO_TCP;
4379                 f->fs.mask.proto = ~0;
4380         }
4381
4382         f->fs.dirsteer = 1;
4383         f->fs.iq = queue;
4384         /* Mark filter as locked */
4385         f->locked = 1;
4386         f->fs.rpttid = 1;
4387
4388         ret = set_filter_wr(adap, stid);
4389         if (ret) {
4390                 clear_filter(adap, f);
4391                 return ret;
4392         }
4393
4394         return 0;
4395 }
4396 EXPORT_SYMBOL(cxgb4_create_server_filter);
4397
4398 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4399                 unsigned int queue, bool ipv6)
4400 {
4401         int ret;
4402         struct filter_entry *f;
4403         struct adapter *adap;
4404
4405         adap = netdev2adap(dev);
4406
4407         /* Adjust stid to correct filter index */
4408         stid -= adap->tids.sftid_base;
4409         stid += adap->tids.nftids;
4410
4411         f = &adap->tids.ftid_tab[stid];
4412         /* Unlock the filter */
4413         f->locked = 0;
4414
4415         ret = delete_filter(adap, stid);
4416         if (ret)
4417                 return ret;
4418
4419         return 0;
4420 }
4421 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4422
4423 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4424                                                 struct rtnl_link_stats64 *ns)
4425 {
4426         struct port_stats stats;
4427         struct port_info *p = netdev_priv(dev);
4428         struct adapter *adapter = p->adapter;
4429
4430         /* Block retrieving statistics during EEH error
4431          * recovery. Otherwise, the recovery might fail
4432          * and the PCI device will be removed permanently
4433          */
4434         spin_lock(&adapter->stats_lock);
4435         if (!netif_device_present(dev)) {
4436                 spin_unlock(&adapter->stats_lock);
4437                 return ns;
4438         }
4439         t4_get_port_stats(adapter, p->tx_chan, &stats);
4440         spin_unlock(&adapter->stats_lock);
4441
4442         ns->tx_bytes   = stats.tx_octets;
4443         ns->tx_packets = stats.tx_frames;
4444         ns->rx_bytes   = stats.rx_octets;
4445         ns->rx_packets = stats.rx_frames;
4446         ns->multicast  = stats.rx_mcast_frames;
4447
4448         /* detailed rx_errors */
4449         ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4450                                stats.rx_runt;
4451         ns->rx_over_errors   = 0;
4452         ns->rx_crc_errors    = stats.rx_fcs_err;
4453         ns->rx_frame_errors  = stats.rx_symbol_err;
4454         ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
4455                                stats.rx_ovflow2 + stats.rx_ovflow3 +
4456                                stats.rx_trunc0 + stats.rx_trunc1 +
4457                                stats.rx_trunc2 + stats.rx_trunc3;
4458         ns->rx_missed_errors = 0;
4459
4460         /* detailed tx_errors */
4461         ns->tx_aborted_errors   = 0;
4462         ns->tx_carrier_errors   = 0;
4463         ns->tx_fifo_errors      = 0;
4464         ns->tx_heartbeat_errors = 0;
4465         ns->tx_window_errors    = 0;
4466
4467         ns->tx_errors = stats.tx_error_frames;
4468         ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4469                 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4470         return ns;
4471 }
4472
4473 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4474 {
4475         unsigned int mbox;
4476         int ret = 0, prtad, devad;
4477         struct port_info *pi = netdev_priv(dev);
4478         struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4479
4480         switch (cmd) {
4481         case SIOCGMIIPHY:
4482                 if (pi->mdio_addr < 0)
4483                         return -EOPNOTSUPP;
4484                 data->phy_id = pi->mdio_addr;
4485                 break;
4486         case SIOCGMIIREG:
4487         case SIOCSMIIREG:
4488                 if (mdio_phy_id_is_c45(data->phy_id)) {
4489                         prtad = mdio_phy_id_prtad(data->phy_id);
4490                         devad = mdio_phy_id_devad(data->phy_id);
4491                 } else if (data->phy_id < 32) {
4492                         prtad = data->phy_id;
4493                         devad = 0;
4494                         data->reg_num &= 0x1f;
4495                 } else
4496                         return -EINVAL;
4497
4498                 mbox = pi->adapter->fn;
4499                 if (cmd == SIOCGMIIREG)
4500                         ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4501                                          data->reg_num, &data->val_out);
4502                 else
4503                         ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4504                                          data->reg_num, data->val_in);
4505                 break;
4506         default:
4507                 return -EOPNOTSUPP;
4508         }
4509         return ret;
4510 }
4511
4512 static void cxgb_set_rxmode(struct net_device *dev)
4513 {
4514         /* unfortunately we can't return errors to the stack */
4515         set_rxmode(dev, -1, false);
4516 }
4517
4518 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4519 {
4520         int ret;
4521         struct port_info *pi = netdev_priv(dev);
4522
4523         if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
4524                 return -EINVAL;
4525         ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4526                             -1, -1, -1, true);
4527         if (!ret)
4528                 dev->mtu = new_mtu;
4529         return ret;
4530 }
4531
4532 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4533 {
4534         int ret;
4535         struct sockaddr *addr = p;
4536         struct port_info *pi = netdev_priv(dev);
4537
4538         if (!is_valid_ether_addr(addr->sa_data))
4539                 return -EADDRNOTAVAIL;
4540
4541         ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4542                             pi->xact_addr_filt, addr->sa_data, true, true);
4543         if (ret < 0)
4544                 return ret;
4545
4546         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4547         pi->xact_addr_filt = ret;
4548         return 0;
4549 }
4550
4551 #ifdef CONFIG_NET_POLL_CONTROLLER
4552 static void cxgb_netpoll(struct net_device *dev)
4553 {
4554         struct port_info *pi = netdev_priv(dev);
4555         struct adapter *adap = pi->adapter;
4556
4557         if (adap->flags & USING_MSIX) {
4558                 int i;
4559                 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4560
4561                 for (i = pi->nqsets; i; i--, rx++)
4562                         t4_sge_intr_msix(0, &rx->rspq);
4563         } else
4564                 t4_intr_handler(adap)(0, adap);
4565 }
4566 #endif
4567
4568 static const struct net_device_ops cxgb4_netdev_ops = {
4569         .ndo_open             = cxgb_open,
4570         .ndo_stop             = cxgb_close,
4571         .ndo_start_xmit       = t4_eth_xmit,
4572         .ndo_select_queue     = cxgb_select_queue,
4573         .ndo_get_stats64      = cxgb_get_stats,
4574         .ndo_set_rx_mode      = cxgb_set_rxmode,
4575         .ndo_set_mac_address  = cxgb_set_mac_addr,
4576         .ndo_set_features     = cxgb_set_features,
4577         .ndo_validate_addr    = eth_validate_addr,
4578         .ndo_do_ioctl         = cxgb_ioctl,
4579         .ndo_change_mtu       = cxgb_change_mtu,
4580 #ifdef CONFIG_NET_POLL_CONTROLLER
4581         .ndo_poll_controller  = cxgb_netpoll,
4582 #endif
4583 #ifdef CONFIG_NET_RX_BUSY_POLL
4584         .ndo_busy_poll        = cxgb_busy_poll,
4585 #endif
4586
4587 };
4588
4589 void t4_fatal_err(struct adapter *adap)
4590 {
4591         t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0);
4592         t4_intr_disable(adap);
4593         dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4594 }
4595
4596 /* Return the specified PCI-E Configuration Space register from our Physical
4597  * Function.  We try first via a Firmware LDST Command since we prefer to let
4598  * the firmware own all of these registers, but if that fails we go for it
4599  * directly ourselves.
4600  */
4601 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4602 {
4603         struct fw_ldst_cmd ldst_cmd;
4604         u32 val;
4605         int ret;
4606
4607         /* Construct and send the Firmware LDST Command to retrieve the
4608          * specified PCI-E Configuration Space register.
4609          */
4610         memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4611         ldst_cmd.op_to_addrspace =
4612                 htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4613                       FW_CMD_REQUEST_F |
4614                       FW_CMD_READ_F |
4615                       FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
4616         ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
4617         ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
4618         ldst_cmd.u.pcie.ctrl_to_fn =
4619                 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
4620         ldst_cmd.u.pcie.r = reg;
4621         ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4622                          &ldst_cmd);
4623
4624         /* If the LDST Command suucceeded, exctract the returned register
4625          * value.  Otherwise read it directly ourself.
4626          */
4627         if (ret == 0)
4628                 val = ntohl(ldst_cmd.u.pcie.data[0]);
4629         else
4630                 t4_hw_pci_read_cfg4(adap, reg, &val);
4631
4632         return val;
4633 }
4634
4635 static void setup_memwin(struct adapter *adap)
4636 {
4637         u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
4638
4639         if (is_t4(adap->params.chip)) {
4640                 u32 bar0;
4641
4642                 /* Truncation intentional: we only read the bottom 32-bits of
4643                  * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
4644                  * mechanism to read BAR0 instead of using
4645                  * pci_resource_start() because we could be operating from
4646                  * within a Virtual Machine which is trapping our accesses to
4647                  * our Configuration Space and we need to set up the PCI-E
4648                  * Memory Window decoders with the actual addresses which will
4649                  * be coming across the PCI-E link.
4650                  */
4651                 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4652                 bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4653                 adap->t4_bar0 = bar0;
4654
4655                 mem_win0_base = bar0 + MEMWIN0_BASE;
4656                 mem_win1_base = bar0 + MEMWIN1_BASE;
4657                 mem_win2_base = bar0 + MEMWIN2_BASE;
4658                 mem_win2_aperture = MEMWIN2_APERTURE;
4659         } else {
4660                 /* For T5, only relative offset inside the PCIe BAR is passed */
4661                 mem_win0_base = MEMWIN0_BASE;
4662                 mem_win1_base = MEMWIN1_BASE;
4663                 mem_win2_base = MEMWIN2_BASE_T5;
4664                 mem_win2_aperture = MEMWIN2_APERTURE_T5;
4665         }
4666         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0),
4667                      mem_win0_base | BIR_V(0) |
4668                      WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10));
4669         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1),
4670                      mem_win1_base | BIR_V(0) |
4671                      WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10));
4672         t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2),
4673                      mem_win2_base | BIR_V(0) |
4674                      WINDOW_V(ilog2(mem_win2_aperture) - 10));
4675         t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2));
4676 }
4677
4678 static void setup_memwin_rdma(struct adapter *adap)
4679 {
4680         if (adap->vres.ocq.size) {
4681                 u32 start;
4682                 unsigned int sz_kb;
4683
4684                 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4685                 start &= PCI_BASE_ADDRESS_MEM_MASK;
4686                 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4687                 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4688                 t4_write_reg(adap,
4689                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3),
4690                              start | BIR_V(1) | WINDOW_V(ilog2(sz_kb)));
4691                 t4_write_reg(adap,
4692                              PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3),
4693                              adap->vres.ocq.start);
4694                 t4_read_reg(adap,
4695                             PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3));
4696         }
4697 }
4698
4699 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4700 {
4701         u32 v;
4702         int ret;
4703
4704         /* get device capabilities */
4705         memset(c, 0, sizeof(*c));
4706         c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4707                                FW_CMD_REQUEST_F | FW_CMD_READ_F);
4708         c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4709         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4710         if (ret < 0)
4711                 return ret;
4712
4713         /* select capabilities we'll be using */
4714         if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4715                 if (!vf_acls)
4716                         c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4717                 else
4718                         c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4719         } else if (vf_acls) {
4720                 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4721                 return ret;
4722         }
4723         c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4724                                FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
4725         ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4726         if (ret < 0)
4727                 return ret;
4728
4729         ret = t4_config_glbl_rss(adap, adap->fn,
4730                                  FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4731                                  FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4732                                  FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
4733         if (ret < 0)
4734                 return ret;
4735
4736         ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4737                           0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4738         if (ret < 0)
4739                 return ret;
4740
4741         t4_sge_init(adap);
4742
4743         /* tweak some settings */
4744         t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849);
4745         t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12));
4746         t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A);
4747         v = t4_read_reg(adap, TP_PIO_DATA_A);
4748         t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F);
4749
4750         /* first 4 Tx modulation queues point to consecutive Tx channels */
4751         adap->params.tp.tx_modq_map = 0xE4;
4752         t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A,
4753                      TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map));
4754
4755         /* associate each Tx modulation queue with consecutive Tx channels */
4756         v = 0x84218421;
4757         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4758                           &v, 1, TP_TX_SCHED_HDR_A);
4759         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4760                           &v, 1, TP_TX_SCHED_FIFO_A);
4761         t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A,
4762                           &v, 1, TP_TX_SCHED_PCMD_A);
4763
4764 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4765         if (is_offload(adap)) {
4766                 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A,
4767                              TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4768                              TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4769                              TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4770                              TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4771                 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A,
4772                              TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4773                              TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4774                              TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4775                              TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4776         }
4777
4778         /* get basic stuff going */
4779         return t4_early_init(adap, adap->fn);
4780 }
4781
4782 /*
4783  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
4784  */
4785 #define MAX_ATIDS 8192U
4786
4787 /*
4788  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4789  *
4790  * If the firmware we're dealing with has Configuration File support, then
4791  * we use that to perform all configuration
4792  */
4793
4794 /*
4795  * Tweak configuration based on module parameters, etc.  Most of these have
4796  * defaults assigned to them by Firmware Configuration Files (if we're using
4797  * them) but need to be explicitly set if we're using hard-coded
4798  * initialization.  But even in the case of using Firmware Configuration
4799  * Files, we'd like to expose the ability to change these via module
4800  * parameters so these are essentially common tweaks/settings for
4801  * Configuration Files and hard-coded initialization ...
4802  */
4803 static int adap_init0_tweaks(struct adapter *adapter)
4804 {
4805         /*
4806          * Fix up various Host-Dependent Parameters like Page Size, Cache
4807          * Line Size, etc.  The firmware default is for a 4KB Page Size and
4808          * 64B Cache Line Size ...
4809          */
4810         t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
4811
4812         /*
4813          * Process module parameters which affect early initialization.
4814          */
4815         if (rx_dma_offset != 2 && rx_dma_offset != 0) {
4816                 dev_err(&adapter->pdev->dev,
4817                         "Ignoring illegal rx_dma_offset=%d, using 2\n",
4818                         rx_dma_offset);
4819                 rx_dma_offset = 2;
4820         }
4821         t4_set_reg_field(adapter, SGE_CONTROL_A,
4822                          PKTSHIFT_V(PKTSHIFT_M),
4823                          PKTSHIFT_V(rx_dma_offset));
4824
4825         /*
4826          * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
4827          * adds the pseudo header itself.
4828          */
4829         t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A,
4830                                CSUM_HAS_PSEUDO_HDR_F, 0);
4831
4832         return 0;
4833 }
4834
4835 /*
4836  * Attempt to initialize the adapter via a Firmware Configuration File.
4837  */
4838 static int adap_init0_config(struct adapter *adapter, int reset)
4839 {
4840         struct fw_caps_config_cmd caps_cmd;
4841         const struct firmware *cf;
4842         unsigned long mtype = 0, maddr = 0;
4843         u32 finiver, finicsum, cfcsum;
4844         int ret;
4845         int config_issued = 0;
4846         char *fw_config_file, fw_config_file_path[256];
4847         char *config_name = NULL;
4848
4849         /*
4850          * Reset device if necessary.
4851          */
4852         if (reset) {
4853                 ret = t4_fw_reset(adapter, adapter->mbox,
4854                                   PIORSTMODE_F | PIORST_F);
4855                 if (ret < 0)
4856                         goto bye;
4857         }
4858
4859         /*
4860          * If we have a T4 configuration file under /lib/firmware/cxgb4/,
4861          * then use that.  Otherwise, use the configuration file stored
4862          * in the adapter flash ...
4863          */
4864         switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
4865         case CHELSIO_T4:
4866                 fw_config_file = FW4_CFNAME;
4867                 break;
4868         case CHELSIO_T5:
4869                 fw_config_file = FW5_CFNAME;
4870                 break;
4871         default:
4872                 dev_err(adapter->pdev_dev, "Device %d is not supported\n",
4873                        adapter->pdev->device);
4874                 ret = -EINVAL;
4875                 goto bye;
4876         }
4877
4878         ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
4879         if (ret < 0) {
4880                 config_name = "On FLASH";
4881                 mtype = FW_MEMTYPE_CF_FLASH;
4882                 maddr = t4_flash_cfg_addr(adapter);
4883         } else {
4884                 u32 params[7], val[7];
4885
4886                 sprintf(fw_config_file_path,
4887                         "/lib/firmware/%s", fw_config_file);
4888                 config_name = fw_config_file_path;
4889
4890                 if (cf->size >= FLASH_CFG_MAX_SIZE)
4891                         ret = -ENOMEM;
4892                 else {
4893                         params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
4894                              FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
4895                         ret = t4_query_params(adapter, adapter->mbox,
4896                                               adapter->fn, 0, 1, params, val);
4897                         if (ret == 0) {
4898                                 /*
4899                                  * For t4_memory_rw() below addresses and
4900                                  * sizes have to be in terms of multiples of 4
4901                                  * bytes.  So, if the Configuration File isn't
4902                                  * a multiple of 4 bytes in length we'll have
4903                                  * to write that out separately since we can't
4904                                  * guarantee that the bytes following the
4905                                  * residual byte in the buffer returned by
4906                                  * request_firmware() are zeroed out ...
4907                                  */
4908                                 size_t resid = cf->size & 0x3;
4909                                 size_t size = cf->size & ~0x3;
4910                                 __be32 *data = (__be32 *)cf->data;
4911
4912                                 mtype = FW_PARAMS_PARAM_Y_G(val[0]);
4913                                 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
4914
4915                                 spin_lock(&adapter->win0_lock);
4916                                 ret = t4_memory_rw(adapter, 0, mtype, maddr,
4917                                                    size, data, T4_MEMORY_WRITE);
4918                                 if (ret == 0 && resid != 0) {
4919                                         union {
4920                                                 __be32 word;
4921                                                 char buf[4];
4922                                         } last;
4923                                         int i;
4924
4925                                         last.word = data[size >> 2];
4926                                         for (i = resid; i < 4; i++)
4927                                                 last.buf[i] = 0;
4928                                         ret = t4_memory_rw(adapter, 0, mtype,
4929                                                            maddr + size,
4930                                                            4, &last.word,
4931                                                            T4_MEMORY_WRITE);
4932                                 }
4933                                 spin_unlock(&adapter->win0_lock);
4934                         }
4935                 }
4936
4937                 release_firmware(cf);
4938                 if (ret)
4939                         goto bye;
4940         }
4941
4942         /*
4943          * Issue a Capability Configuration command to the firmware to get it
4944          * to parse the Configuration File.  We don't use t4_fw_config_file()
4945          * because we want the ability to modify various features after we've
4946          * processed the configuration file ...
4947          */
4948         memset(&caps_cmd, 0, sizeof(caps_cmd));
4949         caps_cmd.op_to_write =
4950                 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4951                       FW_CMD_REQUEST_F |
4952                       FW_CMD_READ_F);
4953         caps_cmd.cfvalid_to_len16 =
4954                 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
4955                       FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
4956                       FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
4957                       FW_LEN16(caps_cmd));
4958         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
4959                          &caps_cmd);
4960
4961         /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
4962          * Configuration File in FLASH), our last gasp effort is to use the
4963          * Firmware Configuration File which is embedded in the firmware.  A
4964          * very few early versions of the firmware didn't have one embedded
4965          * but we can ignore those.
4966          */
4967         if (ret == -ENOENT) {
4968                 memset(&caps_cmd, 0, sizeof(caps_cmd));
4969                 caps_cmd.op_to_write =
4970                         htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4971                                         FW_CMD_REQUEST_F |
4972                                         FW_CMD_READ_F);
4973                 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4974                 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
4975                                 sizeof(caps_cmd), &caps_cmd);
4976                 config_name = "Firmware Default";
4977         }
4978
4979         config_issued = 1;
4980         if (ret < 0)
4981                 goto bye;
4982
4983         finiver = ntohl(caps_cmd.finiver);
4984         finicsum = ntohl(caps_cmd.finicsum);
4985         cfcsum = ntohl(caps_cmd.cfcsum);
4986         if (finicsum != cfcsum)
4987                 dev_warn(adapter->pdev_dev, "Configuration File checksum "\
4988                          "mismatch: [fini] csum=%#x, computed csum=%#x\n",
4989                          finicsum, cfcsum);
4990
4991         /*
4992          * And now tell the firmware to use the configuration we just loaded.
4993          */
4994         caps_cmd.op_to_write =
4995                 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4996                       FW_CMD_REQUEST_F |
4997                       FW_CMD_WRITE_F);
4998         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
4999         ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5000                          NULL);
5001         if (ret < 0)
5002                 goto bye;
5003
5004         /*
5005          * Tweak configuration based on system architecture, module
5006          * parameters, etc.
5007          */
5008         ret = adap_init0_tweaks(adapter);
5009         if (ret < 0)
5010                 goto bye;
5011
5012         /*
5013          * And finally tell the firmware to initialize itself using the
5014          * parameters from the Configuration File.
5015          */
5016         ret = t4_fw_initialize(adapter, adapter->mbox);
5017         if (ret < 0)
5018                 goto bye;
5019
5020         /* Emit Firmware Configuration File information and return
5021          * successfully.
5022          */
5023         dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
5024                  "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5025                  config_name, finiver, cfcsum);
5026         return 0;
5027
5028         /*
5029          * Something bad happened.  Return the error ...  (If the "error"
5030          * is that there's no Configuration File on the adapter we don't
5031          * want to issue a warning since this is fairly common.)
5032          */
5033 bye:
5034         if (config_issued && ret != -ENOENT)
5035                 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5036                          config_name, -ret);
5037         return ret;
5038 }
5039
5040 static struct fw_info fw_info_array[] = {
5041         {
5042                 .chip = CHELSIO_T4,
5043                 .fs_name = FW4_CFNAME,
5044                 .fw_mod_name = FW4_FNAME,
5045                 .fw_hdr = {
5046                         .chip = FW_HDR_CHIP_T4,
5047                         .fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5048                         .intfver_nic = FW_INTFVER(T4, NIC),
5049                         .intfver_vnic = FW_INTFVER(T4, VNIC),
5050                         .intfver_ri = FW_INTFVER(T4, RI),
5051                         .intfver_iscsi = FW_INTFVER(T4, ISCSI),
5052                         .intfver_fcoe = FW_INTFVER(T4, FCOE),
5053                 },
5054         }, {
5055                 .chip = CHELSIO_T5,
5056                 .fs_name = FW5_CFNAME,
5057                 .fw_mod_name = FW5_FNAME,
5058                 .fw_hdr = {
5059                         .chip = FW_HDR_CHIP_T5,
5060                         .fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5061                         .intfver_nic = FW_INTFVER(T5, NIC),
5062                         .intfver_vnic = FW_INTFVER(T5, VNIC),
5063                         .intfver_ri = FW_INTFVER(T5, RI),
5064                         .intfver_iscsi = FW_INTFVER(T5, ISCSI),
5065                         .intfver_fcoe = FW_INTFVER(T5, FCOE),
5066                 },
5067         }
5068 };
5069
5070 static struct fw_info *find_fw_info(int chip)
5071 {
5072         int i;
5073
5074         for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5075                 if (fw_info_array[i].chip == chip)
5076                         return &fw_info_array[i];
5077         }
5078         return NULL;
5079 }
5080
5081 /*
5082  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5083  */
5084 static int adap_init0(struct adapter *adap)
5085 {
5086         int ret;
5087         u32 v, port_vec;
5088         enum dev_state state;
5089         u32 params[7], val[7];
5090         struct fw_caps_config_cmd caps_cmd;
5091         struct fw_devlog_cmd devlog_cmd;
5092         u32 devlog_meminfo;
5093         int reset = 1;
5094
5095         /* Contact FW, advertising Master capability */
5096         ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
5097         if (ret < 0) {
5098                 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5099                         ret);
5100                 return ret;
5101         }
5102         if (ret == adap->mbox)
5103                 adap->flags |= MASTER_PF;
5104
5105         /*
5106          * If we're the Master PF Driver and the device is uninitialized,
5107          * then let's consider upgrading the firmware ...  (We always want
5108          * to check the firmware version number in order to A. get it for
5109          * later reporting and B. to warn if the currently loaded firmware
5110          * is excessively mismatched relative to the driver.)
5111          */
5112         t4_get_fw_version(adap, &adap->params.fw_vers);
5113         t4_get_tp_version(adap, &adap->params.tp_vers);
5114         if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5115                 struct fw_info *fw_info;
5116                 struct fw_hdr *card_fw;
5117                 const struct firmware *fw;
5118                 const u8 *fw_data = NULL;
5119                 unsigned int fw_size = 0;
5120
5121                 /* This is the firmware whose headers the driver was compiled
5122                  * against
5123                  */
5124                 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5125                 if (fw_info == NULL) {
5126                         dev_err(adap->pdev_dev,
5127                                 "unable to get firmware info for chip %d.\n",
5128                                 CHELSIO_CHIP_VERSION(adap->params.chip));
5129                         return -EINVAL;
5130                 }
5131
5132                 /* allocate memory to read the header of the firmware on the
5133                  * card
5134                  */
5135                 card_fw = t4_alloc_mem(sizeof(*card_fw));
5136
5137                 /* Get FW from from /lib/firmware/ */
5138                 ret = request_firmware(&fw, fw_info->fw_mod_name,
5139                                        adap->pdev_dev);
5140                 if (ret < 0) {
5141                         dev_err(adap->pdev_dev,
5142                                 "unable to load firmware image %s, error %d\n",
5143                                 fw_info->fw_mod_name, ret);
5144                 } else {
5145                         fw_data = fw->data;
5146                         fw_size = fw->size;
5147                 }
5148
5149                 /* upgrade FW logic */
5150                 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5151                                  state, &reset);
5152
5153                 /* Cleaning up */
5154                 release_firmware(fw);
5155                 t4_free_mem(card_fw);
5156
5157                 if (ret < 0)
5158                         goto bye;
5159         }
5160
5161         /*
5162          * Grab VPD parameters.  This should be done after we establish a
5163          * connection to the firmware since some of the VPD parameters
5164          * (notably the Core Clock frequency) are retrieved via requests to
5165          * the firmware.  On the other hand, we need these fairly early on
5166          * so we do this right after getting ahold of the firmware.
5167          */
5168         ret = get_vpd_params(adap, &adap->params.vpd);
5169         if (ret < 0)
5170                 goto bye;
5171
5172         /* Read firmware device log parameters.  We really need to find a way
5173          * to get these parameters initialized with some default values (which
5174          * are likely to be correct) for the case where we either don't
5175          * attache to the firmware or it's crashed when we probe the adapter.
5176          * That way we'll still be able to perform early firmware startup
5177          * debugging ...  If the request to get the Firmware's Device Log
5178          * parameters fails, we'll live so we don't make that a fatal error.
5179          */
5180         memset(&devlog_cmd, 0, sizeof(devlog_cmd));
5181         devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) |
5182                                        FW_CMD_REQUEST_F | FW_CMD_READ_F);
5183         devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd));
5184         ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd),
5185                          &devlog_cmd);
5186         if (ret == 0) {
5187                 devlog_meminfo =
5188                         ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog);
5189                 adap->params.devlog.memtype =
5190                         FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo);
5191                 adap->params.devlog.start =
5192                         FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4;
5193                 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog);
5194         }
5195
5196         /*
5197          * Find out what ports are available to us.  Note that we need to do
5198          * this before calling adap_init0_no_config() since it needs nports
5199          * and portvec ...
5200          */
5201         v =
5202             FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5203             FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
5204         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5205         if (ret < 0)
5206                 goto bye;
5207
5208         adap->params.nports = hweight32(port_vec);
5209         adap->params.portvec = port_vec;
5210
5211         /* If the firmware is initialized already, emit a simply note to that
5212          * effect. Otherwise, it's time to try initializing the adapter.
5213          */
5214         if (state == DEV_STATE_INIT) {
5215                 dev_info(adap->pdev_dev, "Coming up as %s: "\
5216                          "Adapter already initialized\n",
5217                          adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5218         } else {
5219                 dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5220                          "Initializing adapter\n");
5221
5222                 /* Find out whether we're dealing with a version of the
5223                  * firmware which has configuration file support.
5224                  */
5225                 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5226                              FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5227                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5228                                       params, val);
5229
5230                 /* If the firmware doesn't support Configuration Files,
5231                  * return an error.
5232                  */
5233                 if (ret < 0) {
5234                         dev_err(adap->pdev_dev, "firmware doesn't support "
5235                                 "Firmware Configuration Files\n");
5236                         goto bye;
5237                 }
5238
5239                 /* The firmware provides us with a memory buffer where we can
5240                  * load a Configuration File from the host if we want to
5241                  * override the Configuration File in flash.
5242                  */
5243                 ret = adap_init0_config(adap, reset);
5244                 if (ret == -ENOENT) {
5245                         dev_err(adap->pdev_dev, "no Configuration File "
5246                                 "present on adapter.\n");
5247                         goto bye;
5248                 }
5249                 if (ret < 0) {
5250                         dev_err(adap->pdev_dev, "could not initialize "
5251                                 "adapter, error %d\n", -ret);
5252                         goto bye;
5253                 }
5254         }
5255
5256         /* Give the SGE code a chance to pull in anything that it needs ...
5257          * Note that this must be called after we retrieve our VPD parameters
5258          * in order to know how to convert core ticks to seconds, etc.
5259          */
5260         ret = t4_sge_init(adap);
5261         if (ret < 0)
5262                 goto bye;
5263
5264         if (is_bypass_device(adap->pdev->device))
5265                 adap->params.bypass = 1;
5266
5267         /*
5268          * Grab some of our basic fundamental operating parameters.
5269          */
5270 #define FW_PARAM_DEV(param) \
5271         (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5272         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
5273
5274 #define FW_PARAM_PFVF(param) \
5275         FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5276         FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)|  \
5277         FW_PARAMS_PARAM_Y_V(0) | \
5278         FW_PARAMS_PARAM_Z_V(0)
5279
5280         params[0] = FW_PARAM_PFVF(EQ_START);
5281         params[1] = FW_PARAM_PFVF(L2T_START);
5282         params[2] = FW_PARAM_PFVF(L2T_END);
5283         params[3] = FW_PARAM_PFVF(FILTER_START);
5284         params[4] = FW_PARAM_PFVF(FILTER_END);
5285         params[5] = FW_PARAM_PFVF(IQFLINT_START);
5286         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5287         if (ret < 0)
5288                 goto bye;
5289         adap->sge.egr_start = val[0];
5290         adap->l2t_start = val[1];
5291         adap->l2t_end = val[2];
5292         adap->tids.ftid_base = val[3];
5293         adap->tids.nftids = val[4] - val[3] + 1;
5294         adap->sge.ingr_start = val[5];
5295
5296         params[0] = FW_PARAM_PFVF(CLIP_START);
5297         params[1] = FW_PARAM_PFVF(CLIP_END);
5298         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5299         if (ret < 0)
5300                 goto bye;
5301         adap->clipt_start = val[0];
5302         adap->clipt_end = val[1];
5303
5304         /* query params related to active filter region */
5305         params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5306         params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5307         ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5308         /* If Active filter size is set we enable establishing
5309          * offload connection through firmware work request
5310          */
5311         if ((val[0] != val[1]) && (ret >= 0)) {
5312                 adap->flags |= FW_OFLD_CONN;
5313                 adap->tids.aftid_base = val[0];
5314                 adap->tids.aftid_end = val[1];
5315         }
5316
5317         /* If we're running on newer firmware, let it know that we're
5318          * prepared to deal with encapsulated CPL messages.  Older
5319          * firmware won't understand this and we'll just get
5320          * unencapsulated messages ...
5321          */
5322         params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5323         val[0] = 1;
5324         (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5325
5326         /*
5327          * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5328          * capability.  Earlier versions of the firmware didn't have the
5329          * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5330          * permission to use ULPTX MEMWRITE DSGL.
5331          */
5332         if (is_t4(adap->params.chip)) {
5333                 adap->params.ulptx_memwrite_dsgl = false;
5334         } else {
5335                 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5336                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5337                                       1, params, val);
5338                 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5339         }
5340
5341         /*
5342          * Get device capabilities so we can determine what resources we need
5343          * to manage.
5344          */
5345         memset(&caps_cmd, 0, sizeof(caps_cmd));
5346         caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5347                                      FW_CMD_REQUEST_F | FW_CMD_READ_F);
5348         caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5349         ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5350                          &caps_cmd);
5351         if (ret < 0)
5352                 goto bye;
5353
5354         if (caps_cmd.ofldcaps) {
5355                 /* query offload-related parameters */
5356                 params[0] = FW_PARAM_DEV(NTID);
5357                 params[1] = FW_PARAM_PFVF(SERVER_START);
5358                 params[2] = FW_PARAM_PFVF(SERVER_END);
5359                 params[3] = FW_PARAM_PFVF(TDDP_START);
5360                 params[4] = FW_PARAM_PFVF(TDDP_END);
5361                 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5362                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5363                                       params, val);
5364                 if (ret < 0)
5365                         goto bye;
5366                 adap->tids.ntids = val[0];
5367                 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5368                 adap->tids.stid_base = val[1];
5369                 adap->tids.nstids = val[2] - val[1] + 1;
5370                 /*
5371                  * Setup server filter region. Divide the availble filter
5372                  * region into two parts. Regular filters get 1/3rd and server
5373                  * filters get 2/3rd part. This is only enabled if workarond
5374                  * path is enabled.
5375                  * 1. For regular filters.
5376                  * 2. Server filter: This are special filters which are used
5377                  * to redirect SYN packets to offload queue.
5378                  */
5379                 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5380                         adap->tids.sftid_base = adap->tids.ftid_base +
5381                                         DIV_ROUND_UP(adap->tids.nftids, 3);
5382                         adap->tids.nsftids = adap->tids.nftids -
5383                                          DIV_ROUND_UP(adap->tids.nftids, 3);
5384                         adap->tids.nftids = adap->tids.sftid_base -
5385                                                 adap->tids.ftid_base;
5386                 }
5387                 adap->vres.ddp.start = val[3];
5388                 adap->vres.ddp.size = val[4] - val[3] + 1;
5389                 adap->params.ofldq_wr_cred = val[5];
5390
5391                 adap->params.offload = 1;
5392         }
5393         if (caps_cmd.rdmacaps) {
5394                 params[0] = FW_PARAM_PFVF(STAG_START);
5395                 params[1] = FW_PARAM_PFVF(STAG_END);
5396                 params[2] = FW_PARAM_PFVF(RQ_START);
5397                 params[3] = FW_PARAM_PFVF(RQ_END);
5398                 params[4] = FW_PARAM_PFVF(PBL_START);
5399                 params[5] = FW_PARAM_PFVF(PBL_END);
5400                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5401                                       params, val);
5402                 if (ret < 0)
5403                         goto bye;
5404                 adap->vres.stag.start = val[0];
5405                 adap->vres.stag.size = val[1] - val[0] + 1;
5406                 adap->vres.rq.start = val[2];
5407                 adap->vres.rq.size = val[3] - val[2] + 1;
5408                 adap->vres.pbl.start = val[4];
5409                 adap->vres.pbl.size = val[5] - val[4] + 1;
5410
5411                 params[0] = FW_PARAM_PFVF(SQRQ_START);
5412                 params[1] = FW_PARAM_PFVF(SQRQ_END);
5413                 params[2] = FW_PARAM_PFVF(CQ_START);
5414                 params[3] = FW_PARAM_PFVF(CQ_END);
5415                 params[4] = FW_PARAM_PFVF(OCQ_START);
5416                 params[5] = FW_PARAM_PFVF(OCQ_END);
5417                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5418                                       val);
5419                 if (ret < 0)
5420                         goto bye;
5421                 adap->vres.qp.start = val[0];
5422                 adap->vres.qp.size = val[1] - val[0] + 1;
5423                 adap->vres.cq.start = val[2];
5424                 adap->vres.cq.size = val[3] - val[2] + 1;
5425                 adap->vres.ocq.start = val[4];
5426                 adap->vres.ocq.size = val[5] - val[4] + 1;
5427
5428                 params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5429                 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5430                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5431                                       val);
5432                 if (ret < 0) {
5433                         adap->params.max_ordird_qp = 8;
5434                         adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5435                         ret = 0;
5436                 } else {
5437                         adap->params.max_ordird_qp = val[0];
5438                         adap->params.max_ird_adapter = val[1];
5439                 }
5440                 dev_info(adap->pdev_dev,
5441                          "max_ordird_qp %d max_ird_adapter %d\n",
5442                          adap->params.max_ordird_qp,
5443                          adap->params.max_ird_adapter);
5444         }
5445         if (caps_cmd.iscsicaps) {
5446                 params[0] = FW_PARAM_PFVF(ISCSI_START);
5447                 params[1] = FW_PARAM_PFVF(ISCSI_END);
5448                 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5449                                       params, val);
5450                 if (ret < 0)
5451                         goto bye;
5452                 adap->vres.iscsi.start = val[0];
5453                 adap->vres.iscsi.size = val[1] - val[0] + 1;
5454         }
5455 #undef FW_PARAM_PFVF
5456 #undef FW_PARAM_DEV
5457
5458         /* The MTU/MSS Table is initialized by now, so load their values.  If
5459          * we're initializing the adapter, then we'll make any modifications
5460          * we want to the MTU/MSS Table and also initialize the congestion
5461          * parameters.
5462          */
5463         t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5464         if (state != DEV_STATE_INIT) {
5465                 int i;
5466
5467                 /* The default MTU Table contains values 1492 and 1500.
5468                  * However, for TCP, it's better to have two values which are
5469                  * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5470                  * This allows us to have a TCP Data Payload which is a
5471                  * multiple of 8 regardless of what combination of TCP Options
5472                  * are in use (always a multiple of 4 bytes) which is
5473                  * important for performance reasons.  For instance, if no
5474                  * options are in use, then we have a 20-byte IP header and a
5475                  * 20-byte TCP header.  In this case, a 1500-byte MSS would
5476                  * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5477                  * which is not a multiple of 8.  So using an MSS of 1488 in
5478                  * this case results in a TCP Data Payload of 1448 bytes which
5479                  * is a multiple of 8.  On the other hand, if 12-byte TCP Time
5480                  * Stamps have been negotiated, then an MTU of 1500 bytes
5481                  * results in a TCP Data Payload of 1448 bytes which, as
5482                  * above, is a multiple of 8 bytes ...
5483                  */
5484                 for (i = 0; i < NMTUS; i++)
5485                         if (adap->params.mtus[i] == 1492) {
5486                                 adap->params.mtus[i] = 1488;
5487                                 break;
5488                         }
5489
5490                 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5491                              adap->params.b_wnd);
5492         }
5493         t4_init_sge_params(adap);
5494         t4_init_tp_params(adap);
5495         adap->flags |= FW_OK;
5496         return 0;
5497
5498         /*
5499          * Something bad happened.  If a command timed out or failed with EIO
5500          * FW does not operate within its spec or something catastrophic
5501          * happened to HW/FW, stop issuing commands.
5502          */
5503 bye:
5504         if (ret != -ETIMEDOUT && ret != -EIO)
5505                 t4_fw_bye(adap, adap->mbox);
5506         return ret;
5507 }
5508
5509 /* EEH callbacks */
5510
5511 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5512                                          pci_channel_state_t state)
5513 {
5514         int i;
5515         struct adapter *adap = pci_get_drvdata(pdev);
5516
5517         if (!adap)
5518                 goto out;
5519
5520         rtnl_lock();
5521         adap->flags &= ~FW_OK;
5522         notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5523         spin_lock(&adap->stats_lock);
5524         for_each_port(adap, i) {
5525                 struct net_device *dev = adap->port[i];
5526
5527                 netif_device_detach(dev);
5528                 netif_carrier_off(dev);
5529         }
5530         spin_unlock(&adap->stats_lock);
5531         if (adap->flags & FULL_INIT_DONE)
5532                 cxgb_down(adap);
5533         rtnl_unlock();
5534         if ((adap->flags & DEV_ENABLED)) {
5535                 pci_disable_device(pdev);
5536                 adap->flags &= ~DEV_ENABLED;
5537         }
5538 out:    return state == pci_channel_io_perm_failure ?
5539                 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5540 }
5541
5542 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5543 {
5544         int i, ret;
5545         struct fw_caps_config_cmd c;
5546         struct adapter *adap = pci_get_drvdata(pdev);
5547
5548         if (!adap) {
5549                 pci_restore_state(pdev);
5550                 pci_save_state(pdev);
5551                 return PCI_ERS_RESULT_RECOVERED;
5552         }
5553
5554         if (!(adap->flags & DEV_ENABLED)) {
5555                 if (pci_enable_device(pdev)) {
5556                         dev_err(&pdev->dev, "Cannot reenable PCI "
5557                                             "device after reset\n");
5558                         return PCI_ERS_RESULT_DISCONNECT;
5559                 }
5560                 adap->flags |= DEV_ENABLED;
5561         }
5562
5563         pci_set_master(pdev);
5564         pci_restore_state(pdev);
5565         pci_save_state(pdev);
5566         pci_cleanup_aer_uncorrect_error_status(pdev);
5567
5568         if (t4_wait_dev_ready(adap->regs) < 0)
5569                 return PCI_ERS_RESULT_DISCONNECT;
5570         if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
5571                 return PCI_ERS_RESULT_DISCONNECT;
5572         adap->flags |= FW_OK;
5573         if (adap_init1(adap, &c))
5574                 return PCI_ERS_RESULT_DISCONNECT;
5575
5576         for_each_port(adap, i) {
5577                 struct port_info *p = adap2pinfo(adap, i);
5578
5579                 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
5580                                   NULL, NULL);
5581                 if (ret < 0)
5582                         return PCI_ERS_RESULT_DISCONNECT;
5583                 p->viid = ret;
5584                 p->xact_addr_filt = -1;
5585         }
5586
5587         t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5588                      adap->params.b_wnd);
5589         setup_memwin(adap);
5590         if (cxgb_up(adap))
5591                 return PCI_ERS_RESULT_DISCONNECT;
5592         return PCI_ERS_RESULT_RECOVERED;
5593 }
5594
5595 static void eeh_resume(struct pci_dev *pdev)
5596 {
5597         int i;
5598         struct adapter *adap = pci_get_drvdata(pdev);
5599
5600         if (!adap)
5601                 return;
5602
5603         rtnl_lock();
5604         for_each_port(adap, i) {
5605                 struct net_device *dev = adap->port[i];
5606
5607                 if (netif_running(dev)) {
5608                         link_start(dev);
5609                         cxgb_set_rxmode(dev);
5610                 }
5611                 netif_device_attach(dev);
5612         }
5613         rtnl_unlock();
5614 }
5615
5616 static const struct pci_error_handlers cxgb4_eeh = {
5617         .error_detected = eeh_err_detected,
5618         .slot_reset     = eeh_slot_reset,
5619         .resume         = eeh_resume,
5620 };
5621
5622 static inline bool is_x_10g_port(const struct link_config *lc)
5623 {
5624         return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
5625                (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
5626 }
5627
5628 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
5629                              unsigned int us, unsigned int cnt,
5630                              unsigned int size, unsigned int iqe_size)
5631 {
5632         q->adap = adap;
5633         set_rspq_intr_params(q, us, cnt);
5634         q->iqe_len = iqe_size;
5635         q->size = size;
5636 }
5637
5638 /*
5639  * Perform default configuration of DMA queues depending on the number and type
5640  * of ports we found and the number of available CPUs.  Most settings can be
5641  * modified by the admin prior to actual use.
5642  */
5643 static void cfg_queues(struct adapter *adap)
5644 {
5645         struct sge *s = &adap->sge;
5646         int i, n10g = 0, qidx = 0;
5647 #ifndef CONFIG_CHELSIO_T4_DCB
5648         int q10g = 0;
5649 #endif
5650         int ciq_size;
5651
5652         for_each_port(adap, i)
5653                 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
5654 #ifdef CONFIG_CHELSIO_T4_DCB
5655         /* For Data Center Bridging support we need to be able to support up
5656          * to 8 Traffic Priorities; each of which will be assigned to its
5657          * own TX Queue in order to prevent Head-Of-Line Blocking.
5658          */
5659         if (adap->params.nports * 8 > MAX_ETH_QSETS) {
5660                 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
5661                         MAX_ETH_QSETS, adap->params.nports * 8);
5662                 BUG_ON(1);
5663         }
5664
5665         for_each_port(adap, i) {
5666                 struct port_info *pi = adap2pinfo(adap, i);
5667
5668                 pi->first_qset = qidx;
5669                 pi->nqsets = 8;
5670                 qidx += pi->nqsets;
5671         }
5672 #else /* !CONFIG_CHELSIO_T4_DCB */
5673         /*
5674          * We default to 1 queue per non-10G port and up to # of cores queues
5675          * per 10G port.
5676          */
5677         if (n10g)
5678                 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
5679         if (q10g > netif_get_num_default_rss_queues())
5680                 q10g = netif_get_num_default_rss_queues();
5681
5682         for_each_port(adap, i) {
5683                 struct port_info *pi = adap2pinfo(adap, i);
5684
5685                 pi->first_qset = qidx;
5686                 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
5687                 qidx += pi->nqsets;
5688         }
5689 #endif /* !CONFIG_CHELSIO_T4_DCB */
5690
5691         s->ethqsets = qidx;
5692         s->max_ethqsets = qidx;   /* MSI-X may lower it later */
5693
5694         if (is_offload(adap)) {
5695                 /*
5696                  * For offload we use 1 queue/channel if all ports are up to 1G,
5697                  * otherwise we divide all available queues amongst the channels
5698                  * capped by the number of available cores.
5699                  */
5700                 if (n10g) {
5701                         i = min_t(int, ARRAY_SIZE(s->ofldrxq),
5702                                   num_online_cpus());
5703                         s->ofldqsets = roundup(i, adap->params.nports);
5704                 } else
5705                         s->ofldqsets = adap->params.nports;
5706                 /* For RDMA one Rx queue per channel suffices */
5707                 s->rdmaqs = adap->params.nports;
5708                 s->rdmaciqs = adap->params.nports;
5709         }
5710
5711         for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
5712                 struct sge_eth_rxq *r = &s->ethrxq[i];
5713
5714                 init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
5715                 r->fl.size = 72;
5716         }
5717
5718         for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
5719                 s->ethtxq[i].q.size = 1024;
5720
5721         for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
5722                 s->ctrlq[i].q.size = 512;
5723
5724         for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
5725                 s->ofldtxq[i].q.size = 1024;
5726
5727         for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
5728                 struct sge_ofld_rxq *r = &s->ofldrxq[i];
5729
5730                 init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
5731                 r->rspq.uld = CXGB4_ULD_ISCSI;
5732                 r->fl.size = 72;
5733         }
5734
5735         for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
5736                 struct sge_ofld_rxq *r = &s->rdmarxq[i];
5737
5738                 init_rspq(adap, &r->rspq, 5, 1, 511, 64);
5739                 r->rspq.uld = CXGB4_ULD_RDMA;
5740                 r->fl.size = 72;
5741         }
5742
5743         ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
5744         if (ciq_size > SGE_MAX_IQ_SIZE) {
5745                 CH_WARN(adap, "CIQ size too small for available IQs\n");
5746                 ciq_size = SGE_MAX_IQ_SIZE;
5747         }
5748
5749         for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
5750                 struct sge_ofld_rxq *r = &s->rdmaciq[i];
5751
5752                 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
5753                 r->rspq.uld = CXGB4_ULD_RDMA;
5754         }
5755
5756         init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
5757         init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
5758 }
5759
5760 /*
5761  * Reduce the number of Ethernet queues across all ports to at most n.
5762  * n provides at least one queue per port.
5763  */
5764 static void reduce_ethqs(struct adapter *adap, int n)
5765 {
5766         int i;
5767         struct port_info *pi;
5768
5769         while (n < adap->sge.ethqsets)
5770                 for_each_port(adap, i) {
5771                         pi = adap2pinfo(adap, i);
5772                         if (pi->nqsets > 1) {
5773                                 pi->nqsets--;
5774                                 adap->sge.ethqsets--;
5775                                 if (adap->sge.ethqsets <= n)
5776                                         break;
5777                         }
5778                 }
5779
5780         n = 0;
5781         for_each_port(adap, i) {
5782                 pi = adap2pinfo(adap, i);
5783                 pi->first_qset = n;
5784                 n += pi->nqsets;
5785         }
5786 }
5787
5788 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
5789 #define EXTRA_VECS 2
5790
5791 static int enable_msix(struct adapter *adap)
5792 {
5793         int ofld_need = 0;
5794         int i, want, need;
5795         struct sge *s = &adap->sge;
5796         unsigned int nchan = adap->params.nports;
5797         struct msix_entry entries[MAX_INGQ + 1];
5798
5799         for (i = 0; i < ARRAY_SIZE(entries); ++i)
5800                 entries[i].entry = i;
5801
5802         want = s->max_ethqsets + EXTRA_VECS;
5803         if (is_offload(adap)) {
5804                 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
5805                 /* need nchan for each possible ULD */
5806                 ofld_need = 3 * nchan;
5807         }
5808 #ifdef CONFIG_CHELSIO_T4_DCB
5809         /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
5810          * each port.
5811          */
5812         need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
5813 #else
5814         need = adap->params.nports + EXTRA_VECS + ofld_need;
5815 #endif
5816         want = pci_enable_msix_range(adap->pdev, entries, need, want);
5817         if (want < 0)
5818                 return want;
5819
5820         /*
5821          * Distribute available vectors to the various queue groups.
5822          * Every group gets its minimum requirement and NIC gets top
5823          * priority for leftovers.
5824          */
5825         i = want - EXTRA_VECS - ofld_need;
5826         if (i < s->max_ethqsets) {
5827                 s->max_ethqsets = i;
5828                 if (i < s->ethqsets)
5829                         reduce_ethqs(adap, i);
5830         }
5831         if (is_offload(adap)) {
5832                 i = want - EXTRA_VECS - s->max_ethqsets;
5833                 i -= ofld_need - nchan;
5834                 s->ofldqsets = (i / nchan) * nchan;  /* round down */
5835         }
5836         for (i = 0; i < want; ++i)
5837                 adap->msix_info[i].vec = entries[i].vector;
5838
5839         return 0;
5840 }
5841
5842 #undef EXTRA_VECS
5843
5844 static int init_rss(struct adapter *adap)
5845 {
5846         unsigned int i, j;
5847
5848         for_each_port(adap, i) {
5849                 struct port_info *pi = adap2pinfo(adap, i);
5850
5851                 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
5852                 if (!pi->rss)
5853                         return -ENOMEM;
5854                 for (j = 0; j < pi->rss_size; j++)
5855                         pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
5856         }
5857         return 0;
5858 }
5859
5860 static void print_port_info(const struct net_device *dev)
5861 {
5862         char buf[80];
5863         char *bufp = buf;
5864         const char *spd = "";
5865         const struct port_info *pi = netdev_priv(dev);
5866         const struct adapter *adap = pi->adapter;
5867
5868         if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
5869                 spd = " 2.5 GT/s";
5870         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
5871                 spd = " 5 GT/s";
5872         else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
5873                 spd = " 8 GT/s";
5874
5875         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
5876                 bufp += sprintf(bufp, "100/");
5877         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
5878                 bufp += sprintf(bufp, "1000/");
5879         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
5880                 bufp += sprintf(bufp, "10G/");
5881         if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
5882                 bufp += sprintf(bufp, "40G/");
5883         if (bufp != buf)
5884                 --bufp;
5885         sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
5886
5887         netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
5888                     adap->params.vpd.id,
5889                     CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
5890                     is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
5891                     (adap->flags & USING_MSIX) ? " MSI-X" :
5892                     (adap->flags & USING_MSI) ? " MSI" : "");
5893         netdev_info(dev, "S/N: %s, P/N: %s\n",
5894                     adap->params.vpd.sn, adap->params.vpd.pn);
5895 }
5896
5897 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
5898 {
5899         pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
5900 }
5901
5902 /*
5903  * Free the following resources:
5904  * - memory used for tables
5905  * - MSI/MSI-X
5906  * - net devices
5907  * - resources FW is holding for us
5908  */
5909 static void free_some_resources(struct adapter *adapter)
5910 {
5911         unsigned int i;
5912
5913         t4_free_mem(adapter->l2t);
5914         t4_free_mem(adapter->tids.tid_tab);
5915         disable_msi(adapter);
5916
5917         for_each_port(adapter, i)
5918                 if (adapter->port[i]) {
5919                         kfree(adap2pinfo(adapter, i)->rss);
5920                         free_netdev(adapter->port[i]);
5921                 }
5922         if (adapter->flags & FW_OK)
5923                 t4_fw_bye(adapter, adapter->fn);
5924 }
5925
5926 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
5927 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
5928                    NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
5929 #define SEGMENT_SIZE 128
5930
5931 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
5932 {
5933         int func, i, err, s_qpp, qpp, num_seg;
5934         struct port_info *pi;
5935         bool highdma = false;
5936         struct adapter *adapter = NULL;
5937         void __iomem *regs;
5938
5939         printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
5940
5941         err = pci_request_regions(pdev, KBUILD_MODNAME);
5942         if (err) {
5943                 /* Just info, some other driver may have claimed the device. */
5944                 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
5945                 return err;
5946         }
5947
5948         err = pci_enable_device(pdev);
5949         if (err) {
5950                 dev_err(&pdev->dev, "cannot enable PCI device\n");
5951                 goto out_release_regions;
5952         }
5953
5954         regs = pci_ioremap_bar(pdev, 0);
5955         if (!regs) {
5956                 dev_err(&pdev->dev, "cannot map device registers\n");
5957                 err = -ENOMEM;
5958                 goto out_disable_device;
5959         }
5960
5961         err = t4_wait_dev_ready(regs);
5962         if (err < 0)
5963                 goto out_unmap_bar0;
5964
5965         /* We control everything through one PF */
5966         func = SOURCEPF_G(readl(regs + PL_WHOAMI_A));
5967         if (func != ent->driver_data) {
5968                 iounmap(regs);
5969                 pci_disable_device(pdev);
5970                 pci_save_state(pdev);        /* to restore SR-IOV later */
5971                 goto sriov;
5972         }
5973
5974         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
5975                 highdma = true;
5976                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
5977                 if (err) {
5978                         dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
5979                                 "coherent allocations\n");
5980                         goto out_unmap_bar0;
5981                 }
5982         } else {
5983                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
5984                 if (err) {
5985                         dev_err(&pdev->dev, "no usable DMA configuration\n");
5986                         goto out_unmap_bar0;
5987                 }
5988         }
5989
5990         pci_enable_pcie_error_reporting(pdev);
5991         enable_pcie_relaxed_ordering(pdev);
5992         pci_set_master(pdev);
5993         pci_save_state(pdev);
5994
5995         adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
5996         if (!adapter) {
5997                 err = -ENOMEM;
5998                 goto out_unmap_bar0;
5999         }
6000
6001         adapter->workq = create_singlethread_workqueue("cxgb4");
6002         if (!adapter->workq) {
6003                 err = -ENOMEM;
6004                 goto out_free_adapter;
6005         }
6006
6007         /* PCI device has been enabled */
6008         adapter->flags |= DEV_ENABLED;
6009
6010         adapter->regs = regs;
6011         adapter->pdev = pdev;
6012         adapter->pdev_dev = &pdev->dev;
6013         adapter->mbox = func;
6014         adapter->fn = func;
6015         adapter->msg_enable = dflt_msg_enable;
6016         memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6017
6018         spin_lock_init(&adapter->stats_lock);
6019         spin_lock_init(&adapter->tid_release_lock);
6020         spin_lock_init(&adapter->win0_lock);
6021
6022         INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6023         INIT_WORK(&adapter->db_full_task, process_db_full);
6024         INIT_WORK(&adapter->db_drop_task, process_db_drop);
6025
6026         err = t4_prep_adapter(adapter);
6027         if (err)
6028                 goto out_free_adapter;
6029
6030
6031         if (!is_t4(adapter->params.chip)) {
6032                 s_qpp = (QUEUESPERPAGEPF0_S +
6033                         (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) *
6034                         adapter->fn);
6035                 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter,
6036                       SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp);
6037                 num_seg = PAGE_SIZE / SEGMENT_SIZE;
6038
6039                 /* Each segment size is 128B. Write coalescing is enabled only
6040                  * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6041                  * queue is less no of segments that can be accommodated in
6042                  * a page size.
6043                  */
6044                 if (qpp > num_seg) {
6045                         dev_err(&pdev->dev,
6046                                 "Incorrect number of egress queues per page\n");
6047                         err = -EINVAL;
6048                         goto out_free_adapter;
6049                 }
6050                 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6051                 pci_resource_len(pdev, 2));
6052                 if (!adapter->bar2) {
6053                         dev_err(&pdev->dev, "cannot map device bar2 region\n");
6054                         err = -ENOMEM;
6055                         goto out_free_adapter;
6056                 }
6057         }
6058
6059         setup_memwin(adapter);
6060         err = adap_init0(adapter);
6061         setup_memwin_rdma(adapter);
6062         if (err)
6063                 goto out_unmap_bar;
6064
6065         for_each_port(adapter, i) {
6066                 struct net_device *netdev;
6067
6068                 netdev = alloc_etherdev_mq(sizeof(struct port_info),
6069                                            MAX_ETH_QSETS);
6070                 if (!netdev) {
6071                         err = -ENOMEM;
6072                         goto out_free_dev;
6073                 }
6074
6075                 SET_NETDEV_DEV(netdev, &pdev->dev);
6076
6077                 adapter->port[i] = netdev;
6078                 pi = netdev_priv(netdev);
6079                 pi->adapter = adapter;
6080                 pi->xact_addr_filt = -1;
6081                 pi->port_id = i;
6082                 netdev->irq = pdev->irq;
6083
6084                 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6085                         NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6086                         NETIF_F_RXCSUM | NETIF_F_RXHASH |
6087                         NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6088                 if (highdma)
6089                         netdev->hw_features |= NETIF_F_HIGHDMA;
6090                 netdev->features |= netdev->hw_features;
6091                 netdev->vlan_features = netdev->features & VLAN_FEAT;
6092
6093                 netdev->priv_flags |= IFF_UNICAST_FLT;
6094
6095                 netdev->netdev_ops = &cxgb4_netdev_ops;
6096 #ifdef CONFIG_CHELSIO_T4_DCB
6097                 netdev->dcbnl_ops = &cxgb4_dcb_ops;
6098                 cxgb4_dcb_state_init(netdev);
6099 #endif
6100                 netdev->ethtool_ops = &cxgb_ethtool_ops;
6101         }
6102
6103         pci_set_drvdata(pdev, adapter);
6104
6105         if (adapter->flags & FW_OK) {
6106                 err = t4_port_init(adapter, func, func, 0);
6107                 if (err)
6108                         goto out_free_dev;
6109         }
6110
6111         /*
6112          * Configure queues and allocate tables now, they can be needed as
6113          * soon as the first register_netdev completes.
6114          */
6115         cfg_queues(adapter);
6116
6117         adapter->l2t = t4_init_l2t();
6118         if (!adapter->l2t) {
6119                 /* We tolerate a lack of L2T, giving up some functionality */
6120                 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6121                 adapter->params.offload = 0;
6122         }
6123
6124 #if IS_ENABLED(CONFIG_IPV6)
6125         adapter->clipt = t4_init_clip_tbl(adapter->clipt_start,
6126                                           adapter->clipt_end);
6127         if (!adapter->clipt) {
6128                 /* We tolerate a lack of clip_table, giving up
6129                  * some functionality
6130                  */
6131                 dev_warn(&pdev->dev,
6132                          "could not allocate Clip table, continuing\n");
6133                 adapter->params.offload = 0;
6134         }
6135 #endif
6136         if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6137                 dev_warn(&pdev->dev, "could not allocate TID table, "
6138                          "continuing\n");
6139                 adapter->params.offload = 0;
6140         }
6141
6142         /* See what interrupts we'll be using */
6143         if (msi > 1 && enable_msix(adapter) == 0)
6144                 adapter->flags |= USING_MSIX;
6145         else if (msi > 0 && pci_enable_msi(pdev) == 0)
6146                 adapter->flags |= USING_MSI;
6147
6148         err = init_rss(adapter);
6149         if (err)
6150                 goto out_free_dev;
6151
6152         /*
6153          * The card is now ready to go.  If any errors occur during device
6154          * registration we do not fail the whole card but rather proceed only
6155          * with the ports we manage to register successfully.  However we must
6156          * register at least one net device.
6157          */
6158         for_each_port(adapter, i) {
6159                 pi = adap2pinfo(adapter, i);
6160                 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6161                 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6162
6163                 err = register_netdev(adapter->port[i]);
6164                 if (err)
6165                         break;
6166                 adapter->chan_map[pi->tx_chan] = i;
6167                 print_port_info(adapter->port[i]);
6168         }
6169         if (i == 0) {
6170                 dev_err(&pdev->dev, "could not register any net devices\n");
6171                 goto out_free_dev;
6172         }
6173         if (err) {
6174                 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6175                 err = 0;
6176         }
6177
6178         if (cxgb4_debugfs_root) {
6179                 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6180                                                            cxgb4_debugfs_root);
6181                 setup_debugfs(adapter);
6182         }
6183
6184         /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6185         pdev->needs_freset = 1;
6186
6187         if (is_offload(adapter))
6188                 attach_ulds(adapter);
6189
6190 sriov:
6191 #ifdef CONFIG_PCI_IOV
6192         if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6193                 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6194                         dev_info(&pdev->dev,
6195                                  "instantiated %u virtual functions\n",
6196                                  num_vf[func]);
6197 #endif
6198         return 0;
6199
6200  out_free_dev:
6201         free_some_resources(adapter);
6202  out_unmap_bar:
6203         if (!is_t4(adapter->params.chip))
6204                 iounmap(adapter->bar2);
6205  out_free_adapter:
6206         if (adapter->workq)
6207                 destroy_workqueue(adapter->workq);
6208
6209         kfree(adapter);
6210  out_unmap_bar0:
6211         iounmap(regs);
6212  out_disable_device:
6213         pci_disable_pcie_error_reporting(pdev);
6214         pci_disable_device(pdev);
6215  out_release_regions:
6216         pci_release_regions(pdev);
6217         return err;
6218 }
6219
6220 static void remove_one(struct pci_dev *pdev)
6221 {
6222         struct adapter *adapter = pci_get_drvdata(pdev);
6223
6224 #ifdef CONFIG_PCI_IOV
6225         pci_disable_sriov(pdev);
6226
6227 #endif
6228
6229         if (adapter) {
6230                 int i;
6231
6232                 /* Tear down per-adapter Work Queue first since it can contain
6233                  * references to our adapter data structure.
6234                  */
6235                 destroy_workqueue(adapter->workq);
6236
6237                 if (is_offload(adapter))
6238                         detach_ulds(adapter);
6239
6240                 for_each_port(adapter, i)
6241                         if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6242                                 unregister_netdev(adapter->port[i]);
6243
6244                 debugfs_remove_recursive(adapter->debugfs_root);
6245
6246                 /* If we allocated filters, free up state associated with any
6247                  * valid filters ...
6248                  */
6249                 if (adapter->tids.ftid_tab) {
6250                         struct filter_entry *f = &adapter->tids.ftid_tab[0];
6251                         for (i = 0; i < (adapter->tids.nftids +
6252                                         adapter->tids.nsftids); i++, f++)
6253                                 if (f->valid)
6254                                         clear_filter(adapter, f);
6255                 }
6256
6257                 if (adapter->flags & FULL_INIT_DONE)
6258                         cxgb_down(adapter);
6259
6260                 free_some_resources(adapter);
6261 #if IS_ENABLED(CONFIG_IPV6)
6262                 t4_cleanup_clip_tbl(adapter);
6263 #endif
6264                 iounmap(adapter->regs);
6265                 if (!is_t4(adapter->params.chip))
6266                         iounmap(adapter->bar2);
6267                 pci_disable_pcie_error_reporting(pdev);
6268                 if ((adapter->flags & DEV_ENABLED)) {
6269                         pci_disable_device(pdev);
6270                         adapter->flags &= ~DEV_ENABLED;
6271                 }
6272                 pci_release_regions(pdev);
6273                 synchronize_rcu();
6274                 kfree(adapter);
6275         } else
6276                 pci_release_regions(pdev);
6277 }
6278
6279 static struct pci_driver cxgb4_driver = {
6280         .name     = KBUILD_MODNAME,
6281         .id_table = cxgb4_pci_tbl,
6282         .probe    = init_one,
6283         .remove   = remove_one,
6284         .shutdown = remove_one,
6285         .err_handler = &cxgb4_eeh,
6286 };
6287
6288 static int __init cxgb4_init_module(void)
6289 {
6290         int ret;
6291
6292         /* Debugfs support is optional, just warn if this fails */
6293         cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6294         if (!cxgb4_debugfs_root)
6295                 pr_warn("could not create debugfs entry, continuing\n");
6296
6297         ret = pci_register_driver(&cxgb4_driver);
6298         if (ret < 0)
6299                 debugfs_remove(cxgb4_debugfs_root);
6300
6301 #if IS_ENABLED(CONFIG_IPV6)
6302         if (!inet6addr_registered) {
6303                 register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6304                 inet6addr_registered = true;
6305         }
6306 #endif
6307
6308         return ret;
6309 }
6310
6311 static void __exit cxgb4_cleanup_module(void)
6312 {
6313 #if IS_ENABLED(CONFIG_IPV6)
6314         if (inet6addr_registered) {
6315                 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6316                 inet6addr_registered = false;
6317         }
6318 #endif
6319         pci_unregister_driver(&cxgb4_driver);
6320         debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
6321 }
6322
6323 module_init(cxgb4_init_module);
6324 module_exit(cxgb4_cleanup_module);