drivers/net/ixgbe/ixgbe_main.c

   1 /*******************************************************************************
   2
   3   Intel 10 Gigabit PCI Express Linux driver
   4   Copyright(c) 1999 - 2011 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/types.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/string.h>
  34 #include <linux/in.h>
  35 #include <linux/ip.h>
  36 #include <linux/tcp.h>
  37 #include <linux/pkt_sched.h>
  38 #include <linux/ipv6.h>
  39 #include <linux/slab.h>
  40 #include <net/checksum.h>
  41 #include <net/ip6_checksum.h>
  42 #include <linux/ethtool.h>
  43 #include <linux/if_vlan.h>
  44 #include <scsi/fc/fc_fcoe.h>
  45
  46 #include "ixgbe.h"
  47 #include "ixgbe_common.h"
  48 #include "ixgbe_dcb_82599.h"
  49 #include "ixgbe_sriov.h"
  50
  51 char ixgbe_driver_name[] = "ixgbe";
  52 static const char ixgbe_driver_string[] =
  53                               "Intel(R) 10 Gigabit PCI Express Network Driver";
  54
  55 #define DRV_VERSION "3.2.9-k2"
  56 const char ixgbe_driver_version[] = DRV_VERSION;
  57 static const char ixgbe_copyright[] =
  58                                 "Copyright (c) 1999-2011 Intel Corporation.";
  59
  60 static const struct ixgbe_info *ixgbe_info_tbl[] = {
  61         [board_82598] = &ixgbe_82598_info,
  62         [board_82599] = &ixgbe_82599_info,
  63         [board_X540] = &ixgbe_X540_info,
  64 };
  65
  66 /* ixgbe_pci_tbl - PCI Device ID Table
  67  *
  68  * Wildcard entries (PCI_ANY_ID) should come last
  69  * Last entry must be all 0s
  70  *
  71  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  72  *   Class, Class Mask, private data (not used) }
  73  */
  74 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
  75         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598),
  76          board_82598 },
  77         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT),
  78          board_82598 },
  79         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT),
  80          board_82598 },
  81         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT),
  82          board_82598 },
  83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2),
  84          board_82598 },
  85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4),
  86          board_82598 },
  87         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT),
  88          board_82598 },
  89         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT),
  90          board_82598 },
  91         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM),
  92          board_82598 },
  93         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR),
  94          board_82598 },
  95         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM),
  96          board_82598 },
  97         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX),
  98          board_82598 },
  99         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4),
 100          board_82599 },
 101         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM),
 102          board_82599 },
 103         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR),
 104          board_82599 },
 105         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP),
 106          board_82599 },
 107         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM),
 108          board_82599 },
 109         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ),
 110          board_82599 },
 111         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4),
 112          board_82599 },
 113         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE),
 114          board_82599 },
 115         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE),
 116          board_82599 },
 117         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM),
 118          board_82599 },
 119         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE),
 120          board_82599 },
 121         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T),
 122          board_X540 },
 123
 124         /* required last entry */
 125         {0, }
 126 };
 127 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
 128
 129 #ifdef CONFIG_IXGBE_DCA
 130 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
 131                             void *p);
 132 static struct notifier_block dca_notifier = {
 133         .notifier_call = ixgbe_notify_dca,
 134         .next          = NULL,
 135         .priority      = 0
 136 };
 137 #endif
 138
 139 #ifdef CONFIG_PCI_IOV
 140 static unsigned int max_vfs;
 141 module_param(max_vfs, uint, 0);
 142 MODULE_PARM_DESC(max_vfs,
 143                  "Maximum number of virtual functions to allocate per physical function");
 144 #endif /* CONFIG_PCI_IOV */
 145
 146 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 147 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
 148 MODULE_LICENSE("GPL");
 149 MODULE_VERSION(DRV_VERSION);
 150
 151 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
 152
 153 static inline void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 154 {
 155         struct ixgbe_hw *hw = &adapter->hw;
 156         u32 gcr;
 157         u32 gpie;
 158         u32 vmdctl;
 159
 160 #ifdef CONFIG_PCI_IOV
 161         /* disable iov and allow time for transactions to clear */
 162         pci_disable_sriov(adapter->pdev);
 163 #endif
 164
 165         /* turn off device IOV mode */
 166         gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
 167         gcr &= ~(IXGBE_GCR_EXT_SRIOV);
 168         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr);
 169         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
 170         gpie &= ~IXGBE_GPIE_VTMODE_MASK;
 171         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 172
 173         /* set default pool back to 0 */
 174         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
 175         vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
 176         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
 177
 178         /* take a breather then clean up driver data */
 179         msleep(100);
 180
 181         kfree(adapter->vfinfo);
 182         adapter->vfinfo = NULL;
 183
 184         adapter->num_vfs = 0;
 185         adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
 186 }
 187
 188 struct ixgbe_reg_info {
 189         u32 ofs;
 190         char *name;
 191 };
 192
 193 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
 194
 195         /* General Registers */
 196         {IXGBE_CTRL, "CTRL"},
 197         {IXGBE_STATUS, "STATUS"},
 198         {IXGBE_CTRL_EXT, "CTRL_EXT"},
 199
 200         /* Interrupt Registers */
 201         {IXGBE_EICR, "EICR"},
 202
 203         /* RX Registers */
 204         {IXGBE_SRRCTL(0), "SRRCTL"},
 205         {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
 206         {IXGBE_RDLEN(0), "RDLEN"},
 207         {IXGBE_RDH(0), "RDH"},
 208         {IXGBE_RDT(0), "RDT"},
 209         {IXGBE_RXDCTL(0), "RXDCTL"},
 210         {IXGBE_RDBAL(0), "RDBAL"},
 211         {IXGBE_RDBAH(0), "RDBAH"},
 212
 213         /* TX Registers */
 214         {IXGBE_TDBAL(0), "TDBAL"},
 215         {IXGBE_TDBAH(0), "TDBAH"},
 216         {IXGBE_TDLEN(0), "TDLEN"},
 217         {IXGBE_TDH(0), "TDH"},
 218         {IXGBE_TDT(0), "TDT"},
 219         {IXGBE_TXDCTL(0), "TXDCTL"},
 220
 221         /* List Terminator */
 222         {}
 223 };
 224
 225
 226 /*
 227  * ixgbe_regdump - register printout routine
 228  */
 229 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
 230 {
 231         int i = 0, j = 0;
 232         char rname[16];
 233         u32 regs[64];
 234
 235         switch (reginfo->ofs) {
 236         case IXGBE_SRRCTL(0):
 237                 for (i = 0; i < 64; i++)
 238                         regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
 239                 break;
 240         case IXGBE_DCA_RXCTRL(0):
 241                 for (i = 0; i < 64; i++)
 242                         regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
 243                 break;
 244         case IXGBE_RDLEN(0):
 245                 for (i = 0; i < 64; i++)
 246                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
 247                 break;
 248         case IXGBE_RDH(0):
 249                 for (i = 0; i < 64; i++)
 250                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
 251                 break;
 252         case IXGBE_RDT(0):
 253                 for (i = 0; i < 64; i++)
 254                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
 255                 break;
 256         case IXGBE_RXDCTL(0):
 257                 for (i = 0; i < 64; i++)
 258                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 259                 break;
 260         case IXGBE_RDBAL(0):
 261                 for (i = 0; i < 64; i++)
 262                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
 263                 break;
 264         case IXGBE_RDBAH(0):
 265                 for (i = 0; i < 64; i++)
 266                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
 267                 break;
 268         case IXGBE_TDBAL(0):
 269                 for (i = 0; i < 64; i++)
 270                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
 271                 break;
 272         case IXGBE_TDBAH(0):
 273                 for (i = 0; i < 64; i++)
 274                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
 275                 break;
 276         case IXGBE_TDLEN(0):
 277                 for (i = 0; i < 64; i++)
 278                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
 279                 break;
 280         case IXGBE_TDH(0):
 281                 for (i = 0; i < 64; i++)
 282                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
 283                 break;
 284         case IXGBE_TDT(0):
 285                 for (i = 0; i < 64; i++)
 286                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
 287                 break;
 288         case IXGBE_TXDCTL(0):
 289                 for (i = 0; i < 64; i++)
 290                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
 291                 break;
 292         default:
 293                 pr_info("%-15s %08x\n", reginfo->name,
 294                         IXGBE_READ_REG(hw, reginfo->ofs));
 295                 return;
 296         }
 297
 298         for (i = 0; i < 8; i++) {
 299                 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
 300                 pr_err("%-15s", rname);
 301                 for (j = 0; j < 8; j++)
 302                         pr_cont(" %08x", regs[i*8+j]);
 303                 pr_cont("\n");
 304         }
 305
 306 }
 307
 308 /*
 309  * ixgbe_dump - Print registers, tx-rings and rx-rings
 310  */
 311 static void ixgbe_dump(struct ixgbe_adapter *adapter)
 312 {
 313         struct net_device *netdev = adapter->netdev;
 314         struct ixgbe_hw *hw = &adapter->hw;
 315         struct ixgbe_reg_info *reginfo;
 316         int n = 0;
 317         struct ixgbe_ring *tx_ring;
 318         struct ixgbe_tx_buffer *tx_buffer_info;
 319         union ixgbe_adv_tx_desc *tx_desc;
 320         struct my_u0 { u64 a; u64 b; } *u0;
 321         struct ixgbe_ring *rx_ring;
 322         union ixgbe_adv_rx_desc *rx_desc;
 323         struct ixgbe_rx_buffer *rx_buffer_info;
 324         u32 staterr;
 325         int i = 0;
 326
 327         if (!netif_msg_hw(adapter))
 328                 return;
 329
 330         /* Print netdevice Info */
 331         if (netdev) {
 332                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 333                 pr_info("Device Name     state            "
 334                         "trans_start      last_rx\n");
 335                 pr_info("%-15s %016lX %016lX %016lX\n",
 336                         netdev->name,
 337                         netdev->state,
 338                         netdev->trans_start,
 339                         netdev->last_rx);
 340         }
 341
 342         /* Print Registers */
 343         dev_info(&adapter->pdev->dev, "Register Dump\n");
 344         pr_info(" Register Name   Value\n");
 345         for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
 346              reginfo->name; reginfo++) {
 347                 ixgbe_regdump(hw, reginfo);
 348         }
 349
 350         /* Print TX Ring Summary */
 351         if (!netdev || !netif_running(netdev))
 352                 goto exit;
 353
 354         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 355         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 356         for (n = 0; n < adapter->num_tx_queues; n++) {
 357                 tx_ring = adapter->tx_ring[n];
 358                 tx_buffer_info =
 359                         &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 360                 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
 361                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
 362                            (u64)tx_buffer_info->dma,
 363                            tx_buffer_info->length,
 364                            tx_buffer_info->next_to_watch,
 365                            (u64)tx_buffer_info->time_stamp);
 366         }
 367
 368         /* Print TX Rings */
 369         if (!netif_msg_tx_done(adapter))
 370                 goto rx_ring_summary;
 371
 372         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 373
 374         /* Transmit Descriptor Formats
 375          *
 376          * Advanced Transmit Descriptor
 377          *   +--------------------------------------------------------------+
 378          * 0 |         Buffer Address [63:0]                                |
 379          *   +--------------------------------------------------------------+
 380          * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
 381          *   +--------------------------------------------------------------+
 382          *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
 383          */
 384
 385         for (n = 0; n < adapter->num_tx_queues; n++) {
 386                 tx_ring = adapter->tx_ring[n];
 387                 pr_info("------------------------------------\n");
 388                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 389                 pr_info("------------------------------------\n");
 390                 pr_info("T [desc]     [address 63:0  ] "
 391                         "[PlPOIdStDDt Ln] [bi->dma       ] "
 392                         "leng  ntw timestamp        bi->skb\n");
 393
 394                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 395                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 396                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 397                         u0 = (struct my_u0 *)tx_desc;
 398                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
 399                                 " %04X  %3X %016llX %p", i,
 400                                 le64_to_cpu(u0->a),
 401                                 le64_to_cpu(u0->b),
 402                                 (u64)tx_buffer_info->dma,
 403                                 tx_buffer_info->length,
 404                                 tx_buffer_info->next_to_watch,
 405                                 (u64)tx_buffer_info->time_stamp,
 406                                 tx_buffer_info->skb);
 407                         if (i == tx_ring->next_to_use &&
 408                                 i == tx_ring->next_to_clean)
 409                                 pr_cont(" NTC/U\n");
 410                         else if (i == tx_ring->next_to_use)
 411                                 pr_cont(" NTU\n");
 412                         else if (i == tx_ring->next_to_clean)
 413                                 pr_cont(" NTC\n");
 414                         else
 415                                 pr_cont("\n");
 416
 417                         if (netif_msg_pktdata(adapter) &&
 418                                 tx_buffer_info->dma != 0)
 419                                 print_hex_dump(KERN_INFO, "",
 420                                         DUMP_PREFIX_ADDRESS, 16, 1,
 421                                         phys_to_virt(tx_buffer_info->dma),
 422                                         tx_buffer_info->length, true);
 423                 }
 424         }
 425
 426         /* Print RX Rings Summary */
 427 rx_ring_summary:
 428         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 429         pr_info("Queue [NTU] [NTC]\n");
 430         for (n = 0; n < adapter->num_rx_queues; n++) {
 431                 rx_ring = adapter->rx_ring[n];
 432                 pr_info("%5d %5X %5X\n",
 433                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 434         }
 435
 436         /* Print RX Rings */
 437         if (!netif_msg_rx_status(adapter))
 438                 goto exit;
 439
 440         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 441
 442         /* Advanced Receive Descriptor (Read) Format
 443          *    63                                           1        0
 444          *    +-----------------------------------------------------+
 445          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 446          *    +----------------------------------------------+------+
 447          *  8 |       Header Buffer Address [63:1]           |  DD  |
 448          *    +-----------------------------------------------------+
 449          *
 450          *
 451          * Advanced Receive Descriptor (Write-Back) Format
 452          *
 453          *   63       48 47    32 31  30      21 20 16 15   4 3     0
 454          *   +------------------------------------------------------+
 455          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 456          *   | Checksum   Ident  |   |           |    | Type | Type |
 457          *   +------------------------------------------------------+
 458          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 459          *   +------------------------------------------------------+
 460          *   63       48 47    32 31            20 19               0
 461          */
 462         for (n = 0; n < adapter->num_rx_queues; n++) {
 463                 rx_ring = adapter->rx_ring[n];
 464                 pr_info("------------------------------------\n");
 465                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 466                 pr_info("------------------------------------\n");
 467                 pr_info("R  [desc]      [ PktBuf     A0] "
 468                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
 469                         "<-- Adv Rx Read format\n");
 470                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
 471                         "[vl er S cks ln] ---------------- [bi->skb] "
 472                         "<-- Adv Rx Write-Back format\n");
 473
 474                 for (i = 0; i < rx_ring->count; i++) {
 475                         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 476                         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
 477                         u0 = (struct my_u0 *)rx_desc;
 478                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 479                         if (staterr & IXGBE_RXD_STAT_DD) {
 480                                 /* Descriptor Done */
 481                                 pr_info("RWB[0x%03X]     %016llX "
 482                                         "%016llX ---------------- %p", i,
 483                                         le64_to_cpu(u0->a),
 484                                         le64_to_cpu(u0->b),
 485                                         rx_buffer_info->skb);
 486                         } else {
 487                                 pr_info("R  [0x%03X]     %016llX "
 488                                         "%016llX %016llX %p", i,
 489                                         le64_to_cpu(u0->a),
 490                                         le64_to_cpu(u0->b),
 491                                         (u64)rx_buffer_info->dma,
 492                                         rx_buffer_info->skb);
 493
 494                                 if (netif_msg_pktdata(adapter)) {
 495                                         print_hex_dump(KERN_INFO, "",
 496                                            DUMP_PREFIX_ADDRESS, 16, 1,
 497                                            phys_to_virt(rx_buffer_info->dma),
 498                                            rx_ring->rx_buf_len, true);
 499
 500                                         if (rx_ring->rx_buf_len
 501                                                 < IXGBE_RXBUFFER_2048)
 502                                                 print_hex_dump(KERN_INFO, "",
 503                                                   DUMP_PREFIX_ADDRESS, 16, 1,
 504                                                   phys_to_virt(
 505                                                     rx_buffer_info->page_dma +
 506                                                     rx_buffer_info->page_offset
 507                                                   ),
 508                                                   PAGE_SIZE/2, true);
 509                                 }
 510                         }
 511
 512                         if (i == rx_ring->next_to_use)
 513                                 pr_cont(" NTU\n");
 514                         else if (i == rx_ring->next_to_clean)
 515                                 pr_cont(" NTC\n");
 516                         else
 517                                 pr_cont("\n");
 518
 519                 }
 520         }
 521
 522 exit:
 523         return;
 524 }
 525
 526 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
 527 {
 528         u32 ctrl_ext;
 529
 530         /* Let firmware take over control of h/w */
 531         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 532         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 533                         ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
 534 }
 535
 536 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
 537 {
 538         u32 ctrl_ext;
 539
 540         /* Let firmware know the driver has taken over */
 541         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 542         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 543                         ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
 544 }
 545
 546 /*
 547  * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 548  * @adapter: pointer to adapter struct
 549  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 550  * @queue: queue to map the corresponding interrupt to
 551  * @msix_vector: the vector to map to the corresponding queue
 552  *
 553  */
 554 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
 555                            u8 queue, u8 msix_vector)
 556 {
 557         u32 ivar, index;
 558         struct ixgbe_hw *hw = &adapter->hw;
 559         switch (hw->mac.type) {
 560         case ixgbe_mac_82598EB:
 561                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 562                 if (direction == -1)
 563                         direction = 0;
 564                 index = (((direction * 64) + queue) >> 2) & 0x1F;
 565                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
 566                 ivar &= ~(0xFF << (8 * (queue & 0x3)));
 567                 ivar |= (msix_vector << (8 * (queue & 0x3)));
 568                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
 569                 break;
 570         case ixgbe_mac_82599EB:
 571         case ixgbe_mac_X540:
 572                 if (direction == -1) {
 573                         /* other causes */
 574                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 575                         index = ((queue & 1) * 8);
 576                         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
 577                         ivar &= ~(0xFF << index);
 578                         ivar |= (msix_vector << index);
 579                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
 580                         break;
 581                 } else {
 582                         /* tx or rx causes */
 583                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 584                         index = ((16 * (queue & 1)) + (8 * direction));
 585                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
 586                         ivar &= ~(0xFF << index);
 587                         ivar |= (msix_vector << index);
 588                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
 589                         break;
 590                 }
 591         default:
 592                 break;
 593         }
 594 }
 595
 596 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
 597                                           u64 qmask)
 598 {
 599         u32 mask;
 600
 601         switch (adapter->hw.mac.type) {
 602         case ixgbe_mac_82598EB:
 603                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
 604                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
 605                 break;
 606         case ixgbe_mac_82599EB:
 607         case ixgbe_mac_X540:
 608                 mask = (qmask & 0xFFFFFFFF);
 609                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
 610                 mask = (qmask >> 32);
 611                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
 612                 break;
 613         default:
 614                 break;
 615         }
 616 }
 617
 618 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
 619                                       struct ixgbe_tx_buffer *tx_buffer_info)
 620 {
 621         if (tx_buffer_info->dma) {
 622                 if (tx_buffer_info->mapped_as_page)
 623                         dma_unmap_page(tx_ring->dev,
 624                                        tx_buffer_info->dma,
 625                                        tx_buffer_info->length,
 626                                        DMA_TO_DEVICE);
 627                 else
 628                         dma_unmap_single(tx_ring->dev,
 629                                          tx_buffer_info->dma,
 630                                          tx_buffer_info->length,
 631                                          DMA_TO_DEVICE);
 632                 tx_buffer_info->dma = 0;
 633         }
 634         if (tx_buffer_info->skb) {
 635                 dev_kfree_skb_any(tx_buffer_info->skb);
 636                 tx_buffer_info->skb = NULL;
 637         }
 638         tx_buffer_info->time_stamp = 0;
 639         /* tx_buffer_info must be completely set up in the transmit path */
 640 }
 641
 642 /**
 643  * ixgbe_dcb_txq_to_tc - convert a reg index to a traffic class
 644  * @adapter: driver private struct
 645  * @index: reg idx of queue to query (0-127)
 646  *
 647  * Helper function to determine the traffic index for a particular
 648  * register index.
 649  *
 650  * Returns : a tc index for use in range 0-7, or 0-3
 651  */
 652 static u8 ixgbe_dcb_txq_to_tc(struct ixgbe_adapter *adapter, u8 reg_idx)
 653 {
 654         int tc = -1;
 655         int dcb_i = netdev_get_num_tc(adapter->netdev);
 656
 657         /* if DCB is not enabled the queues have no TC */
 658         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED))
 659                 return tc;
 660
 661         /* check valid range */
 662         if (reg_idx >= adapter->hw.mac.max_tx_queues)
 663                 return tc;
 664
 665         switch (adapter->hw.mac.type) {
 666         case ixgbe_mac_82598EB:
 667                 tc = reg_idx >> 2;
 668                 break;
 669         default:
 670                 if (dcb_i != 4 && dcb_i != 8)
 671                         break;
 672
 673                 /* if VMDq is enabled the lowest order bits determine TC */
 674                 if (adapter->flags & (IXGBE_FLAG_SRIOV_ENABLED |
 675                                       IXGBE_FLAG_VMDQ_ENABLED)) {
 676                         tc = reg_idx & (dcb_i - 1);
 677                         break;
 678                 }
 679
 680                 /*
 681                  * Convert the reg_idx into the correct TC. This bitmask
 682                  * targets the last full 32 ring traffic class and assigns
 683                  * it a value of 1. From there the rest of the rings are
 684                  * based on shifting the mask further up to include the
 685                  * reg_idx / 16 and then reg_idx / 8. It assumes dcB_i
 686                  * will only ever be 8 or 4 and that reg_idx will never
 687                  * be greater then 128. The code without the power of 2
 688                  * optimizations would be:
 689                  * (((reg_idx % 32) + 32) * dcb_i) >> (9 - reg_idx / 32)
 690                  */
 691                 tc = ((reg_idx & 0X1F) + 0x20) * dcb_i;
 692                 tc >>= 9 - (reg_idx >> 5);
 693         }
 694
 695         return tc;
 696 }
 697
 698 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
 699 {
 700         struct ixgbe_hw *hw = &adapter->hw;
 701         struct ixgbe_hw_stats *hwstats = &adapter->stats;
 702         u32 data = 0;
 703         u32 xoff[8] = {0};
 704         int i;
 705
 706         if ((hw->fc.current_mode == ixgbe_fc_full) ||
 707             (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
 708                 switch (hw->mac.type) {
 709                 case ixgbe_mac_82598EB:
 710                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
 711                         break;
 712                 default:
 713                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
 714                 }
 715                 hwstats->lxoffrxc += data;
 716
 717                 /* refill credits (no tx hang) if we received xoff */
 718                 if (!data)
 719                         return;
 720
 721                 for (i = 0; i < adapter->num_tx_queues; i++)
 722                         clear_bit(__IXGBE_HANG_CHECK_ARMED,
 723                                   &adapter->tx_ring[i]->state);
 724                 return;
 725         } else if (!(adapter->dcb_cfg.pfc_mode_enable))
 726                 return;
 727
 728         /* update stats for each tc, only valid with PFC enabled */
 729         for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
 730                 switch (hw->mac.type) {
 731                 case ixgbe_mac_82598EB:
 732                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
 733                         break;
 734                 default:
 735                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
 736                 }
 737                 hwstats->pxoffrxc[i] += xoff[i];
 738         }
 739
 740         /* disarm tx queues that have received xoff frames */
 741         for (i = 0; i < adapter->num_tx_queues; i++) {
 742                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
 743                 u32 tc = ixgbe_dcb_txq_to_tc(adapter, tx_ring->reg_idx);
 744
 745                 if (xoff[tc])
 746                         clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 747         }
 748 }
 749
 750 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
 751 {
 752         return ring->tx_stats.completed;
 753 }
 754
 755 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
 756 {
 757         struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
 758         struct ixgbe_hw *hw = &adapter->hw;
 759
 760         u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
 761         u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
 762
 763         if (head != tail)
 764                 return (head < tail) ?
 765                         tail - head : (tail + ring->count - head);
 766
 767         return 0;
 768 }
 769
 770 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
 771 {
 772         u32 tx_done = ixgbe_get_tx_completed(tx_ring);
 773         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
 774         u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
 775         bool ret = false;
 776
 777         clear_check_for_tx_hang(tx_ring);
 778
 779         /*
 780          * Check for a hung queue, but be thorough. This verifies
 781          * that a transmit has been completed since the previous
 782          * check AND there is at least one packet pending. The
 783          * ARMED bit is set to indicate a potential hang. The
 784          * bit is cleared if a pause frame is received to remove
 785          * false hang detection due to PFC or 802.3x frames. By
 786          * requiring this to fail twice we avoid races with
 787          * pfc clearing the ARMED bit and conditions where we
 788          * run the check_tx_hang logic with a transmit completion
 789          * pending but without time to complete it yet.
 790          */
 791         if ((tx_done_old == tx_done) && tx_pending) {
 792                 /* make sure it is true for two checks in a row */
 793                 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
 794                                        &tx_ring->state);
 795         } else {
 796                 /* update completed stats and continue */
 797                 tx_ring->tx_stats.tx_done_old = tx_done;
 798                 /* reset the countdown */
 799                 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 800         }
 801
 802         return ret;
 803 }
 804
 805 #define IXGBE_MAX_TXD_PWR       14
 806 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 807
 808 /* Tx Descriptors needed, worst case */
 809 #define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \
 810                          (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0))
 811 #define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \
 812         MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */
 813
 814 static void ixgbe_tx_timeout(struct net_device *netdev);
 815
 816 /**
 817  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
 818  * @q_vector: structure containing interrupt and ring information
 819  * @tx_ring: tx ring to clean
 820  **/
 821 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 822                                struct ixgbe_ring *tx_ring)
 823 {
 824         struct ixgbe_adapter *adapter = q_vector->adapter;
 825         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 826         struct ixgbe_tx_buffer *tx_buffer_info;
 827         unsigned int total_bytes = 0, total_packets = 0;
 828         u16 i, eop, count = 0;
 829
 830         i = tx_ring->next_to_clean;
 831         eop = tx_ring->tx_buffer_info[i].next_to_watch;
 832         eop_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 833
 834         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
 835                (count < tx_ring->work_limit)) {
 836                 bool cleaned = false;
 837                 rmb(); /* read buffer_info after eop_desc */
 838                 for ( ; !cleaned; count++) {
 839                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 840                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 841
 842                         tx_desc->wb.status = 0;
 843                         cleaned = (i == eop);
 844
 845                         i++;
 846                         if (i == tx_ring->count)
 847                                 i = 0;
 848
 849                         if (cleaned && tx_buffer_info->skb) {
 850                                 total_bytes += tx_buffer_info->bytecount;
 851                                 total_packets += tx_buffer_info->gso_segs;
 852                         }
 853
 854                         ixgbe_unmap_and_free_tx_resource(tx_ring,
 855                                                          tx_buffer_info);
 856                 }
 857
 858                 tx_ring->tx_stats.completed++;
 859                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
 860                 eop_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 861         }
 862
 863         tx_ring->next_to_clean = i;
 864         tx_ring->total_bytes += total_bytes;
 865         tx_ring->total_packets += total_packets;
 866         u64_stats_update_begin(&tx_ring->syncp);
 867         tx_ring->stats.packets += total_packets;
 868         tx_ring->stats.bytes += total_bytes;
 869         u64_stats_update_end(&tx_ring->syncp);
 870
 871         if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
 872                 /* schedule immediate reset if we believe we hung */
 873                 struct ixgbe_hw *hw = &adapter->hw;
 874                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 875                 e_err(drv, "Detected Tx Unit Hang\n"
 876                         "  Tx Queue             <%d>\n"
 877                         "  TDH, TDT             <%x>, <%x>\n"
 878                         "  next_to_use          <%x>\n"
 879                         "  next_to_clean        <%x>\n"
 880                         "tx_buffer_info[next_to_clean]\n"
 881                         "  time_stamp           <%lx>\n"
 882                         "  jiffies              <%lx>\n",
 883                         tx_ring->queue_index,
 884                         IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
 885                         IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
 886                         tx_ring->next_to_use, eop,
 887                         tx_ring->tx_buffer_info[eop].time_stamp, jiffies);
 888
 889                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 890
 891                 e_info(probe,
 892                        "tx hang %d detected on queue %d, resetting adapter\n",
 893                         adapter->tx_timeout_count + 1, tx_ring->queue_index);
 894
 895                 /* schedule immediate reset if we believe we hung */
 896                 ixgbe_tx_timeout(adapter->netdev);
 897
 898                 /* the adapter is about to reset, no point in enabling stuff */
 899                 return true;
 900         }
 901
 902 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 903         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
 904                      (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
 905                 /* Make sure that anybody stopping the queue after this
 906                  * sees the new next_to_clean.
 907                  */
 908                 smp_mb();
 909                 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
 910                     !test_bit(__IXGBE_DOWN, &adapter->state)) {
 911                         netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
 912                         ++tx_ring->tx_stats.restart_queue;
 913                 }
 914         }
 915
 916         return count < tx_ring->work_limit;
 917 }
 918
 919 #ifdef CONFIG_IXGBE_DCA
 920 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
 921                                 struct ixgbe_ring *rx_ring,
 922                                 int cpu)
 923 {
 924         struct ixgbe_hw *hw = &adapter->hw;
 925         u32 rxctrl;
 926         u8 reg_idx = rx_ring->reg_idx;
 927
 928         rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
 929         switch (hw->mac.type) {
 930         case ixgbe_mac_82598EB:
 931                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
 932                 rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
 933                 break;
 934         case ixgbe_mac_82599EB:
 935         case ixgbe_mac_X540:
 936                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
 937                 rxctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
 938                            IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
 939                 break;
 940         default:
 941                 break;
 942         }
 943         rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
 944         rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
 945         rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
 946         rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_WRO_EN |
 947                     IXGBE_DCA_RXCTRL_DESC_HSRO_EN);
 948         IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
 949 }
 950
 951 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
 952                                 struct ixgbe_ring *tx_ring,
 953                                 int cpu)
 954 {
 955         struct ixgbe_hw *hw = &adapter->hw;
 956         u32 txctrl;
 957         u8 reg_idx = tx_ring->reg_idx;
 958
 959         switch (hw->mac.type) {
 960         case ixgbe_mac_82598EB:
 961                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
 962                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
 963                 txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
 964                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 965                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
 966                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
 967                 break;
 968         case ixgbe_mac_82599EB:
 969         case ixgbe_mac_X540:
 970                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
 971                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
 972                 txctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
 973                            IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
 974                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 975                 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN;
 976                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
 977                 break;
 978         default:
 979                 break;
 980         }
 981 }
 982
 983 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
 984 {
 985         struct ixgbe_adapter *adapter = q_vector->adapter;
 986         int cpu = get_cpu();
 987         long r_idx;
 988         int i;
 989
 990         if (q_vector->cpu == cpu)
 991                 goto out_no_update;
 992
 993         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
 994         for (i = 0; i < q_vector->txr_count; i++) {
 995                 ixgbe_update_tx_dca(adapter, adapter->tx_ring[r_idx], cpu);
 996                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
 997                                       r_idx + 1);
 998         }
 999
1000         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
1001         for (i = 0; i < q_vector->rxr_count; i++) {
1002                 ixgbe_update_rx_dca(adapter, adapter->rx_ring[r_idx], cpu);
1003                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
1004                                       r_idx + 1);
1005         }
1006
1007         q_vector->cpu = cpu;
1008 out_no_update:
1009         put_cpu();
1010 }
1011
1012 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
1013 {
1014         int num_q_vectors;
1015         int i;
1016
1017         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
1018                 return;
1019
1020         /* always use CB2 mode, difference is masked in the CB driver */
1021         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
1022
1023         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
1024                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1025         else
1026                 num_q_vectors = 1;
1027
1028         for (i = 0; i < num_q_vectors; i++) {
1029                 adapter->q_vector[i]->cpu = -1;
1030                 ixgbe_update_dca(adapter->q_vector[i]);
1031         }
1032 }
1033
1034 static int __ixgbe_notify_dca(struct device *dev, void *data)
1035 {
1036         struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1037         unsigned long event = *(unsigned long *)data;
1038
1039         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
1040                 return 0;
1041
1042         switch (event) {
1043         case DCA_PROVIDER_ADD:
1044                 /* if we're already enabled, don't do it again */
1045                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1046                         break;
1047                 if (dca_add_requester(dev) == 0) {
1048                         adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1049                         ixgbe_setup_dca(adapter);
1050                         break;
1051                 }
1052                 /* Fall Through since DCA is disabled. */
1053         case DCA_PROVIDER_REMOVE:
1054                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1055                         dca_remove_requester(dev);
1056                         adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1057                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
1058                 }
1059                 break;
1060         }
1061
1062         return 0;
1063 }
1064
1065 #endif /* CONFIG_IXGBE_DCA */
1066 /**
1067  * ixgbe_receive_skb - Send a completed packet up the stack
1068  * @adapter: board private structure
1069  * @skb: packet to send up
1070  * @status: hardware indication of status of receive
1071  * @rx_ring: rx descriptor ring (for a specific queue) to setup
1072  * @rx_desc: rx descriptor
1073  **/
1074 static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1075                               struct sk_buff *skb, u8 status,
1076                               struct ixgbe_ring *ring,
1077                               union ixgbe_adv_rx_desc *rx_desc)
1078 {
1079         struct ixgbe_adapter *adapter = q_vector->adapter;
1080         struct napi_struct *napi = &q_vector->napi;
1081         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1082         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1083
1084         if (is_vlan && (tag & VLAN_VID_MASK))
1085                 __vlan_hwaccel_put_tag(skb, tag);
1086
1087         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1088                 napi_gro_receive(napi, skb);
1089         else
1090                 netif_rx(skb);
1091 }
1092
1093 /**
1094  * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1095  * @adapter: address of board private structure
1096  * @status_err: hardware indication of status of receive
1097  * @skb: skb currently being received and modified
1098  **/
1099 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1100                                      union ixgbe_adv_rx_desc *rx_desc,
1101                                      struct sk_buff *skb)
1102 {
1103         u32 status_err = le32_to_cpu(rx_desc->wb.upper.status_error);
1104
1105         skb_checksum_none_assert(skb);
1106
1107         /* Rx csum disabled */
1108         if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1109                 return;
1110
1111         /* if IP and error */
1112         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1113             (status_err & IXGBE_RXDADV_ERR_IPE)) {
1114                 adapter->hw_csum_rx_error++;
1115                 return;
1116         }
1117
1118         if (!(status_err & IXGBE_RXD_STAT_L4CS))
1119                 return;
1120
1121         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1122                 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1123
1124                 /*
1125                  * 82599 errata, UDP frames with a 0 checksum can be marked as
1126                  * checksum errors.
1127                  */
1128                 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1129                     (adapter->hw.mac.type == ixgbe_mac_82599EB))
1130                         return;
1131
1132                 adapter->hw_csum_rx_error++;
1133                 return;
1134         }
1135
1136         /* It must be a TCP or UDP packet with a valid checksum */
1137         skb->ip_summed = CHECKSUM_UNNECESSARY;
1138 }
1139
1140 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1141 {
1142         /*
1143          * Force memory writes to complete before letting h/w
1144          * know there are new descriptors to fetch.  (Only
1145          * applicable for weak-ordered memory model archs,
1146          * such as IA-64).
1147          */
1148         wmb();
1149         writel(val, rx_ring->tail);
1150 }
1151
1152 /**
1153  * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1154  * @rx_ring: ring to place buffers on
1155  * @cleaned_count: number of buffers to replace
1156  **/
1157 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1158 {
1159         union ixgbe_adv_rx_desc *rx_desc;
1160         struct ixgbe_rx_buffer *bi;
1161         struct sk_buff *skb;
1162         u16 i = rx_ring->next_to_use;
1163
1164         /* do nothing if no valid netdev defined */
1165         if (!rx_ring->netdev)
1166                 return;
1167
1168         while (cleaned_count--) {
1169                 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1170                 bi = &rx_ring->rx_buffer_info[i];
1171                 skb = bi->skb;
1172
1173                 if (!skb) {
1174                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1175                                                         rx_ring->rx_buf_len);
1176                         if (!skb) {
1177                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1178                                 goto no_buffers;
1179                         }
1180                         /* initialize queue mapping */
1181                         skb_record_rx_queue(skb, rx_ring->queue_index);
1182                         bi->skb = skb;
1183                 }
1184
1185                 if (!bi->dma) {
1186                         bi->dma = dma_map_single(rx_ring->dev,
1187                                                  skb->data,
1188                                                  rx_ring->rx_buf_len,
1189                                                  DMA_FROM_DEVICE);
1190                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1191                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1192                                 bi->dma = 0;
1193                                 goto no_buffers;
1194                         }
1195                 }
1196
1197                 if (ring_is_ps_enabled(rx_ring)) {
1198                         if (!bi->page) {
1199                                 bi->page = netdev_alloc_page(rx_ring->netdev);
1200                                 if (!bi->page) {
1201                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1202                                         goto no_buffers;
1203                                 }
1204                         }
1205
1206                         if (!bi->page_dma) {
1207                                 /* use a half page if we're re-using */
1208                                 bi->page_offset ^= PAGE_SIZE / 2;
1209                                 bi->page_dma = dma_map_page(rx_ring->dev,
1210                                                             bi->page,
1211                                                             bi->page_offset,
1212                                                             PAGE_SIZE / 2,
1213                                                             DMA_FROM_DEVICE);
1214                                 if (dma_mapping_error(rx_ring->dev,
1215                                                       bi->page_dma)) {
1216                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1217                                         bi->page_dma = 0;
1218                                         goto no_buffers;
1219                                 }
1220                         }
1221
1222                         /* Refresh the desc even if buffer_addrs didn't change
1223                          * because each write-back erases this info. */
1224                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1225                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1226                 } else {
1227                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1228                         rx_desc->read.hdr_addr = 0;
1229                 }
1230
1231                 i++;
1232                 if (i == rx_ring->count)
1233                         i = 0;
1234         }
1235
1236 no_buffers:
1237         if (rx_ring->next_to_use != i) {
1238                 rx_ring->next_to_use = i;
1239                 ixgbe_release_rx_desc(rx_ring, i);
1240         }
1241 }
1242
1243 static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1244 {
1245         /* HW will not DMA in data larger than the given buffer, even if it
1246          * parses the (NFS, of course) header to be larger.  In that case, it
1247          * fills the header buffer and spills the rest into the page.
1248          */
1249         u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1250         u16 hlen = (hdr_info &  IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1251                     IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1252         if (hlen > IXGBE_RX_HDR_SIZE)
1253                 hlen = IXGBE_RX_HDR_SIZE;
1254         return hlen;
1255 }
1256
1257 /**
1258  * ixgbe_transform_rsc_queue - change rsc queue into a full packet
1259  * @skb: pointer to the last skb in the rsc queue
1260  *
1261  * This function changes a queue full of hw rsc buffers into a completed
1262  * packet.  It uses the ->prev pointers to find the first packet and then
1263  * turns it into the frag list owner.
1264  **/
1265 static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1266 {
1267         unsigned int frag_list_size = 0;
1268         unsigned int skb_cnt = 1;
1269
1270         while (skb->prev) {
1271                 struct sk_buff *prev = skb->prev;
1272                 frag_list_size += skb->len;
1273                 skb->prev = NULL;
1274                 skb = prev;
1275                 skb_cnt++;
1276         }
1277
1278         skb_shinfo(skb)->frag_list = skb->next;
1279         skb->next = NULL;
1280         skb->len += frag_list_size;
1281         skb->data_len += frag_list_size;
1282         skb->truesize += frag_list_size;
1283         IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
1284
1285         return skb;
1286 }
1287
1288 static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
1289 {
1290         return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
1291                 IXGBE_RXDADV_RSCCNT_MASK);
1292 }
1293
1294 static void ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1295                                struct ixgbe_ring *rx_ring,
1296                                int *work_done, int work_to_do)
1297 {
1298         struct ixgbe_adapter *adapter = q_vector->adapter;
1299         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
1300         struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
1301         struct sk_buff *skb;
1302         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1303         const int current_node = numa_node_id();
1304 #ifdef IXGBE_FCOE
1305         int ddp_bytes = 0;
1306 #endif /* IXGBE_FCOE */
1307         u32 staterr;
1308         u16 i;
1309         u16 cleaned_count = 0;
1310         bool pkt_is_rsc = false;
1311
1312         i = rx_ring->next_to_clean;
1313         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1314         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1315
1316         while (staterr & IXGBE_RXD_STAT_DD) {
1317                 u32 upper_len = 0;
1318
1319                 rmb(); /* read descriptor and rx_buffer_info after status DD */
1320
1321                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1322
1323                 skb = rx_buffer_info->skb;
1324                 rx_buffer_info->skb = NULL;
1325                 prefetch(skb->data);
1326
1327                 if (ring_is_rsc_enabled(rx_ring))
1328                         pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1329
1330                 /* if this is a skb from previous receive DMA will be 0 */
1331                 if (rx_buffer_info->dma) {
1332                         u16 hlen;
1333                         if (pkt_is_rsc &&
1334                             !(staterr & IXGBE_RXD_STAT_EOP) &&
1335                             !skb->prev) {
1336                                 /*
1337                                  * When HWRSC is enabled, delay unmapping
1338                                  * of the first packet. It carries the
1339                                  * header information, HW may still
1340                                  * access the header after the writeback.
1341                                  * Only unmap it when EOP is reached
1342                                  */
1343                                 IXGBE_RSC_CB(skb)->delay_unmap = true;
1344                                 IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1345                         } else {
1346                                 dma_unmap_single(rx_ring->dev,
1347                                                  rx_buffer_info->dma,
1348                                                  rx_ring->rx_buf_len,
1349                                                  DMA_FROM_DEVICE);
1350                         }
1351                         rx_buffer_info->dma = 0;
1352
1353                         if (ring_is_ps_enabled(rx_ring)) {
1354                                 hlen = ixgbe_get_hlen(rx_desc);
1355                                 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1356                         } else {
1357                                 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1358                         }
1359
1360                         skb_put(skb, hlen);
1361                 } else {
1362                         /* assume packet split since header is unmapped */
1363                         upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1364                 }
1365
1366                 if (upper_len) {
1367                         dma_unmap_page(rx_ring->dev,
1368                                        rx_buffer_info->page_dma,
1369                                        PAGE_SIZE / 2,
1370                                        DMA_FROM_DEVICE);
1371                         rx_buffer_info->page_dma = 0;
1372                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1373                                            rx_buffer_info->page,
1374                                            rx_buffer_info->page_offset,
1375                                            upper_len);
1376
1377                         if ((page_count(rx_buffer_info->page) == 1) &&
1378                             (page_to_nid(rx_buffer_info->page) == current_node))
1379                                 get_page(rx_buffer_info->page);
1380                         else
1381                                 rx_buffer_info->page = NULL;
1382
1383                         skb->len += upper_len;
1384                         skb->data_len += upper_len;
1385                         skb->truesize += upper_len;
1386                 }
1387
1388                 i++;
1389                 if (i == rx_ring->count)
1390                         i = 0;
1391
1392                 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1393                 prefetch(next_rxd);
1394                 cleaned_count++;
1395
1396                 if (pkt_is_rsc) {
1397                         u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1398                                      IXGBE_RXDADV_NEXTP_SHIFT;
1399                         next_buffer = &rx_ring->rx_buffer_info[nextp];
1400                 } else {
1401                         next_buffer = &rx_ring->rx_buffer_info[i];
1402                 }
1403
1404                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1405                         if (ring_is_ps_enabled(rx_ring)) {
1406                                 rx_buffer_info->skb = next_buffer->skb;
1407                                 rx_buffer_info->dma = next_buffer->dma;
1408                                 next_buffer->skb = skb;
1409                                 next_buffer->dma = 0;
1410                         } else {
1411                                 skb->next = next_buffer->skb;
1412                                 skb->next->prev = skb;
1413                         }
1414                         rx_ring->rx_stats.non_eop_descs++;
1415                         goto next_desc;
1416                 }
1417
1418                 if (skb->prev) {
1419                         skb = ixgbe_transform_rsc_queue(skb);
1420                         /* if we got here without RSC the packet is invalid */
1421                         if (!pkt_is_rsc) {
1422                                 __pskb_trim(skb, 0);
1423                                 rx_buffer_info->skb = skb;
1424                                 goto next_desc;
1425                         }
1426                 }
1427
1428                 if (ring_is_rsc_enabled(rx_ring)) {
1429                         if (IXGBE_RSC_CB(skb)->delay_unmap) {
1430                                 dma_unmap_single(rx_ring->dev,
1431                                                  IXGBE_RSC_CB(skb)->dma,
1432                                                  rx_ring->rx_buf_len,
1433                                                  DMA_FROM_DEVICE);
1434                                 IXGBE_RSC_CB(skb)->dma = 0;
1435                                 IXGBE_RSC_CB(skb)->delay_unmap = false;
1436                         }
1437                 }
1438                 if (pkt_is_rsc) {
1439                         if (ring_is_ps_enabled(rx_ring))
1440                                 rx_ring->rx_stats.rsc_count +=
1441                                         skb_shinfo(skb)->nr_frags;
1442                         else
1443                                 rx_ring->rx_stats.rsc_count +=
1444                                         IXGBE_RSC_CB(skb)->skb_cnt;
1445                         rx_ring->rx_stats.rsc_flush++;
1446                 }
1447
1448                 /* ERR_MASK will only have valid bits if EOP set */
1449                 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) {
1450                         /* trim packet back to size 0 and recycle it */
1451                         __pskb_trim(skb, 0);
1452                         rx_buffer_info->skb = skb;
1453                         goto next_desc;
1454                 }
1455
1456                 ixgbe_rx_checksum(adapter, rx_desc, skb);
1457
1458                 /* probably a little skewed due to removing CRC */
1459                 total_rx_bytes += skb->len;
1460                 total_rx_packets++;
1461
1462                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1463 #ifdef IXGBE_FCOE
1464                 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1465                 if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) {
1466                         ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);
1467                         if (!ddp_bytes)
1468                                 goto next_desc;
1469                 }
1470 #endif /* IXGBE_FCOE */
1471                 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1472
1473 next_desc:
1474                 rx_desc->wb.upper.status_error = 0;
1475
1476                 (*work_done)++;
1477                 if (*work_done >= work_to_do)
1478                         break;
1479
1480                 /* return some buffers to hardware, one at a time is too slow */
1481                 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1482                         ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1483                         cleaned_count = 0;
1484                 }
1485
1486                 /* use prefetched values */
1487                 rx_desc = next_rxd;
1488                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1489         }
1490
1491         rx_ring->next_to_clean = i;
1492         cleaned_count = IXGBE_DESC_UNUSED(rx_ring);
1493
1494         if (cleaned_count)
1495                 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1496
1497 #ifdef IXGBE_FCOE
1498         /* include DDPed FCoE data */
1499         if (ddp_bytes > 0) {
1500                 unsigned int mss;
1501
1502                 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1503                         sizeof(struct fc_frame_header) -
1504                         sizeof(struct fcoe_crc_eof);
1505                 if (mss > 512)
1506                         mss &= ~511;
1507                 total_rx_bytes += ddp_bytes;
1508                 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1509         }
1510 #endif /* IXGBE_FCOE */
1511
1512         rx_ring->total_packets += total_rx_packets;
1513         rx_ring->total_bytes += total_rx_bytes;
1514         u64_stats_update_begin(&rx_ring->syncp);
1515         rx_ring->stats.packets += total_rx_packets;
1516         rx_ring->stats.bytes += total_rx_bytes;
1517         u64_stats_update_end(&rx_ring->syncp);
1518 }
1519
1520 static int ixgbe_clean_rxonly(struct napi_struct *, int);
1521 /**
1522  * ixgbe_configure_msix - Configure MSI-X hardware
1523  * @adapter: board private structure
1524  *
1525  * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1526  * interrupts.
1527  **/
1528 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1529 {
1530         struct ixgbe_q_vector *q_vector;
1531         int i, q_vectors, v_idx, r_idx;
1532         u32 mask;
1533
1534         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1535
1536         /*
1537          * Populate the IVAR table and set the ITR values to the
1538          * corresponding register.
1539          */
1540         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1541                 q_vector = adapter->q_vector[v_idx];
1542                 /* XXX for_each_set_bit(...) */
1543                 r_idx = find_first_bit(q_vector->rxr_idx,
1544                                        adapter->num_rx_queues);
1545
1546                 for (i = 0; i < q_vector->rxr_count; i++) {
1547                         u8 reg_idx = adapter->rx_ring[r_idx]->reg_idx;
1548                         ixgbe_set_ivar(adapter, 0, reg_idx, v_idx);
1549                         r_idx = find_next_bit(q_vector->rxr_idx,
1550                                               adapter->num_rx_queues,
1551                                               r_idx + 1);
1552                 }
1553                 r_idx = find_first_bit(q_vector->txr_idx,
1554                                        adapter->num_tx_queues);
1555
1556                 for (i = 0; i < q_vector->txr_count; i++) {
1557                         u8 reg_idx = adapter->tx_ring[r_idx]->reg_idx;
1558                         ixgbe_set_ivar(adapter, 1, reg_idx, v_idx);
1559                         r_idx = find_next_bit(q_vector->txr_idx,
1560                                               adapter->num_tx_queues,
1561                                               r_idx + 1);
1562                 }
1563
1564                 if (q_vector->txr_count && !q_vector->rxr_count)
1565                         /* tx only */
1566                         q_vector->eitr = adapter->tx_eitr_param;
1567                 else if (q_vector->rxr_count)
1568                         /* rx or mixed */
1569                         q_vector->eitr = adapter->rx_eitr_param;
1570
1571                 ixgbe_write_eitr(q_vector);
1572                 /* If Flow Director is enabled, set interrupt affinity */
1573                 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ||
1574                     (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
1575                         /*
1576                          * Allocate the affinity_hint cpumask, assign the mask
1577                          * for this vector, and set our affinity_hint for
1578                          * this irq.
1579                          */
1580                         if (!alloc_cpumask_var(&q_vector->affinity_mask,
1581                                                GFP_KERNEL))
1582                                 return;
1583                         cpumask_set_cpu(v_idx, q_vector->affinity_mask);
1584                         irq_set_affinity_hint(adapter->msix_entries[v_idx].vector,
1585                                               q_vector->affinity_mask);
1586                 }
1587         }
1588
1589         switch (adapter->hw.mac.type) {
1590         case ixgbe_mac_82598EB:
1591                 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1592                                v_idx);
1593                 break;
1594         case ixgbe_mac_82599EB:
1595         case ixgbe_mac_X540:
1596                 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1597                 break;
1598
1599         default:
1600                 break;
1601         }
1602         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1603
1604         /* set up to autoclear timer, and the vectors */
1605         mask = IXGBE_EIMS_ENABLE_MASK;
1606         if (adapter->num_vfs)
1607                 mask &= ~(IXGBE_EIMS_OTHER |
1608                           IXGBE_EIMS_MAILBOX |
1609                           IXGBE_EIMS_LSC);
1610         else
1611                 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1612         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1613 }
1614
1615 enum latency_range {
1616         lowest_latency = 0,
1617         low_latency = 1,
1618         bulk_latency = 2,
1619         latency_invalid = 255
1620 };
1621
1622 /**
1623  * ixgbe_update_itr - update the dynamic ITR value based on statistics
1624  * @adapter: pointer to adapter
1625  * @eitr: eitr setting (ints per sec) to give last timeslice
1626  * @itr_setting: current throttle rate in ints/second
1627  * @packets: the number of packets during this measurement interval
1628  * @bytes: the number of bytes during this measurement interval
1629  *
1630  *      Stores a new ITR value based on packets and byte
1631  *      counts during the last interrupt.  The advantage of per interrupt
1632  *      computation is faster updates and more accurate ITR for the current
1633  *      traffic pattern.  Constants in this function were computed
1634  *      based on theoretical maximum wire speed and thresholds were set based
1635  *      on testing data as well as attempting to minimize response time
1636  *      while increasing bulk throughput.
1637  *      this functionality is controlled by the InterruptThrottleRate module
1638  *      parameter (see ixgbe_param.c)
1639  **/
1640 static u8 ixgbe_update_itr(struct ixgbe_adapter *adapter,
1641                            u32 eitr, u8 itr_setting,
1642                            int packets, int bytes)
1643 {
1644         unsigned int retval = itr_setting;
1645         u32 timepassed_us;
1646         u64 bytes_perint;
1647
1648         if (packets == 0)
1649                 goto update_itr_done;
1650
1651
1652         /* simple throttlerate management
1653          *    0-20MB/s lowest (100000 ints/s)
1654          *   20-100MB/s low   (20000 ints/s)
1655          *  100-1249MB/s bulk (8000 ints/s)
1656          */
1657         /* what was last interrupt timeslice? */
1658         timepassed_us = 1000000/eitr;
1659         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1660
1661         switch (itr_setting) {
1662         case lowest_latency:
1663                 if (bytes_perint > adapter->eitr_low)
1664                         retval = low_latency;
1665                 break;
1666         case low_latency:
1667                 if (bytes_perint > adapter->eitr_high)
1668                         retval = bulk_latency;
1669                 else if (bytes_perint <= adapter->eitr_low)
1670                         retval = lowest_latency;
1671                 break;
1672         case bulk_latency:
1673                 if (bytes_perint <= adapter->eitr_high)
1674                         retval = low_latency;
1675                 break;
1676         }
1677
1678 update_itr_done:
1679         return retval;
1680 }
1681
1682 /**
1683  * ixgbe_write_eitr - write EITR register in hardware specific way
1684  * @q_vector: structure containing interrupt and ring information
1685  *
1686  * This function is made to be called by ethtool and by the driver
1687  * when it needs to update EITR registers at runtime.  Hardware
1688  * specific quirks/differences are taken care of here.
1689  */
1690 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1691 {
1692         struct ixgbe_adapter *adapter = q_vector->adapter;
1693         struct ixgbe_hw *hw = &adapter->hw;
1694         int v_idx = q_vector->v_idx;
1695         u32 itr_reg = EITR_INTS_PER_SEC_TO_REG(q_vector->eitr);
1696
1697         switch (adapter->hw.mac.type) {
1698         case ixgbe_mac_82598EB:
1699                 /* must write high and low 16 bits to reset counter */
1700                 itr_reg |= (itr_reg << 16);
1701                 break;
1702         case ixgbe_mac_82599EB:
1703         case ixgbe_mac_X540:
1704                 /*
1705                  * 82599 and X540 can support a value of zero, so allow it for
1706                  * max interrupt rate, but there is an errata where it can
1707                  * not be zero with RSC
1708                  */
1709                 if (itr_reg == 8 &&
1710                     !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
1711                         itr_reg = 0;
1712
1713                 /*
1714                  * set the WDIS bit to not clear the timer bits and cause an
1715                  * immediate assertion of the interrupt
1716                  */
1717                 itr_reg |= IXGBE_EITR_CNT_WDIS;
1718                 break;
1719         default:
1720                 break;
1721         }
1722         IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1723 }
1724
1725 static void ixgbe_set_itr_msix(struct ixgbe_q_vector *q_vector)
1726 {
1727         struct ixgbe_adapter *adapter = q_vector->adapter;
1728         int i, r_idx;
1729         u32 new_itr;
1730         u8 current_itr, ret_itr;
1731
1732         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
1733         for (i = 0; i < q_vector->txr_count; i++) {
1734                 struct ixgbe_ring *tx_ring = adapter->tx_ring[r_idx];
1735                 ret_itr = ixgbe_update_itr(adapter, q_vector->eitr,
1736                                            q_vector->tx_itr,
1737                                            tx_ring->total_packets,
1738                                            tx_ring->total_bytes);
1739                 /* if the result for this queue would decrease interrupt
1740                  * rate for this vector then use that result */
1741                 q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ?
1742                                     q_vector->tx_itr - 1 : ret_itr);
1743                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
1744                                       r_idx + 1);
1745         }
1746
1747         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
1748         for (i = 0; i < q_vector->rxr_count; i++) {
1749                 struct ixgbe_ring *rx_ring = adapter->rx_ring[r_idx];
1750                 ret_itr = ixgbe_update_itr(adapter, q_vector->eitr,
1751                                            q_vector->rx_itr,
1752                                            rx_ring->total_packets,
1753                                            rx_ring->total_bytes);
1754                 /* if the result for this queue would decrease interrupt
1755                  * rate for this vector then use that result */
1756                 q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ?
1757                                     q_vector->rx_itr - 1 : ret_itr);
1758                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
1759                                       r_idx + 1);
1760         }
1761
1762         current_itr = max(q_vector->rx_itr, q_vector->tx_itr);
1763
1764         switch (current_itr) {
1765         /* counts and packets in update_itr are dependent on these numbers */
1766         case lowest_latency:
1767                 new_itr = 100000;
1768                 break;
1769         case low_latency:
1770                 new_itr = 20000; /* aka hwitr = ~200 */
1771                 break;
1772         case bulk_latency:
1773         default:
1774                 new_itr = 8000;
1775                 break;
1776         }
1777
1778         if (new_itr != q_vector->eitr) {
1779                 /* do an exponential smoothing */
1780                 new_itr = ((q_vector->eitr * 9) + new_itr)/10;
1781
1782                 /* save the algorithm value here, not the smoothed one */
1783                 q_vector->eitr = new_itr;
1784
1785                 ixgbe_write_eitr(q_vector);
1786         }
1787 }
1788
1789 /**
1790  * ixgbe_check_overtemp_task - worker thread to check over tempurature
1791  * @work: pointer to work_struct containing our data
1792  **/
1793 static void ixgbe_check_overtemp_task(struct work_struct *work)
1794 {
1795         struct ixgbe_adapter *adapter = container_of(work,
1796                                                      struct ixgbe_adapter,
1797                                                      check_overtemp_task);
1798         struct ixgbe_hw *hw = &adapter->hw;
1799         u32 eicr = adapter->interrupt_event;
1800
1801         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
1802                 return;
1803
1804         switch (hw->device_id) {
1805         case IXGBE_DEV_ID_82599_T3_LOM: {
1806                 u32 autoneg;
1807                 bool link_up = false;
1808
1809                 if (hw->mac.ops.check_link)
1810                         hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1811
1812                 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) ||
1813                     (eicr & IXGBE_EICR_LSC))
1814                         /* Check if this is due to overtemp */
1815                         if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP)
1816                                 break;
1817                 return;
1818         }
1819         default:
1820                 if (!(eicr & IXGBE_EICR_GPI_SDP0))
1821                         return;
1822                 break;
1823         }
1824         e_crit(drv,
1825                "Network adapter has been stopped because it has over heated. "
1826                "Restart the computer. If the problem persists, "
1827                "power off the system and replace the adapter\n");
1828         /* write to clear the interrupt */
1829         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0);
1830 }
1831
1832 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1833 {
1834         struct ixgbe_hw *hw = &adapter->hw;
1835
1836         if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1837             (eicr & IXGBE_EICR_GPI_SDP1)) {
1838                 e_crit(probe, "Fan has stopped, replace the adapter\n");
1839                 /* write to clear the interrupt */
1840                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1841         }
1842 }
1843
1844 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1845 {
1846         struct ixgbe_hw *hw = &adapter->hw;
1847
1848         if (eicr & IXGBE_EICR_GPI_SDP2) {
1849                 /* Clear the interrupt */
1850                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1851                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
1852                         schedule_work(&adapter->sfp_config_module_task);
1853         }
1854
1855         if (eicr & IXGBE_EICR_GPI_SDP1) {
1856                 /* Clear the interrupt */
1857                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1858                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
1859                         schedule_work(&adapter->multispeed_fiber_task);
1860         }
1861 }
1862
1863 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1864 {
1865         struct ixgbe_hw *hw = &adapter->hw;
1866
1867         adapter->lsc_int++;
1868         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1869         adapter->link_check_timeout = jiffies;
1870         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1871                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1872                 IXGBE_WRITE_FLUSH(hw);
1873                 schedule_work(&adapter->watchdog_task);
1874         }
1875 }
1876
1877 static irqreturn_t ixgbe_msix_lsc(int irq, void *data)
1878 {
1879         struct net_device *netdev = data;
1880         struct ixgbe_adapter *adapter = netdev_priv(netdev);
1881         struct ixgbe_hw *hw = &adapter->hw;
1882         u32 eicr;
1883
1884         /*
1885          * Workaround for Silicon errata.  Use clear-by-write instead
1886          * of clear-by-read.  Reading with EICS will return the
1887          * interrupt causes without clearing, which later be done
1888          * with the write to EICR.
1889          */
1890         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1891         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1892
1893         if (eicr & IXGBE_EICR_LSC)
1894                 ixgbe_check_lsc(adapter);
1895
1896         if (eicr & IXGBE_EICR_MAILBOX)
1897                 ixgbe_msg_task(adapter);
1898
1899         switch (hw->mac.type) {
1900         case ixgbe_mac_82599EB:
1901                 ixgbe_check_sfp_event(adapter, eicr);
1902                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1903                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
1904                         adapter->interrupt_event = eicr;
1905                         schedule_work(&adapter->check_overtemp_task);
1906                 }
1907                 /* now fallthrough to handle Flow Director */
1908         case ixgbe_mac_X540:
1909                 /* Handle Flow Director Full threshold interrupt */
1910                 if (eicr & IXGBE_EICR_FLOW_DIR) {
1911                         int i;
1912                         IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_FLOW_DIR);
1913                         /* Disable transmits before FDIR Re-initialization */
1914                         netif_tx_stop_all_queues(netdev);
1915                         for (i = 0; i < adapter->num_tx_queues; i++) {
1916                                 struct ixgbe_ring *tx_ring =
1917                                                             adapter->tx_ring[i];
1918                                 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1919                                                        &tx_ring->state))
1920                                         schedule_work(&adapter->fdir_reinit_task);
1921                         }
1922                 }
1923                 break;
1924         default:
1925                 break;
1926         }
1927
1928         ixgbe_check_fan_failure(adapter, eicr);
1929
1930         if (!test_bit(__IXGBE_DOWN, &adapter->state))
1931                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER);
1932
1933         return IRQ_HANDLED;
1934 }
1935
1936 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
1937                                            u64 qmask)
1938 {
1939         u32 mask;
1940         struct ixgbe_hw *hw = &adapter->hw;
1941
1942         switch (hw->mac.type) {
1943         case ixgbe_mac_82598EB:
1944                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1945                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1946                 break;
1947         case ixgbe_mac_82599EB:
1948         case ixgbe_mac_X540:
1949                 mask = (qmask & 0xFFFFFFFF);
1950                 if (mask)
1951                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1952                 mask = (qmask >> 32);
1953                 if (mask)
1954                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1955                 break;
1956         default:
1957                 break;
1958         }
1959         /* skip the flush */
1960 }
1961
1962 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1963                                             u64 qmask)
1964 {
1965         u32 mask;
1966         struct ixgbe_hw *hw = &adapter->hw;
1967
1968         switch (hw->mac.type) {
1969         case ixgbe_mac_82598EB:
1970                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1971                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1972                 break;
1973         case ixgbe_mac_82599EB:
1974         case ixgbe_mac_X540:
1975                 mask = (qmask & 0xFFFFFFFF);
1976                 if (mask)
1977                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1978                 mask = (qmask >> 32);
1979                 if (mask)
1980                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1981                 break;
1982         default:
1983                 break;
1984         }
1985         /* skip the flush */
1986 }
1987
1988 static irqreturn_t ixgbe_msix_clean_tx(int irq, void *data)
1989 {
1990         struct ixgbe_q_vector *q_vector = data;
1991         struct ixgbe_adapter  *adapter = q_vector->adapter;
1992         struct ixgbe_ring     *tx_ring;
1993         int i, r_idx;
1994
1995         if (!q_vector->txr_count)
1996                 return IRQ_HANDLED;
1997
1998         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
1999         for (i = 0; i < q_vector->txr_count; i++) {
2000                 tx_ring = adapter->tx_ring[r_idx];
2001                 tx_ring->total_bytes = 0;
2002                 tx_ring->total_packets = 0;
2003                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
2004                                       r_idx + 1);
2005         }
2006
2007         /* EIAM disabled interrupts (on this vector) for us */
2008         napi_schedule(&q_vector->napi);
2009
2010         return IRQ_HANDLED;
2011 }
2012
2013 /**
2014  * ixgbe_msix_clean_rx - single unshared vector rx clean (all queues)
2015  * @irq: unused
2016  * @data: pointer to our q_vector struct for this interrupt vector
2017  **/
2018 static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data)
2019 {
2020         struct ixgbe_q_vector *q_vector = data;
2021         struct ixgbe_adapter  *adapter = q_vector->adapter;
2022         struct ixgbe_ring  *rx_ring;
2023         int r_idx;
2024         int i;
2025
2026 #ifdef CONFIG_IXGBE_DCA
2027         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2028                 ixgbe_update_dca(q_vector);
2029 #endif
2030
2031         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
2032         for (i = 0; i < q_vector->rxr_count; i++) {
2033                 rx_ring = adapter->rx_ring[r_idx];
2034                 rx_ring->total_bytes = 0;
2035                 rx_ring->total_packets = 0;
2036                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
2037                                       r_idx + 1);
2038         }
2039
2040         if (!q_vector->rxr_count)
2041                 return IRQ_HANDLED;
2042
2043         /* EIAM disabled interrupts (on this vector) for us */
2044         napi_schedule(&q_vector->napi);
2045
2046         return IRQ_HANDLED;
2047 }
2048
2049 static irqreturn_t ixgbe_msix_clean_many(int irq, void *data)
2050 {
2051         struct ixgbe_q_vector *q_vector = data;
2052         struct ixgbe_adapter  *adapter = q_vector->adapter;
2053         struct ixgbe_ring  *ring;
2054         int r_idx;
2055         int i;
2056
2057         if (!q_vector->txr_count && !q_vector->rxr_count)
2058                 return IRQ_HANDLED;
2059
2060         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
2061         for (i = 0; i < q_vector->txr_count; i++) {
2062                 ring = adapter->tx_ring[r_idx];
2063                 ring->total_bytes = 0;
2064                 ring->total_packets = 0;
2065                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
2066                                       r_idx + 1);
2067         }
2068
2069         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
2070         for (i = 0; i < q_vector->rxr_count; i++) {
2071                 ring = adapter->rx_ring[r_idx];
2072                 ring->total_bytes = 0;
2073                 ring->total_packets = 0;
2074                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
2075                                       r_idx + 1);
2076         }
2077
2078         /* EIAM disabled interrupts (on this vector) for us */
2079         napi_schedule(&q_vector->napi);
2080
2081         return IRQ_HANDLED;
2082 }
2083
2084 /**
2085  * ixgbe_clean_rxonly - msix (aka one shot) rx clean routine
2086  * @napi: napi struct with our devices info in it
2087  * @budget: amount of work driver is allowed to do this pass, in packets
2088  *
2089  * This function is optimized for cleaning one queue only on a single
2090  * q_vector!!!
2091  **/
2092 static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget)
2093 {
2094         struct ixgbe_q_vector *q_vector =
2095                                container_of(napi, struct ixgbe_q_vector, napi);
2096         struct ixgbe_adapter *adapter = q_vector->adapter;
2097         struct ixgbe_ring *rx_ring = NULL;
2098         int work_done = 0;
2099         long r_idx;
2100
2101 #ifdef CONFIG_IXGBE_DCA
2102         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2103                 ixgbe_update_dca(q_vector);
2104 #endif
2105
2106         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
2107         rx_ring = adapter->rx_ring[r_idx];
2108
2109         ixgbe_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
2110
2111         /* If all Rx work done, exit the polling mode */
2112         if (work_done < budget) {
2113                 napi_complete(napi);
2114                 if (adapter->rx_itr_setting & 1)
2115                         ixgbe_set_itr_msix(q_vector);
2116                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2117                         ixgbe_irq_enable_queues(adapter,
2118                                                 ((u64)1 << q_vector->v_idx));
2119         }
2120
2121         return work_done;
2122 }
2123
2124 /**
2125  * ixgbe_clean_rxtx_many - msix (aka one shot) rx clean routine
2126  * @napi: napi struct with our devices info in it
2127  * @budget: amount of work driver is allowed to do this pass, in packets
2128  *
2129  * This function will clean more than one rx queue associated with a
2130  * q_vector.
2131  **/
2132 static int ixgbe_clean_rxtx_many(struct napi_struct *napi, int budget)
2133 {
2134         struct ixgbe_q_vector *q_vector =
2135                                container_of(napi, struct ixgbe_q_vector, napi);
2136         struct ixgbe_adapter *adapter = q_vector->adapter;
2137         struct ixgbe_ring *ring = NULL;
2138         int work_done = 0, i;
2139         long r_idx;
2140         bool tx_clean_complete = true;
2141
2142 #ifdef CONFIG_IXGBE_DCA
2143         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2144                 ixgbe_update_dca(q_vector);
2145 #endif
2146
2147         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
2148         for (i = 0; i < q_vector->txr_count; i++) {
2149                 ring = adapter->tx_ring[r_idx];
2150                 tx_clean_complete &= ixgbe_clean_tx_irq(q_vector, ring);
2151                 r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues,
2152                                       r_idx + 1);
2153         }
2154
2155         /* attempt to distribute budget to each queue fairly, but don't allow
2156          * the budget to go below 1 because we'll exit polling */
2157         budget /= (q_vector->rxr_count ?: 1);
2158         budget = max(budget, 1);
2159         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
2160         for (i = 0; i < q_vector->rxr_count; i++) {
2161                 ring = adapter->rx_ring[r_idx];
2162                 ixgbe_clean_rx_irq(q_vector, ring, &work_done, budget);
2163                 r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues,
2164                                       r_idx + 1);
2165         }
2166
2167         r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues);
2168         ring = adapter->rx_ring[r_idx];
2169         /* If all Rx work done, exit the polling mode */
2170         if (work_done < budget) {
2171                 napi_complete(napi);
2172                 if (adapter->rx_itr_setting & 1)
2173                         ixgbe_set_itr_msix(q_vector);
2174                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2175                         ixgbe_irq_enable_queues(adapter,
2176                                                 ((u64)1 << q_vector->v_idx));
2177                 return 0;
2178         }
2179
2180         return work_done;
2181 }
2182
2183 /**
2184  * ixgbe_clean_txonly - msix (aka one shot) tx clean routine
2185  * @napi: napi struct with our devices info in it
2186  * @budget: amount of work driver is allowed to do this pass, in packets
2187  *
2188  * This function is optimized for cleaning one queue only on a single
2189  * q_vector!!!
2190  **/
2191 static int ixgbe_clean_txonly(struct napi_struct *napi, int budget)
2192 {
2193         struct ixgbe_q_vector *q_vector =
2194                                container_of(napi, struct ixgbe_q_vector, napi);
2195         struct ixgbe_adapter *adapter = q_vector->adapter;
2196         struct ixgbe_ring *tx_ring = NULL;
2197         int work_done = 0;
2198         long r_idx;
2199
2200 #ifdef CONFIG_IXGBE_DCA
2201         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2202                 ixgbe_update_dca(q_vector);
2203 #endif
2204
2205         r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues);
2206         tx_ring = adapter->tx_ring[r_idx];
2207
2208         if (!ixgbe_clean_tx_irq(q_vector, tx_ring))
2209                 work_done = budget;
2210
2211         /* If all Tx work done, exit the polling mode */
2212         if (work_done < budget) {
2213                 napi_complete(napi);
2214                 if (adapter->tx_itr_setting & 1)
2215                         ixgbe_set_itr_msix(q_vector);
2216                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2217                         ixgbe_irq_enable_queues(adapter,
2218                                                 ((u64)1 << q_vector->v_idx));
2219         }
2220
2221         return work_done;
2222 }
2223
2224 static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
2225                                      int r_idx)
2226 {
2227         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2228         struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
2229
2230         set_bit(r_idx, q_vector->rxr_idx);
2231         q_vector->rxr_count++;
2232         rx_ring->q_vector = q_vector;
2233 }
2234
2235 static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
2236                                      int t_idx)
2237 {
2238         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2239         struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
2240
2241         set_bit(t_idx, q_vector->txr_idx);
2242         q_vector->txr_count++;
2243         tx_ring->q_vector = q_vector;
2244 }
2245
2246 /**
2247  * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
2248  * @adapter: board private structure to initialize
2249  *
2250  * This function maps descriptor rings to the queue-specific vectors
2251  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2252  * one vector per ring/queue, but on a constrained vector budget, we
2253  * group the rings as "efficiently" as possible.  You would add new
2254  * mapping configurations in here.
2255  **/
2256 static int ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2257 {
2258         int q_vectors;
2259         int v_start = 0;
2260         int rxr_idx = 0, txr_idx = 0;
2261         int rxr_remaining = adapter->num_rx_queues;
2262         int txr_remaining = adapter->num_tx_queues;
2263         int i, j;
2264         int rqpv, tqpv;
2265         int err = 0;
2266
2267         /* No mapping required if MSI-X is disabled. */
2268         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2269                 goto out;
2270
2271         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2272
2273         /*
2274          * The ideal configuration...
2275          * We have enough vectors to map one per queue.
2276          */
2277         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
2278                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
2279                         map_vector_to_rxq(adapter, v_start, rxr_idx);
2280
2281                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
2282                         map_vector_to_txq(adapter, v_start, txr_idx);
2283
2284                 goto out;
2285         }
2286
2287         /*
2288          * If we don't have enough vectors for a 1-to-1
2289          * mapping, we'll have to group them so there are
2290          * multiple queues per vector.
2291          */
2292         /* Re-adjusting *qpv takes care of the remainder. */
2293         for (i = v_start; i < q_vectors; i++) {
2294                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
2295                 for (j = 0; j < rqpv; j++) {
2296                         map_vector_to_rxq(adapter, i, rxr_idx);
2297                         rxr_idx++;
2298                         rxr_remaining--;
2299                 }
2300                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
2301                 for (j = 0; j < tqpv; j++) {
2302                         map_vector_to_txq(adapter, i, txr_idx);
2303                         txr_idx++;
2304                         txr_remaining--;
2305                 }
2306         }
2307 out:
2308         return err;
2309 }
2310
2311 /**
2312  * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2313  * @adapter: board private structure
2314  *
2315  * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2316  * interrupts from the kernel.
2317  **/
2318 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2319 {
2320         struct net_device *netdev = adapter->netdev;
2321         irqreturn_t (*handler)(int, void *);
2322         int i, vector, q_vectors, err;
2323         int ri = 0, ti = 0;
2324
2325         /* Decrement for Other and TCP Timer vectors */
2326         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2327
2328         err = ixgbe_map_rings_to_vectors(adapter);
2329         if (err)
2330                 return err;
2331
2332 #define SET_HANDLER(_v) (((_v)->rxr_count && (_v)->txr_count)        \
2333                                           ? &ixgbe_msix_clean_many : \
2334                           (_v)->rxr_count ? &ixgbe_msix_clean_rx   : \
2335                           (_v)->txr_count ? &ixgbe_msix_clean_tx   : \
2336                           NULL)
2337         for (vector = 0; vector < q_vectors; vector++) {
2338                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2339                 handler = SET_HANDLER(q_vector);
2340
2341                 if (handler == &ixgbe_msix_clean_rx) {
2342                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2343                                  "%s-%s-%d", netdev->name, "rx", ri++);
2344                 } else if (handler == &ixgbe_msix_clean_tx) {
2345                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2346                                  "%s-%s-%d", netdev->name, "tx", ti++);
2347                 } else if (handler == &ixgbe_msix_clean_many) {
2348                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2349                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
2350                         ti++;
2351                 } else {
2352                         /* skip this unused q_vector */
2353                         continue;
2354                 }
2355                 err = request_irq(adapter->msix_entries[vector].vector,
2356                                   handler, 0, q_vector->name,
2357                                   q_vector);
2358                 if (err) {
2359                         e_err(probe, "request_irq failed for MSIX interrupt "
2360                               "Error: %d\n", err);
2361                         goto free_queue_irqs;
2362                 }
2363         }
2364
2365         sprintf(adapter->lsc_int_name, "%s:lsc", netdev->name);
2366         err = request_irq(adapter->msix_entries[vector].vector,
2367                           ixgbe_msix_lsc, 0, adapter->lsc_int_name, netdev);
2368         if (err) {
2369                 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2370                 goto free_queue_irqs;
2371         }
2372
2373         return 0;
2374
2375 free_queue_irqs:
2376         for (i = vector - 1; i >= 0; i--)
2377                 free_irq(adapter->msix_entries[--vector].vector,
2378                          adapter->q_vector[i]);
2379         adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2380         pci_disable_msix(adapter->pdev);
2381         kfree(adapter->msix_entries);
2382         adapter->msix_entries = NULL;
2383         return err;
2384 }
2385
2386 static void ixgbe_set_itr(struct ixgbe_adapter *adapter)
2387 {
2388         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2389         struct ixgbe_ring *rx_ring = adapter->rx_ring[0];
2390         struct ixgbe_ring *tx_ring = adapter->tx_ring[0];
2391         u32 new_itr = q_vector->eitr;
2392         u8 current_itr;
2393
2394         q_vector->tx_itr = ixgbe_update_itr(adapter, new_itr,
2395                                             q_vector->tx_itr,
2396                                             tx_ring->total_packets,
2397                                             tx_ring->total_bytes);
2398         q_vector->rx_itr = ixgbe_update_itr(adapter, new_itr,
2399                                             q_vector->rx_itr,
2400                                             rx_ring->total_packets,
2401                                             rx_ring->total_bytes);
2402
2403         current_itr = max(q_vector->rx_itr, q_vector->tx_itr);
2404
2405         switch (current_itr) {
2406         /* counts and packets in update_itr are dependent on these numbers */
2407         case lowest_latency:
2408                 new_itr = 100000;
2409                 break;
2410         case low_latency:
2411                 new_itr = 20000; /* aka hwitr = ~200 */
2412                 break;
2413         case bulk_latency:
2414                 new_itr = 8000;
2415                 break;
2416         default:
2417                 break;
2418         }
2419
2420         if (new_itr != q_vector->eitr) {
2421                 /* do an exponential smoothing */
2422                 new_itr = ((q_vector->eitr * 9) + new_itr)/10;
2423
2424                 /* save the algorithm value here */
2425                 q_vector->eitr = new_itr;
2426
2427                 ixgbe_write_eitr(q_vector);
2428         }
2429 }
2430
2431 /**
2432  * ixgbe_irq_enable - Enable default interrupt generation settings
2433  * @adapter: board private structure
2434  **/
2435 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2436                                     bool flush)
2437 {
2438         u32 mask;
2439
2440         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2441         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
2442                 mask |= IXGBE_EIMS_GPI_SDP0;
2443         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2444                 mask |= IXGBE_EIMS_GPI_SDP1;
2445         switch (adapter->hw.mac.type) {
2446         case ixgbe_mac_82599EB:
2447         case ixgbe_mac_X540:
2448                 mask |= IXGBE_EIMS_ECC;
2449                 mask |= IXGBE_EIMS_GPI_SDP1;
2450                 mask |= IXGBE_EIMS_GPI_SDP2;
2451                 if (adapter->num_vfs)
2452                         mask |= IXGBE_EIMS_MAILBOX;
2453                 break;
2454         default:
2455                 break;
2456         }
2457         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
2458             adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
2459                 mask |= IXGBE_EIMS_FLOW_DIR;
2460
2461         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2462         if (queues)
2463                 ixgbe_irq_enable_queues(adapter, ~0);
2464         if (flush)
2465                 IXGBE_WRITE_FLUSH(&adapter->hw);
2466
2467         if (adapter->num_vfs > 32) {
2468                 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
2469                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2470         }
2471 }
2472
2473 /**
2474  * ixgbe_intr - legacy mode Interrupt Handler
2475  * @irq: interrupt number
2476  * @data: pointer to a network interface device structure
2477  **/
2478 static irqreturn_t ixgbe_intr(int irq, void *data)
2479 {
2480         struct net_device *netdev = data;
2481         struct ixgbe_adapter *adapter = netdev_priv(netdev);
2482         struct ixgbe_hw *hw = &adapter->hw;
2483         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2484         u32 eicr;
2485
2486         /*
2487          * Workaround for silicon errata on 82598.  Mask the interrupts
2488          * before the read of EICR.
2489          */
2490         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2491
2492         /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2493          * therefore no explict interrupt disable is necessary */
2494         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2495         if (!eicr) {
2496                 /*
2497                  * shared interrupt alert!
2498                  * make sure interrupts are enabled because the read will
2499                  * have disabled interrupts due to EIAM
2500                  * finish the workaround of silicon errata on 82598.  Unmask
2501                  * the interrupt that we masked before the EICR read.
2502                  */
2503                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2504                         ixgbe_irq_enable(adapter, true, true);
2505                 return IRQ_NONE;        /* Not our interrupt */
2506         }
2507
2508         if (eicr & IXGBE_EICR_LSC)
2509                 ixgbe_check_lsc(adapter);
2510
2511         switch (hw->mac.type) {
2512         case ixgbe_mac_82599EB:
2513                 ixgbe_check_sfp_event(adapter, eicr);
2514                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2515                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
2516                         adapter->interrupt_event = eicr;
2517                         schedule_work(&adapter->check_overtemp_task);
2518                 }
2519                 break;
2520         default:
2521                 break;
2522         }
2523
2524         ixgbe_check_fan_failure(adapter, eicr);
2525
2526         if (napi_schedule_prep(&(q_vector->napi))) {
2527                 adapter->tx_ring[0]->total_packets = 0;
2528                 adapter->tx_ring[0]->total_bytes = 0;
2529                 adapter->rx_ring[0]->total_packets = 0;
2530                 adapter->rx_ring[0]->total_bytes = 0;
2531                 /* would disable interrupts here but EIAM disabled it */
2532                 __napi_schedule(&(q_vector->napi));
2533         }
2534
2535         /*
2536          * re-enable link(maybe) and non-queue interrupts, no flush.
2537          * ixgbe_poll will re-enable the queue interrupts
2538          */
2539
2540         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2541                 ixgbe_irq_enable(adapter, false, false);
2542
2543         return IRQ_HANDLED;
2544 }
2545
2546 static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2547 {
2548         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2549
2550         for (i = 0; i < q_vectors; i++) {
2551                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2552                 bitmap_zero(q_vector->rxr_idx, MAX_RX_QUEUES);
2553                 bitmap_zero(q_vector->txr_idx, MAX_TX_QUEUES);
2554                 q_vector->rxr_count = 0;
2555                 q_vector->txr_count = 0;
2556         }
2557 }
2558
2559 /**
2560  * ixgbe_request_irq - initialize interrupts
2561  * @adapter: board private structure
2562  *
2563  * Attempts to configure interrupts using the best available
2564  * capabilities of the hardware and kernel.
2565  **/
2566 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2567 {
2568         struct net_device *netdev = adapter->netdev;
2569         int err;
2570
2571         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2572                 err = ixgbe_request_msix_irqs(adapter);
2573         } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
2574                 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2575                                   netdev->name, netdev);
2576         } else {
2577                 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2578                                   netdev->name, netdev);
2579         }
2580
2581         if (err)
2582                 e_err(probe, "request_irq failed, Error %d\n", err);
2583
2584         return err;
2585 }
2586
2587 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2588 {
2589         struct net_device *netdev = adapter->netdev;
2590
2591         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2592                 int i, q_vectors;
2593
2594                 q_vectors = adapter->num_msix_vectors;
2595
2596                 i = q_vectors - 1;
2597                 free_irq(adapter->msix_entries[i].vector, netdev);
2598
2599                 i--;
2600                 for (; i >= 0; i--) {
2601                         /* free only the irqs that were actually requested */
2602                         if (!adapter->q_vector[i]->rxr_count &&
2603                             !adapter->q_vector[i]->txr_count)
2604                                 continue;
2605
2606                         free_irq(adapter->msix_entries[i].vector,
2607                                  adapter->q_vector[i]);
2608                 }
2609
2610                 ixgbe_reset_q_vectors(adapter);
2611         } else {
2612                 free_irq(adapter->pdev->irq, netdev);
2613         }
2614 }
2615
2616 /**
2617  * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2618  * @adapter: board private structure
2619  **/
2620 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2621 {
2622         switch (adapter->hw.mac.type) {
2623         case ixgbe_mac_82598EB:
2624                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2625                 break;
2626         case ixgbe_mac_82599EB:
2627         case ixgbe_mac_X540:
2628                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2629                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2630                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2631                 if (adapter->num_vfs > 32)
2632                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
2633                 break;
2634         default:
2635                 break;
2636         }
2637         IXGBE_WRITE_FLUSH(&adapter->hw);
2638         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2639                 int i;
2640                 for (i = 0; i < adapter->num_msix_vectors; i++)
2641                         synchronize_irq(adapter->msix_entries[i].vector);
2642         } else {
2643                 synchronize_irq(adapter->pdev->irq);
2644         }
2645 }
2646
2647 /**
2648  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2649  *
2650  **/
2651 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2652 {
2653         struct ixgbe_hw *hw = &adapter->hw;
2654
2655         IXGBE_WRITE_REG(hw, IXGBE_EITR(0),
2656                         EITR_INTS_PER_SEC_TO_REG(adapter->rx_eitr_param));
2657
2658         ixgbe_set_ivar(adapter, 0, 0, 0);
2659         ixgbe_set_ivar(adapter, 1, 0, 0);
2660
2661         map_vector_to_rxq(adapter, 0, 0);
2662         map_vector_to_txq(adapter, 0, 0);
2663
2664         e_info(hw, "Legacy interrupt IVAR setup done\n");
2665 }
2666
2667 /**
2668  * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2669  * @adapter: board private structure
2670  * @ring: structure containing ring specific data
2671  *
2672  * Configure the Tx descriptor ring after a reset.
2673  **/
2674 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2675                              struct ixgbe_ring *ring)
2676 {
2677         struct ixgbe_hw *hw = &adapter->hw;
2678         u64 tdba = ring->dma;
2679         int wait_loop = 10;
2680         u32 txdctl;
2681         u8 reg_idx = ring->reg_idx;
2682
2683         /* disable queue to avoid issues while updating state */
2684         txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2685         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx),
2686                         txdctl & ~IXGBE_TXDCTL_ENABLE);
2687         IXGBE_WRITE_FLUSH(hw);
2688
2689         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2690                         (tdba & DMA_BIT_MASK(32)));
2691         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2692         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2693                         ring->count * sizeof(union ixgbe_adv_tx_desc));
2694         IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2695         IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2696         ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2697
2698         /* configure fetching thresholds */
2699         if (adapter->rx_itr_setting == 0) {
2700                 /* cannot set wthresh when itr==0 */
2701                 txdctl &= ~0x007F0000;
2702         } else {
2703                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
2704                 txdctl |= (8 << 16);
2705         }
2706         if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
2707                 /* PThresh workaround for Tx hang with DFP enabled. */
2708                 txdctl |= 32;
2709         }
2710
2711         /* reinitialize flowdirector state */
2712         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2713             adapter->atr_sample_rate) {
2714                 ring->atr_sample_rate = adapter->atr_sample_rate;
2715                 ring->atr_count = 0;
2716                 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2717         } else {
2718                 ring->atr_sample_rate = 0;
2719         }
2720
2721         clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2722
2723         /* enable queue */
2724         txdctl |= IXGBE_TXDCTL_ENABLE;
2725         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2726
2727         /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2728         if (hw->mac.type == ixgbe_mac_82598EB &&
2729             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2730                 return;
2731
2732         /* poll to verify queue is enabled */
2733         do {
2734                 msleep(1);
2735                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2736         } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2737         if (!wait_loop)
2738                 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2739 }
2740
2741 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2742 {
2743         struct ixgbe_hw *hw = &adapter->hw;
2744         u32 rttdcs;
2745         u32 mask;
2746
2747         if (hw->mac.type == ixgbe_mac_82598EB)
2748                 return;
2749
2750         /* disable the arbiter while setting MTQC */
2751         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2752         rttdcs |= IXGBE_RTTDCS_ARBDIS;
2753         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2754
2755         /* set transmit pool layout */
2756         mask = (IXGBE_FLAG_SRIOV_ENABLED | IXGBE_FLAG_DCB_ENABLED);
2757         switch (adapter->flags & mask) {
2758
2759         case (IXGBE_FLAG_SRIOV_ENABLED):
2760                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2761                                 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2762                 break;
2763
2764         case (IXGBE_FLAG_DCB_ENABLED):
2765                 /* We enable 8 traffic classes, DCB only */
2766                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2767                               (IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ));
2768                 break;
2769
2770         default:
2771                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2772                 break;
2773         }
2774
2775         /* re-enable the arbiter */
2776         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2777         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2778 }
2779
2780 /**
2781  * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2782  * @adapter: board private structure
2783  *
2784  * Configure the Tx unit of the MAC after a reset.
2785  **/
2786 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2787 {
2788         struct ixgbe_hw *hw = &adapter->hw;
2789         u32 dmatxctl;
2790         u32 i;
2791
2792         ixgbe_setup_mtqc(adapter);
2793
2794         if (hw->mac.type != ixgbe_mac_82598EB) {
2795                 /* DMATXCTL.EN must be before Tx queues are enabled */
2796                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2797                 dmatxctl |= IXGBE_DMATXCTL_TE;
2798                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2799         }
2800
2801         /* Setup the HW Tx Head and Tail descriptor pointers */
2802         for (i = 0; i < adapter->num_tx_queues; i++)
2803                 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2804 }
2805
2806 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2807
2808 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2809                                    struct ixgbe_ring *rx_ring)
2810 {
2811         u32 srrctl;
2812         u8 reg_idx = rx_ring->reg_idx;
2813
2814         switch (adapter->hw.mac.type) {
2815         case ixgbe_mac_82598EB: {
2816                 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2817                 const int mask = feature[RING_F_RSS].mask;
2818                 reg_idx = reg_idx & mask;
2819         }
2820                 break;
2821         case ixgbe_mac_82599EB:
2822         case ixgbe_mac_X540:
2823         default:
2824                 break;
2825         }
2826
2827         srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2828
2829         srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2830         srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2831         if (adapter->num_vfs)
2832                 srrctl |= IXGBE_SRRCTL_DROP_EN;
2833
2834         srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2835                   IXGBE_SRRCTL_BSIZEHDR_MASK;
2836
2837         if (ring_is_ps_enabled(rx_ring)) {
2838 #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2839                 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2840 #else
2841                 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2842 #endif
2843                 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2844         } else {
2845                 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2846                           IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2847                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2848         }
2849
2850         IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2851 }
2852
2853 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2854 {
2855         struct ixgbe_hw *hw = &adapter->hw;
2856         static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2857                           0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2858                           0x6A3E67EA, 0x14364D17, 0x3BED200D};
2859         u32 mrqc = 0, reta = 0;
2860         u32 rxcsum;
2861         int i, j;
2862         int mask;
2863
2864         /* Fill out hash function seeds */
2865         for (i = 0; i < 10; i++)
2866                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2867
2868         /* Fill out redirection table */
2869         for (i = 0, j = 0; i < 128; i++, j++) {
2870                 if (j == adapter->ring_feature[RING_F_RSS].indices)
2871                         j = 0;
2872                 /* reta = 4-byte sliding window of
2873                  * 0x00..(indices-1)(indices-1)00..etc. */
2874                 reta = (reta << 8) | (j * 0x11);
2875                 if ((i & 3) == 3)
2876                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2877         }
2878
2879         /* Disable indicating checksum in descriptor, enables RSS hash */
2880         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2881         rxcsum |= IXGBE_RXCSUM_PCSD;
2882         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2883
2884         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
2885                 mask = adapter->flags & IXGBE_FLAG_RSS_ENABLED;
2886         else
2887                 mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2888 #ifdef CONFIG_IXGBE_DCB
2889                                          | IXGBE_FLAG_DCB_ENABLED
2890 #endif
2891                                          | IXGBE_FLAG_SRIOV_ENABLED
2892                                         );
2893
2894         switch (mask) {
2895 #ifdef CONFIG_IXGBE_DCB
2896         case (IXGBE_FLAG_DCB_ENABLED | IXGBE_FLAG_RSS_ENABLED):
2897                 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2898                 break;
2899         case (IXGBE_FLAG_DCB_ENABLED):
2900                 mrqc = IXGBE_MRQC_RT8TCEN;
2901                 break;
2902 #endif /* CONFIG_IXGBE_DCB */
2903         case (IXGBE_FLAG_RSS_ENABLED):
2904                 mrqc = IXGBE_MRQC_RSSEN;
2905                 break;
2906         case (IXGBE_FLAG_SRIOV_ENABLED):
2907                 mrqc = IXGBE_MRQC_VMDQEN;
2908                 break;
2909         default:
2910                 break;
2911         }
2912
2913         /* Perform hash on these packet types */
2914         mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2915               | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2916               | IXGBE_MRQC_RSS_FIELD_IPV6
2917               | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2918
2919         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2920 }
2921
2922 /**
2923  * ixgbe_clear_rscctl - disable RSC for the indicated ring
2924  * @adapter: address of board private structure
2925  * @ring: structure containing ring specific data
2926  **/
2927 void ixgbe_clear_rscctl(struct ixgbe_adapter *adapter,
2928                         struct ixgbe_ring *ring)
2929 {
2930         struct ixgbe_hw *hw = &adapter->hw;
2931         u32 rscctrl;
2932         u8 reg_idx = ring->reg_idx;
2933
2934         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2935         rscctrl &= ~IXGBE_RSCCTL_RSCEN;
2936         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2937 }
2938
2939 /**
2940  * ixgbe_configure_rscctl - enable RSC for the indicated ring
2941  * @adapter:    address of board private structure
2942  * @index:      index of ring to set
2943  **/
2944 void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2945                                    struct ixgbe_ring *ring)
2946 {
2947         struct ixgbe_hw *hw = &adapter->hw;
2948         u32 rscctrl;
2949         int rx_buf_len;
2950         u8 reg_idx = ring->reg_idx;
2951
2952         if (!ring_is_rsc_enabled(ring))
2953                 return;
2954
2955         rx_buf_len = ring->rx_buf_len;
2956         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2957         rscctrl |= IXGBE_RSCCTL_RSCEN;
2958         /*
2959          * we must limit the number of descriptors so that the
2960          * total size of max desc * buf_len is not greater
2961          * than 65535
2962          */
2963         if (ring_is_ps_enabled(ring)) {
2964 #if (MAX_SKB_FRAGS > 16)
2965                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2966 #elif (MAX_SKB_FRAGS > 8)
2967                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2968 #elif (MAX_SKB_FRAGS > 4)
2969                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2970 #else
2971                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2972 #endif
2973         } else {
2974                 if (rx_buf_len < IXGBE_RXBUFFER_4096)
2975                         rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2976                 else if (rx_buf_len < IXGBE_RXBUFFER_8192)
2977                         rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2978                 else
2979                         rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2980         }
2981         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2982 }
2983
2984 /**
2985  *  ixgbe_set_uta - Set unicast filter table address
2986  *  @adapter: board private structure
2987  *
2988  *  The unicast table address is a register array of 32-bit registers.
2989  *  The table is meant to be used in a way similar to how the MTA is used
2990  *  however due to certain limitations in the hardware it is necessary to
2991  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2992  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
2993  **/
2994 static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2995 {
2996         struct ixgbe_hw *hw = &adapter->hw;
2997         int i;
2998
2999         /* The UTA table only exists on 82599 hardware and newer */
3000         if (hw->mac.type < ixgbe_mac_82599EB)
3001                 return;
3002
3003         /* we only need to do this if VMDq is enabled */
3004         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3005                 return;
3006
3007         for (i = 0; i < 128; i++)
3008                 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
3009 }
3010
3011 #define IXGBE_MAX_RX_DESC_POLL 10
3012 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
3013                                        struct ixgbe_ring *ring)
3014 {
3015         struct ixgbe_hw *hw = &adapter->hw;
3016         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3017         u32 rxdctl;
3018         u8 reg_idx = ring->reg_idx;
3019
3020         /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
3021         if (hw->mac.type == ixgbe_mac_82598EB &&
3022             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3023                 return;
3024
3025         do {
3026                 msleep(1);
3027                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3028         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
3029
3030         if (!wait_loop) {
3031                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
3032                       "the polling period\n", reg_idx);
3033         }
3034 }
3035
3036 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
3037                             struct ixgbe_ring *ring)
3038 {
3039         struct ixgbe_hw *hw = &adapter->hw;
3040         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
3041         u32 rxdctl;
3042         u8 reg_idx = ring->reg_idx;
3043
3044         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3045         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
3046
3047         /* write value back with RXDCTL.ENABLE bit cleared */
3048         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3049
3050         if (hw->mac.type == ixgbe_mac_82598EB &&
3051             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
3052                 return;
3053
3054         /* the hardware may take up to 100us to really disable the rx queue */
3055         do {
3056                 udelay(10);
3057                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3058         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
3059
3060         if (!wait_loop) {
3061                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
3062                       "the polling period\n", reg_idx);
3063         }
3064 }
3065
3066 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
3067                              struct ixgbe_ring *ring)
3068 {
3069         struct ixgbe_hw *hw = &adapter->hw;
3070         u64 rdba = ring->dma;
3071         u32 rxdctl;
3072         u8 reg_idx = ring->reg_idx;
3073
3074         /* disable queue to avoid issues while updating state */
3075         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3076         ixgbe_disable_rx_queue(adapter, ring);
3077
3078         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
3079         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
3080         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
3081                         ring->count * sizeof(union ixgbe_adv_rx_desc));
3082         IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
3083         IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
3084         ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
3085
3086         ixgbe_configure_srrctl(adapter, ring);
3087         ixgbe_configure_rscctl(adapter, ring);
3088
3089         /* If operating in IOV mode set RLPML for X540 */
3090         if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
3091             hw->mac.type == ixgbe_mac_X540) {
3092                 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
3093                 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
3094                             ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
3095         }
3096
3097         if (hw->mac.type == ixgbe_mac_82598EB) {
3098                 /*
3099                  * enable cache line friendly hardware writes:
3100                  * PTHRESH=32 descriptors (half the internal cache),
3101                  * this also removes ugly rx_no_buffer_count increment
3102                  * HTHRESH=4 descriptors (to minimize latency on fetch)
3103                  * WTHRESH=8 burst writeback up to two cache lines
3104                  */
3105                 rxdctl &= ~0x3FFFFF;
3106                 rxdctl |=  0x080420;
3107         }
3108
3109         /* enable receive descriptor ring */
3110         rxdctl |= IXGBE_RXDCTL_ENABLE;
3111         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3112
3113         ixgbe_rx_desc_queue_enable(adapter, ring);
3114         ixgbe_alloc_rx_buffers(ring, IXGBE_DESC_UNUSED(ring));
3115 }
3116
3117 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
3118 {
3119         struct ixgbe_hw *hw = &adapter->hw;
3120         int p;
3121
3122         /* PSRTYPE must be initialized in non 82598 adapters */
3123         u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3124                       IXGBE_PSRTYPE_UDPHDR |
3125                       IXGBE_PSRTYPE_IPV4HDR |
3126                       IXGBE_PSRTYPE_L2HDR |
3127                       IXGBE_PSRTYPE_IPV6HDR;
3128
3129         if (hw->mac.type == ixgbe_mac_82598EB)
3130                 return;
3131
3132         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
3133                 psrtype |= (adapter->num_rx_queues_per_pool << 29);
3134
3135         for (p = 0; p < adapter->num_rx_pools; p++)
3136                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
3137                                 psrtype);
3138 }
3139
3140 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
3141 {
3142         struct ixgbe_hw *hw = &adapter->hw;
3143         u32 gcr_ext;
3144         u32 vt_reg_bits;
3145         u32 reg_offset, vf_shift;
3146         u32 vmdctl;
3147
3148         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3149                 return;
3150
3151         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3152         vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
3153         vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
3154         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
3155
3156         vf_shift = adapter->num_vfs % 32;
3157         reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
3158
3159         /* Enable only the PF's pool for Tx/Rx */
3160         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
3161         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
3162         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
3163         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
3164         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3165
3166         /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
3167         hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
3168
3169         /*
3170          * Set up VF register offsets for selected VT Mode,
3171          * i.e. 32 or 64 VFs for SR-IOV
3172          */
3173         gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
3174         gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
3175         gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
3176         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3177
3178         /* enable Tx loopback for VF/PF communication */
3179         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3180         /* Enable MAC Anti-Spoofing */
3181         hw->mac.ops.set_mac_anti_spoofing(hw, (adapter->num_vfs != 0),
3182                                           adapter->num_vfs);
3183 }
3184
3185 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
3186 {
3187         struct ixgbe_hw *hw = &adapter->hw;
3188         struct net_device *netdev = adapter->netdev;
3189         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3190         int rx_buf_len;
3191         struct ixgbe_ring *rx_ring;
3192         int i;
3193         u32 mhadd, hlreg0;
3194
3195         /* Decide whether to use packet split mode or not */
3196         /* On by default */
3197         adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
3198
3199         /* Do not use packet split if we're in SR-IOV Mode */
3200         if (adapter->num_vfs)
3201                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3202
3203         /* Disable packet split due to 82599 erratum #45 */
3204         if (hw->mac.type == ixgbe_mac_82599EB)
3205                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3206
3207         /* Set the RX buffer length according to the mode */
3208         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
3209                 rx_buf_len = IXGBE_RX_HDR_SIZE;
3210         } else {
3211                 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
3212                     (netdev->mtu <= ETH_DATA_LEN))
3213                         rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3214                 else
3215                         rx_buf_len = ALIGN(max_frame + VLAN_HLEN, 1024);
3216         }
3217
3218 #ifdef IXGBE_FCOE
3219         /* adjust max frame to be able to do baby jumbo for FCoE */
3220         if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3221             (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3222                 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3223
3224 #endif /* IXGBE_FCOE */
3225         mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3226         if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3227                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3228                 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3229
3230                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3231         }
3232
3233         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3234         /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3235         hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3236         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3237
3238         /*
3239          * Setup the HW Rx Head and Tail Descriptor Pointers and
3240          * the Base and Length of the Rx Descriptor Ring
3241          */
3242         for (i = 0; i < adapter->num_rx_queues; i++) {
3243                 rx_ring = adapter->rx_ring[i];
3244                 rx_ring->rx_buf_len = rx_buf_len;
3245
3246                 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
3247                         set_ring_ps_enabled(rx_ring);
3248                 else
3249                         clear_ring_ps_enabled(rx_ring);
3250
3251                 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3252                         set_ring_rsc_enabled(rx_ring);
3253                 else
3254                         clear_ring_rsc_enabled(rx_ring);
3255
3256 #ifdef IXGBE_FCOE
3257                 if (netdev->features & NETIF_F_FCOE_MTU) {
3258                         struct ixgbe_ring_feature *f;
3259                         f = &adapter->ring_feature[RING_F_FCOE];
3260                         if ((i >= f->mask) && (i < f->mask + f->indices)) {
3261                                 clear_ring_ps_enabled(rx_ring);
3262                                 if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
3263                                         rx_ring->rx_buf_len =
3264                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
3265                         } else if (!ring_is_rsc_enabled(rx_ring) &&
3266                                    !ring_is_ps_enabled(rx_ring)) {
3267                                 rx_ring->rx_buf_len =
3268                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
3269                         }
3270                 }
3271 #endif /* IXGBE_FCOE */
3272         }
3273 }
3274
3275 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3276 {
3277         struct ixgbe_hw *hw = &adapter->hw;
3278         u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3279
3280         switch (hw->mac.type) {
3281         case ixgbe_mac_82598EB:
3282                 /*
3283                  * For VMDq support of different descriptor types or
3284                  * buffer sizes through the use of multiple SRRCTL
3285                  * registers, RDRXCTL.MVMEN must be set to 1
3286                  *
3287                  * also, the manual doesn't mention it clearly but DCA hints
3288                  * will only use queue 0's tags unless this bit is set.  Side
3289                  * effects of setting this bit are only that SRRCTL must be
3290                  * fully programmed [0..15]
3291                  */
3292                 rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3293                 break;
3294         case ixgbe_mac_82599EB:
3295         case ixgbe_mac_X540:
3296                 /* Disable RSC for ACK packets */
3297                 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3298                    (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3299                 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3300                 /* hardware requires some bits to be set by default */
3301                 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
3302                 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3303                 break;
3304         default:
3305                 /* We should do nothing since we don't know this hardware */
3306                 return;
3307         }
3308
3309         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3310 }
3311
3312 /**
3313  * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
3314  * @adapter: board private structure
3315  *
3316  * Configure the Rx unit of the MAC after a reset.
3317  **/
3318 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3319 {
3320         struct ixgbe_hw *hw = &adapter->hw;
3321         int i;
3322         u32 rxctrl;
3323
3324         /* disable receives while setting up the descriptors */
3325         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3326         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
3327
3328         ixgbe_setup_psrtype(adapter);
3329         ixgbe_setup_rdrxctl(adapter);
3330
3331         /* Program registers for the distribution of queues */
3332         ixgbe_setup_mrqc(adapter);
3333
3334         ixgbe_set_uta(adapter);
3335
3336         /* set_rx_buffer_len must be called before ring initialization */
3337         ixgbe_set_rx_buffer_len(adapter);
3338
3339         /*
3340          * Setup the HW Rx Head and Tail Descriptor Pointers and
3341          * the Base and Length of the Rx Descriptor Ring
3342          */
3343         for (i = 0; i < adapter->num_rx_queues; i++)
3344                 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
3345
3346         /* disable drop enable for 82598 parts */
3347         if (hw->mac.type == ixgbe_mac_82598EB)
3348                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3349
3350         /* enable all receives */
3351         rxctrl |= IXGBE_RXCTRL_RXEN;
3352         hw->mac.ops.enable_rx_dma(hw, rxctrl);
3353 }
3354
3355 static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
3356 {
3357         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3358         struct ixgbe_hw *hw = &adapter->hw;
3359         int pool_ndx = adapter->num_vfs;
3360
3361         /* add VID to filter table */
3362         hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true);
3363         set_bit(vid, adapter->active_vlans);
3364 }
3365
3366 static void ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
3367 {
3368         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3369         struct ixgbe_hw *hw = &adapter->hw;
3370         int pool_ndx = adapter->num_vfs;
3371
3372         /* remove VID from filter table */
3373         hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false);
3374         clear_bit(vid, adapter->active_vlans);
3375 }
3376
3377 /**
3378  * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
3379  * @adapter: driver data
3380  */
3381 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
3382 {
3383         struct ixgbe_hw *hw = &adapter->hw;
3384         u32 vlnctrl;
3385
3386         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3387         vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
3388         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3389 }
3390
3391 /**
3392  * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
3393  * @adapter: driver data
3394  */
3395 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
3396 {
3397         struct ixgbe_hw *hw = &adapter->hw;
3398         u32 vlnctrl;
3399
3400         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3401         vlnctrl |= IXGBE_VLNCTRL_VFE;
3402         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
3403         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3404 }
3405
3406 /**
3407  * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
3408  * @adapter: driver data
3409  */
3410 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
3411 {
3412         struct ixgbe_hw *hw = &adapter->hw;
3413         u32 vlnctrl;
3414         int i, j;
3415
3416         switch (hw->mac.type) {
3417         case ixgbe_mac_82598EB:
3418                 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3419                 vlnctrl &= ~IXGBE_VLNCTRL_VME;
3420                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3421                 break;
3422         case ixgbe_mac_82599EB:
3423         case ixgbe_mac_X540:
3424                 for (i = 0; i < adapter->num_rx_queues; i++) {
3425                         j = adapter->rx_ring[i]->reg_idx;
3426                         vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3427                         vlnctrl &= ~IXGBE_RXDCTL_VME;
3428                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3429                 }
3430                 break;
3431         default:
3432                 break;
3433         }
3434 }
3435
3436 /**
3437  * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
3438  * @adapter: driver data
3439  */
3440 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
3441 {
3442         struct ixgbe_hw *hw = &adapter->hw;
3443         u32 vlnctrl;
3444         int i, j;
3445
3446         switch (hw->mac.type) {
3447         case ixgbe_mac_82598EB:
3448                 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3449                 vlnctrl |= IXGBE_VLNCTRL_VME;
3450                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3451                 break;
3452         case ixgbe_mac_82599EB:
3453         case ixgbe_mac_X540:
3454                 for (i = 0; i < adapter->num_rx_queues; i++) {
3455                         j = adapter->rx_ring[i]->reg_idx;
3456                         vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3457                         vlnctrl |= IXGBE_RXDCTL_VME;
3458                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3459                 }
3460                 break;
3461         default:
3462                 break;
3463         }
3464 }
3465
3466 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
3467 {
3468         u16 vid;
3469
3470         ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
3471
3472         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
3473                 ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
3474 }
3475
3476 /**
3477  * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
3478  * @netdev: network interface device structure
3479  *
3480  * Writes unicast address list to the RAR table.
3481  * Returns: -ENOMEM on failure/insufficient address space
3482  *                0 on no addresses written
3483  *                X on writing X addresses to the RAR table
3484  **/
3485 static int ixgbe_write_uc_addr_list(struct net_device *netdev)
3486 {
3487         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3488         struct ixgbe_hw *hw = &adapter->hw;
3489         unsigned int vfn = adapter->num_vfs;
3490         unsigned int rar_entries = hw->mac.num_rar_entries - (vfn + 1);
3491         int count = 0;
3492
3493         /* return ENOMEM indicating insufficient memory for addresses */
3494         if (netdev_uc_count(netdev) > rar_entries)
3495                 return -ENOMEM;
3496
3497         if (!netdev_uc_empty(netdev) && rar_entries) {
3498                 struct netdev_hw_addr *ha;
3499                 /* return error if we do not support writing to RAR table */
3500                 if (!hw->mac.ops.set_rar)
3501                         return -ENOMEM;
3502
3503                 netdev_for_each_uc_addr(ha, netdev) {
3504                         if (!rar_entries)
3505                                 break;
3506                         hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
3507                                             vfn, IXGBE_RAH_AV);
3508                         count++;
3509                 }
3510         }
3511         /* write the addresses in reverse order to avoid write combining */
3512         for (; rar_entries > 0 ; rar_entries--)
3513                 hw->mac.ops.clear_rar(hw, rar_entries);
3514
3515         return count;
3516 }
3517
3518 /**
3519  * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
3520  * @netdev: network interface device structure
3521  *
3522  * The set_rx_method entry point is called whenever the unicast/multicast
3523  * address list or the network interface flags are updated.  This routine is
3524  * responsible for configuring the hardware for proper unicast, multicast and
3525  * promiscuous mode.
3526  **/
3527 void ixgbe_set_rx_mode(struct net_device *netdev)
3528 {
3529         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3530         struct ixgbe_hw *hw = &adapter->hw;
3531         u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
3532         int count;
3533
3534         /* Check for Promiscuous and All Multicast modes */
3535
3536         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3537
3538         /* set all bits that we expect to always be set */
3539         fctrl |= IXGBE_FCTRL_BAM;
3540         fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
3541         fctrl |= IXGBE_FCTRL_PMCF;
3542
3543         /* clear the bits we are changing the status of */
3544         fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3545
3546         if (netdev->flags & IFF_PROMISC) {
3547                 hw->addr_ctrl.user_set_promisc = true;
3548                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3549                 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
3550                 /* don't hardware filter vlans in promisc mode */
3551                 ixgbe_vlan_filter_disable(adapter);
3552         } else {
3553                 if (netdev->flags & IFF_ALLMULTI) {
3554                         fctrl |= IXGBE_FCTRL_MPE;
3555                         vmolr |= IXGBE_VMOLR_MPE;
3556                 } else {
3557                         /*
3558                          * Write addresses to the MTA, if the attempt fails
3559                          * then we should just turn on promiscuous mode so
3560                          * that we can at least receive multicast traffic
3561                          */
3562                         hw->mac.ops.update_mc_addr_list(hw, netdev);
3563                         vmolr |= IXGBE_VMOLR_ROMPE;
3564                 }
3565                 ixgbe_vlan_filter_enable(adapter);
3566                 hw->addr_ctrl.user_set_promisc = false;
3567                 /*
3568                  * Write addresses to available RAR registers, if there is not
3569                  * sufficient space to store all the addresses then enable
3570                  * unicast promiscuous mode
3571                  */
3572                 count = ixgbe_write_uc_addr_list(netdev);
3573                 if (count < 0) {
3574                         fctrl |= IXGBE_FCTRL_UPE;
3575                         vmolr |= IXGBE_VMOLR_ROPE;
3576                 }
3577         }
3578
3579         if (adapter->num_vfs) {
3580                 ixgbe_restore_vf_multicasts(adapter);
3581                 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) &
3582                          ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
3583                            IXGBE_VMOLR_ROPE);
3584                 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr);
3585         }
3586
3587         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3588
3589         if (netdev->features & NETIF_F_HW_VLAN_RX)
3590                 ixgbe_vlan_strip_enable(adapter);
3591         else
3592                 ixgbe_vlan_strip_disable(adapter);
3593 }
3594
3595 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
3596 {
3597         int q_idx;
3598         struct ixgbe_q_vector *q_vector;
3599         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3600
3601         /* legacy and MSI only use one vector */
3602         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3603                 q_vectors = 1;
3604
3605         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3606                 struct napi_struct *napi;
3607                 q_vector = adapter->q_vector[q_idx];
3608                 napi = &q_vector->napi;
3609                 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3610                         if (!q_vector->rxr_count || !q_vector->txr_count) {
3611                                 if (q_vector->txr_count == 1)
3612                                         napi->poll = &ixgbe_clean_txonly;
3613                                 else if (q_vector->rxr_count == 1)
3614                                         napi->poll = &ixgbe_clean_rxonly;
3615                         }
3616                 }
3617
3618                 napi_enable(napi);
3619         }
3620 }
3621
3622 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
3623 {
3624         int q_idx;
3625         struct ixgbe_q_vector *q_vector;
3626         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3627
3628         /* legacy and MSI only use one vector */
3629         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3630                 q_vectors = 1;
3631
3632         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3633                 q_vector = adapter->q_vector[q_idx];
3634                 napi_disable(&q_vector->napi);
3635         }
3636 }
3637
3638 #ifdef CONFIG_IXGBE_DCB
3639 /*
3640  * ixgbe_configure_dcb - Configure DCB hardware
3641  * @adapter: ixgbe adapter struct
3642  *
3643  * This is called by the driver on open to configure the DCB hardware.
3644  * This is also called by the gennetlink interface when reconfiguring
3645  * the DCB state.
3646  */
3647 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
3648 {
3649         struct ixgbe_hw *hw = &adapter->hw;
3650         int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3651
3652         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
3653                 if (hw->mac.type == ixgbe_mac_82598EB)
3654                         netif_set_gso_max_size(adapter->netdev, 65536);
3655                 return;
3656         }
3657
3658         if (hw->mac.type == ixgbe_mac_82598EB)
3659                 netif_set_gso_max_size(adapter->netdev, 32768);
3660
3661
3662         /* Enable VLAN tag insert/strip */
3663         adapter->netdev->features |= NETIF_F_HW_VLAN_RX;
3664
3665         hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true);
3666
3667         /* reconfigure the hardware */
3668         if (adapter->dcbx_cap & (DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE)) {
3669 #ifdef CONFIG_FCOE
3670                 if (adapter->netdev->features & NETIF_F_FCOE_MTU)
3671                         max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
3672 #endif
3673                 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3674                                                 DCB_TX_CONFIG);
3675                 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3676                                                 DCB_RX_CONFIG);
3677                 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
3678         } else {
3679                 struct net_device *dev = adapter->netdev;
3680
3681                 if (adapter->ixgbe_ieee_ets)
3682                         dev->dcbnl_ops->ieee_setets(dev,
3683                                                     adapter->ixgbe_ieee_ets);
3684                 if (adapter->ixgbe_ieee_pfc)
3685                         dev->dcbnl_ops->ieee_setpfc(dev,
3686                                                     adapter->ixgbe_ieee_pfc);
3687         }
3688
3689         /* Enable RSS Hash per TC */
3690         if (hw->mac.type != ixgbe_mac_82598EB) {
3691                 int i;
3692                 u32 reg = 0;
3693
3694                 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
3695                         u8 msb = 0;
3696                         u8 cnt = adapter->netdev->tc_to_txq[i].count;
3697
3698                         while (cnt >>= 1)
3699                                 msb++;
3700
3701                         reg |= msb << IXGBE_RQTC_SHIFT_TC(i);
3702                 }
3703                 IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg);
3704         }
3705 }
3706
3707 #endif
3708 static void ixgbe_configure(struct ixgbe_adapter *adapter)
3709 {
3710         struct net_device *netdev = adapter->netdev;
3711         struct ixgbe_hw *hw = &adapter->hw;
3712         int i;
3713
3714 #ifdef CONFIG_IXGBE_DCB
3715         ixgbe_configure_dcb(adapter);
3716 #endif
3717
3718         ixgbe_set_rx_mode(netdev);
3719         ixgbe_restore_vlan(adapter);
3720
3721 #ifdef IXGBE_FCOE
3722         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
3723                 ixgbe_configure_fcoe(adapter);
3724
3725 #endif /* IXGBE_FCOE */
3726         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3727                 for (i = 0; i < adapter->num_tx_queues; i++)
3728                         adapter->tx_ring[i]->atr_sample_rate =
3729                                                        adapter->atr_sample_rate;
3730                 ixgbe_init_fdir_signature_82599(hw, adapter->fdir_pballoc);
3731         } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
3732                 ixgbe_init_fdir_perfect_82599(hw, adapter->fdir_pballoc);
3733         }
3734         ixgbe_configure_virtualization(adapter);
3735
3736         ixgbe_configure_tx(adapter);
3737         ixgbe_configure_rx(adapter);
3738 }
3739
3740 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
3741 {
3742         switch (hw->phy.type) {
3743         case ixgbe_phy_sfp_avago:
3744         case ixgbe_phy_sfp_ftl:
3745         case ixgbe_phy_sfp_intel:
3746         case ixgbe_phy_sfp_unknown:
3747         case ixgbe_phy_sfp_passive_tyco:
3748         case ixgbe_phy_sfp_passive_unknown:
3749         case ixgbe_phy_sfp_active_unknown:
3750         case ixgbe_phy_sfp_ftl_active:
3751                 return true;
3752         default:
3753                 return false;
3754         }
3755 }
3756
3757 /**
3758  * ixgbe_sfp_link_config - set up SFP+ link
3759  * @adapter: pointer to private adapter struct
3760  **/
3761 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
3762 {
3763         struct ixgbe_hw *hw = &adapter->hw;
3764
3765                 if (hw->phy.multispeed_fiber) {
3766                         /*
3767                          * In multispeed fiber setups, the device may not have
3768                          * had a physical connection when the driver loaded.
3769                          * If that's the case, the initial link configuration
3770                          * couldn't get the MAC into 10G or 1G mode, so we'll
3771                          * never have a link status change interrupt fire.
3772                          * We need to try and force an autonegotiation
3773                          * session, then bring up link.
3774                          */
3775                         if (hw->mac.ops.setup_sfp)
3776                                 hw->mac.ops.setup_sfp(hw);
3777                         if (!(adapter->flags & IXGBE_FLAG_IN_SFP_LINK_TASK))
3778                                 schedule_work(&adapter->multispeed_fiber_task);
3779                 } else {
3780                         /*
3781                          * Direct Attach Cu and non-multispeed fiber modules
3782                          * still need to be configured properly prior to
3783                          * attempting link.
3784                          */
3785                         if (!(adapter->flags & IXGBE_FLAG_IN_SFP_MOD_TASK))
3786                                 schedule_work(&adapter->sfp_config_module_task);
3787                 }
3788 }
3789
3790 /**
3791  * ixgbe_non_sfp_link_config - set up non-SFP+ link
3792  * @hw: pointer to private hardware struct
3793  *
3794  * Returns 0 on success, negative on failure
3795  **/
3796 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
3797 {
3798         u32 autoneg;
3799         bool negotiation, link_up = false;
3800         u32 ret = IXGBE_ERR_LINK_SETUP;
3801
3802         if (hw->mac.ops.check_link)
3803                 ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
3804
3805         if (ret)
3806                 goto link_cfg_out;
3807
3808         autoneg = hw->phy.autoneg_advertised;
3809         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3810                 ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
3811                                                         &negotiation);
3812         if (ret)
3813                 goto link_cfg_out;
3814
3815         if (hw->mac.ops.setup_link)
3816                 ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
3817 link_cfg_out:
3818         return ret;
3819 }
3820
3821 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
3822 {
3823         struct ixgbe_hw *hw = &adapter->hw;
3824         u32 gpie = 0;
3825
3826         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3827                 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
3828                        IXGBE_GPIE_OCD;
3829                 gpie |= IXGBE_GPIE_EIAME;
3830                 /*
3831                  * use EIAM to auto-mask when MSI-X interrupt is asserted
3832                  * this saves a register write for every interrupt
3833                  */
3834                 switch (hw->mac.type) {
3835                 case ixgbe_mac_82598EB:
3836                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3837                         break;
3838                 case ixgbe_mac_82599EB:
3839                 case ixgbe_mac_X540:
3840                 default:
3841                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
3842                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
3843                         break;
3844                 }
3845         } else {
3846                 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
3847                  * specifically only auto mask tx and rx interrupts */
3848                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3849         }
3850
3851         /* XXX: to interrupt immediately for EICS writes, enable this */
3852         /* gpie |= IXGBE_GPIE_EIMEN; */
3853
3854         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3855                 gpie &= ~IXGBE_GPIE_VTMODE_MASK;
3856                 gpie |= IXGBE_GPIE_VTMODE_64;
3857         }
3858
3859         /* Enable fan failure interrupt */
3860         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
3861                 gpie |= IXGBE_SDP1_GPIEN;
3862
3863         if (hw->mac.type == ixgbe_mac_82599EB)
3864                 gpie |= IXGBE_SDP1_GPIEN;
3865                 gpie |= IXGBE_SDP2_GPIEN;
3866
3867         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3868 }
3869
3870 static int ixgbe_up_complete(struct ixgbe_adapter *adapter)
3871 {
3872         struct ixgbe_hw *hw = &adapter->hw;
3873         int err;
3874         u32 ctrl_ext;
3875
3876         ixgbe_get_hw_control(adapter);
3877         ixgbe_setup_gpie(adapter);
3878
3879         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3880                 ixgbe_configure_msix(adapter);
3881         else
3882                 ixgbe_configure_msi_and_legacy(adapter);
3883
3884         /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
3885         if (hw->mac.ops.enable_tx_laser &&
3886             ((hw->phy.multispeed_fiber) ||
3887              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
3888               (hw->mac.type == ixgbe_mac_82599EB))))
3889                 hw->mac.ops.enable_tx_laser(hw);
3890
3891         clear_bit(__IXGBE_DOWN, &adapter->state);
3892         ixgbe_napi_enable_all(adapter);
3893
3894         if (ixgbe_is_sfp(hw)) {
3895                 ixgbe_sfp_link_config(adapter);
3896         } else {
3897                 err = ixgbe_non_sfp_link_config(hw);
3898                 if (err)
3899                         e_err(probe, "link_config FAILED %d\n", err);
3900         }
3901
3902         /* clear any pending interrupts, may auto mask */
3903         IXGBE_READ_REG(hw, IXGBE_EICR);
3904         ixgbe_irq_enable(adapter, true, true);
3905
3906         /*
3907          * If this adapter has a fan, check to see if we had a failure
3908          * before we enabled the interrupt.
3909          */
3910         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
3911                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
3912                 if (esdp & IXGBE_ESDP_SDP1)
3913                         e_crit(drv, "Fan has stopped, replace the adapter\n");
3914         }
3915
3916         /*
3917          * For hot-pluggable SFP+ devices, a new SFP+ module may have
3918          * arrived before interrupts were enabled but after probe.  Such
3919          * devices wouldn't have their type identified yet. We need to
3920          * kick off the SFP+ module setup first, then try to bring up link.
3921          * If we're not hot-pluggable SFP+, we just need to configure link
3922          * and bring it up.
3923          */
3924         if (hw->phy.type == ixgbe_phy_none)
3925                 schedule_work(&adapter->sfp_config_module_task);
3926
3927         /* enable transmits */
3928         netif_tx_start_all_queues(adapter->netdev);
3929
3930         /* bring the link up in the watchdog, this could race with our first
3931          * link up interrupt but shouldn't be a problem */
3932         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3933         adapter->link_check_timeout = jiffies;
3934         mod_timer(&adapter->watchdog_timer, jiffies);
3935
3936         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
3937         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3938         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
3939         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3940
3941         return 0;
3942 }
3943
3944 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
3945 {
3946         WARN_ON(in_interrupt());
3947         while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
3948                 msleep(1);
3949         ixgbe_down(adapter);
3950         /*
3951          * If SR-IOV enabled then wait a bit before bringing the adapter
3952          * back up to give the VFs time to respond to the reset.  The
3953          * two second wait is based upon the watchdog timer cycle in
3954          * the VF driver.
3955          */
3956         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3957                 msleep(2000);
3958         ixgbe_up(adapter);
3959         clear_bit(__IXGBE_RESETTING, &adapter->state);
3960 }
3961
3962 int ixgbe_up(struct ixgbe_adapter *adapter)
3963 {
3964         /* hardware has been reset, we need to reload some things */
3965         ixgbe_configure(adapter);
3966
3967         return ixgbe_up_complete(adapter);
3968 }
3969
3970 void ixgbe_reset(struct ixgbe_adapter *adapter)
3971 {
3972         struct ixgbe_hw *hw = &adapter->hw;
3973         int err;
3974
3975         err = hw->mac.ops.init_hw(hw);
3976         switch (err) {
3977         case 0:
3978         case IXGBE_ERR_SFP_NOT_PRESENT:
3979                 break;
3980         case IXGBE_ERR_MASTER_REQUESTS_PENDING:
3981                 e_dev_err("master disable timed out\n");
3982                 break;
3983         case IXGBE_ERR_EEPROM_VERSION:
3984                 /* We are running on a pre-production device, log a warning */
3985                 e_dev_warn("This device is a pre-production adapter/LOM. "
3986                            "Please be aware there may be issuesassociated with "
3987                            "your hardware.  If you are experiencing problems "
3988                            "please contact your Intel or hardware "
3989                            "representative who provided you with this "
3990                            "hardware.\n");
3991                 break;
3992         default:
3993                 e_dev_err("Hardware Error: %d\n", err);
3994         }
3995
3996         /* reprogram the RAR[0] in case user changed it. */
3997         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
3998                             IXGBE_RAH_AV);
3999 }
4000
4001 /**
4002  * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
4003  * @rx_ring: ring to free buffers from
4004  **/
4005 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
4006 {
4007         struct device *dev = rx_ring->dev;
4008         unsigned long size;
4009         u16 i;
4010
4011         /* ring already cleared, nothing to do */
4012         if (!rx_ring->rx_buffer_info)
4013                 return;
4014
4015         /* Free all the Rx ring sk_buffs */
4016         for (i = 0; i < rx_ring->count; i++) {
4017                 struct ixgbe_rx_buffer *rx_buffer_info;
4018
4019                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
4020                 if (rx_buffer_info->dma) {
4021                         dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
4022                                          rx_ring->rx_buf_len,
4023                                          DMA_FROM_DEVICE);
4024                         rx_buffer_info->dma = 0;
4025                 }
4026                 if (rx_buffer_info->skb) {
4027                         struct sk_buff *skb = rx_buffer_info->skb;
4028                         rx_buffer_info->skb = NULL;
4029                         do {
4030                                 struct sk_buff *this = skb;
4031                                 if (IXGBE_RSC_CB(this)->delay_unmap) {
4032                                         dma_unmap_single(dev,
4033                                                          IXGBE_RSC_CB(this)->dma,
4034                                                          rx_ring->rx_buf_len,
4035                                                          DMA_FROM_DEVICE);
4036                                         IXGBE_RSC_CB(this)->dma = 0;
4037                                         IXGBE_RSC_CB(skb)->delay_unmap = false;
4038                                 }
4039                                 skb = skb->prev;
4040                                 dev_kfree_skb(this);
4041                         } while (skb);
4042                 }
4043                 if (!rx_buffer_info->page)
4044                         continue;
4045                 if (rx_buffer_info->page_dma) {
4046                         dma_unmap_page(dev, rx_buffer_info->page_dma,
4047                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
4048                         rx_buffer_info->page_dma = 0;
4049                 }
4050                 put_page(rx_buffer_info->page);
4051                 rx_buffer_info->page = NULL;
4052                 rx_buffer_info->page_offset = 0;
4053         }
4054
4055         size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4056         memset(rx_ring->rx_buffer_info, 0, size);
4057
4058         /* Zero out the descriptor ring */
4059         memset(rx_ring->desc, 0, rx_ring->size);
4060
4061         rx_ring->next_to_clean = 0;
4062         rx_ring->next_to_use = 0;
4063 }
4064
4065 /**
4066  * ixgbe_clean_tx_ring - Free Tx Buffers
4067  * @tx_ring: ring to be cleaned
4068  **/
4069 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
4070 {
4071         struct ixgbe_tx_buffer *tx_buffer_info;
4072         unsigned long size;
4073         u16 i;
4074
4075         /* ring already cleared, nothing to do */
4076         if (!tx_ring->tx_buffer_info)
4077                 return;
4078
4079         /* Free all the Tx ring sk_buffs */
4080         for (i = 0; i < tx_ring->count; i++) {
4081                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4082                 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4083         }
4084
4085         size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
4086         memset(tx_ring->tx_buffer_info, 0, size);
4087
4088         /* Zero out the descriptor ring */
4089         memset(tx_ring->desc, 0, tx_ring->size);
4090
4091         tx_ring->next_to_use = 0;
4092         tx_ring->next_to_clean = 0;
4093 }
4094
4095 /**
4096  * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
4097  * @adapter: board private structure
4098  **/
4099 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
4100 {
4101         int i;
4102
4103         for (i = 0; i < adapter->num_rx_queues; i++)
4104                 ixgbe_clean_rx_ring(adapter->rx_ring[i]);
4105 }
4106
4107 /**
4108  * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
4109  * @adapter: board private structure
4110  **/
4111 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
4112 {
4113         int i;
4114
4115         for (i = 0; i < adapter->num_tx_queues; i++)
4116                 ixgbe_clean_tx_ring(adapter->tx_ring[i]);
4117 }
4118
4119 void ixgbe_down(struct ixgbe_adapter *adapter)
4120 {
4121         struct net_device *netdev = adapter->netdev;
4122         struct ixgbe_hw *hw = &adapter->hw;
4123         u32 rxctrl;
4124         u32 txdctl;
4125         int i;
4126         int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4127
4128         /* signal that we are down to the interrupt handler */
4129         set_bit(__IXGBE_DOWN, &adapter->state);
4130
4131         /* disable receive for all VFs and wait one second */
4132         if (adapter->num_vfs) {
4133                 /* ping all the active vfs to let them know we are going down */
4134                 ixgbe_ping_all_vfs(adapter);
4135
4136                 /* Disable all VFTE/VFRE TX/RX */
4137                 ixgbe_disable_tx_rx(adapter);
4138
4139                 /* Mark all the VFs as inactive */
4140                 for (i = 0 ; i < adapter->num_vfs; i++)
4141                         adapter->vfinfo[i].clear_to_send = 0;
4142         }
4143
4144         /* disable receives */
4145         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4146         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
4147
4148         /* disable all enabled rx queues */
4149         for (i = 0; i < adapter->num_rx_queues; i++)
4150                 /* this call also flushes the previous write */
4151                 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
4152
4153         msleep(10);
4154
4155         netif_tx_stop_all_queues(netdev);
4156
4157         clear_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state);
4158         del_timer_sync(&adapter->sfp_timer);
4159         del_timer_sync(&adapter->watchdog_timer);
4160         cancel_work_sync(&adapter->watchdog_task);
4161
4162         netif_carrier_off(netdev);
4163         netif_tx_disable(netdev);
4164
4165         ixgbe_irq_disable(adapter);
4166
4167         ixgbe_napi_disable_all(adapter);
4168
4169         /* Cleanup the affinity_hint CPU mask memory and callback */
4170         for (i = 0; i < num_q_vectors; i++) {
4171                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
4172                 /* clear the affinity_mask in the IRQ descriptor */
4173                 irq_set_affinity_hint(adapter->msix_entries[i]. vector, NULL);
4174                 /* release the CPU mask memory */
4175                 free_cpumask_var(q_vector->affinity_mask);
4176         }
4177
4178         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
4179             adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
4180                 cancel_work_sync(&adapter->fdir_reinit_task);
4181
4182         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
4183                 cancel_work_sync(&adapter->check_overtemp_task);
4184
4185         /* disable transmits in the hardware now that interrupts are off */
4186         for (i = 0; i < adapter->num_tx_queues; i++) {
4187                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
4188                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
4189                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx),
4190                                 (txdctl & ~IXGBE_TXDCTL_ENABLE));
4191         }
4192         /* Disable the Tx DMA engine on 82599 */
4193         switch (hw->mac.type) {
4194         case ixgbe_mac_82599EB:
4195         case ixgbe_mac_X540:
4196                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
4197                                 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
4198                                  ~IXGBE_DMATXCTL_TE));
4199                 break;
4200         default:
4201                 break;
4202         }
4203
4204         /* clear n-tuple filters that are cached */
4205         ethtool_ntuple_flush(netdev);
4206
4207         if (!pci_channel_offline(adapter->pdev))
4208                 ixgbe_reset(adapter);
4209
4210         /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
4211         if (hw->mac.ops.disable_tx_laser &&
4212             ((hw->phy.multispeed_fiber) ||
4213              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
4214               (hw->mac.type == ixgbe_mac_82599EB))))
4215                 hw->mac.ops.disable_tx_laser(hw);
4216
4217         ixgbe_clean_all_tx_rings(adapter);
4218         ixgbe_clean_all_rx_rings(adapter);
4219
4220 #ifdef CONFIG_IXGBE_DCA
4221         /* since we reset the hardware DCA settings were cleared */
4222         ixgbe_setup_dca(adapter);
4223 #endif
4224 }
4225
4226 /**
4227  * ixgbe_poll - NAPI Rx polling callback
4228  * @napi: structure for representing this polling device
4229  * @budget: how many packets driver is allowed to clean
4230  *
4231  * This function is used for legacy and MSI, NAPI mode
4232  **/
4233 static int ixgbe_poll(struct napi_struct *napi, int budget)
4234 {
4235         struct ixgbe_q_vector *q_vector =
4236                                 container_of(napi, struct ixgbe_q_vector, napi);
4237         struct ixgbe_adapter *adapter = q_vector->adapter;
4238         int tx_clean_complete, work_done = 0;
4239
4240 #ifdef CONFIG_IXGBE_DCA
4241         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
4242                 ixgbe_update_dca(q_vector);
4243 #endif
4244
4245         tx_clean_complete = ixgbe_clean_tx_irq(q_vector, adapter->tx_ring[0]);
4246         ixgbe_clean_rx_irq(q_vector, adapter->rx_ring[0], &work_done, budget);
4247
4248         if (!tx_clean_complete)
4249                 work_done = budget;
4250
4251         /* If budget not fully consumed, exit the polling mode */
4252         if (work_done < budget) {
4253                 napi_complete(napi);
4254                 if (adapter->rx_itr_setting & 1)
4255                         ixgbe_set_itr(adapter);
4256                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
4257                         ixgbe_irq_enable_queues(adapter, IXGBE_EIMS_RTX_QUEUE);
4258         }
4259         return work_done;
4260 }
4261
4262 /**
4263  * ixgbe_tx_timeout - Respond to a Tx Hang
4264  * @netdev: network interface device structure
4265  **/
4266 static void ixgbe_tx_timeout(struct net_device *netdev)
4267 {
4268         struct ixgbe_adapter *adapter = netdev_priv(netdev);
4269
4270         adapter->tx_timeout_count++;
4271
4272         /* Do the reset outside of interrupt context */
4273         schedule_work(&adapter->reset_task);
4274 }
4275
4276 static void ixgbe_reset_task(struct work_struct *work)
4277 {
4278         struct ixgbe_adapter *adapter;
4279         adapter = container_of(work, struct ixgbe_adapter, reset_task);
4280
4281         /* If we're already down or resetting, just bail */
4282         if (test_bit(__IXGBE_DOWN, &adapter->state) ||
4283             test_bit(__IXGBE_RESETTING, &adapter->state))
4284                 return;
4285
4286         ixgbe_dump(adapter);
4287         netdev_err(adapter->netdev, "Reset adapter\n");
4288         ixgbe_reinit_locked(adapter);
4289 }
4290
4291 /**
4292  * ixgbe_set_rss_queues: Allocate queues for RSS
4293  * @adapter: board private structure to initialize
4294  *
4295  * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
4296  * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
4297  *
4298  **/
4299 static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter)
4300 {
4301         bool ret = false;
4302         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS];
4303
4304         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4305                 f->mask = 0xF;
4306                 adapter->num_rx_queues = f->indices;
4307                 adapter->num_tx_queues = f->indices;
4308                 ret = true;
4309         } else {
4310                 ret = false;
4311         }
4312
4313         return ret;
4314 }
4315
4316 /**
4317  * ixgbe_set_fdir_queues: Allocate queues for Flow Director
4318  * @adapter: board private structure to initialize
4319  *
4320  * Flow Director is an advanced Rx filter, attempting to get Rx flows back
4321  * to the original CPU that initiated the Tx session.  This runs in addition
4322  * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the
4323  * Rx load across CPUs using RSS.
4324  *
4325  **/
4326 static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter)
4327 {
4328         bool ret = false;
4329         struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR];
4330
4331         f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices);
4332         f_fdir->mask = 0;
4333
4334         /* Flow Director must have RSS enabled */
4335         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED &&
4336             ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
4337              (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)))) {
4338                 adapter->num_tx_queues = f_fdir->indices;
4339                 adapter->num_rx_queues = f_fdir->indices;
4340                 ret = true;
4341         } else {
4342                 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
4343                 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
4344         }
4345         return ret;
4346 }
4347
4348 #ifdef IXGBE_FCOE
4349 /**
4350  * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE)
4351  * @adapter: board private structure to initialize
4352  *
4353  * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges.
4354  * The ring feature mask is not used as a mask for FCoE, as it can take any 8
4355  * rx queues out of the max number of rx queues, instead, it is used as the
4356  * index of the first rx queue used by FCoE.
4357  *
4358  **/
4359 static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter)
4360 {
4361         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4362
4363         if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4364                 return false;
4365
4366         if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
4367 #ifdef CONFIG_IXGBE_DCB
4368                 int tc;
4369                 struct net_device *dev = adapter->netdev;
4370
4371                 tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
4372                 f->indices = dev->tc_to_txq[tc].count;
4373                 f->mask = dev->tc_to_txq[tc].offset;
4374 #endif
4375         } else {
4376                 f->indices = min((int)num_online_cpus(), f->indices);
4377
4378                 adapter->num_rx_queues = 1;
4379                 adapter->num_tx_queues = 1;
4380
4381                 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4382                         e_info(probe, "FCoE enabled with RSS\n");
4383                         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ||
4384                             (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
4385                                 ixgbe_set_fdir_queues(adapter);
4386                         else
4387                                 ixgbe_set_rss_queues(adapter);
4388                 }
4389                 /* adding FCoE rx rings to the end */
4390                 f->mask = adapter->num_rx_queues;
4391                 adapter->num_rx_queues += f->indices;
4392                 adapter->num_tx_queues += f->indices;
4393         }
4394
4395         return true;
4396 }
4397 #endif /* IXGBE_FCOE */
4398
4399 #ifdef CONFIG_IXGBE_DCB
4400 static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
4401 {
4402         bool ret = false;
4403         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_DCB];
4404         int i, q;
4405
4406         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED))
4407                 return ret;
4408
4409         f->indices = 0;
4410         for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
4411                 q = min((int)num_online_cpus(), MAX_TRAFFIC_CLASS);
4412                 f->indices += q;
4413         }
4414
4415         f->mask = 0x7 << 3;
4416         adapter->num_rx_queues = f->indices;
4417         adapter->num_tx_queues = f->indices;
4418         ret = true;
4419
4420 #ifdef IXGBE_FCOE
4421         /* FCoE enabled queues require special configuration done through
4422          * configure_fcoe() and others. Here we map FCoE indices onto the
4423          * DCB queue pairs allowing FCoE to own configuration later.
4424          */
4425         ixgbe_set_fcoe_queues(adapter);
4426 #endif
4427
4428         return ret;
4429 }
4430 #endif
4431
4432 /**
4433  * ixgbe_set_sriov_queues: Allocate queues for IOV use
4434  * @adapter: board private structure to initialize
4435  *
4436  * IOV doesn't actually use anything, so just NAK the
4437  * request for now and let the other queue routines
4438  * figure out what to do.
4439  */
4440 static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
4441 {
4442         return false;
4443 }
4444
4445 /*
4446  * ixgbe_set_num_queues: Allocate queues for device, feature dependent
4447  * @adapter: board private structure to initialize
4448  *
4449  * This is the top level queue allocation routine.  The order here is very
4450  * important, starting with the "most" number of features turned on at once,
4451  * and ending with the smallest set of features.  This way large combinations
4452  * can be allocated if they're turned on, and smaller combinations are the
4453  * fallthrough conditions.
4454  *
4455  **/
4456 static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
4457 {
4458         /* Start with base case */
4459         adapter->num_rx_queues = 1;
4460         adapter->num_tx_queues = 1;
4461         adapter->num_rx_pools = adapter->num_rx_queues;
4462         adapter->num_rx_queues_per_pool = 1;
4463
4464         if (ixgbe_set_sriov_queues(adapter))
4465                 goto done;
4466
4467 #ifdef CONFIG_IXGBE_DCB
4468         if (ixgbe_set_dcb_queues(adapter))
4469                 goto done;
4470
4471 #endif
4472 #ifdef IXGBE_FCOE
4473         if (ixgbe_set_fcoe_queues(adapter))
4474                 goto done;
4475
4476 #endif /* IXGBE_FCOE */
4477         if (ixgbe_set_fdir_queues(adapter))
4478                 goto done;
4479
4480         if (ixgbe_set_rss_queues(adapter))
4481                 goto done;
4482
4483         /* fallback to base case */
4484         adapter->num_rx_queues = 1;
4485         adapter->num_tx_queues = 1;
4486
4487 done:
4488         /* Notify the stack of the (possibly) reduced queue counts. */
4489         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
4490         return netif_set_real_num_rx_queues(adapter->netdev,
4491                                             adapter->num_rx_queues);
4492 }
4493
4494 static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter,
4495                                        int vectors)
4496 {
4497         int err, vector_threshold;
4498
4499         /* We'll want at least 3 (vector_threshold):
4500          * 1) TxQ[0] Cleanup
4501          * 2) RxQ[0] Cleanup
4502          * 3) Other (Link Status Change, etc.)
4503          * 4) TCP Timer (optional)
4504          */
4505         vector_threshold = MIN_MSIX_COUNT;
4506
4507         /* The more we get, the more we will assign to Tx/Rx Cleanup
4508          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
4509          * Right now, we simply care about how many we'll get; we'll
4510          * set them up later while requesting irq's.
4511          */
4512         while (vectors >= vector_threshold) {
4513                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
4514                                       vectors);
4515                 if (!err) /* Success in acquiring all requested vectors. */
4516                         break;
4517                 else if (err < 0)
4518                         vectors = 0; /* Nasty failure, quit now */
4519                 else /* err == number of vectors we should try again with */
4520                         vectors = err;
4521         }
4522
4523         if (vectors < vector_threshold) {
4524                 /* Can't allocate enough MSI-X interrupts?  Oh well.
4525                  * This just means we'll go with either a single MSI
4526                  * vector or fall back to legacy interrupts.
4527                  */
4528                 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4529                              "Unable to allocate MSI-X interrupts\n");
4530                 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
4531                 kfree(adapter->msix_entries);
4532                 adapter->msix_entries = NULL;
4533         } else {
4534                 adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */
4535                 /*
4536                  * Adjust for only the vectors we'll use, which is minimum
4537                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
4538                  * vectors we were allocated.
4539                  */
4540                 adapter->num_msix_vectors = min(vectors,
4541                                    adapter->max_msix_q_vectors + NON_Q_VECTORS);
4542         }
4543 }
4544
4545 /**
4546  * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS
4547  * @adapter: board private structure to initialize
4548  *
4549  * Cache the descriptor ring offsets for RSS to the assigned rings.
4550  *
4551  **/
4552 static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
4553 {
4554         int i;
4555
4556         if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
4557                 return false;
4558
4559         for (i = 0; i < adapter->num_rx_queues; i++)
4560                 adapter->rx_ring[i]->reg_idx = i;
4561         for (i = 0; i < adapter->num_tx_queues; i++)
4562                 adapter->tx_ring[i]->reg_idx = i;
4563
4564         return true;
4565 }
4566
4567 #ifdef CONFIG_IXGBE_DCB
4568
4569 /* ixgbe_get_first_reg_idx - Return first register index associated with ring */
4570 void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
4571                              unsigned int *tx, unsigned int *rx)
4572 {
4573         struct net_device *dev = adapter->netdev;
4574         struct ixgbe_hw *hw = &adapter->hw;
4575         u8 num_tcs = netdev_get_num_tc(dev);
4576
4577         *tx = 0;
4578         *rx = 0;
4579
4580         switch (hw->mac.type) {
4581         case ixgbe_mac_82598EB:
4582                 *tx = tc << 3;
4583                 *rx = tc << 2;
4584                 break;
4585         case ixgbe_mac_82599EB:
4586         case ixgbe_mac_X540:
4587                 if (num_tcs == 8) {
4588                         if (tc < 3) {
4589                                 *tx = tc << 5;
4590                                 *rx = tc << 4;
4591                         } else if (tc <  5) {
4592                                 *tx = ((tc + 2) << 4);
4593                                 *rx = tc << 4;
4594                         } else if (tc < num_tcs) {
4595                                 *tx = ((tc + 8) << 3);
4596                                 *rx = tc << 4;
4597                         }
4598                 } else if (num_tcs == 4) {
4599                         *rx =  tc << 5;
4600                         switch (tc) {
4601                         case 0:
4602                                 *tx =  0;
4603                                 break;
4604                         case 1:
4605                                 *tx = 64;
4606                                 break;
4607                         case 2:
4608                                 *tx = 96;
4609                                 break;
4610                         case 3:
4611                                 *tx = 112;
4612                                 break;
4613                         default:
4614                                 break;
4615                         }
4616                 }
4617                 break;
4618         default:
4619                 break;
4620         }
4621 }
4622
4623 #define IXGBE_MAX_Q_PER_TC      (IXGBE_MAX_DCB_INDICES / MAX_TRAFFIC_CLASS)
4624
4625 /* ixgbe_setup_tc - routine to configure net_device for multiple traffic
4626  * classes.
4627  *
4628  * @netdev: net device to configure
4629  * @tc: number of traffic classes to enable
4630  */
4631 int ixgbe_setup_tc(struct net_device *dev, u8 tc)
4632 {
4633         int i;
4634         unsigned int q, offset = 0;
4635
4636         if (!tc) {
4637                 netdev_reset_tc(dev);
4638         } else {
4639                 struct ixgbe_adapter *adapter = netdev_priv(dev);
4640
4641                 /* Hardware supports up to 8 traffic classes */
4642                 if (tc > MAX_TRAFFIC_CLASS || netdev_set_num_tc(dev, tc))
4643                         return -EINVAL;
4644
4645                 /* Partition Tx queues evenly amongst traffic classes */
4646                 for (i = 0; i < tc; i++) {
4647                         q = min((int)num_online_cpus(), IXGBE_MAX_Q_PER_TC);
4648                         netdev_set_prio_tc_map(dev, i, i);
4649                         netdev_set_tc_queue(dev, i, q, offset);
4650                         offset += q;
4651                 }
4652
4653                 /* This enables multiple traffic class support in the hardware
4654                  * which defaults to strict priority transmission by default.
4655                  * If traffic classes are already enabled perhaps through DCB
4656                  * code path then existing configuration will be used.
4657                  */
4658                 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&
4659                     dev->dcbnl_ops && dev->dcbnl_ops->setdcbx) {
4660                         struct ieee_ets ets = {
4661                                         .prio_tc = {0, 1, 2, 3, 4, 5, 6, 7},
4662                                               };
4663                         u8 mode = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_IEEE;
4664
4665                         dev->dcbnl_ops->setdcbx(dev, mode);
4666                         dev->dcbnl_ops->ieee_setets(dev, &ets);
4667                 }
4668         }
4669         return 0;
4670 }
4671
4672 /**
4673  * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB
4674  * @adapter: board private structure to initialize
4675  *
4676  * Cache the descriptor ring offsets for DCB to the assigned rings.
4677  *
4678  **/
4679 static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
4680 {
4681         struct net_device *dev = adapter->netdev;
4682         int i, j, k;
4683         u8 num_tcs = netdev_get_num_tc(dev);
4684
4685         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED))
4686                 return false;
4687
4688         for (i = 0, k = 0; i < num_tcs; i++) {
4689                 unsigned int tx_s, rx_s;
4690                 u16 count = dev->tc_to_txq[i].count;
4691
4692                 ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s);
4693                 for (j = 0; j < count; j++, k++) {
4694                         adapter->tx_ring[k]->reg_idx = tx_s + j;
4695                         adapter->rx_ring[k]->reg_idx = rx_s + j;
4696                         adapter->tx_ring[k]->dcb_tc = i;
4697                         adapter->rx_ring[k]->dcb_tc = i;
4698                 }
4699         }
4700
4701         return true;
4702 }
4703 #endif
4704
4705 /**
4706  * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director
4707  * @adapter: board private structure to initialize
4708  *
4709  * Cache the descriptor ring offsets for Flow Director to the assigned rings.
4710  *
4711  **/
4712 static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter)
4713 {
4714         int i;
4715         bool ret = false;
4716
4717         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED &&
4718             ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ||
4719              (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))) {
4720                 for (i = 0; i < adapter->num_rx_queues; i++)
4721                         adapter->rx_ring[i]->reg_idx = i;
4722                 for (i = 0; i < adapter->num_tx_queues; i++)
4723                         adapter->tx_ring[i]->reg_idx = i;
4724                 ret = true;
4725         }
4726
4727         return ret;
4728 }
4729
4730 #ifdef IXGBE_FCOE
4731 /**
4732  * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE
4733  * @adapter: board private structure to initialize
4734  *
4735  * Cache the descriptor ring offsets for FCoE mode to the assigned rings.
4736  *
4737  */
4738 static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter)
4739 {
4740         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4741         int i;
4742         u8 fcoe_rx_i = 0, fcoe_tx_i = 0;
4743
4744         if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4745                 return false;
4746
4747         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4748                 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) ||
4749                     (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))
4750                         ixgbe_cache_ring_fdir(adapter);
4751                 else
4752                         ixgbe_cache_ring_rss(adapter);
4753
4754                 fcoe_rx_i = f->mask;
4755                 fcoe_tx_i = f->mask;
4756         }
4757         for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) {
4758                 adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i;
4759                 adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i;
4760         }
4761         return true;
4762 }
4763
4764 #endif /* IXGBE_FCOE */
4765 /**
4766  * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov
4767  * @adapter: board private structure to initialize
4768  *
4769  * SR-IOV doesn't use any descriptor rings but changes the default if
4770  * no other mapping is used.
4771  *
4772  */
4773 static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
4774 {
4775         adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2;
4776         adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2;
4777         if (adapter->num_vfs)
4778                 return true;
4779         else
4780                 return false;
4781 }
4782
4783 /**
4784  * ixgbe_cache_ring_register - Descriptor ring to register mapping
4785  * @adapter: board private structure to initialize
4786  *
4787  * Once we know the feature-set enabled for the device, we'll cache
4788  * the register offset the descriptor ring is assigned to.
4789  *
4790  * Note, the order the various feature calls is important.  It must start with
4791  * the "most" features enabled at the same time, then trickle down to the
4792  * least amount of features turned on at once.
4793  **/
4794 static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter)
4795 {
4796         /* start with default case */
4797         adapter->rx_ring[0]->reg_idx = 0;
4798         adapter->tx_ring[0]->reg_idx = 0;
4799
4800         if (ixgbe_cache_ring_sriov(adapter))
4801                 return;
4802
4803 #ifdef CONFIG_IXGBE_DCB
4804         if (ixgbe_cache_ring_dcb(adapter))
4805                 return;
4806 #endif
4807
4808 #ifdef IXGBE_FCOE
4809         if (ixgbe_cache_ring_fcoe(adapter))
4810                 return;
4811 #endif /* IXGBE_FCOE */
4812
4813         if (ixgbe_cache_ring_fdir(adapter))
4814                 return;
4815
4816         if (ixgbe_cache_ring_rss(adapter))
4817                 return;
4818 }
4819
4820 /**
4821  * ixgbe_alloc_queues - Allocate memory for all rings
4822  * @adapter: board private structure to initialize
4823  *
4824  * We allocate one ring per queue at run-time since we don't know the
4825  * number of queues at compile-time.  The polling_netdev array is
4826  * intended for Multiqueue, but should work fine with a single queue.
4827  **/
4828 static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter)
4829 {
4830         int rx = 0, tx = 0, nid = adapter->node;
4831
4832         if (nid < 0 || !node_online(nid))
4833                 nid = first_online_node;
4834
4835         for (; tx < adapter->num_tx_queues; tx++) {
4836                 struct ixgbe_ring *ring;
4837
4838                 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4839                 if (!ring)
4840                         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4841                 if (!ring)
4842                         goto err_allocation;
4843                 ring->count = adapter->tx_ring_count;
4844                 ring->queue_index = tx;
4845                 ring->numa_node = nid;
4846                 ring->dev = &adapter->pdev->dev;
4847                 ring->netdev = adapter->netdev;
4848
4849                 adapter->tx_ring[tx] = ring;
4850         }
4851
4852         for (; rx < adapter->num_rx_queues; rx++) {
4853                 struct ixgbe_ring *ring;
4854
4855                 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4856                 if (!ring)
4857                         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4858                 if (!ring)
4859                         goto err_allocation;
4860                 ring->count = adapter->rx_ring_count;
4861                 ring->queue_index = rx;
4862                 ring->numa_node = nid;
4863                 ring->dev = &adapter->pdev->dev;
4864                 ring->netdev = adapter->netdev;
4865
4866                 adapter->rx_ring[rx] = ring;
4867         }
4868
4869         ixgbe_cache_ring_register(adapter);
4870
4871         return 0;
4872
4873 err_allocation:
4874         while (tx)
4875                 kfree(adapter->tx_ring[--tx]);
4876
4877         while (rx)
4878                 kfree(adapter->rx_ring[--rx]);
4879         return -ENOMEM;
4880 }
4881
4882 /**
4883  * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported
4884  * @adapter: board private structure to initialize
4885  *
4886  * Attempt to configure the interrupts using the best available
4887  * capabilities of the hardware and the kernel.
4888  **/
4889 static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
4890 {
4891         struct ixgbe_hw *hw = &adapter->hw;
4892         int err = 0;
4893         int vector, v_budget;
4894
4895         /*
4896          * It's easy to be greedy for MSI-X vectors, but it really
4897          * doesn't do us much good if we have a lot more vectors
4898          * than CPU's.  So let's be conservative and only ask for
4899          * (roughly) the same number of vectors as there are CPU's.
4900          */
4901         v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
4902                        (int)num_online_cpus()) + NON_Q_VECTORS;
4903
4904         /*
4905          * At the same time, hardware can only support a maximum of
4906          * hw.mac->max_msix_vectors vectors.  With features
4907          * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
4908          * descriptor queues supported by our device.  Thus, we cap it off in
4909          * those rare cases where the cpu count also exceeds our vector limit.
4910          */
4911         v_budget = min(v_budget, (int)hw->mac.max_msix_vectors);
4912
4913         /* A failure in MSI-X entry allocation isn't fatal, but it does
4914          * mean we disable MSI-X capabilities of the adapter. */
4915         adapter->msix_entries = kcalloc(v_budget,
4916                                         sizeof(struct msix_entry), GFP_KERNEL);
4917         if (adapter->msix_entries) {
4918                 for (vector = 0; vector < v_budget; vector++)
4919                         adapter->msix_entries[vector].entry = vector;
4920
4921                 ixgbe_acquire_msix_vectors(adapter, v_budget);
4922
4923                 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4924                         goto out;
4925         }
4926
4927         adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
4928         adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED;
4929         if (adapter->flags & (IXGBE_FLAG_FDIR_HASH_CAPABLE |
4930                               IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
4931                 e_err(probe,
4932                       "Flow Director is not supported while multiple "
4933                       "queues are disabled.  Disabling Flow Director\n");
4934         }
4935         adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
4936         adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
4937         adapter->atr_sample_rate = 0;
4938         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4939                 ixgbe_disable_sriov(adapter);
4940
4941         err = ixgbe_set_num_queues(adapter);
4942         if (err)
4943                 return err;
4944
4945         err = pci_enable_msi(adapter->pdev);
4946         if (!err) {
4947                 adapter->flags |= IXGBE_FLAG_MSI_ENABLED;
4948         } else {
4949                 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4950                              "Unable to allocate MSI interrupt, "
4951                              "falling back to legacy.  Error: %d\n", err);
4952                 /* reset err */
4953                 err = 0;
4954         }
4955
4956 out:
4957         return err;
4958 }
4959
4960 /**
4961  * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors
4962  * @adapter: board private structure to initialize
4963  *
4964  * We allocate one q_vector per queue interrupt.  If allocation fails we
4965  * return -ENOMEM.
4966  **/
4967 static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter)
4968 {
4969         int q_idx, num_q_vectors;
4970         struct ixgbe_q_vector *q_vector;
4971         int (*poll)(struct napi_struct *, int);
4972
4973         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
4974                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4975                 poll = &ixgbe_clean_rxtx_many;
4976         } else {
4977                 num_q_vectors = 1;
4978                 poll = &ixgbe_poll;
4979         }
4980
4981         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
4982                 q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector),
4983                                         GFP_KERNEL, adapter->node);
4984                 if (!q_vector)
4985                         q_vector = kzalloc(sizeof(struct ixgbe_q_vector),
4986                                            GFP_KERNEL);
4987                 if (!q_vector)
4988                         goto err_out;
4989                 q_vector->adapter = adapter;
4990                 if (q_vector->txr_count && !q_vector->rxr_count)
4991                         q_vector->eitr = adapter->tx_eitr_param;
4992                 else
4993                         q_vector->eitr = adapter->rx_eitr_param;
4994                 q_vector->v_idx = q_idx;
4995                 netif_napi_add(adapter->netdev, &q_vector->napi, (*poll), 64);
4996                 adapter->q_vector[q_idx] = q_vector;
4997         }
4998
4999         return 0;
5000
5001 err_out:
5002         while (q_idx) {
5003                 q_idx--;
5004                 q_vector = adapter->q_vector[q_idx];
5005                 netif_napi_del(&q_vector->napi);
5006                 kfree(q_vector);
5007                 adapter->q_vector[q_idx] = NULL;
5008         }
5009         return -ENOMEM;
5010 }
5011
5012 /**
5013  * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors
5014  * @adapter: board private structure to initialize
5015  *
5016  * This function frees the memory allocated to the q_vectors.  In addition if
5017  * NAPI is enabled it will delete any references to the NAPI struct prior
5018  * to freeing the q_vector.
5019  **/
5020 static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter)
5021 {
5022         int q_idx, num_q_vectors;
5023
5024         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
5025                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
5026         else
5027                 num_q_vectors = 1;
5028
5029         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
5030                 struct ixgbe_q_vector *q_vector = adapter->q_vector[q_idx];
5031                 adapter->q_vector[q_idx] = NULL;
5032                 netif_napi_del(&q_vector->napi);
5033                 kfree(q_vector);
5034         }
5035 }
5036
5037 static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter)
5038 {
5039         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
5040                 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
5041                 pci_disable_msix(adapter->pdev);
5042                 kfree(adapter->msix_entries);
5043                 adapter->msix_entries = NULL;
5044         } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
5045                 adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED;
5046                 pci_disable_msi(adapter->pdev);
5047         }
5048 }
5049
5050 /**
5051  * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme
5052  * @adapter: board private structure to initialize
5053  *
5054  * We determine which interrupt scheme to use based on...
5055  * - Kernel support (MSI, MSI-X)
5056  *   - which can be user-defined (via MODULE_PARAM)
5057  * - Hardware queue count (num_*_queues)
5058  *   - defined by miscellaneous hardware support/features (RSS, etc.)
5059  **/
5060 int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter)
5061 {
5062         int err;
5063
5064         /* Number of supported queues */
5065         err = ixgbe_set_num_queues(adapter);
5066         if (err)
5067                 return err;
5068
5069         err = ixgbe_set_interrupt_capability(adapter);
5070         if (err) {
5071                 e_dev_err("Unable to setup interrupt capabilities\n");
5072                 goto err_set_interrupt;
5073         }
5074
5075         err = ixgbe_alloc_q_vectors(adapter);
5076         if (err) {
5077                 e_dev_err("Unable to allocate memory for queue vectors\n");
5078                 goto err_alloc_q_vectors;
5079         }
5080
5081         err = ixgbe_alloc_queues(adapter);
5082         if (err) {
5083                 e_dev_err("Unable to allocate memory for queues\n");
5084                 goto err_alloc_queues;
5085         }
5086
5087         e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n",
5088                    (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
5089                    adapter->num_rx_queues, adapter->num_tx_queues);
5090
5091         set_bit(__IXGBE_DOWN, &adapter->state);
5092
5093         return 0;
5094
5095 err_alloc_queues:
5096         ixgbe_free_q_vectors(adapter);
5097 err_alloc_q_vectors:
5098         ixgbe_reset_interrupt_capability(adapter);
5099 err_set_interrupt:
5100         return err;
5101 }
5102
5103 static void ring_free_rcu(struct rcu_head *head)
5104 {
5105         kfree(container_of(head, struct ixgbe_ring, rcu));
5106 }
5107
5108 /**
5109  * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
5110  * @adapter: board private structure to clear interrupt scheme on
5111  *
5112  * We go through and clear interrupt specific resources and reset the structure
5113  * to pre-load conditions
5114  **/
5115 void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
5116 {
5117         int i;
5118
5119         for (i = 0; i < adapter->num_tx_queues; i++) {
5120                 kfree(adapter->tx_ring[i]);
5121                 adapter->tx_ring[i] = NULL;
5122         }
5123         for (i = 0; i < adapter->num_rx_queues; i++) {
5124                 struct ixgbe_ring *ring = adapter->rx_ring[i];
5125
5126                 /* ixgbe_get_stats64() might access this ring, we must wait
5127                  * a grace period before freeing it.
5128                  */
5129                 call_rcu(&ring->rcu, ring_free_rcu);
5130                 adapter->rx_ring[i] = NULL;
5131         }
5132
5133         adapter->num_tx_queues = 0;
5134         adapter->num_rx_queues = 0;
5135
5136         ixgbe_free_q_vectors(adapter);
5137         ixgbe_reset_interrupt_capability(adapter);
5138 }
5139
5140 /**
5141  * ixgbe_sfp_timer - worker thread to find a missing module
5142  * @data: pointer to our adapter struct
5143  **/
5144 static void ixgbe_sfp_timer(unsigned long data)
5145 {
5146         struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
5147
5148         /*
5149          * Do the sfp_timer outside of interrupt context due to the
5150          * delays that sfp+ detection requires
5151          */
5152         schedule_work(&adapter->sfp_task);
5153 }
5154
5155 /**
5156  * ixgbe_sfp_task - worker thread to find a missing module
5157  * @work: pointer to work_struct containing our data
5158  **/
5159 static void ixgbe_sfp_task(struct work_struct *work)
5160 {
5161         struct ixgbe_adapter *adapter = container_of(work,
5162                                                      struct ixgbe_adapter,
5163                                                      sfp_task);
5164         struct ixgbe_hw *hw = &adapter->hw;
5165
5166         if ((hw->phy.type == ixgbe_phy_nl) &&
5167             (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) {
5168                 s32 ret = hw->phy.ops.identify_sfp(hw);
5169                 if (ret == IXGBE_ERR_SFP_NOT_PRESENT)
5170                         goto reschedule;
5171                 ret = hw->phy.ops.reset(hw);
5172                 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) {
5173                         e_dev_err("failed to initialize because an unsupported "
5174                                   "SFP+ module type was detected.\n");
5175                         e_dev_err("Reload the driver after installing a "
5176                                   "supported module.\n");
5177                         unregister_netdev(adapter->netdev);
5178                 } else {
5179                         e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
5180                 }
5181                 /* don't need this routine any more */
5182                 clear_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state);
5183         }
5184         return;
5185 reschedule:
5186         if (test_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state))
5187                 mod_timer(&adapter->sfp_timer,
5188                           round_jiffies(jiffies + (2 * HZ)));
5189 }
5190
5191 /**
5192  * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
5193  * @adapter: board private structure to initialize
5194  *
5195  * ixgbe_sw_init initializes the Adapter private data structure.
5196  * Fields are initialized based on PCI device information and
5197  * OS network device settings (MTU size).
5198  **/
5199 static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
5200 {
5201         struct ixgbe_hw *hw = &adapter->hw;
5202         struct pci_dev *pdev = adapter->pdev;
5203         struct net_device *dev = adapter->netdev;
5204         unsigned int rss;
5205 #ifdef CONFIG_IXGBE_DCB
5206         int j;
5207         struct tc_configuration *tc;
5208 #endif
5209         int max_frame = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
5210
5211         /* PCI config space info */
5212
5213         hw->vendor_id = pdev->vendor;
5214         hw->device_id = pdev->device;
5215         hw->revision_id = pdev->revision;
5216         hw->subsystem_vendor_id = pdev->subsystem_vendor;
5217         hw->subsystem_device_id = pdev->subsystem_device;
5218
5219         /* Set capability flags */
5220         rss = min(IXGBE_MAX_RSS_INDICES, (int)num_online_cpus());
5221         adapter->ring_feature[RING_F_RSS].indices = rss;
5222         adapter->flags |= IXGBE_FLAG_RSS_ENABLED;
5223         adapter->ring_feature[RING_F_DCB].indices = IXGBE_MAX_DCB_INDICES;
5224         switch (hw->mac.type) {
5225         case ixgbe_mac_82598EB:
5226                 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5227                         adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
5228                 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
5229                 break;
5230         case ixgbe_mac_82599EB:
5231         case ixgbe_mac_X540:
5232                 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
5233                 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
5234                 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
5235                 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
5236                         adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5237                 /* n-tuple support exists, always init our spinlock */
5238                 spin_lock_init(&adapter->fdir_perfect_lock);
5239                 /* Flow Director hash filters enabled */
5240                 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
5241                 adapter->atr_sample_rate = 20;
5242                 adapter->ring_feature[RING_F_FDIR].indices =
5243                                                          IXGBE_MAX_FDIR_INDICES;
5244                 adapter->fdir_pballoc = 0;
5245 #ifdef IXGBE_FCOE
5246                 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
5247                 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5248                 adapter->ring_feature[RING_F_FCOE].indices = 0;
5249 #ifdef CONFIG_IXGBE_DCB
5250                 /* Default traffic class to use for FCoE */
5251                 adapter->fcoe.tc = IXGBE_FCOE_DEFTC;
5252                 adapter->fcoe.up = IXGBE_FCOE_DEFTC;
5253 #endif
5254 #endif /* IXGBE_FCOE */
5255                 break;
5256         default:
5257                 break;
5258         }
5259
5260 #ifdef CONFIG_IXGBE_DCB
5261         /* Configure DCB traffic classes */
5262         for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
5263                 tc = &adapter->dcb_cfg.tc_config[j];
5264                 tc->path[DCB_TX_CONFIG].bwg_id = 0;
5265                 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
5266                 tc->path[DCB_RX_CONFIG].bwg_id = 0;
5267                 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
5268                 tc->dcb_pfc = pfc_disabled;
5269         }
5270         adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
5271         adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
5272         adapter->dcb_cfg.rx_pba_cfg = pba_equal;
5273         adapter->dcb_cfg.pfc_mode_enable = false;
5274         adapter->dcb_set_bitmap = 0x00;
5275         adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
5276         ixgbe_copy_dcb_cfg(&adapter->dcb_cfg, &adapter->temp_dcb_cfg,
5277                            MAX_TRAFFIC_CLASS);
5278
5279 #endif
5280
5281         /* default flow control settings */
5282         hw->fc.requested_mode = ixgbe_fc_full;
5283         hw->fc.current_mode = ixgbe_fc_full;    /* init for ethtool output */
5284 #ifdef CONFIG_DCB
5285         adapter->last_lfc_mode = hw->fc.current_mode;
5286 #endif
5287         hw->fc.high_water = FC_HIGH_WATER(max_frame);
5288         hw->fc.low_water = FC_LOW_WATER(max_frame);
5289         hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
5290         hw->fc.send_xon = true;
5291         hw->fc.disable_fc_autoneg = false;
5292
5293         /* enable itr by default in dynamic mode */
5294         adapter->rx_itr_setting = 1;
5295         adapter->rx_eitr_param = 20000;
5296         adapter->tx_itr_setting = 1;
5297         adapter->tx_eitr_param = 10000;
5298
5299         /* set defaults for eitr in MegaBytes */
5300         adapter->eitr_low = 10;
5301         adapter->eitr_high = 20;
5302
5303         /* set default ring sizes */
5304         adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
5305         adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
5306
5307         /* initialize eeprom parameters */
5308         if (ixgbe_init_eeprom_params_generic(hw)) {
5309                 e_dev_err("EEPROM initialization failed\n");
5310                 return -EIO;
5311         }
5312
5313         /* enable rx csum by default */
5314         adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
5315
5316         /* get assigned NUMA node */
5317         adapter->node = dev_to_node(&pdev->dev);
5318
5319         set_bit(__IXGBE_DOWN, &adapter->state);
5320
5321         return 0;
5322 }
5323
5324 /**
5325  * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
5326  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
5327  *
5328  * Return 0 on success, negative on failure
5329  **/
5330 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
5331 {
5332         struct device *dev = tx_ring->dev;
5333         int size;
5334
5335         size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
5336         tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
5337         if (!tx_ring->tx_buffer_info)
5338                 tx_ring->tx_buffer_info = vzalloc(size);
5339         if (!tx_ring->tx_buffer_info)
5340                 goto err;
5341
5342         /* round up to nearest 4K */
5343         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
5344         tx_ring->size = ALIGN(tx_ring->size, 4096);
5345
5346         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
5347                                            &tx_ring->dma, GFP_KERNEL);
5348         if (!tx_ring->desc)
5349                 goto err;
5350
5351         tx_ring->next_to_use = 0;
5352         tx_ring->next_to_clean = 0;
5353         tx_ring->work_limit = tx_ring->count;
5354         return 0;
5355
5356 err:
5357         vfree(tx_ring->tx_buffer_info);
5358         tx_ring->tx_buffer_info = NULL;
5359         dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
5360         return -ENOMEM;
5361 }
5362
5363 /**
5364  * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
5365  * @adapter: board private structure
5366  *
5367  * If this function returns with an error, then it's possible one or
5368  * more of the rings is populated (while the rest are not).  It is the
5369  * callers duty to clean those orphaned rings.
5370  *
5371  * Return 0 on success, negative on failure
5372  **/
5373 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
5374 {
5375         int i, err = 0;
5376
5377         for (i = 0; i < adapter->num_tx_queues; i++) {
5378                 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
5379                 if (!err)
5380                         continue;
5381                 e_err(probe, "Allocation for Tx Queue %u failed\n", i);
5382                 break;
5383         }
5384
5385         return err;
5386 }
5387
5388 /**
5389  * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
5390  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
5391  *
5392  * Returns 0 on success, negative on failure
5393  **/
5394 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
5395 {
5396         struct device *dev = rx_ring->dev;
5397         int size;
5398
5399         size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
5400         rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
5401         if (!rx_ring->rx_buffer_info)
5402                 rx_ring->rx_buffer_info = vzalloc(size);
5403         if (!rx_ring->rx_buffer_info)
5404                 goto err;
5405
5406         /* Round up to nearest 4K */
5407         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5408         rx_ring->size = ALIGN(rx_ring->size, 4096);
5409
5410         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
5411                                            &rx_ring->dma, GFP_KERNEL);
5412
5413         if (!rx_ring->desc)
5414                 goto err;
5415
5416         rx_ring->next_to_clean = 0;
5417         rx_ring->next_to_use = 0;
5418
5419         return 0;
5420 err:
5421         vfree(rx_ring->rx_buffer_info);
5422         rx_ring->rx_buffer_info = NULL;
5423         dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
5424         return -ENOMEM;
5425 }
5426
5427 /**
5428  * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
5429  * @adapter: board private structure
5430  *
5431  * If this function returns with an error, then it's possible one or
5432  * more of the rings is populated (while the rest are not).  It is the
5433  * callers duty to clean those orphaned rings.
5434  *
5435  * Return 0 on success, negative on failure
5436  **/
5437 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
5438 {
5439         int i, err = 0;
5440
5441         for (i = 0; i < adapter->num_rx_queues; i++) {
5442                 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
5443                 if (!err)
5444                         continue;
5445                 e_err(probe, "Allocation for Rx Queue %u failed\n", i);
5446                 break;
5447         }
5448
5449         return err;
5450 }
5451
5452 /**
5453  * ixgbe_free_tx_resources - Free Tx Resources per Queue
5454  * @tx_ring: Tx descriptor ring for a specific queue
5455  *
5456  * Free all transmit software resources
5457  **/
5458 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
5459 {
5460         ixgbe_clean_tx_ring(tx_ring);
5461
5462         vfree(tx_ring->tx_buffer_info);
5463         tx_ring->tx_buffer_info = NULL;
5464
5465         /* if not set, then don't free */
5466         if (!tx_ring->desc)
5467                 return;
5468
5469         dma_free_coherent(tx_ring->dev, tx_ring->size,
5470                           tx_ring->desc, tx_ring->dma);
5471
5472         tx_ring->desc = NULL;
5473 }
5474
5475 /**
5476  * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
5477  * @adapter: board private structure
5478  *
5479  * Free all transmit software resources
5480  **/
5481 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
5482 {
5483         int i;
5484
5485         for (i = 0; i < adapter->num_tx_queues; i++)
5486                 if (adapter->tx_ring[i]->desc)
5487                         ixgbe_free_tx_resources(adapter->tx_ring[i]);
5488 }
5489
5490 /**
5491  * ixgbe_free_rx_resources - Free Rx Resources
5492  * @rx_ring: ring to clean the resources from
5493  *
5494  * Free all receive software resources
5495  **/
5496 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
5497 {
5498         ixgbe_clean_rx_ring(rx_ring);
5499
5500         vfree(rx_ring->rx_buffer_info);
5501         rx_ring->rx_buffer_info = NULL;
5502
5503         /* if not set, then don't free */
5504         if (!rx_ring->desc)
5505                 return;
5506
5507         dma_free_coherent(rx_ring->dev, rx_ring->size,
5508                           rx_ring->desc, rx_ring->dma);
5509
5510         rx_ring->desc = NULL;
5511 }
5512
5513 /**
5514  * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
5515  * @adapter: board private structure
5516  *
5517  * Free all receive software resources
5518  **/
5519 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
5520 {
5521         int i;
5522
5523         for (i = 0; i < adapter->num_rx_queues; i++)
5524                 if (adapter->rx_ring[i]->desc)
5525                         ixgbe_free_rx_resources(adapter->rx_ring[i]);
5526 }
5527
5528 /**
5529  * ixgbe_change_mtu - Change the Maximum Transfer Unit
5530  * @netdev: network interface device structure
5531  * @new_mtu: new value for maximum frame size
5532  *
5533  * Returns 0 on success, negative on failure
5534  **/
5535 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
5536 {
5537         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5538         struct ixgbe_hw *hw = &adapter->hw;
5539         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5540
5541         /* MTU < 68 is an error and causes problems on some kernels */
5542         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED &&
5543             hw->mac.type != ixgbe_mac_X540) {
5544                 if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
5545                         return -EINVAL;
5546         } else {
5547                 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
5548                         return -EINVAL;
5549         }
5550
5551         e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5552         /* must set new MTU before calling down or up */
5553         netdev->mtu = new_mtu;
5554
5555         hw->fc.high_water = FC_HIGH_WATER(max_frame);
5556         hw->fc.low_water = FC_LOW_WATER(max_frame);
5557
5558         if (netif_running(netdev))
5559                 ixgbe_reinit_locked(adapter);
5560
5561         return 0;
5562 }
5563
5564 /**
5565  * ixgbe_open - Called when a network interface is made active
5566  * @netdev: network interface device structure
5567  *
5568  * Returns 0 on success, negative value on failure
5569  *
5570  * The open entry point is called when a network interface is made
5571  * active by the system (IFF_UP).  At this point all resources needed
5572  * for transmit and receive operations are allocated, the interrupt
5573  * handler is registered with the OS, the watchdog timer is started,
5574  * and the stack is notified that the interface is ready.
5575  **/
5576 static int ixgbe_open(struct net_device *netdev)
5577 {
5578         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5579         int err;
5580
5581         /* disallow open during test */
5582         if (test_bit(__IXGBE_TESTING, &adapter->state))
5583                 return -EBUSY;
5584
5585         netif_carrier_off(netdev);
5586
5587         /* allocate transmit descriptors */
5588         err = ixgbe_setup_all_tx_resources(adapter);
5589         if (err)
5590                 goto err_setup_tx;
5591
5592         /* allocate receive descriptors */
5593         err = ixgbe_setup_all_rx_resources(adapter);
5594         if (err)
5595                 goto err_setup_rx;
5596
5597         ixgbe_configure(adapter);
5598
5599         err = ixgbe_request_irq(adapter);
5600         if (err)
5601                 goto err_req_irq;
5602
5603         err = ixgbe_up_complete(adapter);
5604         if (err)
5605                 goto err_up;
5606
5607         netif_tx_start_all_queues(netdev);
5608
5609         return 0;
5610
5611 err_up:
5612         ixgbe_release_hw_control(adapter);
5613         ixgbe_free_irq(adapter);
5614 err_req_irq:
5615 err_setup_rx:
5616         ixgbe_free_all_rx_resources(adapter);
5617 err_setup_tx:
5618         ixgbe_free_all_tx_resources(adapter);
5619         ixgbe_reset(adapter);
5620
5621         return err;
5622 }
5623
5624 /**
5625  * ixgbe_close - Disables a network interface
5626  * @netdev: network interface device structure
5627  *
5628  * Returns 0, this is not allowed to fail
5629  *
5630  * The close entry point is called when an interface is de-activated
5631  * by the OS.  The hardware is still under the drivers control, but
5632  * needs to be disabled.  A global MAC reset is issued to stop the
5633  * hardware, and all transmit and receive resources are freed.
5634  **/
5635 static int ixgbe_close(struct net_device *netdev)
5636 {
5637         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5638
5639         ixgbe_down(adapter);
5640         ixgbe_free_irq(adapter);
5641
5642         ixgbe_free_all_tx_resources(adapter);
5643         ixgbe_free_all_rx_resources(adapter);
5644
5645         ixgbe_release_hw_control(adapter);
5646
5647         return 0;
5648 }
5649
5650 #ifdef CONFIG_PM
5651 static int ixgbe_resume(struct pci_dev *pdev)
5652 {
5653         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5654         struct net_device *netdev = adapter->netdev;
5655         u32 err;
5656
5657         pci_set_power_state(pdev, PCI_D0);
5658         pci_restore_state(pdev);
5659         /*
5660          * pci_restore_state clears dev->state_saved so call
5661          * pci_save_state to restore it.
5662          */
5663         pci_save_state(pdev);
5664
5665         err = pci_enable_device_mem(pdev);
5666         if (err) {
5667                 e_dev_err("Cannot enable PCI device from suspend\n");
5668                 return err;
5669         }
5670         pci_set_master(pdev);
5671
5672         pci_wake_from_d3(pdev, false);
5673
5674         err = ixgbe_init_interrupt_scheme(adapter);
5675         if (err) {
5676                 e_dev_err("Cannot initialize interrupts for device\n");
5677                 return err;
5678         }
5679
5680         ixgbe_reset(adapter);
5681
5682         IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
5683
5684         if (netif_running(netdev)) {
5685                 err = ixgbe_open(netdev);
5686                 if (err)
5687                         return err;
5688         }
5689
5690         netif_device_attach(netdev);
5691
5692         return 0;
5693 }
5694 #endif /* CONFIG_PM */
5695
5696 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
5697 {
5698         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5699         struct net_device *netdev = adapter->netdev;
5700         struct ixgbe_hw *hw = &adapter->hw;
5701         u32 ctrl, fctrl;
5702         u32 wufc = adapter->wol;
5703 #ifdef CONFIG_PM
5704         int retval = 0;
5705 #endif
5706
5707         netif_device_detach(netdev);
5708
5709         if (netif_running(netdev)) {
5710                 ixgbe_down(adapter);
5711                 ixgbe_free_irq(adapter);
5712                 ixgbe_free_all_tx_resources(adapter);
5713                 ixgbe_free_all_rx_resources(adapter);
5714         }
5715
5716         ixgbe_clear_interrupt_scheme(adapter);
5717 #ifdef CONFIG_DCB
5718         kfree(adapter->ixgbe_ieee_pfc);
5719         kfree(adapter->ixgbe_ieee_ets);
5720 #endif
5721
5722 #ifdef CONFIG_PM
5723         retval = pci_save_state(pdev);
5724         if (retval)
5725                 return retval;
5726
5727 #endif
5728         if (wufc) {
5729                 ixgbe_set_rx_mode(netdev);
5730
5731                 /* turn on all-multi mode if wake on multicast is enabled */
5732                 if (wufc & IXGBE_WUFC_MC) {
5733                         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5734                         fctrl |= IXGBE_FCTRL_MPE;
5735                         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5736                 }
5737
5738                 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
5739                 ctrl |= IXGBE_CTRL_GIO_DIS;
5740                 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
5741
5742                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
5743         } else {
5744                 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
5745                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
5746         }
5747
5748         switch (hw->mac.type) {
5749         case ixgbe_mac_82598EB:
5750                 pci_wake_from_d3(pdev, false);
5751                 break;
5752         case ixgbe_mac_82599EB:
5753         case ixgbe_mac_X540:
5754                 pci_wake_from_d3(pdev, !!wufc);
5755                 break;
5756         default:
5757                 break;
5758         }
5759
5760         *enable_wake = !!wufc;
5761
5762         ixgbe_release_hw_control(adapter);
5763
5764         pci_disable_device(pdev);
5765
5766         return 0;
5767 }
5768
5769 #ifdef CONFIG_PM
5770 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
5771 {
5772         int retval;
5773         bool wake;
5774
5775         retval = __ixgbe_shutdown(pdev, &wake);
5776         if (retval)
5777                 return retval;
5778
5779         if (wake) {
5780                 pci_prepare_to_sleep(pdev);
5781         } else {
5782                 pci_wake_from_d3(pdev, false);
5783                 pci_set_power_state(pdev, PCI_D3hot);
5784         }
5785
5786         return 0;
5787 }
5788 #endif /* CONFIG_PM */
5789
5790 static void ixgbe_shutdown(struct pci_dev *pdev)
5791 {
5792         bool wake;
5793
5794         __ixgbe_shutdown(pdev, &wake);
5795
5796         if (system_state == SYSTEM_POWER_OFF) {
5797                 pci_wake_from_d3(pdev, wake);
5798                 pci_set_power_state(pdev, PCI_D3hot);
5799         }
5800 }
5801
5802 /**
5803  * ixgbe_update_stats - Update the board statistics counters.
5804  * @adapter: board private structure
5805  **/
5806 void ixgbe_update_stats(struct ixgbe_adapter *adapter)
5807 {
5808         struct net_device *netdev = adapter->netdev;
5809         struct ixgbe_hw *hw = &adapter->hw;
5810         struct ixgbe_hw_stats *hwstats = &adapter->stats;
5811         u64 total_mpc = 0;
5812         u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5813         u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
5814         u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
5815         u64 bytes = 0, packets = 0;
5816
5817         if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5818             test_bit(__IXGBE_RESETTING, &adapter->state))
5819                 return;
5820
5821         if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
5822                 u64 rsc_count = 0;
5823                 u64 rsc_flush = 0;
5824                 for (i = 0; i < 16; i++)
5825                         adapter->hw_rx_no_dma_resources +=
5826                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5827                 for (i = 0; i < adapter->num_rx_queues; i++) {
5828                         rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
5829                         rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
5830                 }
5831                 adapter->rsc_total_count = rsc_count;
5832                 adapter->rsc_total_flush = rsc_flush;
5833         }
5834
5835         for (i = 0; i < adapter->num_rx_queues; i++) {
5836                 struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
5837                 non_eop_descs += rx_ring->rx_stats.non_eop_descs;
5838                 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
5839                 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
5840                 bytes += rx_ring->stats.bytes;
5841                 packets += rx_ring->stats.packets;
5842         }
5843         adapter->non_eop_descs = non_eop_descs;
5844         adapter->alloc_rx_page_failed = alloc_rx_page_failed;
5845         adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
5846         netdev->stats.rx_bytes = bytes;
5847         netdev->stats.rx_packets = packets;
5848
5849         bytes = 0;
5850         packets = 0;
5851         /* gather some stats to the adapter struct that are per queue */
5852         for (i = 0; i < adapter->num_tx_queues; i++) {
5853                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
5854                 restart_queue += tx_ring->tx_stats.restart_queue;
5855                 tx_busy += tx_ring->tx_stats.tx_busy;
5856                 bytes += tx_ring->stats.bytes;
5857                 packets += tx_ring->stats.packets;
5858         }
5859         adapter->restart_queue = restart_queue;
5860         adapter->tx_busy = tx_busy;
5861         netdev->stats.tx_bytes = bytes;
5862         netdev->stats.tx_packets = packets;
5863
5864         hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5865         for (i = 0; i < 8; i++) {
5866                 /* for packet buffers not used, the register should read 0 */
5867                 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5868                 missed_rx += mpc;
5869                 hwstats->mpc[i] += mpc;
5870                 total_mpc += hwstats->mpc[i];
5871                 if (hw->mac.type == ixgbe_mac_82598EB)
5872                         hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5873                 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5874                 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5875                 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5876                 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5877                 switch (hw->mac.type) {
5878                 case ixgbe_mac_82598EB:
5879                         hwstats->pxonrxc[i] +=
5880                                 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5881                         break;
5882                 case ixgbe_mac_82599EB:
5883                 case ixgbe_mac_X540:
5884                         hwstats->pxonrxc[i] +=
5885                                 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5886                         break;
5887                 default:
5888                         break;
5889                 }
5890                 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5891                 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5892         }
5893         hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5894         /* work around hardware counting issue */
5895         hwstats->gprc -= missed_rx;
5896
5897         ixgbe_update_xoff_received(adapter);
5898
5899         /* 82598 hardware only has a 32 bit counter in the high register */
5900         switch (hw->mac.type) {
5901         case ixgbe_mac_82598EB:
5902                 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5903                 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5904                 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5905                 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5906                 break;
5907         case ixgbe_mac_82599EB:
5908         case ixgbe_mac_X540:
5909                 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
5910                 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
5911                 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
5912                 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
5913                 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
5914                 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
5915                 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5916                 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
5917                 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
5918 #ifdef IXGBE_FCOE
5919                 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5920                 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5921                 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5922                 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5923                 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5924                 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5925 #endif /* IXGBE_FCOE */
5926                 break;
5927         default:
5928                 break;
5929         }
5930         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5931         hwstats->bprc += bprc;
5932         hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5933         if (hw->mac.type == ixgbe_mac_82598EB)
5934                 hwstats->mprc -= bprc;
5935         hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5936         hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5937         hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5938         hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5939         hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5940         hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5941         hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5942         hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5943         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5944         hwstats->lxontxc += lxon;
5945         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5946         hwstats->lxofftxc += lxoff;
5947         hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5948         hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5949         hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5950         /*
5951          * 82598 errata - tx of flow control packets is included in tx counters
5952          */
5953         xon_off_tot = lxon + lxoff;
5954         hwstats->gptc -= xon_off_tot;
5955         hwstats->mptc -= xon_off_tot;
5956         hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
5957         hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5958         hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5959         hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5960         hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5961         hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5962         hwstats->ptc64 -= xon_off_tot;
5963         hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5964         hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5965         hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5966         hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5967         hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5968         hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5969
5970         /* Fill out the OS statistics structure */
5971         netdev->stats.multicast = hwstats->mprc;
5972
5973         /* Rx Errors */
5974         netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
5975         netdev->stats.rx_dropped = 0;
5976         netdev->stats.rx_length_errors = hwstats->rlec;
5977         netdev->stats.rx_crc_errors = hwstats->crcerrs;
5978         netdev->stats.rx_missed_errors = total_mpc;
5979 }
5980
5981 /**
5982  * ixgbe_watchdog - Timer Call-back
5983  * @data: pointer to adapter cast into an unsigned long
5984  **/
5985 static void ixgbe_watchdog(unsigned long data)
5986 {
5987         struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
5988         struct ixgbe_hw *hw = &adapter->hw;
5989         u64 eics = 0;
5990         int i;
5991
5992         /*
5993          *  Do the watchdog outside of interrupt context due to the lovely
5994          * delays that some of the newer hardware requires
5995          */
5996
5997         if (test_bit(__IXGBE_DOWN, &adapter->state))
5998                 goto watchdog_short_circuit;
5999
6000         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
6001                 /*
6002                  * for legacy and MSI interrupts don't set any bits
6003                  * that are enabled for EIAM, because this operation
6004                  * would set *both* EIMS and EICS for any bit in EIAM
6005                  */
6006                 IXGBE_WRITE_REG(hw, IXGBE_EICS,
6007                         (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
6008                 goto watchdog_reschedule;
6009         }
6010
6011         /* get one bit for every active tx/rx interrupt vector */
6012         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
6013                 struct ixgbe_q_vector *qv = adapter->q_vector[i];
6014                 if (qv->rxr_count || qv->txr_count)
6015                         eics |= ((u64)1 << i);
6016         }
6017
6018         /* Cause software interrupt to ensure rx rings are cleaned */
6019         ixgbe_irq_rearm_queues(adapter, eics);
6020
6021 watchdog_reschedule:
6022         /* Reset the timer */
6023         mod_timer(&adapter->watchdog_timer, round_jiffies(jiffies + 2 * HZ));
6024
6025 watchdog_short_circuit:
6026         schedule_work(&adapter->watchdog_task);
6027 }
6028
6029 /**
6030  * ixgbe_multispeed_fiber_task - worker thread to configure multispeed fiber
6031  * @work: pointer to work_struct containing our data
6032  **/
6033 static void ixgbe_multispeed_fiber_task(struct work_struct *work)
6034 {
6035         struct ixgbe_adapter *adapter = container_of(work,
6036                                                      struct ixgbe_adapter,
6037                                                      multispeed_fiber_task);
6038         struct ixgbe_hw *hw = &adapter->hw;
6039         u32 autoneg;
6040         bool negotiation;
6041
6042         adapter->flags |= IXGBE_FLAG_IN_SFP_LINK_TASK;
6043         autoneg = hw->phy.autoneg_advertised;
6044         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
6045                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
6046         hw->mac.autotry_restart = false;
6047         if (hw->mac.ops.setup_link)
6048                 hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
6049         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
6050         adapter->flags &= ~IXGBE_FLAG_IN_SFP_LINK_TASK;
6051 }
6052
6053 /**
6054  * ixgbe_sfp_config_module_task - worker thread to configure a new SFP+ module
6055  * @work: pointer to work_struct containing our data
6056  **/
6057 static void ixgbe_sfp_config_module_task(struct work_struct *work)
6058 {
6059         struct ixgbe_adapter *adapter = container_of(work,
6060                                                      struct ixgbe_adapter,
6061                                                      sfp_config_module_task);
6062         struct ixgbe_hw *hw = &adapter->hw;
6063         u32 err;
6064
6065         adapter->flags |= IXGBE_FLAG_IN_SFP_MOD_TASK;
6066
6067         /* Time for electrical oscillations to settle down */
6068         msleep(100);
6069         err = hw->phy.ops.identify_sfp(hw);
6070
6071         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
6072                 e_dev_err("failed to initialize because an unsupported SFP+ "
6073                           "module type was detected.\n");
6074                 e_dev_err("Reload the driver after installing a supported "
6075                           "module.\n");
6076                 unregister_netdev(adapter->netdev);
6077                 return;
6078         }
6079         if (hw->mac.ops.setup_sfp)
6080                 hw->mac.ops.setup_sfp(hw);
6081
6082         if (!(adapter->flags & IXGBE_FLAG_IN_SFP_LINK_TASK))
6083                 /* This will also work for DA Twinax connections */
6084                 schedule_work(&adapter->multispeed_fiber_task);
6085         adapter->flags &= ~IXGBE_FLAG_IN_SFP_MOD_TASK;
6086 }
6087
6088 /**
6089  * ixgbe_fdir_reinit_task - worker thread to reinit FDIR filter table
6090  * @work: pointer to work_struct containing our data
6091  **/
6092 static void ixgbe_fdir_reinit_task(struct work_struct *work)
6093 {
6094         struct ixgbe_adapter *adapter = container_of(work,
6095                                                      struct ixgbe_adapter,
6096                                                      fdir_reinit_task);
6097         struct ixgbe_hw *hw = &adapter->hw;
6098         int i;
6099
6100         if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
6101                 for (i = 0; i < adapter->num_tx_queues; i++)
6102                         set_bit(__IXGBE_TX_FDIR_INIT_DONE,
6103                                 &(adapter->tx_ring[i]->state));
6104         } else {
6105                 e_err(probe, "failed to finish FDIR re-initialization, "
6106                       "ignored adding FDIR ATR filters\n");
6107         }
6108         /* Done FDIR Re-initialization, enable transmits */
6109         netif_tx_start_all_queues(adapter->netdev);
6110 }
6111
6112 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
6113 {
6114         u32 ssvpc;
6115
6116         /* Do not perform spoof check for 82598 */
6117         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
6118                 return;
6119
6120         ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
6121
6122         /*
6123          * ssvpc register is cleared on read, if zero then no
6124          * spoofed packets in the last interval.
6125          */
6126         if (!ssvpc)
6127                 return;
6128
6129         e_warn(drv, "%d Spoofed packets detected\n", ssvpc);
6130 }
6131
6132 static DEFINE_MUTEX(ixgbe_watchdog_lock);
6133
6134 /**
6135  * ixgbe_watchdog_task - worker thread to bring link up
6136  * @work: pointer to work_struct containing our data
6137  **/
6138 static void ixgbe_watchdog_task(struct work_struct *work)
6139 {
6140         struct ixgbe_adapter *adapter = container_of(work,
6141                                                      struct ixgbe_adapter,
6142                                                      watchdog_task);
6143         struct net_device *netdev = adapter->netdev;
6144         struct ixgbe_hw *hw = &adapter->hw;
6145         u32 link_speed;
6146         bool link_up;
6147         int i;
6148         struct ixgbe_ring *tx_ring;
6149         int some_tx_pending = 0;
6150
6151         mutex_lock(&ixgbe_watchdog_lock);
6152
6153         link_up = adapter->link_up;
6154         link_speed = adapter->link_speed;
6155
6156         if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
6157                 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
6158                 if (link_up) {
6159 #ifdef CONFIG_DCB
6160                         if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
6161                                 for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
6162                                         hw->mac.ops.fc_enable(hw, i);
6163                         } else {
6164                                 hw->mac.ops.fc_enable(hw, 0);
6165                         }
6166 #else
6167                         hw->mac.ops.fc_enable(hw, 0);
6168 #endif
6169                 }
6170
6171                 if (link_up ||
6172                     time_after(jiffies, (adapter->link_check_timeout +
6173                                          IXGBE_TRY_LINK_TIMEOUT))) {
6174                         adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
6175                         IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
6176                 }
6177                 adapter->link_up = link_up;
6178                 adapter->link_speed = link_speed;
6179         }
6180
6181         if (link_up) {
6182                 if (!netif_carrier_ok(netdev)) {
6183                         bool flow_rx, flow_tx;
6184
6185                         switch (hw->mac.type) {
6186                         case ixgbe_mac_82598EB: {
6187                                 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
6188                                 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
6189                                 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
6190                                 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
6191                         }
6192                                 break;
6193                         case ixgbe_mac_82599EB:
6194                         case ixgbe_mac_X540: {
6195                                 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
6196                                 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
6197                                 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
6198                                 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
6199                         }
6200                                 break;
6201                         default:
6202                                 flow_tx = false;
6203                                 flow_rx = false;
6204                                 break;
6205                         }
6206
6207                         e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
6208                                (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
6209                                "10 Gbps" :
6210                                (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
6211                                "1 Gbps" :
6212                                (link_speed == IXGBE_LINK_SPEED_100_FULL ?
6213                                "100 Mbps" :
6214                                "unknown speed"))),
6215                                ((flow_rx && flow_tx) ? "RX/TX" :
6216                                (flow_rx ? "RX" :
6217                                (flow_tx ? "TX" : "None"))));
6218
6219                         netif_carrier_on(netdev);
6220                         ixgbe_check_vf_rate_limit(adapter);
6221                 } else {
6222                         /* Force detection of hung controller */
6223                         for (i = 0; i < adapter->num_tx_queues; i++) {
6224                                 tx_ring = adapter->tx_ring[i];
6225                                 set_check_for_tx_hang(tx_ring);
6226                         }
6227                 }
6228         } else {
6229                 adapter->link_up = false;
6230                 adapter->link_speed = 0;
6231                 if (netif_carrier_ok(netdev)) {
6232                         e_info(drv, "NIC Link is Down\n");
6233                         netif_carrier_off(netdev);
6234                 }
6235         }
6236
6237         if (!netif_carrier_ok(netdev)) {
6238                 for (i = 0; i < adapter->num_tx_queues; i++) {
6239                         tx_ring = adapter->tx_ring[i];
6240                         if (tx_ring->next_to_use != tx_ring->next_to_clean) {
6241                                 some_tx_pending = 1;
6242                                 break;
6243                         }
6244                 }
6245
6246                 if (some_tx_pending) {
6247                         /* We've lost link, so the controller stops DMA,
6248                          * but we've got queued Tx work that's never going
6249                          * to get done, so reset controller to flush Tx.
6250                          * (Do the reset outside of interrupt context).
6251                          */
6252                          schedule_work(&adapter->reset_task);
6253                 }
6254         }
6255
6256         ixgbe_spoof_check(adapter);
6257         ixgbe_update_stats(adapter);
6258         mutex_unlock(&ixgbe_watchdog_lock);
6259 }
6260
6261 static int ixgbe_tso(struct ixgbe_adapter *adapter,
6262                      struct ixgbe_ring *tx_ring, struct sk_buff *skb,
6263                      u32 tx_flags, u8 *hdr_len, __be16 protocol)
6264 {
6265         struct ixgbe_adv_tx_context_desc *context_desc;
6266         unsigned int i;
6267         int err;
6268         struct ixgbe_tx_buffer *tx_buffer_info;
6269         u32 vlan_macip_lens = 0, type_tucmd_mlhl;
6270         u32 mss_l4len_idx, l4len;
6271
6272         if (skb_is_gso(skb)) {
6273                 if (skb_header_cloned(skb)) {
6274                         err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6275                         if (err)
6276                                 return err;
6277                 }
6278                 l4len = tcp_hdrlen(skb);
6279                 *hdr_len += l4len;
6280
6281                 if (protocol == htons(ETH_P_IP)) {
6282                         struct iphdr *iph = ip_hdr(skb);
6283                         iph->tot_len = 0;
6284                         iph->check = 0;
6285                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
6286                                                                  iph->daddr, 0,
6287                                                                  IPPROTO_TCP,
6288                                                                  0);
6289                 } else if (skb_is_gso_v6(skb)) {
6290                         ipv6_hdr(skb)->payload_len = 0;
6291                         tcp_hdr(skb)->check =
6292                             ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
6293                                              &ipv6_hdr(skb)->daddr,
6294                                              0, IPPROTO_TCP, 0);
6295                 }
6296
6297                 i = tx_ring->next_to_use;
6298
6299                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6300                 context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
6301
6302                 /* VLAN MACLEN IPLEN */
6303                 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
6304                         vlan_macip_lens |=
6305                             (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK);
6306                 vlan_macip_lens |= ((skb_network_offset(skb)) <<
6307                                     IXGBE_ADVTXD_MACLEN_SHIFT);
6308                 *hdr_len += skb_network_offset(skb);
6309                 vlan_macip_lens |=
6310                     (skb_transport_header(skb) - skb_network_header(skb));
6311                 *hdr_len +=
6312                     (skb_transport_header(skb) - skb_network_header(skb));
6313                 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
6314                 context_desc->seqnum_seed = 0;
6315
6316                 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6317                 type_tucmd_mlhl = (IXGBE_TXD_CMD_DEXT |
6318                                    IXGBE_ADVTXD_DTYP_CTXT);
6319
6320                 if (protocol == htons(ETH_P_IP))
6321                         type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
6322                 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
6323                 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
6324
6325                 /* MSS L4LEN IDX */
6326                 mss_l4len_idx =
6327                     (skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT);
6328                 mss_l4len_idx |= (l4len << IXGBE_ADVTXD_L4LEN_SHIFT);
6329                 /* use index 1 for TSO */
6330                 mss_l4len_idx |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
6331                 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
6332
6333                 tx_buffer_info->time_stamp = jiffies;
6334                 tx_buffer_info->next_to_watch = i;
6335
6336                 i++;
6337                 if (i == tx_ring->count)
6338                         i = 0;
6339                 tx_ring->next_to_use = i;
6340
6341                 return true;
6342         }
6343         return false;
6344 }
6345
6346 static u32 ixgbe_psum(struct ixgbe_adapter *adapter, struct sk_buff *skb,
6347                       __be16 protocol)
6348 {
6349         u32 rtn = 0;
6350
6351         switch (protocol) {
6352         case cpu_to_be16(ETH_P_IP):
6353                 rtn |= IXGBE_ADVTXD_TUCMD_IPV4;
6354                 switch (ip_hdr(skb)->protocol) {
6355                 case IPPROTO_TCP:
6356                         rtn |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
6357                         break;
6358                 case IPPROTO_SCTP:
6359                         rtn |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
6360                         break;
6361                 }
6362                 break;
6363         case cpu_to_be16(ETH_P_IPV6):
6364                 /* XXX what about other V6 headers?? */
6365                 switch (ipv6_hdr(skb)->nexthdr) {
6366                 case IPPROTO_TCP:
6367                         rtn |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
6368                         break;
6369                 case IPPROTO_SCTP:
6370                         rtn |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
6371                         break;
6372                 }
6373                 break;
6374         default:
6375                 if (unlikely(net_ratelimit()))
6376                         e_warn(probe, "partial checksum but proto=%x!\n",
6377                                protocol);
6378                 break;
6379         }
6380
6381         return rtn;
6382 }
6383
6384 static bool ixgbe_tx_csum(struct ixgbe_adapter *adapter,
6385                           struct ixgbe_ring *tx_ring,
6386                           struct sk_buff *skb, u32 tx_flags,
6387                           __be16 protocol)
6388 {
6389         struct ixgbe_adv_tx_context_desc *context_desc;
6390         unsigned int i;
6391         struct ixgbe_tx_buffer *tx_buffer_info;
6392         u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0;
6393
6394         if (skb->ip_summed == CHECKSUM_PARTIAL ||
6395             (tx_flags & IXGBE_TX_FLAGS_VLAN)) {
6396                 i = tx_ring->next_to_use;
6397                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6398                 context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
6399
6400                 if (tx_flags & IXGBE_TX_FLAGS_VLAN)
6401                         vlan_macip_lens |=
6402                             (tx_flags & IXGBE_TX_FLAGS_VLAN_MASK);
6403                 vlan_macip_lens |= (skb_network_offset(skb) <<
6404                                     IXGBE_ADVTXD_MACLEN_SHIFT);
6405                 if (skb->ip_summed == CHECKSUM_PARTIAL)
6406                         vlan_macip_lens |= (skb_transport_header(skb) -
6407                                             skb_network_header(skb));
6408
6409                 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
6410                 context_desc->seqnum_seed = 0;
6411
6412                 type_tucmd_mlhl |= (IXGBE_TXD_CMD_DEXT |
6413                                     IXGBE_ADVTXD_DTYP_CTXT);
6414
6415                 if (skb->ip_summed == CHECKSUM_PARTIAL)
6416                         type_tucmd_mlhl |= ixgbe_psum(adapter, skb, protocol);
6417
6418                 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd_mlhl);
6419                 /* use index zero for tx checksum offload */
6420                 context_desc->mss_l4len_idx = 0;
6421
6422                 tx_buffer_info->time_stamp = jiffies;
6423                 tx_buffer_info->next_to_watch = i;
6424
6425                 i++;
6426                 if (i == tx_ring->count)
6427                         i = 0;
6428                 tx_ring->next_to_use = i;
6429
6430                 return true;
6431         }
6432
6433         return false;
6434 }
6435
6436 static int ixgbe_tx_map(struct ixgbe_adapter *adapter,
6437                         struct ixgbe_ring *tx_ring,
6438                         struct sk_buff *skb, u32 tx_flags,
6439                         unsigned int first, const u8 hdr_len)
6440 {
6441         struct device *dev = tx_ring->dev;
6442         struct ixgbe_tx_buffer *tx_buffer_info;
6443         unsigned int len;
6444         unsigned int total = skb->len;
6445         unsigned int offset = 0, size, count = 0, i;
6446         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
6447         unsigned int f;
6448         unsigned int bytecount = skb->len;
6449         u16 gso_segs = 1;
6450
6451         i = tx_ring->next_to_use;
6452
6453         if (tx_flags & IXGBE_TX_FLAGS_FCOE)
6454                 /* excluding fcoe_crc_eof for FCoE */
6455                 total -= sizeof(struct fcoe_crc_eof);
6456
6457         len = min(skb_headlen(skb), total);
6458         while (len) {
6459                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6460                 size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
6461
6462                 tx_buffer_info->length = size;
6463                 tx_buffer_info->mapped_as_page = false;
6464                 tx_buffer_info->dma = dma_map_single(dev,
6465                                                      skb->data + offset,
6466                                                      size, DMA_TO_DEVICE);
6467                 if (dma_mapping_error(dev, tx_buffer_info->dma))
6468                         goto dma_error;
6469                 tx_buffer_info->time_stamp = jiffies;
6470                 tx_buffer_info->next_to_watch = i;
6471
6472                 len -= size;
6473                 total -= size;
6474                 offset += size;
6475                 count++;
6476
6477                 if (len) {
6478                         i++;
6479                         if (i == tx_ring->count)
6480                                 i = 0;
6481                 }
6482         }
6483
6484         for (f = 0; f < nr_frags; f++) {
6485                 struct skb_frag_struct *frag;
6486
6487                 frag = &skb_shinfo(skb)->frags[f];
6488                 len = min((unsigned int)frag->size, total);
6489                 offset = frag->page_offset;
6490
6491                 while (len) {
6492                         i++;
6493                         if (i == tx_ring->count)
6494                                 i = 0;
6495
6496                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
6497                         size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
6498
6499                         tx_buffer_info->length = size;
6500                         tx_buffer_info->dma = dma_map_page(dev,
6501                                                            frag->page,
6502                                                            offset, size,
6503                                                            DMA_TO_DEVICE);
6504                         tx_buffer_info->mapped_as_page = true;
6505                         if (dma_mapping_error(dev, tx_buffer_info->dma))
6506                                 goto dma_error;
6507                         tx_buffer_info->time_stamp = jiffies;
6508                         tx_buffer_info->next_to_watch = i;
6509
6510                         len -= size;
6511                         total -= size;
6512                         offset += size;
6513                         count++;
6514                 }
6515                 if (total == 0)
6516                         break;
6517         }
6518
6519         if (tx_flags & IXGBE_TX_FLAGS_TSO)
6520                 gso_segs = skb_shinfo(skb)->gso_segs;
6521 #ifdef IXGBE_FCOE
6522         /* adjust for FCoE Sequence Offload */
6523         else if (tx_flags & IXGBE_TX_FLAGS_FSO)
6524                 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
6525                                         skb_shinfo(skb)->gso_size);
6526 #endif /* IXGBE_FCOE */
6527         bytecount += (gso_segs - 1) * hdr_len;
6528
6529         /* multiply data chunks by size of headers */
6530         tx_ring->tx_buffer_info[i].bytecount = bytecount;
6531         tx_ring->tx_buffer_info[i].gso_segs = gso_segs;
6532         tx_ring->tx_buffer_info[i].skb = skb;
6533         tx_ring->tx_buffer_info[first].next_to_watch = i;
6534
6535         return count;
6536
6537 dma_error:
6538         e_dev_err("TX DMA map failed\n");
6539
6540         /* clear timestamp and dma mappings for failed tx_buffer_info map */
6541         tx_buffer_info->dma = 0;
6542         tx_buffer_info->time_stamp = 0;
6543         tx_buffer_info->next_to_watch = 0;
6544         if (count)
6545                 count--;
6546
6547         /* clear timestamp and dma mappings for remaining portion of packet */
6548         while (count--) {
6549                 if (i == 0)
6550                         i += tx_ring->count;
6551                 i--;
6552                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6553                 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
6554         }
6555
6556         return 0;
6557 }
6558
6559 static void ixgbe_tx_queue(struct ixgbe_ring *tx_ring,
6560                            int tx_flags, int count, u32 paylen, u8 hdr_len)
6561 {
6562         union ixgbe_adv_tx_desc *tx_desc = NULL;
6563         struct ixgbe_tx_buffer *tx_buffer_info;
6564         u32 olinfo_status = 0, cmd_type_len = 0;
6565         unsigned int i;
6566         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
6567
6568         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
6569
6570         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
6571
6572         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
6573                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
6574
6575         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
6576                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
6577
6578                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
6579                                  IXGBE_ADVTXD_POPTS_SHIFT;
6580
6581                 /* use index 1 context for tso */
6582                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
6583                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
6584                         olinfo_status |= IXGBE_TXD_POPTS_IXSM <<
6585                                          IXGBE_ADVTXD_POPTS_SHIFT;
6586
6587         } else if (tx_flags & IXGBE_TX_FLAGS_CSUM)
6588                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
6589                                  IXGBE_ADVTXD_POPTS_SHIFT;
6590
6591         if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
6592                 olinfo_status |= IXGBE_ADVTXD_CC;
6593                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
6594                 if (tx_flags & IXGBE_TX_FLAGS_FSO)
6595                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
6596         }
6597
6598         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
6599
6600         i = tx_ring->next_to_use;
6601         while (count--) {
6602                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6603                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
6604                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
6605                 tx_desc->read.cmd_type_len =
6606                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
6607                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6608                 i++;
6609                 if (i == tx_ring->count)
6610                         i = 0;
6611         }
6612
6613         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
6614
6615         /*
6616          * Force memory writes to complete before letting h/w
6617          * know there are new descriptors to fetch.  (Only
6618          * applicable for weak-ordered memory model archs,
6619          * such as IA-64).
6620          */
6621         wmb();
6622
6623         tx_ring->next_to_use = i;
6624         writel(i, tx_ring->tail);
6625 }
6626
6627 static void ixgbe_atr(struct ixgbe_ring *ring, struct sk_buff *skb,
6628                       u32 tx_flags, __be16 protocol)
6629 {
6630         struct ixgbe_q_vector *q_vector = ring->q_vector;
6631         union ixgbe_atr_hash_dword input = { .dword = 0 };
6632         union ixgbe_atr_hash_dword common = { .dword = 0 };
6633         union {
6634                 unsigned char *network;
6635                 struct iphdr *ipv4;
6636                 struct ipv6hdr *ipv6;
6637         } hdr;
6638         struct tcphdr *th;
6639         __be16 vlan_id;
6640
6641         /* if ring doesn't have a interrupt vector, cannot perform ATR */
6642         if (!q_vector)
6643                 return;
6644
6645         /* do nothing if sampling is disabled */
6646         if (!ring->atr_sample_rate)
6647                 return;
6648
6649         ring->atr_count++;
6650
6651         /* snag network header to get L4 type and address */
6652         hdr.network = skb_network_header(skb);
6653
6654         /* Currently only IPv4/IPv6 with TCP is supported */
6655         if ((protocol != __constant_htons(ETH_P_IPV6) ||
6656              hdr.ipv6->nexthdr != IPPROTO_TCP) &&
6657             (protocol != __constant_htons(ETH_P_IP) ||
6658              hdr.ipv4->protocol != IPPROTO_TCP))
6659                 return;
6660
6661         th = tcp_hdr(skb);
6662
6663         /* skip this packet since the socket is closing */
6664         if (th->fin)
6665                 return;
6666
6667         /* sample on all syn packets or once every atr sample count */
6668         if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
6669                 return;
6670
6671         /* reset sample count */
6672         ring->atr_count = 0;
6673
6674         vlan_id = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
6675
6676         /*
6677          * src and dst are inverted, think how the receiver sees them
6678          *
6679          * The input is broken into two sections, a non-compressed section
6680          * containing vm_pool, vlan_id, and flow_type.  The rest of the data
6681          * is XORed together and stored in the compressed dword.
6682          */
6683         input.formatted.vlan_id = vlan_id;
6684
6685         /*
6686          * since src port and flex bytes occupy the same word XOR them together
6687          * and write the value to source port portion of compressed dword
6688          */
6689         if (vlan_id)
6690                 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
6691         else
6692                 common.port.src ^= th->dest ^ protocol;
6693         common.port.dst ^= th->source;
6694
6695         if (protocol == __constant_htons(ETH_P_IP)) {
6696                 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
6697                 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
6698         } else {
6699                 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
6700                 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
6701                              hdr.ipv6->saddr.s6_addr32[1] ^
6702                              hdr.ipv6->saddr.s6_addr32[2] ^
6703                              hdr.ipv6->saddr.s6_addr32[3] ^
6704                              hdr.ipv6->daddr.s6_addr32[0] ^
6705                              hdr.ipv6->daddr.s6_addr32[1] ^
6706                              hdr.ipv6->daddr.s6_addr32[2] ^
6707                              hdr.ipv6->daddr.s6_addr32[3];
6708         }
6709
6710         /* This assumes the Rx queue and Tx queue are bound to the same CPU */
6711         ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
6712                                               input, common, ring->queue_index);
6713 }
6714
6715 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, int size)
6716 {
6717         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
6718         /* Herbert's original patch had:
6719          *  smp_mb__after_netif_stop_queue();
6720          * but since that doesn't exist yet, just open code it. */
6721         smp_mb();
6722
6723         /* We need to check again in a case another CPU has just
6724          * made room available. */
6725         if (likely(IXGBE_DESC_UNUSED(tx_ring) < size))
6726                 return -EBUSY;
6727
6728         /* A reprieve! - use start_queue because it doesn't call schedule */
6729         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
6730         ++tx_ring->tx_stats.restart_queue;
6731         return 0;
6732 }
6733
6734 static int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, int size)
6735 {
6736         if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size))
6737                 return 0;
6738         return __ixgbe_maybe_stop_tx(tx_ring, size);
6739 }
6740
6741 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
6742 {
6743         struct ixgbe_adapter *adapter = netdev_priv(dev);
6744         int txq = smp_processor_id();
6745 #ifdef IXGBE_FCOE
6746         __be16 protocol;
6747
6748         protocol = vlan_get_protocol(skb);
6749
6750         if (((protocol == htons(ETH_P_FCOE)) ||
6751             (protocol == htons(ETH_P_FIP))) &&
6752             (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
6753                 txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1);
6754                 txq += adapter->ring_feature[RING_F_FCOE].mask;
6755                 return txq;
6756         }
6757 #endif
6758
6759         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
6760                 while (unlikely(txq >= dev->real_num_tx_queues))
6761                         txq -= dev->real_num_tx_queues;
6762                 return txq;
6763         }
6764
6765         return skb_tx_hash(dev, skb);
6766 }
6767
6768 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
6769                           struct ixgbe_adapter *adapter,
6770                           struct ixgbe_ring *tx_ring)
6771 {
6772         unsigned int first;
6773         unsigned int tx_flags = 0;
6774         u8 hdr_len = 0;
6775         int tso;
6776         int count = 0;
6777         unsigned int f;
6778         __be16 protocol;
6779
6780         protocol = vlan_get_protocol(skb);
6781
6782         if (vlan_tx_tag_present(skb)) {
6783                 tx_flags |= vlan_tx_tag_get(skb);
6784                 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
6785                         tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
6786                         tx_flags |= tx_ring->dcb_tc << 13;
6787                 }
6788                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
6789                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
6790         } else if (adapter->flags & IXGBE_FLAG_DCB_ENABLED &&
6791                    skb->priority != TC_PRIO_CONTROL) {
6792                 tx_flags |= tx_ring->dcb_tc << 13;
6793                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
6794                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
6795         }
6796
6797 #ifdef IXGBE_FCOE
6798         /* for FCoE with DCB, we force the priority to what
6799          * was specified by the switch */
6800         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED &&
6801             (protocol == htons(ETH_P_FCOE)))
6802                 tx_flags |= IXGBE_TX_FLAGS_FCOE;
6803 #endif
6804
6805         /* four things can cause us to need a context descriptor */
6806         if (skb_is_gso(skb) ||
6807             (skb->ip_summed == CHECKSUM_PARTIAL) ||
6808             (tx_flags & IXGBE_TX_FLAGS_VLAN) ||
6809             (tx_flags & IXGBE_TX_FLAGS_FCOE))
6810                 count++;
6811
6812         count += TXD_USE_COUNT(skb_headlen(skb));
6813         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6814                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
6815
6816         if (ixgbe_maybe_stop_tx(tx_ring, count)) {
6817                 tx_ring->tx_stats.tx_busy++;
6818                 return NETDEV_TX_BUSY;
6819         }
6820
6821         first = tx_ring->next_to_use;
6822         if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
6823 #ifdef IXGBE_FCOE
6824                 /* setup tx offload for FCoE */
6825                 tso = ixgbe_fso(adapter, tx_ring, skb, tx_flags, &hdr_len);
6826                 if (tso < 0) {
6827                         dev_kfree_skb_any(skb);
6828                         return NETDEV_TX_OK;
6829                 }
6830                 if (tso)
6831                         tx_flags |= IXGBE_TX_FLAGS_FSO;
6832 #endif /* IXGBE_FCOE */
6833         } else {
6834                 if (protocol == htons(ETH_P_IP))
6835                         tx_flags |= IXGBE_TX_FLAGS_IPV4;
6836                 tso = ixgbe_tso(adapter, tx_ring, skb, tx_flags, &hdr_len,
6837                                 protocol);
6838                 if (tso < 0) {
6839                         dev_kfree_skb_any(skb);
6840                         return NETDEV_TX_OK;
6841                 }
6842
6843                 if (tso)
6844                         tx_flags |= IXGBE_TX_FLAGS_TSO;
6845                 else if (ixgbe_tx_csum(adapter, tx_ring, skb, tx_flags,
6846                                        protocol) &&
6847                          (skb->ip_summed == CHECKSUM_PARTIAL))
6848                         tx_flags |= IXGBE_TX_FLAGS_CSUM;
6849         }
6850
6851         count = ixgbe_tx_map(adapter, tx_ring, skb, tx_flags, first, hdr_len);
6852         if (count) {
6853                 /* add the ATR filter if ATR is on */
6854                 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
6855                         ixgbe_atr(tx_ring, skb, tx_flags, protocol);
6856                 ixgbe_tx_queue(tx_ring, tx_flags, count, skb->len, hdr_len);
6857                 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
6858
6859         } else {
6860                 dev_kfree_skb_any(skb);
6861                 tx_ring->tx_buffer_info[first].time_stamp = 0;
6862                 tx_ring->next_to_use = first;
6863         }
6864
6865         return NETDEV_TX_OK;
6866 }
6867
6868 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
6869 {
6870         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6871         struct ixgbe_ring *tx_ring;
6872
6873         tx_ring = adapter->tx_ring[skb->queue_mapping];
6874         return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
6875 }
6876
6877 /**
6878  * ixgbe_set_mac - Change the Ethernet Address of the NIC
6879  * @netdev: network interface device structure
6880  * @p: pointer to an address structure
6881  *
6882  * Returns 0 on success, negative on failure
6883  **/
6884 static int ixgbe_set_mac(struct net_device *netdev, void *p)
6885 {
6886         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6887         struct ixgbe_hw *hw = &adapter->hw;
6888         struct sockaddr *addr = p;
6889
6890         if (!is_valid_ether_addr(addr->sa_data))
6891                 return -EADDRNOTAVAIL;
6892
6893         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
6894         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
6895
6896         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
6897                             IXGBE_RAH_AV);
6898
6899         return 0;
6900 }
6901
6902 static int
6903 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
6904 {
6905         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6906         struct ixgbe_hw *hw = &adapter->hw;
6907         u16 value;
6908         int rc;
6909
6910         if (prtad != hw->phy.mdio.prtad)
6911                 return -EINVAL;
6912         rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
6913         if (!rc)
6914                 rc = value;
6915         return rc;
6916 }
6917
6918 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
6919                             u16 addr, u16 value)
6920 {
6921         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6922         struct ixgbe_hw *hw = &adapter->hw;
6923
6924         if (prtad != hw->phy.mdio.prtad)
6925                 return -EINVAL;
6926         return hw->phy.ops.write_reg(hw, addr, devad, value);
6927 }
6928
6929 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
6930 {
6931         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6932
6933         return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
6934 }
6935
6936 /**
6937  * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
6938  * netdev->dev_addrs
6939  * @netdev: network interface device structure
6940  *
6941  * Returns non-zero on failure
6942  **/
6943 static int ixgbe_add_sanmac_netdev(struct net_device *dev)
6944 {
6945         int err = 0;
6946         struct ixgbe_adapter *adapter = netdev_priv(dev);
6947         struct ixgbe_mac_info *mac = &adapter->hw.mac;
6948
6949         if (is_valid_ether_addr(mac->san_addr)) {
6950                 rtnl_lock();
6951                 err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
6952                 rtnl_unlock();
6953         }
6954         return err;
6955 }
6956
6957 /**
6958  * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
6959  * netdev->dev_addrs
6960  * @netdev: network interface device structure
6961  *
6962  * Returns non-zero on failure
6963  **/
6964 static int ixgbe_del_sanmac_netdev(struct net_device *dev)
6965 {
6966         int err = 0;
6967         struct ixgbe_adapter *adapter = netdev_priv(dev);
6968         struct ixgbe_mac_info *mac = &adapter->hw.mac;
6969
6970         if (is_valid_ether_addr(mac->san_addr)) {
6971                 rtnl_lock();
6972                 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
6973                 rtnl_unlock();
6974         }
6975         return err;
6976 }
6977
6978 #ifdef CONFIG_NET_POLL_CONTROLLER
6979 /*
6980  * Polling 'interrupt' - used by things like netconsole to send skbs
6981  * without having to re-enable interrupts. It's not called while
6982  * the interrupt routine is executing.
6983  */
6984 static void ixgbe_netpoll(struct net_device *netdev)
6985 {
6986         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6987         int i;
6988
6989         /* if interface is down do nothing */
6990         if (test_bit(__IXGBE_DOWN, &adapter->state))
6991                 return;
6992
6993         adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
6994         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
6995                 int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
6996                 for (i = 0; i < num_q_vectors; i++) {
6997                         struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
6998                         ixgbe_msix_clean_many(0, q_vector);
6999                 }
7000         } else {
7001                 ixgbe_intr(adapter->pdev->irq, netdev);
7002         }
7003         adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
7004 }
7005 #endif
7006
7007 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
7008                                                    struct rtnl_link_stats64 *stats)
7009 {
7010         struct ixgbe_adapter *adapter = netdev_priv(netdev);
7011         int i;
7012
7013         rcu_read_lock();
7014         for (i = 0; i < adapter->num_rx_queues; i++) {
7015                 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
7016                 u64 bytes, packets;
7017                 unsigned int start;
7018
7019                 if (ring) {
7020                         do {
7021                                 start = u64_stats_fetch_begin_bh(&ring->syncp);
7022                                 packets = ring->stats.packets;
7023                                 bytes   = ring->stats.bytes;
7024                         } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
7025                         stats->rx_packets += packets;
7026                         stats->rx_bytes   += bytes;
7027                 }
7028         }
7029
7030         for (i = 0; i < adapter->num_tx_queues; i++) {
7031                 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
7032                 u64 bytes, packets;
7033                 unsigned int start;
7034
7035                 if (ring) {
7036                         do {
7037                                 start = u64_stats_fetch_begin_bh(&ring->syncp);
7038                                 packets = ring->stats.packets;
7039                                 bytes   = ring->stats.bytes;
7040                         } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
7041                         stats->tx_packets += packets;
7042                         stats->tx_bytes   += bytes;
7043                 }
7044         }
7045         rcu_read_unlock();
7046         /* following stats updated by ixgbe_watchdog_task() */
7047         stats->multicast        = netdev->stats.multicast;
7048         stats->rx_errors        = netdev->stats.rx_errors;
7049         stats->rx_length_errors = netdev->stats.rx_length_errors;
7050         stats->rx_crc_errors    = netdev->stats.rx_crc_errors;
7051         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
7052         return stats;
7053 }
7054
7055
7056 static const struct net_device_ops ixgbe_netdev_ops = {
7057         .ndo_open               = ixgbe_open,
7058         .ndo_stop               = ixgbe_close,
7059         .ndo_start_xmit         = ixgbe_xmit_frame,
7060         .ndo_select_queue       = ixgbe_select_queue,
7061         .ndo_set_rx_mode        = ixgbe_set_rx_mode,
7062         .ndo_set_multicast_list = ixgbe_set_rx_mode,
7063         .ndo_validate_addr      = eth_validate_addr,
7064         .ndo_set_mac_address    = ixgbe_set_mac,
7065         .ndo_change_mtu         = ixgbe_change_mtu,
7066         .ndo_tx_timeout         = ixgbe_tx_timeout,
7067         .ndo_vlan_rx_add_vid    = ixgbe_vlan_rx_add_vid,
7068         .ndo_vlan_rx_kill_vid   = ixgbe_vlan_rx_kill_vid,
7069         .ndo_do_ioctl           = ixgbe_ioctl,
7070         .ndo_set_vf_mac         = ixgbe_ndo_set_vf_mac,
7071         .ndo_set_vf_vlan        = ixgbe_ndo_set_vf_vlan,
7072         .ndo_set_vf_tx_rate     = ixgbe_ndo_set_vf_bw,
7073         .ndo_get_vf_config      = ixgbe_ndo_get_vf_config,
7074         .ndo_get_stats64        = ixgbe_get_stats64,
7075 #ifdef CONFIG_IXGBE_DCB
7076         .ndo_setup_tc           = ixgbe_setup_tc,
7077 #endif
7078 #ifdef CONFIG_NET_POLL_CONTROLLER
7079         .ndo_poll_controller    = ixgbe_netpoll,
7080 #endif
7081 #ifdef IXGBE_FCOE
7082         .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
7083         .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
7084         .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
7085         .ndo_fcoe_enable = ixgbe_fcoe_enable,
7086         .ndo_fcoe_disable = ixgbe_fcoe_disable,
7087         .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
7088 #endif /* IXGBE_FCOE */
7089 };
7090
7091 static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
7092                            const struct ixgbe_info *ii)
7093 {
7094 #ifdef CONFIG_PCI_IOV
7095         struct ixgbe_hw *hw = &adapter->hw;
7096         int err;
7097
7098         if (hw->mac.type == ixgbe_mac_82598EB || !max_vfs)
7099                 return;
7100
7101         /* The 82599 supports up to 64 VFs per physical function
7102          * but this implementation limits allocation to 63 so that
7103          * basic networking resources are still available to the
7104          * physical function
7105          */
7106         adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
7107         adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
7108         err = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
7109         if (err) {
7110                 e_err(probe, "Failed to enable PCI sriov: %d\n", err);
7111                 goto err_novfs;
7112         }
7113         /* If call to enable VFs succeeded then allocate memory
7114          * for per VF control structures.
7115          */
7116         adapter->vfinfo =
7117                 kcalloc(adapter->num_vfs,
7118                         sizeof(struct vf_data_storage), GFP_KERNEL);
7119         if (adapter->vfinfo) {
7120                 /* Now that we're sure SR-IOV is enabled
7121                  * and memory allocated set up the mailbox parameters
7122                  */
7123                 ixgbe_init_mbx_params_pf(hw);
7124                 memcpy(&hw->mbx.ops, ii->mbx_ops,
7125                        sizeof(hw->mbx.ops));
7126
7127                 /* Disable RSC when in SR-IOV mode */
7128                 adapter->flags2 &= ~(IXGBE_FLAG2_RSC_CAPABLE |
7129                                      IXGBE_FLAG2_RSC_ENABLED);
7130                 return;
7131         }
7132
7133         /* Oh oh */
7134         e_err(probe, "Unable to allocate memory for VF Data Storage - "
7135               "SRIOV disabled\n");
7136         pci_disable_sriov(adapter->pdev);
7137
7138 err_novfs:
7139         adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
7140         adapter->num_vfs = 0;
7141 #endif /* CONFIG_PCI_IOV */
7142 }
7143
7144 /**
7145  * ixgbe_probe - Device Initialization Routine
7146  * @pdev: PCI device information struct
7147  * @ent: entry in ixgbe_pci_tbl
7148  *
7149  * Returns 0 on success, negative on failure
7150  *
7151  * ixgbe_probe initializes an adapter identified by a pci_dev structure.
7152  * The OS initialization, configuring of the adapter private structure,
7153  * and a hardware reset occur.
7154  **/
7155 static int __devinit ixgbe_probe(struct pci_dev *pdev,
7156                                  const struct pci_device_id *ent)
7157 {
7158         struct net_device *netdev;
7159         struct ixgbe_adapter *adapter = NULL;
7160         struct ixgbe_hw *hw;
7161         const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
7162         static int cards_found;
7163         int i, err, pci_using_dac;
7164         u8 part_str[IXGBE_PBANUM_LENGTH];
7165         unsigned int indices = num_possible_cpus();
7166 #ifdef IXGBE_FCOE
7167         u16 device_caps;
7168 #endif
7169         u32 eec;
7170
7171         /* Catch broken hardware that put the wrong VF device ID in
7172          * the PCIe SR-IOV capability.
7173          */
7174         if (pdev->is_virtfn) {
7175                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
7176                      pci_name(pdev), pdev->vendor, pdev->device);
7177                 return -EINVAL;
7178         }
7179
7180         err = pci_enable_device_mem(pdev);
7181         if (err)
7182                 return err;
7183
7184         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
7185             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7186                 pci_using_dac = 1;
7187         } else {
7188                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
7189                 if (err) {
7190                         err = dma_set_coherent_mask(&pdev->dev,
7191                                                     DMA_BIT_MASK(32));
7192                         if (err) {
7193                                 dev_err(&pdev->dev,
7194                                         "No usable DMA configuration, aborting\n");
7195                                 goto err_dma;
7196                         }
7197                 }
7198                 pci_using_dac = 0;
7199         }
7200
7201         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7202                                            IORESOURCE_MEM), ixgbe_driver_name);
7203         if (err) {
7204                 dev_err(&pdev->dev,
7205                         "pci_request_selected_regions failed 0x%x\n", err);
7206                 goto err_pci_reg;
7207         }
7208
7209         pci_enable_pcie_error_reporting(pdev);
7210
7211         pci_set_master(pdev);
7212         pci_save_state(pdev);
7213
7214         if (ii->mac == ixgbe_mac_82598EB)
7215                 indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
7216         else
7217                 indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);
7218
7219 #if defined(CONFIG_DCB)
7220         indices = max_t(unsigned int, indices, IXGBE_MAX_DCB_INDICES);
7221 #elif defined(IXGBE_FCOE)
7222         indices += min_t(unsigned int, num_possible_cpus(),
7223                          IXGBE_MAX_FCOE_INDICES);
7224 #endif
7225         netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
7226         if (!netdev) {
7227                 err = -ENOMEM;
7228                 goto err_alloc_etherdev;
7229         }
7230
7231         SET_NETDEV_DEV(netdev, &pdev->dev);
7232
7233         adapter = netdev_priv(netdev);
7234         pci_set_drvdata(pdev, adapter);
7235
7236         adapter->netdev = netdev;
7237         adapter->pdev = pdev;
7238         hw = &adapter->hw;
7239         hw->back = adapter;
7240         adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
7241
7242         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7243                               pci_resource_len(pdev, 0));
7244         if (!hw->hw_addr) {
7245                 err = -EIO;
7246                 goto err_ioremap;
7247         }
7248
7249         for (i = 1; i <= 5; i++) {
7250                 if (pci_resource_len(pdev, i) == 0)
7251                         continue;
7252         }
7253
7254         netdev->netdev_ops = &ixgbe_netdev_ops;
7255         ixgbe_set_ethtool_ops(netdev);
7256         netdev->watchdog_timeo = 5 * HZ;
7257         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
7258
7259         adapter->bd_number = cards_found;
7260
7261         /* Setup hw api */
7262         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
7263         hw->mac.type  = ii->mac;
7264
7265         /* EEPROM */
7266         memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
7267         eec = IXGBE_READ_REG(hw, IXGBE_EEC);
7268         /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
7269         if (!(eec & (1 << 8)))
7270                 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
7271
7272         /* PHY */
7273         memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
7274         hw->phy.sfp_type = ixgbe_sfp_type_unknown;
7275         /* ixgbe_identify_phy_generic will set prtad and mmds properly */
7276         hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
7277         hw->phy.mdio.mmds = 0;
7278         hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
7279         hw->phy.mdio.dev = netdev;
7280         hw->phy.mdio.mdio_read = ixgbe_mdio_read;
7281         hw->phy.mdio.mdio_write = ixgbe_mdio_write;
7282
7283         /* set up this timer and work struct before calling get_invariants
7284          * which might start the timer
7285          */
7286         init_timer(&adapter->sfp_timer);
7287         adapter->sfp_timer.function = ixgbe_sfp_timer;
7288         adapter->sfp_timer.data = (unsigned long) adapter;
7289
7290         INIT_WORK(&adapter->sfp_task, ixgbe_sfp_task);
7291
7292         /* multispeed fiber has its own tasklet, called from GPI SDP1 context */
7293         INIT_WORK(&adapter->multispeed_fiber_task, ixgbe_multispeed_fiber_task);
7294
7295         /* a new SFP+ module arrival, called from GPI SDP2 context */
7296         INIT_WORK(&adapter->sfp_config_module_task,
7297                   ixgbe_sfp_config_module_task);
7298
7299         ii->get_invariants(hw);
7300
7301         /* setup the private structure */
7302         err = ixgbe_sw_init(adapter);
7303         if (err)
7304                 goto err_sw_init;
7305
7306         /* Make it possible the adapter to be woken up via WOL */
7307         switch (adapter->hw.mac.type) {
7308         case ixgbe_mac_82599EB:
7309         case ixgbe_mac_X540:
7310                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7311                 break;
7312         default:
7313                 break;
7314         }
7315
7316         /*
7317          * If there is a fan on this device and it has failed log the
7318          * failure.
7319          */
7320         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
7321                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
7322                 if (esdp & IXGBE_ESDP_SDP1)
7323                         e_crit(probe, "Fan has stopped, replace the adapter\n");
7324         }
7325
7326         /* reset_hw fills in the perm_addr as well */
7327         hw->phy.reset_if_overtemp = true;
7328         err = hw->mac.ops.reset_hw(hw);
7329         hw->phy.reset_if_overtemp = false;
7330         if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
7331             hw->mac.type == ixgbe_mac_82598EB) {
7332                 /*
7333                  * Start a kernel thread to watch for a module to arrive.
7334                  * Only do this for 82598, since 82599 will generate
7335                  * interrupts on module arrival.
7336                  */
7337                 set_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state);
7338                 mod_timer(&adapter->sfp_timer,
7339                           round_jiffies(jiffies + (2 * HZ)));
7340                 err = 0;
7341         } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
7342                 e_dev_err("failed to initialize because an unsupported SFP+ "
7343                           "module type was detected.\n");
7344                 e_dev_err("Reload the driver after installing a supported "
7345                           "module.\n");
7346                 goto err_sw_init;
7347         } else if (err) {
7348                 e_dev_err("HW Init failed: %d\n", err);
7349                 goto err_sw_init;
7350         }
7351
7352         ixgbe_probe_vf(adapter, ii);
7353
7354         netdev->features = NETIF_F_SG |
7355                            NETIF_F_IP_CSUM |
7356                            NETIF_F_HW_VLAN_TX |
7357                            NETIF_F_HW_VLAN_RX |
7358                            NETIF_F_HW_VLAN_FILTER;
7359
7360         netdev->features |= NETIF_F_IPV6_CSUM;
7361         netdev->features |= NETIF_F_TSO;
7362         netdev->features |= NETIF_F_TSO6;
7363         netdev->features |= NETIF_F_GRO;
7364
7365         if (adapter->hw.mac.type == ixgbe_mac_82599EB)
7366                 netdev->features |= NETIF_F_SCTP_CSUM;
7367
7368         netdev->vlan_features |= NETIF_F_TSO;
7369         netdev->vlan_features |= NETIF_F_TSO6;
7370         netdev->vlan_features |= NETIF_F_IP_CSUM;
7371         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
7372         netdev->vlan_features |= NETIF_F_SG;
7373
7374         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7375                 adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED |
7376                                     IXGBE_FLAG_DCB_ENABLED);
7377
7378 #ifdef CONFIG_IXGBE_DCB
7379         netdev->dcbnl_ops = &dcbnl_ops;
7380 #endif
7381
7382 #ifdef IXGBE_FCOE
7383         if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7384                 if (hw->mac.ops.get_device_caps) {
7385                         hw->mac.ops.get_device_caps(hw, &device_caps);
7386                         if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
7387                                 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
7388                 }
7389         }
7390         if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7391                 netdev->vlan_features |= NETIF_F_FCOE_CRC;
7392                 netdev->vlan_features |= NETIF_F_FSO;
7393                 netdev->vlan_features |= NETIF_F_FCOE_MTU;
7394         }
7395 #endif /* IXGBE_FCOE */
7396         if (pci_using_dac) {
7397                 netdev->features |= NETIF_F_HIGHDMA;
7398                 netdev->vlan_features |= NETIF_F_HIGHDMA;
7399         }
7400
7401         if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
7402                 netdev->features |= NETIF_F_LRO;
7403
7404         /* make sure the EEPROM is good */
7405         if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
7406                 e_dev_err("The EEPROM Checksum Is Not Valid\n");
7407                 err = -EIO;
7408                 goto err_eeprom;
7409         }
7410
7411         memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
7412         memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);
7413
7414         if (ixgbe_validate_mac_addr(netdev->perm_addr)) {
7415                 e_dev_err("invalid MAC address\n");
7416                 err = -EIO;
7417                 goto err_eeprom;
7418         }
7419
7420         /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
7421         if (hw->mac.ops.disable_tx_laser &&
7422             ((hw->phy.multispeed_fiber) ||
7423              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
7424               (hw->mac.type == ixgbe_mac_82599EB))))
7425                 hw->mac.ops.disable_tx_laser(hw);
7426
7427         init_timer(&adapter->watchdog_timer);
7428         adapter->watchdog_timer.function = ixgbe_watchdog;
7429         adapter->watchdog_timer.data = (unsigned long)adapter;
7430
7431         INIT_WORK(&adapter->reset_task, ixgbe_reset_task);
7432         INIT_WORK(&adapter->watchdog_task, ixgbe_watchdog_task);
7433
7434         err = ixgbe_init_interrupt_scheme(adapter);
7435         if (err)
7436                 goto err_sw_init;
7437
7438         switch (pdev->device) {
7439         case IXGBE_DEV_ID_82599_SFP:
7440                 /* Only this subdevice supports WOL */
7441                 if (pdev->subsystem_device == IXGBE_SUBDEV_ID_82599_SFP)
7442                         adapter->wol = (IXGBE_WUFC_MAG | IXGBE_WUFC_EX |
7443                                         IXGBE_WUFC_MC | IXGBE_WUFC_BC);
7444                 break;
7445         case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
7446                 /* All except this subdevice support WOL */
7447                 if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
7448                         adapter->wol = (IXGBE_WUFC_MAG | IXGBE_WUFC_EX |
7449                                         IXGBE_WUFC_MC | IXGBE_WUFC_BC);
7450                 break;
7451         case IXGBE_DEV_ID_82599_KX4:
7452                 adapter->wol = (IXGBE_WUFC_MAG | IXGBE_WUFC_EX |
7453                                 IXGBE_WUFC_MC | IXGBE_WUFC_BC);
7454                 break;
7455         default:
7456                 adapter->wol = 0;
7457                 break;
7458         }
7459         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
7460
7461         /* pick up the PCI bus settings for reporting later */
7462         hw->mac.ops.get_bus_info(hw);
7463
7464         /* print bus type/speed/width info */
7465         e_dev_info("(PCI Express:%s:%s) %pM\n",
7466                    (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0Gb/s" :
7467                     hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5Gb/s" :
7468                     "Unknown"),
7469                    (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
7470                     hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
7471                     hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
7472                     "Unknown"),
7473                    netdev->dev_addr);
7474
7475         err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
7476         if (err)
7477                 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
7478         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
7479                 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
7480                            hw->mac.type, hw->phy.type, hw->phy.sfp_type,
7481                            part_str);
7482         else
7483                 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
7484                            hw->mac.type, hw->phy.type, part_str);
7485
7486         if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
7487                 e_dev_warn("PCI-Express bandwidth available for this card is "
7488                            "not sufficient for optimal performance.\n");
7489                 e_dev_warn("For optimal performance a x8 PCI-Express slot "
7490                            "is required.\n");
7491         }
7492
7493         /* save off EEPROM version number */
7494         hw->eeprom.ops.read(hw, 0x29, &adapter->eeprom_version);
7495
7496         /* reset the hardware with the new settings */
7497         err = hw->mac.ops.start_hw(hw);
7498
7499         if (err == IXGBE_ERR_EEPROM_VERSION) {
7500                 /* We are running on a pre-production device, log a warning */
7501                 e_dev_warn("This device is a pre-production adapter/LOM. "
7502                            "Please be aware there may be issues associated "
7503                            "with your hardware.  If you are experiencing "
7504                            "problems please contact your Intel or hardware "
7505                            "representative who provided you with this "
7506                            "hardware.\n");
7507         }
7508         strcpy(netdev->name, "eth%d");
7509         err = register_netdev(netdev);
7510         if (err)
7511                 goto err_register;
7512
7513         /* carrier off reporting is important to ethtool even BEFORE open */
7514         netif_carrier_off(netdev);
7515
7516         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
7517             adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
7518                 INIT_WORK(&adapter->fdir_reinit_task, ixgbe_fdir_reinit_task);
7519
7520         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
7521                 INIT_WORK(&adapter->check_overtemp_task,
7522                           ixgbe_check_overtemp_task);
7523 #ifdef CONFIG_IXGBE_DCA
7524         if (dca_add_requester(&pdev->dev) == 0) {
7525                 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
7526                 ixgbe_setup_dca(adapter);
7527         }
7528 #endif
7529         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
7530                 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
7531                 for (i = 0; i < adapter->num_vfs; i++)
7532                         ixgbe_vf_configuration(pdev, (i | 0x10000000));
7533         }
7534
7535         /* add san mac addr to netdev */
7536         ixgbe_add_sanmac_netdev(netdev);
7537
7538         e_dev_info("Intel(R) 10 Gigabit Network Connection\n");
7539         cards_found++;
7540         return 0;
7541
7542 err_register:
7543         ixgbe_release_hw_control(adapter);
7544         ixgbe_clear_interrupt_scheme(adapter);
7545 err_sw_init:
7546 err_eeprom:
7547         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7548                 ixgbe_disable_sriov(adapter);
7549         clear_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state);
7550         del_timer_sync(&adapter->sfp_timer);
7551         cancel_work_sync(&adapter->sfp_task);
7552         cancel_work_sync(&adapter->multispeed_fiber_task);
7553         cancel_work_sync(&adapter->sfp_config_module_task);
7554         iounmap(hw->hw_addr);
7555 err_ioremap:
7556         free_netdev(netdev);
7557 err_alloc_etherdev:
7558         pci_release_selected_regions(pdev,
7559                                      pci_select_bars(pdev, IORESOURCE_MEM));
7560 err_pci_reg:
7561 err_dma:
7562         pci_disable_device(pdev);
7563         return err;
7564 }
7565
7566 /**
7567  * ixgbe_remove - Device Removal Routine
7568  * @pdev: PCI device information struct
7569  *
7570  * ixgbe_remove is called by the PCI subsystem to alert the driver
7571  * that it should release a PCI device.  The could be caused by a
7572  * Hot-Plug event, or because the driver is going to be removed from
7573  * memory.
7574  **/
7575 static void __devexit ixgbe_remove(struct pci_dev *pdev)
7576 {
7577         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7578         struct net_device *netdev = adapter->netdev;
7579
7580         set_bit(__IXGBE_DOWN, &adapter->state);
7581
7582         /*
7583          * The timers may be rescheduled, so explicitly disable them
7584          * from being rescheduled.
7585          */
7586         clear_bit(__IXGBE_SFP_MODULE_NOT_FOUND, &adapter->state);
7587         del_timer_sync(&adapter->watchdog_timer);
7588         del_timer_sync(&adapter->sfp_timer);
7589
7590         cancel_work_sync(&adapter->watchdog_task);
7591         cancel_work_sync(&adapter->sfp_task);
7592         cancel_work_sync(&adapter->multispeed_fiber_task);
7593         cancel_work_sync(&adapter->sfp_config_module_task);
7594         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
7595             adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
7596                 cancel_work_sync(&adapter->fdir_reinit_task);
7597         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
7598                 cancel_work_sync(&adapter->check_overtemp_task);
7599
7600 #ifdef CONFIG_IXGBE_DCA
7601         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
7602                 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
7603                 dca_remove_requester(&pdev->dev);
7604                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
7605         }
7606
7607 #endif
7608 #ifdef IXGBE_FCOE
7609         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
7610                 ixgbe_cleanup_fcoe(adapter);
7611
7612 #endif /* IXGBE_FCOE */
7613
7614         /* remove the added san mac */
7615         ixgbe_del_sanmac_netdev(netdev);
7616
7617         if (netdev->reg_state == NETREG_REGISTERED)
7618                 unregister_netdev(netdev);
7619
7620         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7621                 ixgbe_disable_sriov(adapter);
7622
7623         ixgbe_clear_interrupt_scheme(adapter);
7624
7625         ixgbe_release_hw_control(adapter);
7626
7627         iounmap(adapter->hw.hw_addr);
7628         pci_release_selected_regions(pdev, pci_select_bars(pdev,
7629                                      IORESOURCE_MEM));
7630
7631         e_dev_info("complete\n");
7632
7633         free_netdev(netdev);
7634
7635         pci_disable_pcie_error_reporting(pdev);
7636
7637         pci_disable_device(pdev);
7638 }
7639
7640 /**
7641  * ixgbe_io_error_detected - called when PCI error is detected
7642  * @pdev: Pointer to PCI device
7643  * @state: The current pci connection state
7644  *
7645  * This function is called after a PCI bus error affecting
7646  * this device has been detected.
7647  */
7648 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
7649                                                 pci_channel_state_t state)
7650 {
7651         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7652         struct net_device *netdev = adapter->netdev;
7653
7654         netif_device_detach(netdev);
7655
7656         if (state == pci_channel_io_perm_failure)
7657                 return PCI_ERS_RESULT_DISCONNECT;
7658
7659         if (netif_running(netdev))
7660                 ixgbe_down(adapter);
7661         pci_disable_device(pdev);
7662
7663         /* Request a slot reset. */
7664         return PCI_ERS_RESULT_NEED_RESET;
7665 }
7666
7667 /**
7668  * ixgbe_io_slot_reset - called after the pci bus has been reset.
7669  * @pdev: Pointer to PCI device
7670  *
7671  * Restart the card from scratch, as if from a cold-boot.
7672  */
7673 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
7674 {
7675         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7676         pci_ers_result_t result;
7677         int err;
7678
7679         if (pci_enable_device_mem(pdev)) {
7680                 e_err(probe, "Cannot re-enable PCI device after reset.\n");
7681                 result = PCI_ERS_RESULT_DISCONNECT;
7682         } else {
7683                 pci_set_master(pdev);
7684                 pci_restore_state(pdev);
7685                 pci_save_state(pdev);
7686
7687                 pci_wake_from_d3(pdev, false);
7688
7689                 ixgbe_reset(adapter);
7690                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7691                 result = PCI_ERS_RESULT_RECOVERED;
7692         }
7693
7694         err = pci_cleanup_aer_uncorrect_error_status(pdev);
7695         if (err) {
7696                 e_dev_err("pci_cleanup_aer_uncorrect_error_status "
7697                           "failed 0x%0x\n", err);
7698                 /* non-fatal, continue */
7699         }
7700
7701         return result;
7702 }
7703
7704 /**
7705  * ixgbe_io_resume - called when traffic can start flowing again.
7706  * @pdev: Pointer to PCI device
7707  *
7708  * This callback is called when the error recovery driver tells us that
7709  * its OK to resume normal operation.
7710  */
7711 static void ixgbe_io_resume(struct pci_dev *pdev)
7712 {
7713         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7714         struct net_device *netdev = adapter->netdev;
7715
7716         if (netif_running(netdev)) {
7717                 if (ixgbe_up(adapter)) {
7718                         e_info(probe, "ixgbe_up failed after reset\n");
7719                         return;
7720                 }
7721         }
7722
7723         netif_device_attach(netdev);
7724 }
7725
7726 static struct pci_error_handlers ixgbe_err_handler = {
7727         .error_detected = ixgbe_io_error_detected,
7728         .slot_reset = ixgbe_io_slot_reset,
7729         .resume = ixgbe_io_resume,
7730 };
7731
7732 static struct pci_driver ixgbe_driver = {
7733         .name     = ixgbe_driver_name,
7734         .id_table = ixgbe_pci_tbl,
7735         .probe    = ixgbe_probe,
7736         .remove   = __devexit_p(ixgbe_remove),
7737 #ifdef CONFIG_PM
7738         .suspend  = ixgbe_suspend,
7739         .resume   = ixgbe_resume,
7740 #endif
7741         .shutdown = ixgbe_shutdown,
7742         .err_handler = &ixgbe_err_handler
7743 };
7744
7745 /**
7746  * ixgbe_init_module - Driver Registration Routine
7747  *
7748  * ixgbe_init_module is the first routine called when the driver is
7749  * loaded. All it does is register with the PCI subsystem.
7750  **/
7751 static int __init ixgbe_init_module(void)
7752 {
7753         int ret;
7754         pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
7755         pr_info("%s\n", ixgbe_copyright);
7756
7757 #ifdef CONFIG_IXGBE_DCA
7758         dca_register_notify(&dca_notifier);
7759 #endif
7760
7761         ret = pci_register_driver(&ixgbe_driver);
7762         return ret;
7763 }
7764
7765 module_init(ixgbe_init_module);
7766
7767 /**
7768  * ixgbe_exit_module - Driver Exit Cleanup Routine
7769  *
7770  * ixgbe_exit_module is called just before the driver is removed
7771  * from memory.
7772  **/
7773 static void __exit ixgbe_exit_module(void)
7774 {
7775 #ifdef CONFIG_IXGBE_DCA
7776         dca_unregister_notify(&dca_notifier);
7777 #endif
7778         pci_unregister_driver(&ixgbe_driver);
7779         rcu_barrier(); /* Wait for completion of call_rcu()'s */
7780 }
7781
7782 #ifdef CONFIG_IXGBE_DCA
7783 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
7784                             void *p)
7785 {
7786         int ret_val;
7787
7788         ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
7789                                          __ixgbe_notify_dca);
7790
7791         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7792 }
7793
7794 #endif /* CONFIG_IXGBE_DCA */
7795
7796 module_exit(ixgbe_exit_module);
7797
7798 /* ixgbe_main.c */