drivers/net/ixgbe/ixgbe_main.c

   1 /*******************************************************************************
   2
   3   Intel 10 Gigabit PCI Express Linux driver
   4   Copyright(c) 1999 - 2011 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/types.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/string.h>
  34 #include <linux/in.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/sctp.h>
  39 #include <linux/pkt_sched.h>
  40 #include <linux/ipv6.h>
  41 #include <linux/slab.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <linux/ethtool.h>
  45 #include <linux/if_vlan.h>
  46 #include <linux/prefetch.h>
  47 #include <scsi/fc/fc_fcoe.h>
  48
  49 #include "ixgbe.h"
  50 #include "ixgbe_common.h"
  51 #include "ixgbe_dcb_82599.h"
  52 #include "ixgbe_sriov.h"
  53
  54 char ixgbe_driver_name[] = "ixgbe";
  55 static const char ixgbe_driver_string[] =
  56                               "Intel(R) 10 Gigabit PCI Express Network Driver";
  57 #define MAJ 3
  58 #define MIN 4
  59 #define BUILD 8
  60 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  61         __stringify(BUILD) "-k"
  62 const char ixgbe_driver_version[] = DRV_VERSION;
  63 static const char ixgbe_copyright[] =
  64                                 "Copyright (c) 1999-2011 Intel Corporation.";
  65
  66 static const struct ixgbe_info *ixgbe_info_tbl[] = {
  67         [board_82598] = &ixgbe_82598_info,
  68         [board_82599] = &ixgbe_82599_info,
  69         [board_X540] = &ixgbe_X540_info,
  70 };
  71
  72 /* ixgbe_pci_tbl - PCI Device ID Table
  73  *
  74  * Wildcard entries (PCI_ANY_ID) should come last
  75  * Last entry must be all 0s
  76  *
  77  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  78  *   Class, Class Mask, private data (not used) }
  79  */
  80 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
  81         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598),
  82          board_82598 },
  83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT),
  84          board_82598 },
  85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT),
  86          board_82598 },
  87         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT),
  88          board_82598 },
  89         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2),
  90          board_82598 },
  91         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4),
  92          board_82598 },
  93         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT),
  94          board_82598 },
  95         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT),
  96          board_82598 },
  97         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM),
  98          board_82598 },
  99         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR),
 100          board_82598 },
 101         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM),
 102          board_82598 },
 103         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX),
 104          board_82598 },
 105         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4),
 106          board_82599 },
 107         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM),
 108          board_82599 },
 109         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR),
 110          board_82599 },
 111         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP),
 112          board_82599 },
 113         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM),
 114          board_82599 },
 115         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ),
 116          board_82599 },
 117         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4),
 118          board_82599 },
 119         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE),
 120          board_82599 },
 121         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE),
 122          board_82599 },
 123         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM),
 124          board_82599 },
 125         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE),
 126          board_82599 },
 127         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T),
 128          board_X540 },
 129         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2),
 130          board_82599 },
 131         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS),
 132          board_82599 },
 133
 134         /* required last entry */
 135         {0, }
 136 };
 137 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
 138
 139 #ifdef CONFIG_IXGBE_DCA
 140 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
 141                             void *p);
 142 static struct notifier_block dca_notifier = {
 143         .notifier_call = ixgbe_notify_dca,
 144         .next          = NULL,
 145         .priority      = 0
 146 };
 147 #endif
 148
 149 #ifdef CONFIG_PCI_IOV
 150 static unsigned int max_vfs;
 151 module_param(max_vfs, uint, 0);
 152 MODULE_PARM_DESC(max_vfs,
 153                  "Maximum number of virtual functions to allocate per physical function");
 154 #endif /* CONFIG_PCI_IOV */
 155
 156 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 157 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
 158 MODULE_LICENSE("GPL");
 159 MODULE_VERSION(DRV_VERSION);
 160
 161 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
 162
 163 static inline void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 164 {
 165         struct ixgbe_hw *hw = &adapter->hw;
 166         u32 gcr;
 167         u32 gpie;
 168         u32 vmdctl;
 169
 170 #ifdef CONFIG_PCI_IOV
 171         /* disable iov and allow time for transactions to clear */
 172         pci_disable_sriov(adapter->pdev);
 173 #endif
 174
 175         /* turn off device IOV mode */
 176         gcr = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
 177         gcr &= ~(IXGBE_GCR_EXT_SRIOV);
 178         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr);
 179         gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
 180         gpie &= ~IXGBE_GPIE_VTMODE_MASK;
 181         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
 182
 183         /* set default pool back to 0 */
 184         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
 185         vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;
 186         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);
 187         IXGBE_WRITE_FLUSH(hw);
 188
 189         /* take a breather then clean up driver data */
 190         msleep(100);
 191
 192         kfree(adapter->vfinfo);
 193         adapter->vfinfo = NULL;
 194
 195         adapter->num_vfs = 0;
 196         adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
 197 }
 198
 199 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
 200 {
 201         if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
 202             !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
 203                 schedule_work(&adapter->service_task);
 204 }
 205
 206 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
 207 {
 208         BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
 209
 210         /* flush memory to make sure state is correct before next watchog */
 211         smp_mb__before_clear_bit();
 212         clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
 213 }
 214
 215 struct ixgbe_reg_info {
 216         u32 ofs;
 217         char *name;
 218 };
 219
 220 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
 221
 222         /* General Registers */
 223         {IXGBE_CTRL, "CTRL"},
 224         {IXGBE_STATUS, "STATUS"},
 225         {IXGBE_CTRL_EXT, "CTRL_EXT"},
 226
 227         /* Interrupt Registers */
 228         {IXGBE_EICR, "EICR"},
 229
 230         /* RX Registers */
 231         {IXGBE_SRRCTL(0), "SRRCTL"},
 232         {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
 233         {IXGBE_RDLEN(0), "RDLEN"},
 234         {IXGBE_RDH(0), "RDH"},
 235         {IXGBE_RDT(0), "RDT"},
 236         {IXGBE_RXDCTL(0), "RXDCTL"},
 237         {IXGBE_RDBAL(0), "RDBAL"},
 238         {IXGBE_RDBAH(0), "RDBAH"},
 239
 240         /* TX Registers */
 241         {IXGBE_TDBAL(0), "TDBAL"},
 242         {IXGBE_TDBAH(0), "TDBAH"},
 243         {IXGBE_TDLEN(0), "TDLEN"},
 244         {IXGBE_TDH(0), "TDH"},
 245         {IXGBE_TDT(0), "TDT"},
 246         {IXGBE_TXDCTL(0), "TXDCTL"},
 247
 248         /* List Terminator */
 249         {}
 250 };
 251
 252
 253 /*
 254  * ixgbe_regdump - register printout routine
 255  */
 256 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
 257 {
 258         int i = 0, j = 0;
 259         char rname[16];
 260         u32 regs[64];
 261
 262         switch (reginfo->ofs) {
 263         case IXGBE_SRRCTL(0):
 264                 for (i = 0; i < 64; i++)
 265                         regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
 266                 break;
 267         case IXGBE_DCA_RXCTRL(0):
 268                 for (i = 0; i < 64; i++)
 269                         regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
 270                 break;
 271         case IXGBE_RDLEN(0):
 272                 for (i = 0; i < 64; i++)
 273                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
 274                 break;
 275         case IXGBE_RDH(0):
 276                 for (i = 0; i < 64; i++)
 277                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
 278                 break;
 279         case IXGBE_RDT(0):
 280                 for (i = 0; i < 64; i++)
 281                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
 282                 break;
 283         case IXGBE_RXDCTL(0):
 284                 for (i = 0; i < 64; i++)
 285                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 286                 break;
 287         case IXGBE_RDBAL(0):
 288                 for (i = 0; i < 64; i++)
 289                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
 290                 break;
 291         case IXGBE_RDBAH(0):
 292                 for (i = 0; i < 64; i++)
 293                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
 294                 break;
 295         case IXGBE_TDBAL(0):
 296                 for (i = 0; i < 64; i++)
 297                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
 298                 break;
 299         case IXGBE_TDBAH(0):
 300                 for (i = 0; i < 64; i++)
 301                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
 302                 break;
 303         case IXGBE_TDLEN(0):
 304                 for (i = 0; i < 64; i++)
 305                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
 306                 break;
 307         case IXGBE_TDH(0):
 308                 for (i = 0; i < 64; i++)
 309                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
 310                 break;
 311         case IXGBE_TDT(0):
 312                 for (i = 0; i < 64; i++)
 313                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
 314                 break;
 315         case IXGBE_TXDCTL(0):
 316                 for (i = 0; i < 64; i++)
 317                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
 318                 break;
 319         default:
 320                 pr_info("%-15s %08x\n", reginfo->name,
 321                         IXGBE_READ_REG(hw, reginfo->ofs));
 322                 return;
 323         }
 324
 325         for (i = 0; i < 8; i++) {
 326                 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
 327                 pr_err("%-15s", rname);
 328                 for (j = 0; j < 8; j++)
 329                         pr_cont(" %08x", regs[i*8+j]);
 330                 pr_cont("\n");
 331         }
 332
 333 }
 334
 335 /*
 336  * ixgbe_dump - Print registers, tx-rings and rx-rings
 337  */
 338 static void ixgbe_dump(struct ixgbe_adapter *adapter)
 339 {
 340         struct net_device *netdev = adapter->netdev;
 341         struct ixgbe_hw *hw = &adapter->hw;
 342         struct ixgbe_reg_info *reginfo;
 343         int n = 0;
 344         struct ixgbe_ring *tx_ring;
 345         struct ixgbe_tx_buffer *tx_buffer_info;
 346         union ixgbe_adv_tx_desc *tx_desc;
 347         struct my_u0 { u64 a; u64 b; } *u0;
 348         struct ixgbe_ring *rx_ring;
 349         union ixgbe_adv_rx_desc *rx_desc;
 350         struct ixgbe_rx_buffer *rx_buffer_info;
 351         u32 staterr;
 352         int i = 0;
 353
 354         if (!netif_msg_hw(adapter))
 355                 return;
 356
 357         /* Print netdevice Info */
 358         if (netdev) {
 359                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 360                 pr_info("Device Name     state            "
 361                         "trans_start      last_rx\n");
 362                 pr_info("%-15s %016lX %016lX %016lX\n",
 363                         netdev->name,
 364                         netdev->state,
 365                         netdev->trans_start,
 366                         netdev->last_rx);
 367         }
 368
 369         /* Print Registers */
 370         dev_info(&adapter->pdev->dev, "Register Dump\n");
 371         pr_info(" Register Name   Value\n");
 372         for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
 373              reginfo->name; reginfo++) {
 374                 ixgbe_regdump(hw, reginfo);
 375         }
 376
 377         /* Print TX Ring Summary */
 378         if (!netdev || !netif_running(netdev))
 379                 goto exit;
 380
 381         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 382         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 383         for (n = 0; n < adapter->num_tx_queues; n++) {
 384                 tx_ring = adapter->tx_ring[n];
 385                 tx_buffer_info =
 386                         &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 387                 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
 388                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
 389                            (u64)tx_buffer_info->dma,
 390                            tx_buffer_info->length,
 391                            tx_buffer_info->next_to_watch,
 392                            (u64)tx_buffer_info->time_stamp);
 393         }
 394
 395         /* Print TX Rings */
 396         if (!netif_msg_tx_done(adapter))
 397                 goto rx_ring_summary;
 398
 399         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 400
 401         /* Transmit Descriptor Formats
 402          *
 403          * Advanced Transmit Descriptor
 404          *   +--------------------------------------------------------------+
 405          * 0 |         Buffer Address [63:0]                                |
 406          *   +--------------------------------------------------------------+
 407          * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
 408          *   +--------------------------------------------------------------+
 409          *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
 410          */
 411
 412         for (n = 0; n < adapter->num_tx_queues; n++) {
 413                 tx_ring = adapter->tx_ring[n];
 414                 pr_info("------------------------------------\n");
 415                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 416                 pr_info("------------------------------------\n");
 417                 pr_info("T [desc]     [address 63:0  ] "
 418                         "[PlPOIdStDDt Ln] [bi->dma       ] "
 419                         "leng  ntw timestamp        bi->skb\n");
 420
 421                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 422                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 423                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 424                         u0 = (struct my_u0 *)tx_desc;
 425                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
 426                                 " %04X  %3X %016llX %p", i,
 427                                 le64_to_cpu(u0->a),
 428                                 le64_to_cpu(u0->b),
 429                                 (u64)tx_buffer_info->dma,
 430                                 tx_buffer_info->length,
 431                                 tx_buffer_info->next_to_watch,
 432                                 (u64)tx_buffer_info->time_stamp,
 433                                 tx_buffer_info->skb);
 434                         if (i == tx_ring->next_to_use &&
 435                                 i == tx_ring->next_to_clean)
 436                                 pr_cont(" NTC/U\n");
 437                         else if (i == tx_ring->next_to_use)
 438                                 pr_cont(" NTU\n");
 439                         else if (i == tx_ring->next_to_clean)
 440                                 pr_cont(" NTC\n");
 441                         else
 442                                 pr_cont("\n");
 443
 444                         if (netif_msg_pktdata(adapter) &&
 445                                 tx_buffer_info->dma != 0)
 446                                 print_hex_dump(KERN_INFO, "",
 447                                         DUMP_PREFIX_ADDRESS, 16, 1,
 448                                         phys_to_virt(tx_buffer_info->dma),
 449                                         tx_buffer_info->length, true);
 450                 }
 451         }
 452
 453         /* Print RX Rings Summary */
 454 rx_ring_summary:
 455         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 456         pr_info("Queue [NTU] [NTC]\n");
 457         for (n = 0; n < adapter->num_rx_queues; n++) {
 458                 rx_ring = adapter->rx_ring[n];
 459                 pr_info("%5d %5X %5X\n",
 460                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 461         }
 462
 463         /* Print RX Rings */
 464         if (!netif_msg_rx_status(adapter))
 465                 goto exit;
 466
 467         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 468
 469         /* Advanced Receive Descriptor (Read) Format
 470          *    63                                           1        0
 471          *    +-----------------------------------------------------+
 472          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 473          *    +----------------------------------------------+------+
 474          *  8 |       Header Buffer Address [63:1]           |  DD  |
 475          *    +-----------------------------------------------------+
 476          *
 477          *
 478          * Advanced Receive Descriptor (Write-Back) Format
 479          *
 480          *   63       48 47    32 31  30      21 20 16 15   4 3     0
 481          *   +------------------------------------------------------+
 482          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 483          *   | Checksum   Ident  |   |           |    | Type | Type |
 484          *   +------------------------------------------------------+
 485          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 486          *   +------------------------------------------------------+
 487          *   63       48 47    32 31            20 19               0
 488          */
 489         for (n = 0; n < adapter->num_rx_queues; n++) {
 490                 rx_ring = adapter->rx_ring[n];
 491                 pr_info("------------------------------------\n");
 492                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 493                 pr_info("------------------------------------\n");
 494                 pr_info("R  [desc]      [ PktBuf     A0] "
 495                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
 496                         "<-- Adv Rx Read format\n");
 497                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
 498                         "[vl er S cks ln] ---------------- [bi->skb] "
 499                         "<-- Adv Rx Write-Back format\n");
 500
 501                 for (i = 0; i < rx_ring->count; i++) {
 502                         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 503                         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
 504                         u0 = (struct my_u0 *)rx_desc;
 505                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 506                         if (staterr & IXGBE_RXD_STAT_DD) {
 507                                 /* Descriptor Done */
 508                                 pr_info("RWB[0x%03X]     %016llX "
 509                                         "%016llX ---------------- %p", i,
 510                                         le64_to_cpu(u0->a),
 511                                         le64_to_cpu(u0->b),
 512                                         rx_buffer_info->skb);
 513                         } else {
 514                                 pr_info("R  [0x%03X]     %016llX "
 515                                         "%016llX %016llX %p", i,
 516                                         le64_to_cpu(u0->a),
 517                                         le64_to_cpu(u0->b),
 518                                         (u64)rx_buffer_info->dma,
 519                                         rx_buffer_info->skb);
 520
 521                                 if (netif_msg_pktdata(adapter)) {
 522                                         print_hex_dump(KERN_INFO, "",
 523                                            DUMP_PREFIX_ADDRESS, 16, 1,
 524                                            phys_to_virt(rx_buffer_info->dma),
 525                                            rx_ring->rx_buf_len, true);
 526
 527                                         if (rx_ring->rx_buf_len
 528                                                 < IXGBE_RXBUFFER_2048)
 529                                                 print_hex_dump(KERN_INFO, "",
 530                                                   DUMP_PREFIX_ADDRESS, 16, 1,
 531                                                   phys_to_virt(
 532                                                     rx_buffer_info->page_dma +
 533                                                     rx_buffer_info->page_offset
 534                                                   ),
 535                                                   PAGE_SIZE/2, true);
 536                                 }
 537                         }
 538
 539                         if (i == rx_ring->next_to_use)
 540                                 pr_cont(" NTU\n");
 541                         else if (i == rx_ring->next_to_clean)
 542                                 pr_cont(" NTC\n");
 543                         else
 544                                 pr_cont("\n");
 545
 546                 }
 547         }
 548
 549 exit:
 550         return;
 551 }
 552
 553 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
 554 {
 555         u32 ctrl_ext;
 556
 557         /* Let firmware take over control of h/w */
 558         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 559         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 560                         ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
 561 }
 562
 563 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
 564 {
 565         u32 ctrl_ext;
 566
 567         /* Let firmware know the driver has taken over */
 568         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 569         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 570                         ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
 571 }
 572
 573 /*
 574  * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 575  * @adapter: pointer to adapter struct
 576  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 577  * @queue: queue to map the corresponding interrupt to
 578  * @msix_vector: the vector to map to the corresponding queue
 579  *
 580  */
 581 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
 582                            u8 queue, u8 msix_vector)
 583 {
 584         u32 ivar, index;
 585         struct ixgbe_hw *hw = &adapter->hw;
 586         switch (hw->mac.type) {
 587         case ixgbe_mac_82598EB:
 588                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 589                 if (direction == -1)
 590                         direction = 0;
 591                 index = (((direction * 64) + queue) >> 2) & 0x1F;
 592                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
 593                 ivar &= ~(0xFF << (8 * (queue & 0x3)));
 594                 ivar |= (msix_vector << (8 * (queue & 0x3)));
 595                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
 596                 break;
 597         case ixgbe_mac_82599EB:
 598         case ixgbe_mac_X540:
 599                 if (direction == -1) {
 600                         /* other causes */
 601                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 602                         index = ((queue & 1) * 8);
 603                         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
 604                         ivar &= ~(0xFF << index);
 605                         ivar |= (msix_vector << index);
 606                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
 607                         break;
 608                 } else {
 609                         /* tx or rx causes */
 610                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 611                         index = ((16 * (queue & 1)) + (8 * direction));
 612                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
 613                         ivar &= ~(0xFF << index);
 614                         ivar |= (msix_vector << index);
 615                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
 616                         break;
 617                 }
 618         default:
 619                 break;
 620         }
 621 }
 622
 623 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
 624                                           u64 qmask)
 625 {
 626         u32 mask;
 627
 628         switch (adapter->hw.mac.type) {
 629         case ixgbe_mac_82598EB:
 630                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
 631                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
 632                 break;
 633         case ixgbe_mac_82599EB:
 634         case ixgbe_mac_X540:
 635                 mask = (qmask & 0xFFFFFFFF);
 636                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
 637                 mask = (qmask >> 32);
 638                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
 639                 break;
 640         default:
 641                 break;
 642         }
 643 }
 644
 645 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
 646                                       struct ixgbe_tx_buffer *tx_buffer_info)
 647 {
 648         if (tx_buffer_info->dma) {
 649                 if (tx_buffer_info->mapped_as_page)
 650                         dma_unmap_page(tx_ring->dev,
 651                                        tx_buffer_info->dma,
 652                                        tx_buffer_info->length,
 653                                        DMA_TO_DEVICE);
 654                 else
 655                         dma_unmap_single(tx_ring->dev,
 656                                          tx_buffer_info->dma,
 657                                          tx_buffer_info->length,
 658                                          DMA_TO_DEVICE);
 659                 tx_buffer_info->dma = 0;
 660         }
 661         if (tx_buffer_info->skb) {
 662                 dev_kfree_skb_any(tx_buffer_info->skb);
 663                 tx_buffer_info->skb = NULL;
 664         }
 665         tx_buffer_info->time_stamp = 0;
 666         /* tx_buffer_info must be completely set up in the transmit path */
 667 }
 668
 669 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
 670 {
 671         struct ixgbe_hw *hw = &adapter->hw;
 672         struct ixgbe_hw_stats *hwstats = &adapter->stats;
 673         u32 data = 0;
 674         u32 xoff[8] = {0};
 675         int i;
 676
 677         if ((hw->fc.current_mode == ixgbe_fc_full) ||
 678             (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
 679                 switch (hw->mac.type) {
 680                 case ixgbe_mac_82598EB:
 681                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
 682                         break;
 683                 default:
 684                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
 685                 }
 686                 hwstats->lxoffrxc += data;
 687
 688                 /* refill credits (no tx hang) if we received xoff */
 689                 if (!data)
 690                         return;
 691
 692                 for (i = 0; i < adapter->num_tx_queues; i++)
 693                         clear_bit(__IXGBE_HANG_CHECK_ARMED,
 694                                   &adapter->tx_ring[i]->state);
 695                 return;
 696         } else if (!(adapter->dcb_cfg.pfc_mode_enable))
 697                 return;
 698
 699         /* update stats for each tc, only valid with PFC enabled */
 700         for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
 701                 switch (hw->mac.type) {
 702                 case ixgbe_mac_82598EB:
 703                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
 704                         break;
 705                 default:
 706                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
 707                 }
 708                 hwstats->pxoffrxc[i] += xoff[i];
 709         }
 710
 711         /* disarm tx queues that have received xoff frames */
 712         for (i = 0; i < adapter->num_tx_queues; i++) {
 713                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
 714                 u8 tc = tx_ring->dcb_tc;
 715
 716                 if (xoff[tc])
 717                         clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 718         }
 719 }
 720
 721 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
 722 {
 723         return ring->tx_stats.completed;
 724 }
 725
 726 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
 727 {
 728         struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
 729         struct ixgbe_hw *hw = &adapter->hw;
 730
 731         u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
 732         u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
 733
 734         if (head != tail)
 735                 return (head < tail) ?
 736                         tail - head : (tail + ring->count - head);
 737
 738         return 0;
 739 }
 740
 741 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
 742 {
 743         u32 tx_done = ixgbe_get_tx_completed(tx_ring);
 744         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
 745         u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
 746         bool ret = false;
 747
 748         clear_check_for_tx_hang(tx_ring);
 749
 750         /*
 751          * Check for a hung queue, but be thorough. This verifies
 752          * that a transmit has been completed since the previous
 753          * check AND there is at least one packet pending. The
 754          * ARMED bit is set to indicate a potential hang. The
 755          * bit is cleared if a pause frame is received to remove
 756          * false hang detection due to PFC or 802.3x frames. By
 757          * requiring this to fail twice we avoid races with
 758          * pfc clearing the ARMED bit and conditions where we
 759          * run the check_tx_hang logic with a transmit completion
 760          * pending but without time to complete it yet.
 761          */
 762         if ((tx_done_old == tx_done) && tx_pending) {
 763                 /* make sure it is true for two checks in a row */
 764                 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
 765                                        &tx_ring->state);
 766         } else {
 767                 /* update completed stats and continue */
 768                 tx_ring->tx_stats.tx_done_old = tx_done;
 769                 /* reset the countdown */
 770                 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 771         }
 772
 773         return ret;
 774 }
 775
 776 /**
 777  * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 778  * @adapter: driver private struct
 779  **/
 780 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
 781 {
 782
 783         /* Do the reset outside of interrupt context */
 784         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
 785                 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
 786                 ixgbe_service_event_schedule(adapter);
 787         }
 788 }
 789
 790 /**
 791  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
 792  * @q_vector: structure containing interrupt and ring information
 793  * @tx_ring: tx ring to clean
 794  **/
 795 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 796                                struct ixgbe_ring *tx_ring)
 797 {
 798         struct ixgbe_adapter *adapter = q_vector->adapter;
 799         union ixgbe_adv_tx_desc *tx_desc, *eop_desc;
 800         struct ixgbe_tx_buffer *tx_buffer_info;
 801         unsigned int total_bytes = 0, total_packets = 0;
 802         u16 i, eop, count = 0;
 803
 804         i = tx_ring->next_to_clean;
 805         eop = tx_ring->tx_buffer_info[i].next_to_watch;
 806         eop_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 807
 808         while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
 809                (count < q_vector->tx.work_limit)) {
 810                 bool cleaned = false;
 811                 rmb(); /* read buffer_info after eop_desc */
 812                 for ( ; !cleaned; count++) {
 813                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 814                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 815
 816                         tx_desc->wb.status = 0;
 817                         cleaned = (i == eop);
 818
 819                         i++;
 820                         if (i == tx_ring->count)
 821                                 i = 0;
 822
 823                         if (cleaned && tx_buffer_info->skb) {
 824                                 total_bytes += tx_buffer_info->bytecount;
 825                                 total_packets += tx_buffer_info->gso_segs;
 826                         }
 827
 828                         ixgbe_unmap_and_free_tx_resource(tx_ring,
 829                                                          tx_buffer_info);
 830                 }
 831
 832                 tx_ring->tx_stats.completed++;
 833                 eop = tx_ring->tx_buffer_info[i].next_to_watch;
 834                 eop_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 835         }
 836
 837         tx_ring->next_to_clean = i;
 838         tx_ring->stats.bytes += total_bytes;
 839         tx_ring->stats.packets += total_packets;
 840         u64_stats_update_begin(&tx_ring->syncp);
 841         q_vector->tx.total_bytes += total_bytes;
 842         q_vector->tx.total_packets += total_packets;
 843         u64_stats_update_end(&tx_ring->syncp);
 844
 845         if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
 846                 /* schedule immediate reset if we believe we hung */
 847                 struct ixgbe_hw *hw = &adapter->hw;
 848                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, eop);
 849                 e_err(drv, "Detected Tx Unit Hang\n"
 850                         "  Tx Queue             <%d>\n"
 851                         "  TDH, TDT             <%x>, <%x>\n"
 852                         "  next_to_use          <%x>\n"
 853                         "  next_to_clean        <%x>\n"
 854                         "tx_buffer_info[next_to_clean]\n"
 855                         "  time_stamp           <%lx>\n"
 856                         "  jiffies              <%lx>\n",
 857                         tx_ring->queue_index,
 858                         IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
 859                         IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
 860                         tx_ring->next_to_use, eop,
 861                         tx_ring->tx_buffer_info[eop].time_stamp, jiffies);
 862
 863                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 864
 865                 e_info(probe,
 866                        "tx hang %d detected on queue %d, resetting adapter\n",
 867                         adapter->tx_timeout_count + 1, tx_ring->queue_index);
 868
 869                 /* schedule immediate reset if we believe we hung */
 870                 ixgbe_tx_timeout_reset(adapter);
 871
 872                 /* the adapter is about to reset, no point in enabling stuff */
 873                 return true;
 874         }
 875
 876 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 877         if (unlikely(count && netif_carrier_ok(tx_ring->netdev) &&
 878                      (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 879                 /* Make sure that anybody stopping the queue after this
 880                  * sees the new next_to_clean.
 881                  */
 882                 smp_mb();
 883                 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
 884                     !test_bit(__IXGBE_DOWN, &adapter->state)) {
 885                         netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
 886                         ++tx_ring->tx_stats.restart_queue;
 887                 }
 888         }
 889
 890         return count < q_vector->tx.work_limit;
 891 }
 892
 893 #ifdef CONFIG_IXGBE_DCA
 894 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
 895                                 struct ixgbe_ring *rx_ring,
 896                                 int cpu)
 897 {
 898         struct ixgbe_hw *hw = &adapter->hw;
 899         u32 rxctrl;
 900         u8 reg_idx = rx_ring->reg_idx;
 901
 902         rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
 903         switch (hw->mac.type) {
 904         case ixgbe_mac_82598EB:
 905                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
 906                 rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
 907                 break;
 908         case ixgbe_mac_82599EB:
 909         case ixgbe_mac_X540:
 910                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
 911                 rxctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
 912                            IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
 913                 break;
 914         default:
 915                 break;
 916         }
 917         rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
 918         rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
 919         rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
 920         IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
 921 }
 922
 923 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
 924                                 struct ixgbe_ring *tx_ring,
 925                                 int cpu)
 926 {
 927         struct ixgbe_hw *hw = &adapter->hw;
 928         u32 txctrl;
 929         u8 reg_idx = tx_ring->reg_idx;
 930
 931         switch (hw->mac.type) {
 932         case ixgbe_mac_82598EB:
 933                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
 934                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
 935                 txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
 936                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 937                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
 938                 break;
 939         case ixgbe_mac_82599EB:
 940         case ixgbe_mac_X540:
 941                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
 942                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
 943                 txctrl |= (dca3_get_tag(&adapter->pdev->dev, cpu) <<
 944                            IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
 945                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 946                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
 947                 break;
 948         default:
 949                 break;
 950         }
 951 }
 952
 953 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
 954 {
 955         struct ixgbe_adapter *adapter = q_vector->adapter;
 956         int cpu = get_cpu();
 957         long r_idx;
 958         int i;
 959
 960         if (q_vector->cpu == cpu)
 961                 goto out_no_update;
 962
 963         r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
 964         for (i = 0; i < q_vector->tx.count; i++) {
 965                 ixgbe_update_tx_dca(adapter, adapter->tx_ring[r_idx], cpu);
 966                 r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
 967                                       r_idx + 1);
 968         }
 969
 970         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
 971         for (i = 0; i < q_vector->rx.count; i++) {
 972                 ixgbe_update_rx_dca(adapter, adapter->rx_ring[r_idx], cpu);
 973                 r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
 974                                       r_idx + 1);
 975         }
 976
 977         q_vector->cpu = cpu;
 978 out_no_update:
 979         put_cpu();
 980 }
 981
 982 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
 983 {
 984         int num_q_vectors;
 985         int i;
 986
 987         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
 988                 return;
 989
 990         /* always use CB2 mode, difference is masked in the CB driver */
 991         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
 992
 993         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
 994                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 995         else
 996                 num_q_vectors = 1;
 997
 998         for (i = 0; i < num_q_vectors; i++) {
 999                 adapter->q_vector[i]->cpu = -1;
1000                 ixgbe_update_dca(adapter->q_vector[i]);
1001         }
1002 }
1003
1004 static int __ixgbe_notify_dca(struct device *dev, void *data)
1005 {
1006         struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
1007         unsigned long event = *(unsigned long *)data;
1008
1009         if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
1010                 return 0;
1011
1012         switch (event) {
1013         case DCA_PROVIDER_ADD:
1014                 /* if we're already enabled, don't do it again */
1015                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
1016                         break;
1017                 if (dca_add_requester(dev) == 0) {
1018                         adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
1019                         ixgbe_setup_dca(adapter);
1020                         break;
1021                 }
1022                 /* Fall Through since DCA is disabled. */
1023         case DCA_PROVIDER_REMOVE:
1024                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
1025                         dca_remove_requester(dev);
1026                         adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
1027                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
1028                 }
1029                 break;
1030         }
1031
1032         return 0;
1033 }
1034 #endif /* CONFIG_IXGBE_DCA */
1035
1036 static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
1037                                  struct sk_buff *skb)
1038 {
1039         skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1040 }
1041
1042 /**
1043  * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
1044  * @adapter: address of board private structure
1045  * @rx_desc: advanced rx descriptor
1046  *
1047  * Returns : true if it is FCoE pkt
1048  */
1049 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
1050                                     union ixgbe_adv_rx_desc *rx_desc)
1051 {
1052         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1053
1054         return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
1055                ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1056                 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1057                              IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1058 }
1059
1060 /**
1061  * ixgbe_receive_skb - Send a completed packet up the stack
1062  * @adapter: board private structure
1063  * @skb: packet to send up
1064  * @status: hardware indication of status of receive
1065  * @rx_ring: rx descriptor ring (for a specific queue) to setup
1066  * @rx_desc: rx descriptor
1067  **/
1068 static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1069                               struct sk_buff *skb, u8 status,
1070                               struct ixgbe_ring *ring,
1071                               union ixgbe_adv_rx_desc *rx_desc)
1072 {
1073         struct ixgbe_adapter *adapter = q_vector->adapter;
1074         struct napi_struct *napi = &q_vector->napi;
1075         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1076         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1077
1078         if (is_vlan && (tag & VLAN_VID_MASK))
1079                 __vlan_hwaccel_put_tag(skb, tag);
1080
1081         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1082                 napi_gro_receive(napi, skb);
1083         else
1084                 netif_rx(skb);
1085 }
1086
1087 /**
1088  * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1089  * @adapter: address of board private structure
1090  * @status_err: hardware indication of status of receive
1091  * @skb: skb currently being received and modified
1092  * @status_err: status error value of last descriptor in packet
1093  **/
1094 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1095                                      union ixgbe_adv_rx_desc *rx_desc,
1096                                      struct sk_buff *skb,
1097                                      u32 status_err)
1098 {
1099         skb->ip_summed = CHECKSUM_NONE;
1100
1101         /* Rx csum disabled */
1102         if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1103                 return;
1104
1105         /* if IP and error */
1106         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1107             (status_err & IXGBE_RXDADV_ERR_IPE)) {
1108                 adapter->hw_csum_rx_error++;
1109                 return;
1110         }
1111
1112         if (!(status_err & IXGBE_RXD_STAT_L4CS))
1113                 return;
1114
1115         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1116                 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1117
1118                 /*
1119                  * 82599 errata, UDP frames with a 0 checksum can be marked as
1120                  * checksum errors.
1121                  */
1122                 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1123                     (adapter->hw.mac.type == ixgbe_mac_82599EB))
1124                         return;
1125
1126                 adapter->hw_csum_rx_error++;
1127                 return;
1128         }
1129
1130         /* It must be a TCP or UDP packet with a valid checksum */
1131         skb->ip_summed = CHECKSUM_UNNECESSARY;
1132 }
1133
1134 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1135 {
1136         /*
1137          * Force memory writes to complete before letting h/w
1138          * know there are new descriptors to fetch.  (Only
1139          * applicable for weak-ordered memory model archs,
1140          * such as IA-64).
1141          */
1142         wmb();
1143         writel(val, rx_ring->tail);
1144 }
1145
1146 /**
1147  * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1148  * @rx_ring: ring to place buffers on
1149  * @cleaned_count: number of buffers to replace
1150  **/
1151 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1152 {
1153         union ixgbe_adv_rx_desc *rx_desc;
1154         struct ixgbe_rx_buffer *bi;
1155         struct sk_buff *skb;
1156         u16 i = rx_ring->next_to_use;
1157
1158         /* do nothing if no valid netdev defined */
1159         if (!rx_ring->netdev)
1160                 return;
1161
1162         while (cleaned_count--) {
1163                 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1164                 bi = &rx_ring->rx_buffer_info[i];
1165                 skb = bi->skb;
1166
1167                 if (!skb) {
1168                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1169                                                         rx_ring->rx_buf_len);
1170                         if (!skb) {
1171                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1172                                 goto no_buffers;
1173                         }
1174                         /* initialize queue mapping */
1175                         skb_record_rx_queue(skb, rx_ring->queue_index);
1176                         bi->skb = skb;
1177                 }
1178
1179                 if (!bi->dma) {
1180                         bi->dma = dma_map_single(rx_ring->dev,
1181                                                  skb->data,
1182                                                  rx_ring->rx_buf_len,
1183                                                  DMA_FROM_DEVICE);
1184                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1185                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1186                                 bi->dma = 0;
1187                                 goto no_buffers;
1188                         }
1189                 }
1190
1191                 if (ring_is_ps_enabled(rx_ring)) {
1192                         if (!bi->page) {
1193                                 bi->page = netdev_alloc_page(rx_ring->netdev);
1194                                 if (!bi->page) {
1195                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1196                                         goto no_buffers;
1197                                 }
1198                         }
1199
1200                         if (!bi->page_dma) {
1201                                 /* use a half page if we're re-using */
1202                                 bi->page_offset ^= PAGE_SIZE / 2;
1203                                 bi->page_dma = dma_map_page(rx_ring->dev,
1204                                                             bi->page,
1205                                                             bi->page_offset,
1206                                                             PAGE_SIZE / 2,
1207                                                             DMA_FROM_DEVICE);
1208                                 if (dma_mapping_error(rx_ring->dev,
1209                                                       bi->page_dma)) {
1210                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1211                                         bi->page_dma = 0;
1212                                         goto no_buffers;
1213                                 }
1214                         }
1215
1216                         /* Refresh the desc even if buffer_addrs didn't change
1217                          * because each write-back erases this info. */
1218                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1219                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1220                 } else {
1221                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1222                         rx_desc->read.hdr_addr = 0;
1223                 }
1224
1225                 i++;
1226                 if (i == rx_ring->count)
1227                         i = 0;
1228         }
1229
1230 no_buffers:
1231         if (rx_ring->next_to_use != i) {
1232                 rx_ring->next_to_use = i;
1233                 ixgbe_release_rx_desc(rx_ring, i);
1234         }
1235 }
1236
1237 static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1238 {
1239         /* HW will not DMA in data larger than the given buffer, even if it
1240          * parses the (NFS, of course) header to be larger.  In that case, it
1241          * fills the header buffer and spills the rest into the page.
1242          */
1243         u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1244         u16 hlen = (hdr_info &  IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1245                     IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1246         if (hlen > IXGBE_RX_HDR_SIZE)
1247                 hlen = IXGBE_RX_HDR_SIZE;
1248         return hlen;
1249 }
1250
1251 /**
1252  * ixgbe_transform_rsc_queue - change rsc queue into a full packet
1253  * @skb: pointer to the last skb in the rsc queue
1254  *
1255  * This function changes a queue full of hw rsc buffers into a completed
1256  * packet.  It uses the ->prev pointers to find the first packet and then
1257  * turns it into the frag list owner.
1258  **/
1259 static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1260 {
1261         unsigned int frag_list_size = 0;
1262         unsigned int skb_cnt = 1;
1263
1264         while (skb->prev) {
1265                 struct sk_buff *prev = skb->prev;
1266                 frag_list_size += skb->len;
1267                 skb->prev = NULL;
1268                 skb = prev;
1269                 skb_cnt++;
1270         }
1271
1272         skb_shinfo(skb)->frag_list = skb->next;
1273         skb->next = NULL;
1274         skb->len += frag_list_size;
1275         skb->data_len += frag_list_size;
1276         skb->truesize += frag_list_size;
1277         IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
1278
1279         return skb;
1280 }
1281
1282 static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
1283 {
1284         return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
1285                 IXGBE_RXDADV_RSCCNT_MASK);
1286 }
1287
1288 static void ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1289                                struct ixgbe_ring *rx_ring,
1290                                int *work_done, int work_to_do)
1291 {
1292         struct ixgbe_adapter *adapter = q_vector->adapter;
1293         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
1294         struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
1295         struct sk_buff *skb;
1296         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1297         const int current_node = numa_node_id();
1298 #ifdef IXGBE_FCOE
1299         int ddp_bytes = 0;
1300 #endif /* IXGBE_FCOE */
1301         u32 staterr;
1302         u16 i;
1303         u16 cleaned_count = 0;
1304         bool pkt_is_rsc = false;
1305
1306         i = rx_ring->next_to_clean;
1307         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1308         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1309
1310         while (staterr & IXGBE_RXD_STAT_DD) {
1311                 u32 upper_len = 0;
1312
1313                 rmb(); /* read descriptor and rx_buffer_info after status DD */
1314
1315                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1316
1317                 skb = rx_buffer_info->skb;
1318                 rx_buffer_info->skb = NULL;
1319                 prefetch(skb->data);
1320
1321                 if (ring_is_rsc_enabled(rx_ring))
1322                         pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1323
1324                 /* if this is a skb from previous receive DMA will be 0 */
1325                 if (rx_buffer_info->dma) {
1326                         u16 hlen;
1327                         if (pkt_is_rsc &&
1328                             !(staterr & IXGBE_RXD_STAT_EOP) &&
1329                             !skb->prev) {
1330                                 /*
1331                                  * When HWRSC is enabled, delay unmapping
1332                                  * of the first packet. It carries the
1333                                  * header information, HW may still
1334                                  * access the header after the writeback.
1335                                  * Only unmap it when EOP is reached
1336                                  */
1337                                 IXGBE_RSC_CB(skb)->delay_unmap = true;
1338                                 IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1339                         } else {
1340                                 dma_unmap_single(rx_ring->dev,
1341                                                  rx_buffer_info->dma,
1342                                                  rx_ring->rx_buf_len,
1343                                                  DMA_FROM_DEVICE);
1344                         }
1345                         rx_buffer_info->dma = 0;
1346
1347                         if (ring_is_ps_enabled(rx_ring)) {
1348                                 hlen = ixgbe_get_hlen(rx_desc);
1349                                 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1350                         } else {
1351                                 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1352                         }
1353
1354                         skb_put(skb, hlen);
1355                 } else {
1356                         /* assume packet split since header is unmapped */
1357                         upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1358                 }
1359
1360                 if (upper_len) {
1361                         dma_unmap_page(rx_ring->dev,
1362                                        rx_buffer_info->page_dma,
1363                                        PAGE_SIZE / 2,
1364                                        DMA_FROM_DEVICE);
1365                         rx_buffer_info->page_dma = 0;
1366                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1367                                            rx_buffer_info->page,
1368                                            rx_buffer_info->page_offset,
1369                                            upper_len);
1370
1371                         if ((page_count(rx_buffer_info->page) == 1) &&
1372                             (page_to_nid(rx_buffer_info->page) == current_node))
1373                                 get_page(rx_buffer_info->page);
1374                         else
1375                                 rx_buffer_info->page = NULL;
1376
1377                         skb->len += upper_len;
1378                         skb->data_len += upper_len;
1379                         skb->truesize += upper_len;
1380                 }
1381
1382                 i++;
1383                 if (i == rx_ring->count)
1384                         i = 0;
1385
1386                 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1387                 prefetch(next_rxd);
1388                 cleaned_count++;
1389
1390                 if (pkt_is_rsc) {
1391                         u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1392                                      IXGBE_RXDADV_NEXTP_SHIFT;
1393                         next_buffer = &rx_ring->rx_buffer_info[nextp];
1394                 } else {
1395                         next_buffer = &rx_ring->rx_buffer_info[i];
1396                 }
1397
1398                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1399                         if (ring_is_ps_enabled(rx_ring)) {
1400                                 rx_buffer_info->skb = next_buffer->skb;
1401                                 rx_buffer_info->dma = next_buffer->dma;
1402                                 next_buffer->skb = skb;
1403                                 next_buffer->dma = 0;
1404                         } else {
1405                                 skb->next = next_buffer->skb;
1406                                 skb->next->prev = skb;
1407                         }
1408                         rx_ring->rx_stats.non_eop_descs++;
1409                         goto next_desc;
1410                 }
1411
1412                 if (skb->prev) {
1413                         skb = ixgbe_transform_rsc_queue(skb);
1414                         /* if we got here without RSC the packet is invalid */
1415                         if (!pkt_is_rsc) {
1416                                 __pskb_trim(skb, 0);
1417                                 rx_buffer_info->skb = skb;
1418                                 goto next_desc;
1419                         }
1420                 }
1421
1422                 if (ring_is_rsc_enabled(rx_ring)) {
1423                         if (IXGBE_RSC_CB(skb)->delay_unmap) {
1424                                 dma_unmap_single(rx_ring->dev,
1425                                                  IXGBE_RSC_CB(skb)->dma,
1426                                                  rx_ring->rx_buf_len,
1427                                                  DMA_FROM_DEVICE);
1428                                 IXGBE_RSC_CB(skb)->dma = 0;
1429                                 IXGBE_RSC_CB(skb)->delay_unmap = false;
1430                         }
1431                 }
1432                 if (pkt_is_rsc) {
1433                         if (ring_is_ps_enabled(rx_ring))
1434                                 rx_ring->rx_stats.rsc_count +=
1435                                         skb_shinfo(skb)->nr_frags;
1436                         else
1437                                 rx_ring->rx_stats.rsc_count +=
1438                                         IXGBE_RSC_CB(skb)->skb_cnt;
1439                         rx_ring->rx_stats.rsc_flush++;
1440                 }
1441
1442                 /* ERR_MASK will only have valid bits if EOP set */
1443                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
1444                         dev_kfree_skb_any(skb);
1445                         goto next_desc;
1446                 }
1447
1448                 ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
1449                 if (adapter->netdev->features & NETIF_F_RXHASH)
1450                         ixgbe_rx_hash(rx_desc, skb);
1451
1452                 /* probably a little skewed due to removing CRC */
1453                 total_rx_bytes += skb->len;
1454                 total_rx_packets++;
1455
1456                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1457 #ifdef IXGBE_FCOE
1458                 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1459                 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
1460                         ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
1461                                                    staterr);
1462                         if (!ddp_bytes)
1463                                 goto next_desc;
1464                 }
1465 #endif /* IXGBE_FCOE */
1466                 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1467
1468 next_desc:
1469                 rx_desc->wb.upper.status_error = 0;
1470
1471                 (*work_done)++;
1472                 if (*work_done >= work_to_do)
1473                         break;
1474
1475                 /* return some buffers to hardware, one at a time is too slow */
1476                 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1477                         ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1478                         cleaned_count = 0;
1479                 }
1480
1481                 /* use prefetched values */
1482                 rx_desc = next_rxd;
1483                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1484         }
1485
1486         rx_ring->next_to_clean = i;
1487         cleaned_count = ixgbe_desc_unused(rx_ring);
1488
1489         if (cleaned_count)
1490                 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1491
1492 #ifdef IXGBE_FCOE
1493         /* include DDPed FCoE data */
1494         if (ddp_bytes > 0) {
1495                 unsigned int mss;
1496
1497                 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1498                         sizeof(struct fc_frame_header) -
1499                         sizeof(struct fcoe_crc_eof);
1500                 if (mss > 512)
1501                         mss &= ~511;
1502                 total_rx_bytes += ddp_bytes;
1503                 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1504         }
1505 #endif /* IXGBE_FCOE */
1506
1507         u64_stats_update_begin(&rx_ring->syncp);
1508         rx_ring->stats.packets += total_rx_packets;
1509         rx_ring->stats.bytes += total_rx_bytes;
1510         u64_stats_update_end(&rx_ring->syncp);
1511         q_vector->rx.total_packets += total_rx_packets;
1512         q_vector->rx.total_bytes += total_rx_bytes;
1513 }
1514
1515 static int ixgbe_clean_rxonly(struct napi_struct *, int);
1516 /**
1517  * ixgbe_configure_msix - Configure MSI-X hardware
1518  * @adapter: board private structure
1519  *
1520  * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1521  * interrupts.
1522  **/
1523 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1524 {
1525         struct ixgbe_q_vector *q_vector;
1526         int i, q_vectors, v_idx, r_idx;
1527         u32 mask;
1528
1529         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1530
1531         /*
1532          * Populate the IVAR table and set the ITR values to the
1533          * corresponding register.
1534          */
1535         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1536                 q_vector = adapter->q_vector[v_idx];
1537                 /* XXX for_each_set_bit(...) */
1538                 r_idx = find_first_bit(q_vector->rx.idx,
1539                                        adapter->num_rx_queues);
1540
1541                 for (i = 0; i < q_vector->rx.count; i++) {
1542                         u8 reg_idx = adapter->rx_ring[r_idx]->reg_idx;
1543                         ixgbe_set_ivar(adapter, 0, reg_idx, v_idx);
1544                         r_idx = find_next_bit(q_vector->rx.idx,
1545                                               adapter->num_rx_queues,
1546                                               r_idx + 1);
1547                 }
1548                 r_idx = find_first_bit(q_vector->tx.idx,
1549                                        adapter->num_tx_queues);
1550
1551                 for (i = 0; i < q_vector->tx.count; i++) {
1552                         u8 reg_idx = adapter->tx_ring[r_idx]->reg_idx;
1553                         ixgbe_set_ivar(adapter, 1, reg_idx, v_idx);
1554                         r_idx = find_next_bit(q_vector->tx.idx,
1555                                               adapter->num_tx_queues,
1556                                               r_idx + 1);
1557                 }
1558
1559                 if (q_vector->tx.count && !q_vector->rx.count)
1560                         /* tx only */
1561                         q_vector->eitr = adapter->tx_eitr_param;
1562                 else if (q_vector->rx.count)
1563                         /* rx or mixed */
1564                         q_vector->eitr = adapter->rx_eitr_param;
1565
1566                 ixgbe_write_eitr(q_vector);
1567                 /* If ATR is enabled, set interrupt affinity */
1568                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
1569                         /*
1570                          * Allocate the affinity_hint cpumask, assign the mask
1571                          * for this vector, and set our affinity_hint for
1572                          * this irq.
1573                          */
1574                         if (!alloc_cpumask_var(&q_vector->affinity_mask,
1575                                                GFP_KERNEL))
1576                                 return;
1577                         cpumask_set_cpu(v_idx, q_vector->affinity_mask);
1578                         irq_set_affinity_hint(adapter->msix_entries[v_idx].vector,
1579                                               q_vector->affinity_mask);
1580                 }
1581         }
1582
1583         switch (adapter->hw.mac.type) {
1584         case ixgbe_mac_82598EB:
1585                 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1586                                v_idx);
1587                 break;
1588         case ixgbe_mac_82599EB:
1589         case ixgbe_mac_X540:
1590                 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1591                 break;
1592
1593         default:
1594                 break;
1595         }
1596         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1597
1598         /* set up to autoclear timer, and the vectors */
1599         mask = IXGBE_EIMS_ENABLE_MASK;
1600         if (adapter->num_vfs)
1601                 mask &= ~(IXGBE_EIMS_OTHER |
1602                           IXGBE_EIMS_MAILBOX |
1603                           IXGBE_EIMS_LSC);
1604         else
1605                 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1606         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1607 }
1608
1609 enum latency_range {
1610         lowest_latency = 0,
1611         low_latency = 1,
1612         bulk_latency = 2,
1613         latency_invalid = 255
1614 };
1615
1616 /**
1617  * ixgbe_update_itr - update the dynamic ITR value based on statistics
1618  * @q_vector: structure containing interrupt and ring information
1619  * @ring_container: structure containing ring performance data
1620  *
1621  *      Stores a new ITR value based on packets and byte
1622  *      counts during the last interrupt.  The advantage of per interrupt
1623  *      computation is faster updates and more accurate ITR for the current
1624  *      traffic pattern.  Constants in this function were computed
1625  *      based on theoretical maximum wire speed and thresholds were set based
1626  *      on testing data as well as attempting to minimize response time
1627  *      while increasing bulk throughput.
1628  *      this functionality is controlled by the InterruptThrottleRate module
1629  *      parameter (see ixgbe_param.c)
1630  **/
1631 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1632                              struct ixgbe_ring_container *ring_container)
1633 {
1634         u64 bytes_perint;
1635         struct ixgbe_adapter *adapter = q_vector->adapter;
1636         int bytes = ring_container->total_bytes;
1637         int packets = ring_container->total_packets;
1638         u32 timepassed_us;
1639         u8 itr_setting = ring_container->itr;
1640
1641         if (packets == 0)
1642                 return;
1643
1644         /* simple throttlerate management
1645          *    0-20MB/s lowest (100000 ints/s)
1646          *   20-100MB/s low   (20000 ints/s)
1647          *  100-1249MB/s bulk (8000 ints/s)
1648          */
1649         /* what was last interrupt timeslice? */
1650         timepassed_us = 1000000/q_vector->eitr;
1651         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1652
1653         switch (itr_setting) {
1654         case lowest_latency:
1655                 if (bytes_perint > adapter->eitr_low)
1656                         itr_setting = low_latency;
1657                 break;
1658         case low_latency:
1659                 if (bytes_perint > adapter->eitr_high)
1660                         itr_setting = bulk_latency;
1661                 else if (bytes_perint <= adapter->eitr_low)
1662                         itr_setting = lowest_latency;
1663                 break;
1664         case bulk_latency:
1665                 if (bytes_perint <= adapter->eitr_high)
1666                         itr_setting = low_latency;
1667                 break;
1668         }
1669
1670         /* clear work counters since we have the values we need */
1671         ring_container->total_bytes = 0;
1672         ring_container->total_packets = 0;
1673
1674         /* write updated itr to ring container */
1675         ring_container->itr = itr_setting;
1676 }
1677
1678 /**
1679  * ixgbe_write_eitr - write EITR register in hardware specific way
1680  * @q_vector: structure containing interrupt and ring information
1681  *
1682  * This function is made to be called by ethtool and by the driver
1683  * when it needs to update EITR registers at runtime.  Hardware
1684  * specific quirks/differences are taken care of here.
1685  */
1686 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1687 {
1688         struct ixgbe_adapter *adapter = q_vector->adapter;
1689         struct ixgbe_hw *hw = &adapter->hw;
1690         int v_idx = q_vector->v_idx;
1691         u32 itr_reg = EITR_INTS_PER_SEC_TO_REG(q_vector->eitr);
1692
1693         switch (adapter->hw.mac.type) {
1694         case ixgbe_mac_82598EB:
1695                 /* must write high and low 16 bits to reset counter */
1696                 itr_reg |= (itr_reg << 16);
1697                 break;
1698         case ixgbe_mac_82599EB:
1699         case ixgbe_mac_X540:
1700                 /*
1701                  * 82599 and X540 can support a value of zero, so allow it for
1702                  * max interrupt rate, but there is an errata where it can
1703                  * not be zero with RSC
1704                  */
1705                 if (itr_reg == 8 &&
1706                     !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
1707                         itr_reg = 0;
1708
1709                 /*
1710                  * set the WDIS bit to not clear the timer bits and cause an
1711                  * immediate assertion of the interrupt
1712                  */
1713                 itr_reg |= IXGBE_EITR_CNT_WDIS;
1714                 break;
1715         default:
1716                 break;
1717         }
1718         IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1719 }
1720
1721 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
1722 {
1723         u32 new_itr = q_vector->eitr;
1724         u8 current_itr;
1725
1726         ixgbe_update_itr(q_vector, &q_vector->tx);
1727         ixgbe_update_itr(q_vector, &q_vector->rx);
1728
1729         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1730
1731         switch (current_itr) {
1732         /* counts and packets in update_itr are dependent on these numbers */
1733         case lowest_latency:
1734                 new_itr = 100000;
1735                 break;
1736         case low_latency:
1737                 new_itr = 20000; /* aka hwitr = ~200 */
1738                 break;
1739         case bulk_latency:
1740                 new_itr = 8000;
1741                 break;
1742         default:
1743                 break;
1744         }
1745
1746         if (new_itr != q_vector->eitr) {
1747                 /* do an exponential smoothing */
1748                 new_itr = ((q_vector->eitr * 9) + new_itr)/10;
1749
1750                 /* save the algorithm value here */
1751                 q_vector->eitr = new_itr;
1752
1753                 ixgbe_write_eitr(q_vector);
1754         }
1755 }
1756
1757 /**
1758  * ixgbe_check_overtemp_subtask - check for over tempurature
1759  * @adapter: pointer to adapter
1760  **/
1761 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
1762 {
1763         struct ixgbe_hw *hw = &adapter->hw;
1764         u32 eicr = adapter->interrupt_event;
1765
1766         if (test_bit(__IXGBE_DOWN, &adapter->state))
1767                 return;
1768
1769         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1770             !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
1771                 return;
1772
1773         adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1774
1775         switch (hw->device_id) {
1776         case IXGBE_DEV_ID_82599_T3_LOM:
1777                 /*
1778                  * Since the warning interrupt is for both ports
1779                  * we don't have to check if:
1780                  *  - This interrupt wasn't for our port.
1781                  *  - We may have missed the interrupt so always have to
1782                  *    check if we  got a LSC
1783                  */
1784                 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
1785                     !(eicr & IXGBE_EICR_LSC))
1786                         return;
1787
1788                 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
1789                         u32 autoneg;
1790                         bool link_up = false;
1791
1792                         hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1793
1794                         if (link_up)
1795                                 return;
1796                 }
1797
1798                 /* Check if this is not due to overtemp */
1799                 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
1800                         return;
1801
1802                 break;
1803         default:
1804                 if (!(eicr & IXGBE_EICR_GPI_SDP0))
1805                         return;
1806                 break;
1807         }
1808         e_crit(drv,
1809                "Network adapter has been stopped because it has over heated. "
1810                "Restart the computer. If the problem persists, "
1811                "power off the system and replace the adapter\n");
1812
1813         adapter->interrupt_event = 0;
1814 }
1815
1816 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1817 {
1818         struct ixgbe_hw *hw = &adapter->hw;
1819
1820         if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1821             (eicr & IXGBE_EICR_GPI_SDP1)) {
1822                 e_crit(probe, "Fan has stopped, replace the adapter\n");
1823                 /* write to clear the interrupt */
1824                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1825         }
1826 }
1827
1828 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1829 {
1830         struct ixgbe_hw *hw = &adapter->hw;
1831
1832         if (eicr & IXGBE_EICR_GPI_SDP2) {
1833                 /* Clear the interrupt */
1834                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1835                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1836                         adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
1837                         ixgbe_service_event_schedule(adapter);
1838                 }
1839         }
1840
1841         if (eicr & IXGBE_EICR_GPI_SDP1) {
1842                 /* Clear the interrupt */
1843                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1844                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1845                         adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
1846                         ixgbe_service_event_schedule(adapter);
1847                 }
1848         }
1849 }
1850
1851 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1852 {
1853         struct ixgbe_hw *hw = &adapter->hw;
1854
1855         adapter->lsc_int++;
1856         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1857         adapter->link_check_timeout = jiffies;
1858         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1859                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1860                 IXGBE_WRITE_FLUSH(hw);
1861                 ixgbe_service_event_schedule(adapter);
1862         }
1863 }
1864
1865 static irqreturn_t ixgbe_msix_lsc(int irq, void *data)
1866 {
1867         struct ixgbe_adapter *adapter = data;
1868         struct ixgbe_hw *hw = &adapter->hw;
1869         u32 eicr;
1870
1871         /*
1872          * Workaround for Silicon errata.  Use clear-by-write instead
1873          * of clear-by-read.  Reading with EICS will return the
1874          * interrupt causes without clearing, which later be done
1875          * with the write to EICR.
1876          */
1877         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1878         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1879
1880         if (eicr & IXGBE_EICR_LSC)
1881                 ixgbe_check_lsc(adapter);
1882
1883         if (eicr & IXGBE_EICR_MAILBOX)
1884                 ixgbe_msg_task(adapter);
1885
1886         switch (hw->mac.type) {
1887         case ixgbe_mac_82599EB:
1888         case ixgbe_mac_X540:
1889                 /* Handle Flow Director Full threshold interrupt */
1890                 if (eicr & IXGBE_EICR_FLOW_DIR) {
1891                         int reinit_count = 0;
1892                         int i;
1893                         for (i = 0; i < adapter->num_tx_queues; i++) {
1894                                 struct ixgbe_ring *ring = adapter->tx_ring[i];
1895                                 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1896                                                        &ring->state))
1897                                         reinit_count++;
1898                         }
1899                         if (reinit_count) {
1900                                 /* no more flow director interrupts until after init */
1901                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
1902                                 eicr &= ~IXGBE_EICR_FLOW_DIR;
1903                                 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
1904                                 ixgbe_service_event_schedule(adapter);
1905                         }
1906                 }
1907                 ixgbe_check_sfp_event(adapter, eicr);
1908                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1909                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
1910                         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1911                                 adapter->interrupt_event = eicr;
1912                                 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1913                                 ixgbe_service_event_schedule(adapter);
1914                         }
1915                 }
1916                 break;
1917         default:
1918                 break;
1919         }
1920
1921         ixgbe_check_fan_failure(adapter, eicr);
1922
1923         /* re-enable the original interrupt state, no lsc, no queues */
1924         if (!test_bit(__IXGBE_DOWN, &adapter->state))
1925                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, eicr &
1926                                 ~(IXGBE_EIMS_LSC | IXGBE_EIMS_RTX_QUEUE));
1927
1928         return IRQ_HANDLED;
1929 }
1930
1931 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
1932                                            u64 qmask)
1933 {
1934         u32 mask;
1935         struct ixgbe_hw *hw = &adapter->hw;
1936
1937         switch (hw->mac.type) {
1938         case ixgbe_mac_82598EB:
1939                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1940                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1941                 break;
1942         case ixgbe_mac_82599EB:
1943         case ixgbe_mac_X540:
1944                 mask = (qmask & 0xFFFFFFFF);
1945                 if (mask)
1946                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1947                 mask = (qmask >> 32);
1948                 if (mask)
1949                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1950                 break;
1951         default:
1952                 break;
1953         }
1954         /* skip the flush */
1955 }
1956
1957 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1958                                             u64 qmask)
1959 {
1960         u32 mask;
1961         struct ixgbe_hw *hw = &adapter->hw;
1962
1963         switch (hw->mac.type) {
1964         case ixgbe_mac_82598EB:
1965                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1966                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1967                 break;
1968         case ixgbe_mac_82599EB:
1969         case ixgbe_mac_X540:
1970                 mask = (qmask & 0xFFFFFFFF);
1971                 if (mask)
1972                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1973                 mask = (qmask >> 32);
1974                 if (mask)
1975                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1976                 break;
1977         default:
1978                 break;
1979         }
1980         /* skip the flush */
1981 }
1982
1983 static irqreturn_t ixgbe_msix_clean_tx(int irq, void *data)
1984 {
1985         struct ixgbe_q_vector *q_vector = data;
1986         struct ixgbe_adapter  *adapter = q_vector->adapter;
1987         struct ixgbe_ring     *tx_ring;
1988         int i, r_idx;
1989
1990         if (!q_vector->tx.count)
1991                 return IRQ_HANDLED;
1992
1993         r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
1994         for (i = 0; i < q_vector->tx.count; i++) {
1995                 tx_ring = adapter->tx_ring[r_idx];
1996                 r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
1997                                       r_idx + 1);
1998         }
1999
2000         /* EIAM disabled interrupts (on this vector) for us */
2001         napi_schedule(&q_vector->napi);
2002
2003         return IRQ_HANDLED;
2004 }
2005
2006 /**
2007  * ixgbe_msix_clean_rx - single unshared vector rx clean (all queues)
2008  * @irq: unused
2009  * @data: pointer to our q_vector struct for this interrupt vector
2010  **/
2011 static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data)
2012 {
2013         struct ixgbe_q_vector *q_vector = data;
2014         struct ixgbe_adapter  *adapter = q_vector->adapter;
2015         struct ixgbe_ring  *rx_ring;
2016         int r_idx;
2017         int i;
2018
2019 #ifdef CONFIG_IXGBE_DCA
2020         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2021                 ixgbe_update_dca(q_vector);
2022 #endif
2023
2024         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2025         for (i = 0; i < q_vector->rx.count; i++) {
2026                 rx_ring = adapter->rx_ring[r_idx];
2027                 r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2028                                       r_idx + 1);
2029         }
2030
2031         if (!q_vector->rx.count)
2032                 return IRQ_HANDLED;
2033
2034         /* EIAM disabled interrupts (on this vector) for us */
2035         napi_schedule(&q_vector->napi);
2036
2037         return IRQ_HANDLED;
2038 }
2039
2040 static irqreturn_t ixgbe_msix_clean_many(int irq, void *data)
2041 {
2042         struct ixgbe_q_vector *q_vector = data;
2043         struct ixgbe_adapter  *adapter = q_vector->adapter;
2044         struct ixgbe_ring  *ring;
2045         int r_idx;
2046         int i;
2047
2048         if (!q_vector->tx.count && !q_vector->rx.count)
2049                 return IRQ_HANDLED;
2050
2051         r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
2052         for (i = 0; i < q_vector->tx.count; i++) {
2053                 ring = adapter->tx_ring[r_idx];
2054                 r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
2055                                       r_idx + 1);
2056         }
2057
2058         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2059         for (i = 0; i < q_vector->rx.count; i++) {
2060                 ring = adapter->rx_ring[r_idx];
2061                 r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2062                                       r_idx + 1);
2063         }
2064
2065         /* EIAM disabled interrupts (on this vector) for us */
2066         napi_schedule(&q_vector->napi);
2067
2068         return IRQ_HANDLED;
2069 }
2070
2071 /**
2072  * ixgbe_clean_rxonly - msix (aka one shot) rx clean routine
2073  * @napi: napi struct with our devices info in it
2074  * @budget: amount of work driver is allowed to do this pass, in packets
2075  *
2076  * This function is optimized for cleaning one queue only on a single
2077  * q_vector!!!
2078  **/
2079 static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget)
2080 {
2081         struct ixgbe_q_vector *q_vector =
2082                                container_of(napi, struct ixgbe_q_vector, napi);
2083         struct ixgbe_adapter *adapter = q_vector->adapter;
2084         struct ixgbe_ring *rx_ring = NULL;
2085         int work_done = 0;
2086         long r_idx;
2087
2088 #ifdef CONFIG_IXGBE_DCA
2089         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2090                 ixgbe_update_dca(q_vector);
2091 #endif
2092
2093         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2094         rx_ring = adapter->rx_ring[r_idx];
2095
2096         ixgbe_clean_rx_irq(q_vector, rx_ring, &work_done, budget);
2097
2098         /* If all Rx work done, exit the polling mode */
2099         if (work_done < budget) {
2100                 napi_complete(napi);
2101                 if (adapter->rx_itr_setting & 1)
2102                         ixgbe_set_itr(q_vector);
2103                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2104                         ixgbe_irq_enable_queues(adapter,
2105                                                 ((u64)1 << q_vector->v_idx));
2106         }
2107
2108         return work_done;
2109 }
2110
2111 /**
2112  * ixgbe_clean_rxtx_many - msix (aka one shot) rx clean routine
2113  * @napi: napi struct with our devices info in it
2114  * @budget: amount of work driver is allowed to do this pass, in packets
2115  *
2116  * This function will clean more than one rx queue associated with a
2117  * q_vector.
2118  **/
2119 static int ixgbe_clean_rxtx_many(struct napi_struct *napi, int budget)
2120 {
2121         struct ixgbe_q_vector *q_vector =
2122                                container_of(napi, struct ixgbe_q_vector, napi);
2123         struct ixgbe_adapter *adapter = q_vector->adapter;
2124         struct ixgbe_ring *ring = NULL;
2125         int work_done = 0, i;
2126         long r_idx;
2127         bool tx_clean_complete = true;
2128
2129 #ifdef CONFIG_IXGBE_DCA
2130         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2131                 ixgbe_update_dca(q_vector);
2132 #endif
2133
2134         r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
2135         for (i = 0; i < q_vector->tx.count; i++) {
2136                 ring = adapter->tx_ring[r_idx];
2137                 tx_clean_complete &= ixgbe_clean_tx_irq(q_vector, ring);
2138                 r_idx = find_next_bit(q_vector->tx.idx, adapter->num_tx_queues,
2139                                       r_idx + 1);
2140         }
2141
2142         /* attempt to distribute budget to each queue fairly, but don't allow
2143          * the budget to go below 1 because we'll exit polling */
2144         budget /= (q_vector->rx.count ?: 1);
2145         budget = max(budget, 1);
2146         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2147         for (i = 0; i < q_vector->rx.count; i++) {
2148                 ring = adapter->rx_ring[r_idx];
2149                 ixgbe_clean_rx_irq(q_vector, ring, &work_done, budget);
2150                 r_idx = find_next_bit(q_vector->rx.idx, adapter->num_rx_queues,
2151                                       r_idx + 1);
2152         }
2153
2154         r_idx = find_first_bit(q_vector->rx.idx, adapter->num_rx_queues);
2155         ring = adapter->rx_ring[r_idx];
2156         /* If all Rx work done, exit the polling mode */
2157         if (work_done < budget) {
2158                 napi_complete(napi);
2159                 if (adapter->rx_itr_setting & 1)
2160                         ixgbe_set_itr(q_vector);
2161                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2162                         ixgbe_irq_enable_queues(adapter,
2163                                                 ((u64)1 << q_vector->v_idx));
2164                 return 0;
2165         }
2166
2167         return work_done;
2168 }
2169
2170 /**
2171  * ixgbe_clean_txonly - msix (aka one shot) tx clean routine
2172  * @napi: napi struct with our devices info in it
2173  * @budget: amount of work driver is allowed to do this pass, in packets
2174  *
2175  * This function is optimized for cleaning one queue only on a single
2176  * q_vector!!!
2177  **/
2178 static int ixgbe_clean_txonly(struct napi_struct *napi, int budget)
2179 {
2180         struct ixgbe_q_vector *q_vector =
2181                                container_of(napi, struct ixgbe_q_vector, napi);
2182         struct ixgbe_adapter *adapter = q_vector->adapter;
2183         struct ixgbe_ring *tx_ring = NULL;
2184         int work_done = 0;
2185         long r_idx;
2186
2187 #ifdef CONFIG_IXGBE_DCA
2188         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
2189                 ixgbe_update_dca(q_vector);
2190 #endif
2191
2192         r_idx = find_first_bit(q_vector->tx.idx, adapter->num_tx_queues);
2193         tx_ring = adapter->tx_ring[r_idx];
2194
2195         if (!ixgbe_clean_tx_irq(q_vector, tx_ring))
2196                 work_done = budget;
2197
2198         /* If all Tx work done, exit the polling mode */
2199         if (work_done < budget) {
2200                 napi_complete(napi);
2201                 if (adapter->tx_itr_setting & 1)
2202                         ixgbe_set_itr(q_vector);
2203                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2204                         ixgbe_irq_enable_queues(adapter,
2205                                                 ((u64)1 << q_vector->v_idx));
2206         }
2207
2208         return work_done;
2209 }
2210
2211 static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
2212                                      int r_idx)
2213 {
2214         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2215         struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
2216
2217         set_bit(r_idx, q_vector->rx.idx);
2218         q_vector->rx.count++;
2219         rx_ring->q_vector = q_vector;
2220 }
2221
2222 static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
2223                                      int t_idx)
2224 {
2225         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2226         struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
2227
2228         set_bit(t_idx, q_vector->tx.idx);
2229         q_vector->tx.count++;
2230         tx_ring->q_vector = q_vector;
2231         q_vector->tx.work_limit = a->tx_work_limit;
2232 }
2233
2234 /**
2235  * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
2236  * @adapter: board private structure to initialize
2237  *
2238  * This function maps descriptor rings to the queue-specific vectors
2239  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2240  * one vector per ring/queue, but on a constrained vector budget, we
2241  * group the rings as "efficiently" as possible.  You would add new
2242  * mapping configurations in here.
2243  **/
2244 static int ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2245 {
2246         int q_vectors;
2247         int v_start = 0;
2248         int rxr_idx = 0, txr_idx = 0;
2249         int rxr_remaining = adapter->num_rx_queues;
2250         int txr_remaining = adapter->num_tx_queues;
2251         int i, j;
2252         int rqpv, tqpv;
2253         int err = 0;
2254
2255         /* No mapping required if MSI-X is disabled. */
2256         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2257                 goto out;
2258
2259         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2260
2261         /*
2262          * The ideal configuration...
2263          * We have enough vectors to map one per queue.
2264          */
2265         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
2266                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
2267                         map_vector_to_rxq(adapter, v_start, rxr_idx);
2268
2269                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
2270                         map_vector_to_txq(adapter, v_start, txr_idx);
2271
2272                 goto out;
2273         }
2274
2275         /*
2276          * If we don't have enough vectors for a 1-to-1
2277          * mapping, we'll have to group them so there are
2278          * multiple queues per vector.
2279          */
2280         /* Re-adjusting *qpv takes care of the remainder. */
2281         for (i = v_start; i < q_vectors; i++) {
2282                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
2283                 for (j = 0; j < rqpv; j++) {
2284                         map_vector_to_rxq(adapter, i, rxr_idx);
2285                         rxr_idx++;
2286                         rxr_remaining--;
2287                 }
2288                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
2289                 for (j = 0; j < tqpv; j++) {
2290                         map_vector_to_txq(adapter, i, txr_idx);
2291                         txr_idx++;
2292                         txr_remaining--;
2293                 }
2294         }
2295 out:
2296         return err;
2297 }
2298
2299 /**
2300  * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2301  * @adapter: board private structure
2302  *
2303  * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2304  * interrupts from the kernel.
2305  **/
2306 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2307 {
2308         struct net_device *netdev = adapter->netdev;
2309         irqreturn_t (*handler)(int, void *);
2310         int i, vector, q_vectors, err;
2311         int ri = 0, ti = 0;
2312
2313         /* Decrement for Other and TCP Timer vectors */
2314         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2315
2316         err = ixgbe_map_rings_to_vectors(adapter);
2317         if (err)
2318                 return err;
2319
2320 #define SET_HANDLER(_v) (((_v)->rx.count && (_v)->tx.count)        \
2321                                           ? &ixgbe_msix_clean_many : \
2322                           (_v)->rx.count ? &ixgbe_msix_clean_rx   : \
2323                           (_v)->tx.count ? &ixgbe_msix_clean_tx   : \
2324                           NULL)
2325         for (vector = 0; vector < q_vectors; vector++) {
2326                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2327                 handler = SET_HANDLER(q_vector);
2328
2329                 if (handler == &ixgbe_msix_clean_rx) {
2330                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2331                                  "%s-%s-%d", netdev->name, "rx", ri++);
2332                 } else if (handler == &ixgbe_msix_clean_tx) {
2333                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2334                                  "%s-%s-%d", netdev->name, "tx", ti++);
2335                 } else if (handler == &ixgbe_msix_clean_many) {
2336                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2337                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
2338                         ti++;
2339                 } else {
2340                         /* skip this unused q_vector */
2341                         continue;
2342                 }
2343                 err = request_irq(adapter->msix_entries[vector].vector,
2344                                   handler, 0, q_vector->name,
2345                                   q_vector);
2346                 if (err) {
2347                         e_err(probe, "request_irq failed for MSIX interrupt "
2348                               "Error: %d\n", err);
2349                         goto free_queue_irqs;
2350                 }
2351         }
2352
2353         sprintf(adapter->lsc_int_name, "%s:lsc", netdev->name);
2354         err = request_irq(adapter->msix_entries[vector].vector,
2355                           ixgbe_msix_lsc, 0, adapter->lsc_int_name, adapter);
2356         if (err) {
2357                 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2358                 goto free_queue_irqs;
2359         }
2360
2361         return 0;
2362
2363 free_queue_irqs:
2364         for (i = vector - 1; i >= 0; i--)
2365                 free_irq(adapter->msix_entries[--vector].vector,
2366                          adapter->q_vector[i]);
2367         adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2368         pci_disable_msix(adapter->pdev);
2369         kfree(adapter->msix_entries);
2370         adapter->msix_entries = NULL;
2371         return err;
2372 }
2373
2374 /**
2375  * ixgbe_irq_enable - Enable default interrupt generation settings
2376  * @adapter: board private structure
2377  **/
2378 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
2379                                     bool flush)
2380 {
2381         u32 mask;
2382
2383         mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
2384         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
2385                 mask |= IXGBE_EIMS_GPI_SDP0;
2386         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
2387                 mask |= IXGBE_EIMS_GPI_SDP1;
2388         switch (adapter->hw.mac.type) {
2389         case ixgbe_mac_82599EB:
2390         case ixgbe_mac_X540:
2391                 mask |= IXGBE_EIMS_ECC;
2392                 mask |= IXGBE_EIMS_GPI_SDP1;
2393                 mask |= IXGBE_EIMS_GPI_SDP2;
2394                 if (adapter->num_vfs)
2395                         mask |= IXGBE_EIMS_MAILBOX;
2396                 break;
2397         default:
2398                 break;
2399         }
2400         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
2401                 mask |= IXGBE_EIMS_FLOW_DIR;
2402
2403         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
2404         if (queues)
2405                 ixgbe_irq_enable_queues(adapter, ~0);
2406         if (flush)
2407                 IXGBE_WRITE_FLUSH(&adapter->hw);
2408
2409         if (adapter->num_vfs > 32) {
2410                 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
2411                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
2412         }
2413 }
2414
2415 /**
2416  * ixgbe_intr - legacy mode Interrupt Handler
2417  * @irq: interrupt number
2418  * @data: pointer to a network interface device structure
2419  **/
2420 static irqreturn_t ixgbe_intr(int irq, void *data)
2421 {
2422         struct ixgbe_adapter *adapter = data;
2423         struct ixgbe_hw *hw = &adapter->hw;
2424         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2425         u32 eicr;
2426
2427         /*
2428          * Workaround for silicon errata on 82598.  Mask the interrupts
2429          * before the read of EICR.
2430          */
2431         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2432
2433         /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2434          * therefore no explict interrupt disable is necessary */
2435         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2436         if (!eicr) {
2437                 /*
2438                  * shared interrupt alert!
2439                  * make sure interrupts are enabled because the read will
2440                  * have disabled interrupts due to EIAM
2441                  * finish the workaround of silicon errata on 82598.  Unmask
2442                  * the interrupt that we masked before the EICR read.
2443                  */
2444                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2445                         ixgbe_irq_enable(adapter, true, true);
2446                 return IRQ_NONE;        /* Not our interrupt */
2447         }
2448
2449         if (eicr & IXGBE_EICR_LSC)
2450                 ixgbe_check_lsc(adapter);
2451
2452         switch (hw->mac.type) {
2453         case ixgbe_mac_82599EB:
2454                 ixgbe_check_sfp_event(adapter, eicr);
2455                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2456                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
2457                         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2458                                 adapter->interrupt_event = eicr;
2459                                 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2460                                 ixgbe_service_event_schedule(adapter);
2461                         }
2462                 }
2463                 break;
2464         default:
2465                 break;
2466         }
2467
2468         ixgbe_check_fan_failure(adapter, eicr);
2469
2470         if (napi_schedule_prep(&(q_vector->napi))) {
2471                 /* would disable interrupts here but EIAM disabled it */
2472                 __napi_schedule(&(q_vector->napi));
2473         }
2474
2475         /*
2476          * re-enable link(maybe) and non-queue interrupts, no flush.
2477          * ixgbe_poll will re-enable the queue interrupts
2478          */
2479
2480         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2481                 ixgbe_irq_enable(adapter, false, false);
2482
2483         return IRQ_HANDLED;
2484 }
2485
2486 static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2487 {
2488         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2489
2490         for (i = 0; i < q_vectors; i++) {
2491                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2492                 bitmap_zero(q_vector->rx.idx, MAX_RX_QUEUES);
2493                 bitmap_zero(q_vector->tx.idx, MAX_TX_QUEUES);
2494                 q_vector->rx.count = 0;
2495                 q_vector->tx.count = 0;
2496         }
2497 }
2498
2499 /**
2500  * ixgbe_request_irq - initialize interrupts
2501  * @adapter: board private structure
2502  *
2503  * Attempts to configure interrupts using the best available
2504  * capabilities of the hardware and kernel.
2505  **/
2506 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2507 {
2508         struct net_device *netdev = adapter->netdev;
2509         int err;
2510
2511         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2512                 err = ixgbe_request_msix_irqs(adapter);
2513         } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
2514                 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2515                                   netdev->name, adapter);
2516         } else {
2517                 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2518                                   netdev->name, adapter);
2519         }
2520
2521         if (err)
2522                 e_err(probe, "request_irq failed, Error %d\n", err);
2523
2524         return err;
2525 }
2526
2527 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2528 {
2529         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2530                 int i, q_vectors;
2531
2532                 q_vectors = adapter->num_msix_vectors;
2533
2534                 i = q_vectors - 1;
2535                 free_irq(adapter->msix_entries[i].vector, adapter);
2536
2537                 i--;
2538                 for (; i >= 0; i--) {
2539                         /* free only the irqs that were actually requested */
2540                         if (!adapter->q_vector[i]->rx.count &&
2541                             !adapter->q_vector[i]->tx.count)
2542                                 continue;
2543
2544                         free_irq(adapter->msix_entries[i].vector,
2545                                  adapter->q_vector[i]);
2546                 }
2547
2548                 ixgbe_reset_q_vectors(adapter);
2549         } else {
2550                 free_irq(adapter->pdev->irq, adapter);
2551         }
2552 }
2553
2554 /**
2555  * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2556  * @adapter: board private structure
2557  **/
2558 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2559 {
2560         switch (adapter->hw.mac.type) {
2561         case ixgbe_mac_82598EB:
2562                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2563                 break;
2564         case ixgbe_mac_82599EB:
2565         case ixgbe_mac_X540:
2566                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2567                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2568                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2569                 if (adapter->num_vfs > 32)
2570                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);
2571                 break;
2572         default:
2573                 break;
2574         }
2575         IXGBE_WRITE_FLUSH(&adapter->hw);
2576         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2577                 int i;
2578                 for (i = 0; i < adapter->num_msix_vectors; i++)
2579                         synchronize_irq(adapter->msix_entries[i].vector);
2580         } else {
2581                 synchronize_irq(adapter->pdev->irq);
2582         }
2583 }
2584
2585 /**
2586  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2587  *
2588  **/
2589 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2590 {
2591         struct ixgbe_hw *hw = &adapter->hw;
2592
2593         IXGBE_WRITE_REG(hw, IXGBE_EITR(0),
2594                         EITR_INTS_PER_SEC_TO_REG(adapter->rx_eitr_param));
2595
2596         ixgbe_set_ivar(adapter, 0, 0, 0);
2597         ixgbe_set_ivar(adapter, 1, 0, 0);
2598
2599         map_vector_to_rxq(adapter, 0, 0);
2600         map_vector_to_txq(adapter, 0, 0);
2601
2602         e_info(hw, "Legacy interrupt IVAR setup done\n");
2603 }
2604
2605 /**
2606  * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2607  * @adapter: board private structure
2608  * @ring: structure containing ring specific data
2609  *
2610  * Configure the Tx descriptor ring after a reset.
2611  **/
2612 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2613                              struct ixgbe_ring *ring)
2614 {
2615         struct ixgbe_hw *hw = &adapter->hw;
2616         u64 tdba = ring->dma;
2617         int wait_loop = 10;
2618         u32 txdctl;
2619         u8 reg_idx = ring->reg_idx;
2620
2621         /* disable queue to avoid issues while updating state */
2622         txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2623         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx),
2624                         txdctl & ~IXGBE_TXDCTL_ENABLE);
2625         IXGBE_WRITE_FLUSH(hw);
2626
2627         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2628                         (tdba & DMA_BIT_MASK(32)));
2629         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2630         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2631                         ring->count * sizeof(union ixgbe_adv_tx_desc));
2632         IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2633         IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2634         ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2635
2636         /* configure fetching thresholds */
2637         if (adapter->rx_itr_setting == 0) {
2638                 /* cannot set wthresh when itr==0 */
2639                 txdctl &= ~0x007F0000;
2640         } else {
2641                 /* enable WTHRESH=8 descriptors, to encourage burst writeback */
2642                 txdctl |= (8 << 16);
2643         }
2644         if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
2645                 /* PThresh workaround for Tx hang with DFP enabled. */
2646                 txdctl |= 32;
2647         }
2648
2649         /* reinitialize flowdirector state */
2650         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2651             adapter->atr_sample_rate) {
2652                 ring->atr_sample_rate = adapter->atr_sample_rate;
2653                 ring->atr_count = 0;
2654                 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2655         } else {
2656                 ring->atr_sample_rate = 0;
2657         }
2658
2659         clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2660
2661         /* enable queue */
2662         txdctl |= IXGBE_TXDCTL_ENABLE;
2663         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2664
2665         /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2666         if (hw->mac.type == ixgbe_mac_82598EB &&
2667             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2668                 return;
2669
2670         /* poll to verify queue is enabled */
2671         do {
2672                 usleep_range(1000, 2000);
2673                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2674         } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2675         if (!wait_loop)
2676                 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2677 }
2678
2679 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2680 {
2681         struct ixgbe_hw *hw = &adapter->hw;
2682         u32 rttdcs;
2683         u32 reg;
2684         u8 tcs = netdev_get_num_tc(adapter->netdev);
2685
2686         if (hw->mac.type == ixgbe_mac_82598EB)
2687                 return;
2688
2689         /* disable the arbiter while setting MTQC */
2690         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2691         rttdcs |= IXGBE_RTTDCS_ARBDIS;
2692         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2693
2694         /* set transmit pool layout */
2695         switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2696         case (IXGBE_FLAG_SRIOV_ENABLED):
2697                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2698                                 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2699                 break;
2700         default:
2701                 if (!tcs)
2702                         reg = IXGBE_MTQC_64Q_1PB;
2703                 else if (tcs <= 4)
2704                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2705                 else
2706                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2707
2708                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2709
2710                 /* Enable Security TX Buffer IFG for multiple pb */
2711                 if (tcs) {
2712                         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2713                         reg |= IXGBE_SECTX_DCB;
2714                         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2715                 }
2716                 break;
2717         }
2718
2719         /* re-enable the arbiter */
2720         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2721         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2722 }
2723
2724 /**
2725  * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2726  * @adapter: board private structure
2727  *
2728  * Configure the Tx unit of the MAC after a reset.
2729  **/
2730 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2731 {
2732         struct ixgbe_hw *hw = &adapter->hw;
2733         u32 dmatxctl;
2734         u32 i;
2735
2736         ixgbe_setup_mtqc(adapter);
2737
2738         if (hw->mac.type != ixgbe_mac_82598EB) {
2739                 /* DMATXCTL.EN must be before Tx queues are enabled */
2740                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2741                 dmatxctl |= IXGBE_DMATXCTL_TE;
2742                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2743         }
2744
2745         /* Setup the HW Tx Head and Tail descriptor pointers */
2746         for (i = 0; i < adapter->num_tx_queues; i++)
2747                 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2748 }
2749
2750 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2751
2752 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2753                                    struct ixgbe_ring *rx_ring)
2754 {
2755         u32 srrctl;
2756         u8 reg_idx = rx_ring->reg_idx;
2757
2758         switch (adapter->hw.mac.type) {
2759         case ixgbe_mac_82598EB: {
2760                 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2761                 const int mask = feature[RING_F_RSS].mask;
2762                 reg_idx = reg_idx & mask;
2763         }
2764                 break;
2765         case ixgbe_mac_82599EB:
2766         case ixgbe_mac_X540:
2767         default:
2768                 break;
2769         }
2770
2771         srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2772
2773         srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2774         srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2775         if (adapter->num_vfs)
2776                 srrctl |= IXGBE_SRRCTL_DROP_EN;
2777
2778         srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2779                   IXGBE_SRRCTL_BSIZEHDR_MASK;
2780
2781         if (ring_is_ps_enabled(rx_ring)) {
2782 #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2783                 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2784 #else
2785                 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2786 #endif
2787                 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2788         } else {
2789                 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2790                           IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2791                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2792         }
2793
2794         IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2795 }
2796
2797 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2798 {
2799         struct ixgbe_hw *hw = &adapter->hw;
2800         static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2801                           0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2802                           0x6A3E67EA, 0x14364D17, 0x3BED200D};
2803         u32 mrqc = 0, reta = 0;
2804         u32 rxcsum;
2805         int i, j;
2806         u8 tcs = netdev_get_num_tc(adapter->netdev);
2807         int maxq = adapter->ring_feature[RING_F_RSS].indices;
2808
2809         if (tcs)
2810                 maxq = min(maxq, adapter->num_tx_queues / tcs);
2811
2812         /* Fill out hash function seeds */
2813         for (i = 0; i < 10; i++)
2814                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2815
2816         /* Fill out redirection table */
2817         for (i = 0, j = 0; i < 128; i++, j++) {
2818                 if (j == maxq)
2819                         j = 0;
2820                 /* reta = 4-byte sliding window of
2821                  * 0x00..(indices-1)(indices-1)00..etc. */
2822                 reta = (reta << 8) | (j * 0x11);
2823                 if ((i & 3) == 3)
2824                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2825         }
2826
2827         /* Disable indicating checksum in descriptor, enables RSS hash */
2828         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2829         rxcsum |= IXGBE_RXCSUM_PCSD;
2830         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2831
2832         if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
2833             (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
2834                 mrqc = IXGBE_MRQC_RSSEN;
2835         } else {
2836                 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2837                                              | IXGBE_FLAG_SRIOV_ENABLED);
2838
2839                 switch (mask) {
2840                 case (IXGBE_FLAG_RSS_ENABLED):
2841                         if (!tcs)
2842                                 mrqc = IXGBE_MRQC_RSSEN;
2843                         else if (tcs <= 4)
2844                                 mrqc = IXGBE_MRQC_RTRSS4TCEN;
2845                         else
2846                                 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2847                         break;
2848                 case (IXGBE_FLAG_SRIOV_ENABLED):
2849                         mrqc = IXGBE_MRQC_VMDQEN;
2850                         break;
2851                 default:
2852                         break;
2853                 }
2854         }
2855
2856         /* Perform hash on these packet types */
2857         mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2858               | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2859               | IXGBE_MRQC_RSS_FIELD_IPV6
2860               | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2861
2862         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2863 }
2864
2865 /**
2866  * ixgbe_configure_rscctl - enable RSC for the indicated ring
2867  * @adapter:    address of board private structure
2868  * @index:      index of ring to set
2869  **/
2870 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2871                                    struct ixgbe_ring *ring)
2872 {
2873         struct ixgbe_hw *hw = &adapter->hw;
2874         u32 rscctrl;
2875         int rx_buf_len;
2876         u8 reg_idx = ring->reg_idx;
2877
2878         if (!ring_is_rsc_enabled(ring))
2879                 return;
2880
2881         rx_buf_len = ring->rx_buf_len;
2882         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2883         rscctrl |= IXGBE_RSCCTL_RSCEN;
2884         /*
2885          * we must limit the number of descriptors so that the
2886          * total size of max desc * buf_len is not greater
2887          * than 65535
2888          */
2889         if (ring_is_ps_enabled(ring)) {
2890 #if (MAX_SKB_FRAGS > 16)
2891                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2892 #elif (MAX_SKB_FRAGS > 8)
2893                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2894 #elif (MAX_SKB_FRAGS > 4)
2895                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2896 #else
2897                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2898 #endif
2899         } else {
2900                 if (rx_buf_len < IXGBE_RXBUFFER_4096)
2901                         rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2902                 else if (rx_buf_len < IXGBE_RXBUFFER_8192)
2903                         rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2904                 else
2905                         rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2906         }
2907         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2908 }
2909
2910 /**
2911  *  ixgbe_set_uta - Set unicast filter table address
2912  *  @adapter: board private structure
2913  *
2914  *  The unicast table address is a register array of 32-bit registers.
2915  *  The table is meant to be used in a way similar to how the MTA is used
2916  *  however due to certain limitations in the hardware it is necessary to
2917  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2918  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
2919  **/
2920 static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2921 {
2922         struct ixgbe_hw *hw = &adapter->hw;
2923         int i;
2924
2925         /* The UTA table only exists on 82599 hardware and newer */
2926         if (hw->mac.type < ixgbe_mac_82599EB)
2927                 return;
2928
2929         /* we only need to do this if VMDq is enabled */
2930         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2931                 return;
2932
2933         for (i = 0; i < 128; i++)
2934                 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2935 }
2936
2937 #define IXGBE_MAX_RX_DESC_POLL 10
2938 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
2939                                        struct ixgbe_ring *ring)
2940 {
2941         struct ixgbe_hw *hw = &adapter->hw;
2942         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2943         u32 rxdctl;
2944         u8 reg_idx = ring->reg_idx;
2945
2946         /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2947         if (hw->mac.type == ixgbe_mac_82598EB &&
2948             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2949                 return;
2950
2951         do {
2952                 usleep_range(1000, 2000);
2953                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2954         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
2955
2956         if (!wait_loop) {
2957                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
2958                       "the polling period\n", reg_idx);
2959         }
2960 }
2961
2962 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
2963                             struct ixgbe_ring *ring)
2964 {
2965         struct ixgbe_hw *hw = &adapter->hw;
2966         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2967         u32 rxdctl;
2968         u8 reg_idx = ring->reg_idx;
2969
2970         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2971         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
2972
2973         /* write value back with RXDCTL.ENABLE bit cleared */
2974         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2975
2976         if (hw->mac.type == ixgbe_mac_82598EB &&
2977             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2978                 return;
2979
2980         /* the hardware may take up to 100us to really disable the rx queue */
2981         do {
2982                 udelay(10);
2983                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2984         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
2985
2986         if (!wait_loop) {
2987                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
2988                       "the polling period\n", reg_idx);
2989         }
2990 }
2991
2992 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
2993                              struct ixgbe_ring *ring)
2994 {
2995         struct ixgbe_hw *hw = &adapter->hw;
2996         u64 rdba = ring->dma;
2997         u32 rxdctl;
2998         u8 reg_idx = ring->reg_idx;
2999
3000         /* disable queue to avoid issues while updating state */
3001         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
3002         ixgbe_disable_rx_queue(adapter, ring);
3003
3004         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
3005         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
3006         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
3007                         ring->count * sizeof(union ixgbe_adv_rx_desc));
3008         IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
3009         IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
3010         ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
3011
3012         ixgbe_configure_srrctl(adapter, ring);
3013         ixgbe_configure_rscctl(adapter, ring);
3014
3015         /* If operating in IOV mode set RLPML for X540 */
3016         if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
3017             hw->mac.type == ixgbe_mac_X540) {
3018                 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
3019                 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
3020                             ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
3021         }
3022
3023         if (hw->mac.type == ixgbe_mac_82598EB) {
3024                 /*
3025                  * enable cache line friendly hardware writes:
3026                  * PTHRESH=32 descriptors (half the internal cache),
3027                  * this also removes ugly rx_no_buffer_count increment
3028                  * HTHRESH=4 descriptors (to minimize latency on fetch)
3029                  * WTHRESH=8 burst writeback up to two cache lines
3030                  */
3031                 rxdctl &= ~0x3FFFFF;
3032                 rxdctl |=  0x080420;
3033         }
3034
3035         /* enable receive descriptor ring */
3036         rxdctl |= IXGBE_RXDCTL_ENABLE;
3037         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
3038
3039         ixgbe_rx_desc_queue_enable(adapter, ring);
3040         ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
3041 }
3042
3043 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
3044 {
3045         struct ixgbe_hw *hw = &adapter->hw;
3046         int p;
3047
3048         /* PSRTYPE must be initialized in non 82598 adapters */
3049         u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
3050                       IXGBE_PSRTYPE_UDPHDR |
3051                       IXGBE_PSRTYPE_IPV4HDR |
3052                       IXGBE_PSRTYPE_L2HDR |
3053                       IXGBE_PSRTYPE_IPV6HDR;
3054
3055         if (hw->mac.type == ixgbe_mac_82598EB)
3056                 return;
3057
3058         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
3059                 psrtype |= (adapter->num_rx_queues_per_pool << 29);
3060
3061         for (p = 0; p < adapter->num_rx_pools; p++)
3062                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
3063                                 psrtype);
3064 }
3065
3066 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
3067 {
3068         struct ixgbe_hw *hw = &adapter->hw;
3069         u32 gcr_ext;
3070         u32 vt_reg_bits;
3071         u32 reg_offset, vf_shift;
3072         u32 vmdctl;
3073
3074         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
3075                 return;
3076
3077         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
3078         vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
3079         vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
3080         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
3081
3082         vf_shift = adapter->num_vfs % 32;
3083         reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
3084
3085         /* Enable only the PF's pool for Tx/Rx */
3086         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
3087         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
3088         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
3089         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
3090         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3091
3092         /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
3093         hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
3094
3095         /*
3096          * Set up VF register offsets for selected VT Mode,
3097          * i.e. 32 or 64 VFs for SR-IOV
3098          */
3099         gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
3100         gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
3101         gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
3102         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
3103
3104         /* enable Tx loopback for VF/PF communication */
3105         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
3106         /* Enable MAC Anti-Spoofing */
3107         hw->mac.ops.set_mac_anti_spoofing(hw,
3108                                           (adapter->antispoofing_enabled =
3109                                            (adapter->num_vfs != 0)),
3110                                           adapter->num_vfs);
3111 }
3112
3113 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
3114 {
3115         struct ixgbe_hw *hw = &adapter->hw;
3116         struct net_device *netdev = adapter->netdev;
3117         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3118         int rx_buf_len;
3119         struct ixgbe_ring *rx_ring;
3120         int i;
3121         u32 mhadd, hlreg0;
3122
3123         /* Decide whether to use packet split mode or not */
3124         /* On by default */
3125         adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
3126
3127         /* Do not use packet split if we're in SR-IOV Mode */
3128         if (adapter->num_vfs)
3129                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3130
3131         /* Disable packet split due to 82599 erratum #45 */
3132         if (hw->mac.type == ixgbe_mac_82599EB)
3133                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
3134
3135         /* Set the RX buffer length according to the mode */
3136         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
3137                 rx_buf_len = IXGBE_RX_HDR_SIZE;
3138         } else {
3139                 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
3140                     (netdev->mtu <= ETH_DATA_LEN))
3141                         rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3142                 else
3143                         rx_buf_len = ALIGN(max_frame + VLAN_HLEN, 1024);
3144         }
3145
3146 #ifdef IXGBE_FCOE
3147         /* adjust max frame to be able to do baby jumbo for FCoE */
3148         if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
3149             (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
3150                 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
3151
3152 #endif /* IXGBE_FCOE */
3153         mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
3154         if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
3155                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
3156                 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
3157
3158                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
3159         }
3160
3161         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
3162         /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
3163         hlreg0 |= IXGBE_HLREG0_JUMBOEN;
3164         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
3165
3166         /*
3167          * Setup the HW Rx Head and Tail Descriptor Pointers and
3168          * the Base and Length of the Rx Descriptor Ring
3169          */
3170         for (i = 0; i < adapter->num_rx_queues; i++) {
3171                 rx_ring = adapter->rx_ring[i];
3172                 rx_ring->rx_buf_len = rx_buf_len;
3173
3174                 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
3175                         set_ring_ps_enabled(rx_ring);
3176                 else
3177                         clear_ring_ps_enabled(rx_ring);
3178
3179                 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
3180                         set_ring_rsc_enabled(rx_ring);
3181                 else
3182                         clear_ring_rsc_enabled(rx_ring);
3183
3184 #ifdef IXGBE_FCOE
3185                 if (netdev->features & NETIF_F_FCOE_MTU) {
3186                         struct ixgbe_ring_feature *f;
3187                         f = &adapter->ring_feature[RING_F_FCOE];
3188                         if ((i >= f->mask) && (i < f->mask + f->indices)) {
3189                                 clear_ring_ps_enabled(rx_ring);
3190                                 if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
3191                                         rx_ring->rx_buf_len =
3192                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
3193                         } else if (!ring_is_rsc_enabled(rx_ring) &&
3194                                    !ring_is_ps_enabled(rx_ring)) {
3195                                 rx_ring->rx_buf_len =
3196                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
3197                         }
3198                 }
3199 #endif /* IXGBE_FCOE */
3200         }
3201 }
3202
3203 static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)
3204 {
3205         struct ixgbe_hw *hw = &adapter->hw;
3206         u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
3207
3208         switch (hw->mac.type) {
3209         case ixgbe_mac_82598EB:
3210                 /*
3211                  * For VMDq support of different descriptor types or
3212                  * buffer sizes through the use of multiple SRRCTL
3213                  * registers, RDRXCTL.MVMEN must be set to 1
3214                  *
3215                  * also, the manual doesn't mention it clearly but DCA hints
3216                  * will only use queue 0's tags unless this bit is set.  Side
3217                  * effects of setting this bit are only that SRRCTL must be
3218                  * fully programmed [0..15]
3219                  */
3220                 rdrxctl |= IXGBE_RDRXCTL_MVMEN;
3221                 break;
3222         case ixgbe_mac_82599EB:
3223         case ixgbe_mac_X540:
3224                 /* Disable RSC for ACK packets */
3225                 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,
3226                    (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));
3227                 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
3228                 /* hardware requires some bits to be set by default */
3229                 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);
3230                 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;
3231                 break;
3232         default:
3233                 /* We should do nothing since we don't know this hardware */
3234                 return;
3235         }
3236
3237         IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
3238 }
3239
3240 /**
3241  * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset
3242  * @adapter: board private structure
3243  *
3244  * Configure the Rx unit of the MAC after a reset.
3245  **/
3246 static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)
3247 {
3248         struct ixgbe_hw *hw = &adapter->hw;
3249         int i;
3250         u32 rxctrl;
3251
3252         /* disable receives while setting up the descriptors */
3253         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
3254         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
3255
3256         ixgbe_setup_psrtype(adapter);
3257         ixgbe_setup_rdrxctl(adapter);
3258
3259         /* Program registers for the distribution of queues */
3260         ixgbe_setup_mrqc(adapter);
3261
3262         ixgbe_set_uta(adapter);
3263
3264         /* set_rx_buffer_len must be called before ring initialization */
3265         ixgbe_set_rx_buffer_len(adapter);
3266
3267         /*
3268          * Setup the HW Rx Head and Tail Descriptor Pointers and
3269          * the Base and Length of the Rx Descriptor Ring
3270          */
3271         for (i = 0; i < adapter->num_rx_queues; i++)
3272                 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);
3273
3274         /* disable drop enable for 82598 parts */
3275         if (hw->mac.type == ixgbe_mac_82598EB)
3276                 rxctrl |= IXGBE_RXCTRL_DMBYPS;
3277
3278         /* enable all receives */
3279         rxctrl |= IXGBE_RXCTRL_RXEN;
3280         hw->mac.ops.enable_rx_dma(hw, rxctrl);
3281 }
3282
3283 static void ixgbe_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
3284 {
3285         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3286         struct ixgbe_hw *hw = &adapter->hw;
3287         int pool_ndx = adapter->num_vfs;
3288
3289         /* add VID to filter table */
3290         hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, true);
3291         set_bit(vid, adapter->active_vlans);
3292 }
3293
3294 static void ixgbe_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
3295 {
3296         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3297         struct ixgbe_hw *hw = &adapter->hw;
3298         int pool_ndx = adapter->num_vfs;
3299
3300         /* remove VID from filter table */
3301         hw->mac.ops.set_vfta(&adapter->hw, vid, pool_ndx, false);
3302         clear_bit(vid, adapter->active_vlans);
3303 }
3304
3305 /**
3306  * ixgbe_vlan_filter_disable - helper to disable hw vlan filtering
3307  * @adapter: driver data
3308  */
3309 static void ixgbe_vlan_filter_disable(struct ixgbe_adapter *adapter)
3310 {
3311         struct ixgbe_hw *hw = &adapter->hw;
3312         u32 vlnctrl;
3313
3314         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3315         vlnctrl &= ~(IXGBE_VLNCTRL_VFE | IXGBE_VLNCTRL_CFIEN);
3316         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3317 }
3318
3319 /**
3320  * ixgbe_vlan_filter_enable - helper to enable hw vlan filtering
3321  * @adapter: driver data
3322  */
3323 static void ixgbe_vlan_filter_enable(struct ixgbe_adapter *adapter)
3324 {
3325         struct ixgbe_hw *hw = &adapter->hw;
3326         u32 vlnctrl;
3327
3328         vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3329         vlnctrl |= IXGBE_VLNCTRL_VFE;
3330         vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
3331         IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3332 }
3333
3334 /**
3335  * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping
3336  * @adapter: driver data
3337  */
3338 static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)
3339 {
3340         struct ixgbe_hw *hw = &adapter->hw;
3341         u32 vlnctrl;
3342         int i, j;
3343
3344         switch (hw->mac.type) {
3345         case ixgbe_mac_82598EB:
3346                 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3347                 vlnctrl &= ~IXGBE_VLNCTRL_VME;
3348                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3349                 break;
3350         case ixgbe_mac_82599EB:
3351         case ixgbe_mac_X540:
3352                 for (i = 0; i < adapter->num_rx_queues; i++) {
3353                         j = adapter->rx_ring[i]->reg_idx;
3354                         vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3355                         vlnctrl &= ~IXGBE_RXDCTL_VME;
3356                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3357                 }
3358                 break;
3359         default:
3360                 break;
3361         }
3362 }
3363
3364 /**
3365  * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping
3366  * @adapter: driver data
3367  */
3368 static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)
3369 {
3370         struct ixgbe_hw *hw = &adapter->hw;
3371         u32 vlnctrl;
3372         int i, j;
3373
3374         switch (hw->mac.type) {
3375         case ixgbe_mac_82598EB:
3376                 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
3377                 vlnctrl |= IXGBE_VLNCTRL_VME;
3378                 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
3379                 break;
3380         case ixgbe_mac_82599EB:
3381         case ixgbe_mac_X540:
3382                 for (i = 0; i < adapter->num_rx_queues; i++) {
3383                         j = adapter->rx_ring[i]->reg_idx;
3384                         vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));
3385                         vlnctrl |= IXGBE_RXDCTL_VME;
3386                         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);
3387                 }
3388                 break;
3389         default:
3390                 break;
3391         }
3392 }
3393
3394 static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)
3395 {
3396         u16 vid;
3397
3398         ixgbe_vlan_rx_add_vid(adapter->netdev, 0);
3399
3400         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
3401                 ixgbe_vlan_rx_add_vid(adapter->netdev, vid);
3402 }
3403
3404 /**
3405  * ixgbe_write_uc_addr_list - write unicast addresses to RAR table
3406  * @netdev: network interface device structure
3407  *
3408  * Writes unicast address list to the RAR table.
3409  * Returns: -ENOMEM on failure/insufficient address space
3410  *                0 on no addresses written
3411  *                X on writing X addresses to the RAR table
3412  **/
3413 static int ixgbe_write_uc_addr_list(struct net_device *netdev)
3414 {
3415         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3416         struct ixgbe_hw *hw = &adapter->hw;
3417         unsigned int vfn = adapter->num_vfs;
3418         unsigned int rar_entries = IXGBE_MAX_PF_MACVLANS;
3419         int count = 0;
3420
3421         /* return ENOMEM indicating insufficient memory for addresses */
3422         if (netdev_uc_count(netdev) > rar_entries)
3423                 return -ENOMEM;
3424
3425         if (!netdev_uc_empty(netdev) && rar_entries) {
3426                 struct netdev_hw_addr *ha;
3427                 /* return error if we do not support writing to RAR table */
3428                 if (!hw->mac.ops.set_rar)
3429                         return -ENOMEM;
3430
3431                 netdev_for_each_uc_addr(ha, netdev) {
3432                         if (!rar_entries)
3433                                 break;
3434                         hw->mac.ops.set_rar(hw, rar_entries--, ha->addr,
3435                                             vfn, IXGBE_RAH_AV);
3436                         count++;
3437                 }
3438         }
3439         /* write the addresses in reverse order to avoid write combining */
3440         for (; rar_entries > 0 ; rar_entries--)
3441                 hw->mac.ops.clear_rar(hw, rar_entries);
3442
3443         return count;
3444 }
3445
3446 /**
3447  * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set
3448  * @netdev: network interface device structure
3449  *
3450  * The set_rx_method entry point is called whenever the unicast/multicast
3451  * address list or the network interface flags are updated.  This routine is
3452  * responsible for configuring the hardware for proper unicast, multicast and
3453  * promiscuous mode.
3454  **/
3455 void ixgbe_set_rx_mode(struct net_device *netdev)
3456 {
3457         struct ixgbe_adapter *adapter = netdev_priv(netdev);
3458         struct ixgbe_hw *hw = &adapter->hw;
3459         u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;
3460         int count;
3461
3462         /* Check for Promiscuous and All Multicast modes */
3463
3464         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3465
3466         /* set all bits that we expect to always be set */
3467         fctrl |= IXGBE_FCTRL_BAM;
3468         fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */
3469         fctrl |= IXGBE_FCTRL_PMCF;
3470
3471         /* clear the bits we are changing the status of */
3472         fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3473
3474         if (netdev->flags & IFF_PROMISC) {
3475                 hw->addr_ctrl.user_set_promisc = true;
3476                 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3477                 vmolr |= (IXGBE_VMOLR_ROPE | IXGBE_VMOLR_MPE);
3478                 /* don't hardware filter vlans in promisc mode */
3479                 ixgbe_vlan_filter_disable(adapter);
3480         } else {
3481                 if (netdev->flags & IFF_ALLMULTI) {
3482                         fctrl |= IXGBE_FCTRL_MPE;
3483                         vmolr |= IXGBE_VMOLR_MPE;
3484                 } else {
3485                         /*
3486                          * Write addresses to the MTA, if the attempt fails
3487                          * then we should just turn on promiscuous mode so
3488                          * that we can at least receive multicast traffic
3489                          */
3490                         hw->mac.ops.update_mc_addr_list(hw, netdev);
3491                         vmolr |= IXGBE_VMOLR_ROMPE;
3492                 }
3493                 ixgbe_vlan_filter_enable(adapter);
3494                 hw->addr_ctrl.user_set_promisc = false;
3495                 /*
3496                  * Write addresses to available RAR registers, if there is not
3497                  * sufficient space to store all the addresses then enable
3498                  * unicast promiscuous mode
3499                  */
3500                 count = ixgbe_write_uc_addr_list(netdev);
3501                 if (count < 0) {
3502                         fctrl |= IXGBE_FCTRL_UPE;
3503                         vmolr |= IXGBE_VMOLR_ROPE;
3504                 }
3505         }
3506
3507         if (adapter->num_vfs) {
3508                 ixgbe_restore_vf_multicasts(adapter);
3509                 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(adapter->num_vfs)) &
3510                          ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |
3511                            IXGBE_VMOLR_ROPE);
3512                 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(adapter->num_vfs), vmolr);
3513         }
3514
3515         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3516
3517         if (netdev->features & NETIF_F_HW_VLAN_RX)
3518                 ixgbe_vlan_strip_enable(adapter);
3519         else
3520                 ixgbe_vlan_strip_disable(adapter);
3521 }
3522
3523 static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)
3524 {
3525         int q_idx;
3526         struct ixgbe_q_vector *q_vector;
3527         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3528
3529         /* legacy and MSI only use one vector */
3530         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3531                 q_vectors = 1;
3532
3533         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3534                 struct napi_struct *napi;
3535                 q_vector = adapter->q_vector[q_idx];
3536                 napi = &q_vector->napi;
3537                 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3538                         if (!q_vector->rx.count || !q_vector->tx.count) {
3539                                 if (q_vector->tx.count == 1)
3540                                         napi->poll = &ixgbe_clean_txonly;
3541                                 else if (q_vector->rx.count == 1)
3542                                         napi->poll = &ixgbe_clean_rxonly;
3543                         }
3544                 }
3545
3546                 napi_enable(napi);
3547         }
3548 }
3549
3550 static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)
3551 {
3552         int q_idx;
3553         struct ixgbe_q_vector *q_vector;
3554         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
3555
3556         /* legacy and MSI only use one vector */
3557         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
3558                 q_vectors = 1;
3559
3560         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
3561                 q_vector = adapter->q_vector[q_idx];
3562                 napi_disable(&q_vector->napi);
3563         }
3564 }
3565
3566 #ifdef CONFIG_IXGBE_DCB
3567 /*
3568  * ixgbe_configure_dcb - Configure DCB hardware
3569  * @adapter: ixgbe adapter struct
3570  *
3571  * This is called by the driver on open to configure the DCB hardware.
3572  * This is also called by the gennetlink interface when reconfiguring
3573  * the DCB state.
3574  */
3575 static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)
3576 {
3577         struct ixgbe_hw *hw = &adapter->hw;
3578         int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3579
3580         if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {
3581                 if (hw->mac.type == ixgbe_mac_82598EB)
3582                         netif_set_gso_max_size(adapter->netdev, 65536);
3583                 return;
3584         }
3585
3586         if (hw->mac.type == ixgbe_mac_82598EB)
3587                 netif_set_gso_max_size(adapter->netdev, 32768);
3588
3589
3590         /* Enable VLAN tag insert/strip */
3591         adapter->netdev->features |= NETIF_F_HW_VLAN_RX;
3592
3593         hw->mac.ops.set_vfta(&adapter->hw, 0, 0, true);
3594
3595         /* reconfigure the hardware */
3596         if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {
3597 #ifdef CONFIG_FCOE
3598                 if (adapter->netdev->features & NETIF_F_FCOE_MTU)
3599                         max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);
3600 #endif
3601                 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3602                                                 DCB_TX_CONFIG);
3603                 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,
3604                                                 DCB_RX_CONFIG);
3605                 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);
3606         } else {
3607                 struct net_device *dev = adapter->netdev;
3608
3609                 if (adapter->ixgbe_ieee_ets)
3610                         dev->dcbnl_ops->ieee_setets(dev,
3611                                                     adapter->ixgbe_ieee_ets);
3612                 if (adapter->ixgbe_ieee_pfc)
3613                         dev->dcbnl_ops->ieee_setpfc(dev,
3614                                                     adapter->ixgbe_ieee_pfc);
3615         }
3616
3617         /* Enable RSS Hash per TC */
3618         if (hw->mac.type != ixgbe_mac_82598EB) {
3619                 int i;
3620                 u32 reg = 0;
3621
3622                 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
3623                         u8 msb = 0;
3624                         u8 cnt = adapter->netdev->tc_to_txq[i].count;
3625
3626                         while (cnt >>= 1)
3627                                 msb++;
3628
3629                         reg |= msb << IXGBE_RQTC_SHIFT_TC(i);
3630                 }
3631                 IXGBE_WRITE_REG(hw, IXGBE_RQTC, reg);
3632         }
3633 }
3634
3635 #endif
3636
3637 static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)
3638 {
3639         int hdrm = 0;
3640         int num_tc = netdev_get_num_tc(adapter->netdev);
3641         struct ixgbe_hw *hw = &adapter->hw;
3642
3643         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||
3644             adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)
3645                 hdrm = 64 << adapter->fdir_pballoc;
3646
3647         hw->mac.ops.set_rxpba(&adapter->hw, num_tc, hdrm, PBA_STRATEGY_EQUAL);
3648 }
3649
3650 static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)
3651 {
3652         struct ixgbe_hw *hw = &adapter->hw;
3653         struct hlist_node *node, *node2;
3654         struct ixgbe_fdir_filter *filter;
3655
3656         spin_lock(&adapter->fdir_perfect_lock);
3657
3658         if (!hlist_empty(&adapter->fdir_filter_list))
3659                 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);
3660
3661         hlist_for_each_entry_safe(filter, node, node2,
3662                                   &adapter->fdir_filter_list, fdir_node) {
3663                 ixgbe_fdir_write_perfect_filter_82599(hw,
3664                                 &filter->filter,
3665                                 filter->sw_idx,
3666                                 (filter->action == IXGBE_FDIR_DROP_QUEUE) ?
3667                                 IXGBE_FDIR_DROP_QUEUE :
3668                                 adapter->rx_ring[filter->action]->reg_idx);
3669         }
3670
3671         spin_unlock(&adapter->fdir_perfect_lock);
3672 }
3673
3674 static void ixgbe_configure(struct ixgbe_adapter *adapter)
3675 {
3676         struct net_device *netdev = adapter->netdev;
3677         struct ixgbe_hw *hw = &adapter->hw;
3678         int i;
3679
3680         ixgbe_configure_pb(adapter);
3681 #ifdef CONFIG_IXGBE_DCB
3682         ixgbe_configure_dcb(adapter);
3683 #endif
3684
3685         ixgbe_set_rx_mode(netdev);
3686         ixgbe_restore_vlan(adapter);
3687
3688 #ifdef IXGBE_FCOE
3689         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
3690                 ixgbe_configure_fcoe(adapter);
3691
3692 #endif /* IXGBE_FCOE */
3693         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
3694                 for (i = 0; i < adapter->num_tx_queues; i++)
3695                         adapter->tx_ring[i]->atr_sample_rate =
3696                                                        adapter->atr_sample_rate;
3697                 ixgbe_init_fdir_signature_82599(hw, adapter->fdir_pballoc);
3698         } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {
3699                 ixgbe_init_fdir_perfect_82599(&adapter->hw,
3700                                               adapter->fdir_pballoc);
3701                 ixgbe_fdir_filter_restore(adapter);
3702         }
3703         ixgbe_configure_virtualization(adapter);
3704
3705         ixgbe_configure_tx(adapter);
3706         ixgbe_configure_rx(adapter);
3707 }
3708
3709 static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)
3710 {
3711         switch (hw->phy.type) {
3712         case ixgbe_phy_sfp_avago:
3713         case ixgbe_phy_sfp_ftl:
3714         case ixgbe_phy_sfp_intel:
3715         case ixgbe_phy_sfp_unknown:
3716         case ixgbe_phy_sfp_passive_tyco:
3717         case ixgbe_phy_sfp_passive_unknown:
3718         case ixgbe_phy_sfp_active_unknown:
3719         case ixgbe_phy_sfp_ftl_active:
3720                 return true;
3721         default:
3722                 return false;
3723         }
3724 }
3725
3726 /**
3727  * ixgbe_sfp_link_config - set up SFP+ link
3728  * @adapter: pointer to private adapter struct
3729  **/
3730 static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)
3731 {
3732         /*
3733          * We are assuming the worst case scenerio here, and that
3734          * is that an SFP was inserted/removed after the reset
3735          * but before SFP detection was enabled.  As such the best
3736          * solution is to just start searching as soon as we start
3737          */
3738         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
3739                 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
3740
3741         adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
3742 }
3743
3744 /**
3745  * ixgbe_non_sfp_link_config - set up non-SFP+ link
3746  * @hw: pointer to private hardware struct
3747  *
3748  * Returns 0 on success, negative on failure
3749  **/
3750 static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)
3751 {
3752         u32 autoneg;
3753         bool negotiation, link_up = false;
3754         u32 ret = IXGBE_ERR_LINK_SETUP;
3755
3756         if (hw->mac.ops.check_link)
3757                 ret = hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
3758
3759         if (ret)
3760                 goto link_cfg_out;
3761
3762         autoneg = hw->phy.autoneg_advertised;
3763         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
3764                 ret = hw->mac.ops.get_link_capabilities(hw, &autoneg,
3765                                                         &negotiation);
3766         if (ret)
3767                 goto link_cfg_out;
3768
3769         if (hw->mac.ops.setup_link)
3770                 ret = hw->mac.ops.setup_link(hw, autoneg, negotiation, link_up);
3771 link_cfg_out:
3772         return ret;
3773 }
3774
3775 static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)
3776 {
3777         struct ixgbe_hw *hw = &adapter->hw;
3778         u32 gpie = 0;
3779
3780         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
3781                 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
3782                        IXGBE_GPIE_OCD;
3783                 gpie |= IXGBE_GPIE_EIAME;
3784                 /*
3785                  * use EIAM to auto-mask when MSI-X interrupt is asserted
3786                  * this saves a register write for every interrupt
3787                  */
3788                 switch (hw->mac.type) {
3789                 case ixgbe_mac_82598EB:
3790                         IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3791                         break;
3792                 case ixgbe_mac_82599EB:
3793                 case ixgbe_mac_X540:
3794                 default:
3795                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
3796                         IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
3797                         break;
3798                 }
3799         } else {
3800                 /* legacy interrupts, use EIAM to auto-mask when reading EICR,
3801                  * specifically only auto mask tx and rx interrupts */
3802                 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
3803         }
3804
3805         /* XXX: to interrupt immediately for EICS writes, enable this */
3806         /* gpie |= IXGBE_GPIE_EIMEN; */
3807
3808         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
3809                 gpie &= ~IXGBE_GPIE_VTMODE_MASK;
3810                 gpie |= IXGBE_GPIE_VTMODE_64;
3811         }
3812
3813         /* Enable fan failure interrupt */
3814         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
3815                 gpie |= IXGBE_SDP1_GPIEN;
3816
3817         if (hw->mac.type == ixgbe_mac_82599EB) {
3818                 gpie |= IXGBE_SDP1_GPIEN;
3819                 gpie |= IXGBE_SDP2_GPIEN;
3820         }
3821
3822         IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
3823 }
3824
3825 static int ixgbe_up_complete(struct ixgbe_adapter *adapter)
3826 {
3827         struct ixgbe_hw *hw = &adapter->hw;
3828         int err;
3829         u32 ctrl_ext;
3830
3831         ixgbe_get_hw_control(adapter);
3832         ixgbe_setup_gpie(adapter);
3833
3834         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
3835                 ixgbe_configure_msix(adapter);
3836         else
3837                 ixgbe_configure_msi_and_legacy(adapter);
3838
3839         /* enable the optics for both mult-speed fiber and 82599 SFP+ fiber */
3840         if (hw->mac.ops.enable_tx_laser &&
3841             ((hw->phy.multispeed_fiber) ||
3842              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
3843               (hw->mac.type == ixgbe_mac_82599EB))))
3844                 hw->mac.ops.enable_tx_laser(hw);
3845
3846         clear_bit(__IXGBE_DOWN, &adapter->state);
3847         ixgbe_napi_enable_all(adapter);
3848
3849         if (ixgbe_is_sfp(hw)) {
3850                 ixgbe_sfp_link_config(adapter);
3851         } else {
3852                 err = ixgbe_non_sfp_link_config(hw);
3853                 if (err)
3854                         e_err(probe, "link_config FAILED %d\n", err);
3855         }
3856
3857         /* clear any pending interrupts, may auto mask */
3858         IXGBE_READ_REG(hw, IXGBE_EICR);
3859         ixgbe_irq_enable(adapter, true, true);
3860
3861         /*
3862          * If this adapter has a fan, check to see if we had a failure
3863          * before we enabled the interrupt.
3864          */
3865         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
3866                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
3867                 if (esdp & IXGBE_ESDP_SDP1)
3868                         e_crit(drv, "Fan has stopped, replace the adapter\n");
3869         }
3870
3871         /* enable transmits */
3872         netif_tx_start_all_queues(adapter->netdev);
3873
3874         /* bring the link up in the watchdog, this could race with our first
3875          * link up interrupt but shouldn't be a problem */
3876         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3877         adapter->link_check_timeout = jiffies;
3878         mod_timer(&adapter->service_timer, jiffies);
3879
3880         /* Set PF Reset Done bit so PF/VF Mail Ops can work */
3881         ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
3882         ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
3883         IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
3884
3885         return 0;
3886 }
3887
3888 void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)
3889 {
3890         WARN_ON(in_interrupt());
3891         /* put off any impending NetWatchDogTimeout */
3892         adapter->netdev->trans_start = jiffies;
3893
3894         while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))
3895                 usleep_range(1000, 2000);
3896         ixgbe_down(adapter);
3897         /*
3898          * If SR-IOV enabled then wait a bit before bringing the adapter
3899          * back up to give the VFs time to respond to the reset.  The
3900          * two second wait is based upon the watchdog timer cycle in
3901          * the VF driver.
3902          */
3903         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
3904                 msleep(2000);
3905         ixgbe_up(adapter);
3906         clear_bit(__IXGBE_RESETTING, &adapter->state);
3907 }
3908
3909 int ixgbe_up(struct ixgbe_adapter *adapter)
3910 {
3911         /* hardware has been reset, we need to reload some things */
3912         ixgbe_configure(adapter);
3913
3914         return ixgbe_up_complete(adapter);
3915 }
3916
3917 void ixgbe_reset(struct ixgbe_adapter *adapter)
3918 {
3919         struct ixgbe_hw *hw = &adapter->hw;
3920         int err;
3921
3922         /* lock SFP init bit to prevent race conditions with the watchdog */
3923         while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
3924                 usleep_range(1000, 2000);
3925
3926         /* clear all SFP and link config related flags while holding SFP_INIT */
3927         adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |
3928                              IXGBE_FLAG2_SFP_NEEDS_RESET);
3929         adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
3930
3931         err = hw->mac.ops.init_hw(hw);
3932         switch (err) {
3933         case 0:
3934         case IXGBE_ERR_SFP_NOT_PRESENT:
3935         case IXGBE_ERR_SFP_NOT_SUPPORTED:
3936                 break;
3937         case IXGBE_ERR_MASTER_REQUESTS_PENDING:
3938                 e_dev_err("master disable timed out\n");
3939                 break;
3940         case IXGBE_ERR_EEPROM_VERSION:
3941                 /* We are running on a pre-production device, log a warning */
3942                 e_dev_warn("This device is a pre-production adapter/LOM. "
3943                            "Please be aware there may be issuesassociated with "
3944                            "your hardware.  If you are experiencing problems "
3945                            "please contact your Intel or hardware "
3946                            "representative who provided you with this "
3947                            "hardware.\n");
3948                 break;
3949         default:
3950                 e_dev_err("Hardware Error: %d\n", err);
3951         }
3952
3953         clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
3954
3955         /* reprogram the RAR[0] in case user changed it. */
3956         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
3957                             IXGBE_RAH_AV);
3958 }
3959
3960 /**
3961  * ixgbe_clean_rx_ring - Free Rx Buffers per Queue
3962  * @rx_ring: ring to free buffers from
3963  **/
3964 static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)
3965 {
3966         struct device *dev = rx_ring->dev;
3967         unsigned long size;
3968         u16 i;
3969
3970         /* ring already cleared, nothing to do */
3971         if (!rx_ring->rx_buffer_info)
3972                 return;
3973
3974         /* Free all the Rx ring sk_buffs */
3975         for (i = 0; i < rx_ring->count; i++) {
3976                 struct ixgbe_rx_buffer *rx_buffer_info;
3977
3978                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
3979                 if (rx_buffer_info->dma) {
3980                         dma_unmap_single(rx_ring->dev, rx_buffer_info->dma,
3981                                          rx_ring->rx_buf_len,
3982                                          DMA_FROM_DEVICE);
3983                         rx_buffer_info->dma = 0;
3984                 }
3985                 if (rx_buffer_info->skb) {
3986                         struct sk_buff *skb = rx_buffer_info->skb;
3987                         rx_buffer_info->skb = NULL;
3988                         do {
3989                                 struct sk_buff *this = skb;
3990                                 if (IXGBE_RSC_CB(this)->delay_unmap) {
3991                                         dma_unmap_single(dev,
3992                                                          IXGBE_RSC_CB(this)->dma,
3993                                                          rx_ring->rx_buf_len,
3994                                                          DMA_FROM_DEVICE);
3995                                         IXGBE_RSC_CB(this)->dma = 0;
3996                                         IXGBE_RSC_CB(skb)->delay_unmap = false;
3997                                 }
3998                                 skb = skb->prev;
3999                                 dev_kfree_skb(this);
4000                         } while (skb);
4001                 }
4002                 if (!rx_buffer_info->page)
4003                         continue;
4004                 if (rx_buffer_info->page_dma) {
4005                         dma_unmap_page(dev, rx_buffer_info->page_dma,
4006                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
4007                         rx_buffer_info->page_dma = 0;
4008                 }
4009                 put_page(rx_buffer_info->page);
4010                 rx_buffer_info->page = NULL;
4011                 rx_buffer_info->page_offset = 0;
4012         }
4013
4014         size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
4015         memset(rx_ring->rx_buffer_info, 0, size);
4016
4017         /* Zero out the descriptor ring */
4018         memset(rx_ring->desc, 0, rx_ring->size);
4019
4020         rx_ring->next_to_clean = 0;
4021         rx_ring->next_to_use = 0;
4022 }
4023
4024 /**
4025  * ixgbe_clean_tx_ring - Free Tx Buffers
4026  * @tx_ring: ring to be cleaned
4027  **/
4028 static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)
4029 {
4030         struct ixgbe_tx_buffer *tx_buffer_info;
4031         unsigned long size;
4032         u16 i;
4033
4034         /* ring already cleared, nothing to do */
4035         if (!tx_ring->tx_buffer_info)
4036                 return;
4037
4038         /* Free all the Tx ring sk_buffs */
4039         for (i = 0; i < tx_ring->count; i++) {
4040                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4041                 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4042         }
4043
4044         size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
4045         memset(tx_ring->tx_buffer_info, 0, size);
4046
4047         /* Zero out the descriptor ring */
4048         memset(tx_ring->desc, 0, tx_ring->size);
4049
4050         tx_ring->next_to_use = 0;
4051         tx_ring->next_to_clean = 0;
4052 }
4053
4054 /**
4055  * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues
4056  * @adapter: board private structure
4057  **/
4058 static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)
4059 {
4060         int i;
4061
4062         for (i = 0; i < adapter->num_rx_queues; i++)
4063                 ixgbe_clean_rx_ring(adapter->rx_ring[i]);
4064 }
4065
4066 /**
4067  * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues
4068  * @adapter: board private structure
4069  **/
4070 static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)
4071 {
4072         int i;
4073
4074         for (i = 0; i < adapter->num_tx_queues; i++)
4075                 ixgbe_clean_tx_ring(adapter->tx_ring[i]);
4076 }
4077
4078 static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)
4079 {
4080         struct hlist_node *node, *node2;
4081         struct ixgbe_fdir_filter *filter;
4082
4083         spin_lock(&adapter->fdir_perfect_lock);
4084
4085         hlist_for_each_entry_safe(filter, node, node2,
4086                                   &adapter->fdir_filter_list, fdir_node) {
4087                 hlist_del(&filter->fdir_node);
4088                 kfree(filter);
4089         }
4090         adapter->fdir_filter_count = 0;
4091
4092         spin_unlock(&adapter->fdir_perfect_lock);
4093 }
4094
4095 void ixgbe_down(struct ixgbe_adapter *adapter)
4096 {
4097         struct net_device *netdev = adapter->netdev;
4098         struct ixgbe_hw *hw = &adapter->hw;
4099         u32 rxctrl;
4100         int i;
4101         int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4102
4103         /* signal that we are down to the interrupt handler */
4104         set_bit(__IXGBE_DOWN, &adapter->state);
4105
4106         /* disable receives */
4107         rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);
4108         IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl & ~IXGBE_RXCTRL_RXEN);
4109
4110         /* disable all enabled rx queues */
4111         for (i = 0; i < adapter->num_rx_queues; i++)
4112                 /* this call also flushes the previous write */
4113                 ixgbe_disable_rx_queue(adapter, adapter->rx_ring[i]);
4114
4115         usleep_range(10000, 20000);
4116
4117         netif_tx_stop_all_queues(netdev);
4118
4119         /* call carrier off first to avoid false dev_watchdog timeouts */
4120         netif_carrier_off(netdev);
4121         netif_tx_disable(netdev);
4122
4123         ixgbe_irq_disable(adapter);
4124
4125         ixgbe_napi_disable_all(adapter);
4126
4127         adapter->flags2 &= ~(IXGBE_FLAG2_FDIR_REQUIRES_REINIT |
4128                              IXGBE_FLAG2_RESET_REQUESTED);
4129         adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
4130
4131         del_timer_sync(&adapter->service_timer);
4132
4133         /* disable receive for all VFs and wait one second */
4134         if (adapter->num_vfs) {
4135                 /* ping all the active vfs to let them know we are going down */
4136                 ixgbe_ping_all_vfs(adapter);
4137
4138                 /* Disable all VFTE/VFRE TX/RX */
4139                 ixgbe_disable_tx_rx(adapter);
4140
4141                 /* Mark all the VFs as inactive */
4142                 for (i = 0 ; i < adapter->num_vfs; i++)
4143                         adapter->vfinfo[i].clear_to_send = 0;
4144         }
4145
4146         /* Cleanup the affinity_hint CPU mask memory and callback */
4147         for (i = 0; i < num_q_vectors; i++) {
4148                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
4149                 /* clear the affinity_mask in the IRQ descriptor */
4150                 irq_set_affinity_hint(adapter->msix_entries[i]. vector, NULL);
4151                 /* release the CPU mask memory */
4152                 free_cpumask_var(q_vector->affinity_mask);
4153         }
4154
4155         /* disable transmits in the hardware now that interrupts are off */
4156         for (i = 0; i < adapter->num_tx_queues; i++) {
4157                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
4158                 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
4159         }
4160
4161         /* Disable the Tx DMA engine on 82599 and X540 */
4162         switch (hw->mac.type) {
4163         case ixgbe_mac_82599EB:
4164         case ixgbe_mac_X540:
4165                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,
4166                                 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &
4167                                  ~IXGBE_DMATXCTL_TE));
4168                 break;
4169         default:
4170                 break;
4171         }
4172
4173         if (!pci_channel_offline(adapter->pdev))
4174                 ixgbe_reset(adapter);
4175
4176         /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
4177         if (hw->mac.ops.disable_tx_laser &&
4178             ((hw->phy.multispeed_fiber) ||
4179              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
4180               (hw->mac.type == ixgbe_mac_82599EB))))
4181                 hw->mac.ops.disable_tx_laser(hw);
4182
4183         ixgbe_clean_all_tx_rings(adapter);
4184         ixgbe_clean_all_rx_rings(adapter);
4185
4186 #ifdef CONFIG_IXGBE_DCA
4187         /* since we reset the hardware DCA settings were cleared */
4188         ixgbe_setup_dca(adapter);
4189 #endif
4190 }
4191
4192 /**
4193  * ixgbe_poll - NAPI Rx polling callback
4194  * @napi: structure for representing this polling device
4195  * @budget: how many packets driver is allowed to clean
4196  *
4197  * This function is used for legacy and MSI, NAPI mode
4198  **/
4199 static int ixgbe_poll(struct napi_struct *napi, int budget)
4200 {
4201         struct ixgbe_q_vector *q_vector =
4202                                 container_of(napi, struct ixgbe_q_vector, napi);
4203         struct ixgbe_adapter *adapter = q_vector->adapter;
4204         int tx_clean_complete, work_done = 0;
4205
4206 #ifdef CONFIG_IXGBE_DCA
4207         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
4208                 ixgbe_update_dca(q_vector);
4209 #endif
4210
4211         tx_clean_complete = ixgbe_clean_tx_irq(q_vector, adapter->tx_ring[0]);
4212         ixgbe_clean_rx_irq(q_vector, adapter->rx_ring[0], &work_done, budget);
4213
4214         if (!tx_clean_complete)
4215                 work_done = budget;
4216
4217         /* If budget not fully consumed, exit the polling mode */
4218         if (work_done < budget) {
4219                 napi_complete(napi);
4220                 if (adapter->rx_itr_setting & 1)
4221                         ixgbe_set_itr(q_vector);
4222                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
4223                         ixgbe_irq_enable_queues(adapter, IXGBE_EIMS_RTX_QUEUE);
4224         }
4225         return work_done;
4226 }
4227
4228 /**
4229  * ixgbe_tx_timeout - Respond to a Tx Hang
4230  * @netdev: network interface device structure
4231  **/
4232 static void ixgbe_tx_timeout(struct net_device *netdev)
4233 {
4234         struct ixgbe_adapter *adapter = netdev_priv(netdev);
4235
4236         /* Do the reset outside of interrupt context */
4237         ixgbe_tx_timeout_reset(adapter);
4238 }
4239
4240 /**
4241  * ixgbe_set_rss_queues: Allocate queues for RSS
4242  * @adapter: board private structure to initialize
4243  *
4244  * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
4245  * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
4246  *
4247  **/
4248 static inline bool ixgbe_set_rss_queues(struct ixgbe_adapter *adapter)
4249 {
4250         bool ret = false;
4251         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_RSS];
4252
4253         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4254                 f->mask = 0xF;
4255                 adapter->num_rx_queues = f->indices;
4256                 adapter->num_tx_queues = f->indices;
4257                 ret = true;
4258         } else {
4259                 ret = false;
4260         }
4261
4262         return ret;
4263 }
4264
4265 /**
4266  * ixgbe_set_fdir_queues: Allocate queues for Flow Director
4267  * @adapter: board private structure to initialize
4268  *
4269  * Flow Director is an advanced Rx filter, attempting to get Rx flows back
4270  * to the original CPU that initiated the Tx session.  This runs in addition
4271  * to RSS, so if a packet doesn't match an FDIR filter, we can still spread the
4272  * Rx load across CPUs using RSS.
4273  *
4274  **/
4275 static inline bool ixgbe_set_fdir_queues(struct ixgbe_adapter *adapter)
4276 {
4277         bool ret = false;
4278         struct ixgbe_ring_feature *f_fdir = &adapter->ring_feature[RING_F_FDIR];
4279
4280         f_fdir->indices = min((int)num_online_cpus(), f_fdir->indices);
4281         f_fdir->mask = 0;
4282
4283         /* Flow Director must have RSS enabled */
4284         if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
4285             (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
4286                 adapter->num_tx_queues = f_fdir->indices;
4287                 adapter->num_rx_queues = f_fdir->indices;
4288                 ret = true;
4289         } else {
4290                 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
4291         }
4292         return ret;
4293 }
4294
4295 #ifdef IXGBE_FCOE
4296 /**
4297  * ixgbe_set_fcoe_queues: Allocate queues for Fiber Channel over Ethernet (FCoE)
4298  * @adapter: board private structure to initialize
4299  *
4300  * FCoE RX FCRETA can use up to 8 rx queues for up to 8 different exchanges.
4301  * The ring feature mask is not used as a mask for FCoE, as it can take any 8
4302  * rx queues out of the max number of rx queues, instead, it is used as the
4303  * index of the first rx queue used by FCoE.
4304  *
4305  **/
4306 static inline bool ixgbe_set_fcoe_queues(struct ixgbe_adapter *adapter)
4307 {
4308         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4309
4310         if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4311                 return false;
4312
4313         f->indices = min((int)num_online_cpus(), f->indices);
4314
4315         adapter->num_rx_queues = 1;
4316         adapter->num_tx_queues = 1;
4317
4318         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4319                 e_info(probe, "FCoE enabled with RSS\n");
4320                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
4321                         ixgbe_set_fdir_queues(adapter);
4322                 else
4323                         ixgbe_set_rss_queues(adapter);
4324         }
4325
4326         /* adding FCoE rx rings to the end */
4327         f->mask = adapter->num_rx_queues;
4328         adapter->num_rx_queues += f->indices;
4329         adapter->num_tx_queues += f->indices;
4330
4331         return true;
4332 }
4333 #endif /* IXGBE_FCOE */
4334
4335 /* Artificial max queue cap per traffic class in DCB mode */
4336 #define DCB_QUEUE_CAP 8
4337
4338 #ifdef CONFIG_IXGBE_DCB
4339 static inline bool ixgbe_set_dcb_queues(struct ixgbe_adapter *adapter)
4340 {
4341         int per_tc_q, q, i, offset = 0;
4342         struct net_device *dev = adapter->netdev;
4343         int tcs = netdev_get_num_tc(dev);
4344
4345         if (!tcs)
4346                 return false;
4347
4348         /* Map queue offset and counts onto allocated tx queues */
4349         per_tc_q = min(dev->num_tx_queues / tcs, (unsigned int)DCB_QUEUE_CAP);
4350         q = min((int)num_online_cpus(), per_tc_q);
4351
4352         for (i = 0; i < tcs; i++) {
4353                 netdev_set_prio_tc_map(dev, i, i);
4354                 netdev_set_tc_queue(dev, i, q, offset);
4355                 offset += q;
4356         }
4357
4358         adapter->num_tx_queues = q * tcs;
4359         adapter->num_rx_queues = q * tcs;
4360
4361 #ifdef IXGBE_FCOE
4362         /* FCoE enabled queues require special configuration indexed
4363          * by feature specific indices and mask. Here we map FCoE
4364          * indices onto the DCB queue pairs allowing FCoE to own
4365          * configuration later.
4366          */
4367         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) {
4368                 int tc;
4369                 struct ixgbe_ring_feature *f =
4370                                         &adapter->ring_feature[RING_F_FCOE];
4371
4372                 tc = netdev_get_prio_tc_map(dev, adapter->fcoe.up);
4373                 f->indices = dev->tc_to_txq[tc].count;
4374                 f->mask = dev->tc_to_txq[tc].offset;
4375         }
4376 #endif
4377
4378         return true;
4379 }
4380 #endif
4381
4382 /**
4383  * ixgbe_set_sriov_queues: Allocate queues for IOV use
4384  * @adapter: board private structure to initialize
4385  *
4386  * IOV doesn't actually use anything, so just NAK the
4387  * request for now and let the other queue routines
4388  * figure out what to do.
4389  */
4390 static inline bool ixgbe_set_sriov_queues(struct ixgbe_adapter *adapter)
4391 {
4392         return false;
4393 }
4394
4395 /*
4396  * ixgbe_set_num_queues: Allocate queues for device, feature dependent
4397  * @adapter: board private structure to initialize
4398  *
4399  * This is the top level queue allocation routine.  The order here is very
4400  * important, starting with the "most" number of features turned on at once,
4401  * and ending with the smallest set of features.  This way large combinations
4402  * can be allocated if they're turned on, and smaller combinations are the
4403  * fallthrough conditions.
4404  *
4405  **/
4406 static int ixgbe_set_num_queues(struct ixgbe_adapter *adapter)
4407 {
4408         /* Start with base case */
4409         adapter->num_rx_queues = 1;
4410         adapter->num_tx_queues = 1;
4411         adapter->num_rx_pools = adapter->num_rx_queues;
4412         adapter->num_rx_queues_per_pool = 1;
4413
4414         if (ixgbe_set_sriov_queues(adapter))
4415                 goto done;
4416
4417 #ifdef CONFIG_IXGBE_DCB
4418         if (ixgbe_set_dcb_queues(adapter))
4419                 goto done;
4420
4421 #endif
4422 #ifdef IXGBE_FCOE
4423         if (ixgbe_set_fcoe_queues(adapter))
4424                 goto done;
4425
4426 #endif /* IXGBE_FCOE */
4427         if (ixgbe_set_fdir_queues(adapter))
4428                 goto done;
4429
4430         if (ixgbe_set_rss_queues(adapter))
4431                 goto done;
4432
4433         /* fallback to base case */
4434         adapter->num_rx_queues = 1;
4435         adapter->num_tx_queues = 1;
4436
4437 done:
4438         /* Notify the stack of the (possibly) reduced queue counts. */
4439         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
4440         return netif_set_real_num_rx_queues(adapter->netdev,
4441                                             adapter->num_rx_queues);
4442 }
4443
4444 static void ixgbe_acquire_msix_vectors(struct ixgbe_adapter *adapter,
4445                                        int vectors)
4446 {
4447         int err, vector_threshold;
4448
4449         /* We'll want at least 3 (vector_threshold):
4450          * 1) TxQ[0] Cleanup
4451          * 2) RxQ[0] Cleanup
4452          * 3) Other (Link Status Change, etc.)
4453          * 4) TCP Timer (optional)
4454          */
4455         vector_threshold = MIN_MSIX_COUNT;
4456
4457         /* The more we get, the more we will assign to Tx/Rx Cleanup
4458          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
4459          * Right now, we simply care about how many we'll get; we'll
4460          * set them up later while requesting irq's.
4461          */
4462         while (vectors >= vector_threshold) {
4463                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
4464                                       vectors);
4465                 if (!err) /* Success in acquiring all requested vectors. */
4466                         break;
4467                 else if (err < 0)
4468                         vectors = 0; /* Nasty failure, quit now */
4469                 else /* err == number of vectors we should try again with */
4470                         vectors = err;
4471         }
4472
4473         if (vectors < vector_threshold) {
4474                 /* Can't allocate enough MSI-X interrupts?  Oh well.
4475                  * This just means we'll go with either a single MSI
4476                  * vector or fall back to legacy interrupts.
4477                  */
4478                 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4479                              "Unable to allocate MSI-X interrupts\n");
4480                 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
4481                 kfree(adapter->msix_entries);
4482                 adapter->msix_entries = NULL;
4483         } else {
4484                 adapter->flags |= IXGBE_FLAG_MSIX_ENABLED; /* Woot! */
4485                 /*
4486                  * Adjust for only the vectors we'll use, which is minimum
4487                  * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
4488                  * vectors we were allocated.
4489                  */
4490                 adapter->num_msix_vectors = min(vectors,
4491                                    adapter->max_msix_q_vectors + NON_Q_VECTORS);
4492         }
4493 }
4494
4495 /**
4496  * ixgbe_cache_ring_rss - Descriptor ring to register mapping for RSS
4497  * @adapter: board private structure to initialize
4498  *
4499  * Cache the descriptor ring offsets for RSS to the assigned rings.
4500  *
4501  **/
4502 static inline bool ixgbe_cache_ring_rss(struct ixgbe_adapter *adapter)
4503 {
4504         int i;
4505
4506         if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
4507                 return false;
4508
4509         for (i = 0; i < adapter->num_rx_queues; i++)
4510                 adapter->rx_ring[i]->reg_idx = i;
4511         for (i = 0; i < adapter->num_tx_queues; i++)
4512                 adapter->tx_ring[i]->reg_idx = i;
4513
4514         return true;
4515 }
4516
4517 #ifdef CONFIG_IXGBE_DCB
4518
4519 /* ixgbe_get_first_reg_idx - Return first register index associated with ring */
4520 static void ixgbe_get_first_reg_idx(struct ixgbe_adapter *adapter, u8 tc,
4521                                     unsigned int *tx, unsigned int *rx)
4522 {
4523         struct net_device *dev = adapter->netdev;
4524         struct ixgbe_hw *hw = &adapter->hw;
4525         u8 num_tcs = netdev_get_num_tc(dev);
4526
4527         *tx = 0;
4528         *rx = 0;
4529
4530         switch (hw->mac.type) {
4531         case ixgbe_mac_82598EB:
4532                 *tx = tc << 2;
4533                 *rx = tc << 3;
4534                 break;
4535         case ixgbe_mac_82599EB:
4536         case ixgbe_mac_X540:
4537                 if (num_tcs == 8) {
4538                         if (tc < 3) {
4539                                 *tx = tc << 5;
4540                                 *rx = tc << 4;
4541                         } else if (tc <  5) {
4542                                 *tx = ((tc + 2) << 4);
4543                                 *rx = tc << 4;
4544                         } else if (tc < num_tcs) {
4545                                 *tx = ((tc + 8) << 3);
4546                                 *rx = tc << 4;
4547                         }
4548                 } else if (num_tcs == 4) {
4549                         *rx =  tc << 5;
4550                         switch (tc) {
4551                         case 0:
4552                                 *tx =  0;
4553                                 break;
4554                         case 1:
4555                                 *tx = 64;
4556                                 break;
4557                         case 2:
4558                                 *tx = 96;
4559                                 break;
4560                         case 3:
4561                                 *tx = 112;
4562                                 break;
4563                         default:
4564                                 break;
4565                         }
4566                 }
4567                 break;
4568         default:
4569                 break;
4570         }
4571 }
4572
4573 /**
4574  * ixgbe_cache_ring_dcb - Descriptor ring to register mapping for DCB
4575  * @adapter: board private structure to initialize
4576  *
4577  * Cache the descriptor ring offsets for DCB to the assigned rings.
4578  *
4579  **/
4580 static inline bool ixgbe_cache_ring_dcb(struct ixgbe_adapter *adapter)
4581 {
4582         struct net_device *dev = adapter->netdev;
4583         int i, j, k;
4584         u8 num_tcs = netdev_get_num_tc(dev);
4585
4586         if (!num_tcs)
4587                 return false;
4588
4589         for (i = 0, k = 0; i < num_tcs; i++) {
4590                 unsigned int tx_s, rx_s;
4591                 u16 count = dev->tc_to_txq[i].count;
4592
4593                 ixgbe_get_first_reg_idx(adapter, i, &tx_s, &rx_s);
4594                 for (j = 0; j < count; j++, k++) {
4595                         adapter->tx_ring[k]->reg_idx = tx_s + j;
4596                         adapter->rx_ring[k]->reg_idx = rx_s + j;
4597                         adapter->tx_ring[k]->dcb_tc = i;
4598                         adapter->rx_ring[k]->dcb_tc = i;
4599                 }
4600         }
4601
4602         return true;
4603 }
4604 #endif
4605
4606 /**
4607  * ixgbe_cache_ring_fdir - Descriptor ring to register mapping for Flow Director
4608  * @adapter: board private structure to initialize
4609  *
4610  * Cache the descriptor ring offsets for Flow Director to the assigned rings.
4611  *
4612  **/
4613 static inline bool ixgbe_cache_ring_fdir(struct ixgbe_adapter *adapter)
4614 {
4615         int i;
4616         bool ret = false;
4617
4618         if ((adapter->flags & IXGBE_FLAG_RSS_ENABLED) &&
4619             (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)) {
4620                 for (i = 0; i < adapter->num_rx_queues; i++)
4621                         adapter->rx_ring[i]->reg_idx = i;
4622                 for (i = 0; i < adapter->num_tx_queues; i++)
4623                         adapter->tx_ring[i]->reg_idx = i;
4624                 ret = true;
4625         }
4626
4627         return ret;
4628 }
4629
4630 #ifdef IXGBE_FCOE
4631 /**
4632  * ixgbe_cache_ring_fcoe - Descriptor ring to register mapping for the FCoE
4633  * @adapter: board private structure to initialize
4634  *
4635  * Cache the descriptor ring offsets for FCoE mode to the assigned rings.
4636  *
4637  */
4638 static inline bool ixgbe_cache_ring_fcoe(struct ixgbe_adapter *adapter)
4639 {
4640         struct ixgbe_ring_feature *f = &adapter->ring_feature[RING_F_FCOE];
4641         int i;
4642         u8 fcoe_rx_i = 0, fcoe_tx_i = 0;
4643
4644         if (!(adapter->flags & IXGBE_FLAG_FCOE_ENABLED))
4645                 return false;
4646
4647         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED) {
4648                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE)
4649                         ixgbe_cache_ring_fdir(adapter);
4650                 else
4651                         ixgbe_cache_ring_rss(adapter);
4652
4653                 fcoe_rx_i = f->mask;
4654                 fcoe_tx_i = f->mask;
4655         }
4656         for (i = 0; i < f->indices; i++, fcoe_rx_i++, fcoe_tx_i++) {
4657                 adapter->rx_ring[f->mask + i]->reg_idx = fcoe_rx_i;
4658                 adapter->tx_ring[f->mask + i]->reg_idx = fcoe_tx_i;
4659         }
4660         return true;
4661 }
4662
4663 #endif /* IXGBE_FCOE */
4664 /**
4665  * ixgbe_cache_ring_sriov - Descriptor ring to register mapping for sriov
4666  * @adapter: board private structure to initialize
4667  *
4668  * SR-IOV doesn't use any descriptor rings but changes the default if
4669  * no other mapping is used.
4670  *
4671  */
4672 static inline bool ixgbe_cache_ring_sriov(struct ixgbe_adapter *adapter)
4673 {
4674         adapter->rx_ring[0]->reg_idx = adapter->num_vfs * 2;
4675         adapter->tx_ring[0]->reg_idx = adapter->num_vfs * 2;
4676         if (adapter->num_vfs)
4677                 return true;
4678         else
4679                 return false;
4680 }
4681
4682 /**
4683  * ixgbe_cache_ring_register - Descriptor ring to register mapping
4684  * @adapter: board private structure to initialize
4685  *
4686  * Once we know the feature-set enabled for the device, we'll cache
4687  * the register offset the descriptor ring is assigned to.
4688  *
4689  * Note, the order the various feature calls is important.  It must start with
4690  * the "most" features enabled at the same time, then trickle down to the
4691  * least amount of features turned on at once.
4692  **/
4693 static void ixgbe_cache_ring_register(struct ixgbe_adapter *adapter)
4694 {
4695         /* start with default case */
4696         adapter->rx_ring[0]->reg_idx = 0;
4697         adapter->tx_ring[0]->reg_idx = 0;
4698
4699         if (ixgbe_cache_ring_sriov(adapter))
4700                 return;
4701
4702 #ifdef CONFIG_IXGBE_DCB
4703         if (ixgbe_cache_ring_dcb(adapter))
4704                 return;
4705 #endif
4706
4707 #ifdef IXGBE_FCOE
4708         if (ixgbe_cache_ring_fcoe(adapter))
4709                 return;
4710 #endif /* IXGBE_FCOE */
4711
4712         if (ixgbe_cache_ring_fdir(adapter))
4713                 return;
4714
4715         if (ixgbe_cache_ring_rss(adapter))
4716                 return;
4717 }
4718
4719 /**
4720  * ixgbe_alloc_queues - Allocate memory for all rings
4721  * @adapter: board private structure to initialize
4722  *
4723  * We allocate one ring per queue at run-time since we don't know the
4724  * number of queues at compile-time.  The polling_netdev array is
4725  * intended for Multiqueue, but should work fine with a single queue.
4726  **/
4727 static int ixgbe_alloc_queues(struct ixgbe_adapter *adapter)
4728 {
4729         int rx = 0, tx = 0, nid = adapter->node;
4730
4731         if (nid < 0 || !node_online(nid))
4732                 nid = first_online_node;
4733
4734         for (; tx < adapter->num_tx_queues; tx++) {
4735                 struct ixgbe_ring *ring;
4736
4737                 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4738                 if (!ring)
4739                         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4740                 if (!ring)
4741                         goto err_allocation;
4742                 ring->count = adapter->tx_ring_count;
4743                 ring->queue_index = tx;
4744                 ring->numa_node = nid;
4745                 ring->dev = &adapter->pdev->dev;
4746                 ring->netdev = adapter->netdev;
4747
4748                 adapter->tx_ring[tx] = ring;
4749         }
4750
4751         for (; rx < adapter->num_rx_queues; rx++) {
4752                 struct ixgbe_ring *ring;
4753
4754                 ring = kzalloc_node(sizeof(*ring), GFP_KERNEL, nid);
4755                 if (!ring)
4756                         ring = kzalloc(sizeof(*ring), GFP_KERNEL);
4757                 if (!ring)
4758                         goto err_allocation;
4759                 ring->count = adapter->rx_ring_count;
4760                 ring->queue_index = rx;
4761                 ring->numa_node = nid;
4762                 ring->dev = &adapter->pdev->dev;
4763                 ring->netdev = adapter->netdev;
4764
4765                 adapter->rx_ring[rx] = ring;
4766         }
4767
4768         ixgbe_cache_ring_register(adapter);
4769
4770         return 0;
4771
4772 err_allocation:
4773         while (tx)
4774                 kfree(adapter->tx_ring[--tx]);
4775
4776         while (rx)
4777                 kfree(adapter->rx_ring[--rx]);
4778         return -ENOMEM;
4779 }
4780
4781 /**
4782  * ixgbe_set_interrupt_capability - set MSI-X or MSI if supported
4783  * @adapter: board private structure to initialize
4784  *
4785  * Attempt to configure the interrupts using the best available
4786  * capabilities of the hardware and the kernel.
4787  **/
4788 static int ixgbe_set_interrupt_capability(struct ixgbe_adapter *adapter)
4789 {
4790         struct ixgbe_hw *hw = &adapter->hw;
4791         int err = 0;
4792         int vector, v_budget;
4793
4794         /*
4795          * It's easy to be greedy for MSI-X vectors, but it really
4796          * doesn't do us much good if we have a lot more vectors
4797          * than CPU's.  So let's be conservative and only ask for
4798          * (roughly) the same number of vectors as there are CPU's.
4799          */
4800         v_budget = min(adapter->num_rx_queues + adapter->num_tx_queues,
4801                        (int)num_online_cpus()) + NON_Q_VECTORS;
4802
4803         /*
4804          * At the same time, hardware can only support a maximum of
4805          * hw.mac->max_msix_vectors vectors.  With features
4806          * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
4807          * descriptor queues supported by our device.  Thus, we cap it off in
4808          * those rare cases where the cpu count also exceeds our vector limit.
4809          */
4810         v_budget = min(v_budget, (int)hw->mac.max_msix_vectors);
4811
4812         /* A failure in MSI-X entry allocation isn't fatal, but it does
4813          * mean we disable MSI-X capabilities of the adapter. */
4814         adapter->msix_entries = kcalloc(v_budget,
4815                                         sizeof(struct msix_entry), GFP_KERNEL);
4816         if (adapter->msix_entries) {
4817                 for (vector = 0; vector < v_budget; vector++)
4818                         adapter->msix_entries[vector].entry = vector;
4819
4820                 ixgbe_acquire_msix_vectors(adapter, v_budget);
4821
4822                 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4823                         goto out;
4824         }
4825
4826         adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;
4827         adapter->flags &= ~IXGBE_FLAG_RSS_ENABLED;
4828         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
4829                 e_err(probe,
4830                       "ATR is not supported while multiple "
4831                       "queues are disabled.  Disabling Flow Director\n");
4832         }
4833         adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
4834         adapter->atr_sample_rate = 0;
4835         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
4836                 ixgbe_disable_sriov(adapter);
4837
4838         err = ixgbe_set_num_queues(adapter);
4839         if (err)
4840                 return err;
4841
4842         err = pci_enable_msi(adapter->pdev);
4843         if (!err) {
4844                 adapter->flags |= IXGBE_FLAG_MSI_ENABLED;
4845         } else {
4846                 netif_printk(adapter, hw, KERN_DEBUG, adapter->netdev,
4847                              "Unable to allocate MSI interrupt, "
4848                              "falling back to legacy.  Error: %d\n", err);
4849                 /* reset err */
4850                 err = 0;
4851         }
4852
4853 out:
4854         return err;
4855 }
4856
4857 /**
4858  * ixgbe_alloc_q_vectors - Allocate memory for interrupt vectors
4859  * @adapter: board private structure to initialize
4860  *
4861  * We allocate one q_vector per queue interrupt.  If allocation fails we
4862  * return -ENOMEM.
4863  **/
4864 static int ixgbe_alloc_q_vectors(struct ixgbe_adapter *adapter)
4865 {
4866         int q_idx, num_q_vectors;
4867         struct ixgbe_q_vector *q_vector;
4868         int (*poll)(struct napi_struct *, int);
4869
4870         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
4871                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4872                 poll = &ixgbe_clean_rxtx_many;
4873         } else {
4874                 num_q_vectors = 1;
4875                 poll = &ixgbe_poll;
4876         }
4877
4878         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
4879                 q_vector = kzalloc_node(sizeof(struct ixgbe_q_vector),
4880                                         GFP_KERNEL, adapter->node);
4881                 if (!q_vector)
4882                         q_vector = kzalloc(sizeof(struct ixgbe_q_vector),
4883                                            GFP_KERNEL);
4884                 if (!q_vector)
4885                         goto err_out;
4886                 q_vector->adapter = adapter;
4887                 if (q_vector->tx.count && !q_vector->rx.count)
4888                         q_vector->eitr = adapter->tx_eitr_param;
4889                 else
4890                         q_vector->eitr = adapter->rx_eitr_param;
4891                 q_vector->v_idx = q_idx;
4892                 netif_napi_add(adapter->netdev, &q_vector->napi, (*poll), 64);
4893                 adapter->q_vector[q_idx] = q_vector;
4894         }
4895
4896         return 0;
4897
4898 err_out:
4899         while (q_idx) {
4900                 q_idx--;
4901                 q_vector = adapter->q_vector[q_idx];
4902                 netif_napi_del(&q_vector->napi);
4903                 kfree(q_vector);
4904                 adapter->q_vector[q_idx] = NULL;
4905         }
4906         return -ENOMEM;
4907 }
4908
4909 /**
4910  * ixgbe_free_q_vectors - Free memory allocated for interrupt vectors
4911  * @adapter: board private structure to initialize
4912  *
4913  * This function frees the memory allocated to the q_vectors.  In addition if
4914  * NAPI is enabled it will delete any references to the NAPI struct prior
4915  * to freeing the q_vector.
4916  **/
4917 static void ixgbe_free_q_vectors(struct ixgbe_adapter *adapter)
4918 {
4919         int q_idx, num_q_vectors;
4920
4921         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
4922                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
4923         else
4924                 num_q_vectors = 1;
4925
4926         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
4927                 struct ixgbe_q_vector *q_vector = adapter->q_vector[q_idx];
4928                 adapter->q_vector[q_idx] = NULL;
4929                 netif_napi_del(&q_vector->napi);
4930                 kfree(q_vector);
4931         }
4932 }
4933
4934 static void ixgbe_reset_interrupt_capability(struct ixgbe_adapter *adapter)
4935 {
4936         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
4937                 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
4938                 pci_disable_msix(adapter->pdev);
4939                 kfree(adapter->msix_entries);
4940                 adapter->msix_entries = NULL;
4941         } else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED) {
4942                 adapter->flags &= ~IXGBE_FLAG_MSI_ENABLED;
4943                 pci_disable_msi(adapter->pdev);
4944         }
4945 }
4946
4947 /**
4948  * ixgbe_init_interrupt_scheme - Determine proper interrupt scheme
4949  * @adapter: board private structure to initialize
4950  *
4951  * We determine which interrupt scheme to use based on...
4952  * - Kernel support (MSI, MSI-X)
4953  *   - which can be user-defined (via MODULE_PARAM)
4954  * - Hardware queue count (num_*_queues)
4955  *   - defined by miscellaneous hardware support/features (RSS, etc.)
4956  **/
4957 int ixgbe_init_interrupt_scheme(struct ixgbe_adapter *adapter)
4958 {
4959         int err;
4960
4961         /* Number of supported queues */
4962         err = ixgbe_set_num_queues(adapter);
4963         if (err)
4964                 return err;
4965
4966         err = ixgbe_set_interrupt_capability(adapter);
4967         if (err) {
4968                 e_dev_err("Unable to setup interrupt capabilities\n");
4969                 goto err_set_interrupt;
4970         }
4971
4972         err = ixgbe_alloc_q_vectors(adapter);
4973         if (err) {
4974                 e_dev_err("Unable to allocate memory for queue vectors\n");
4975                 goto err_alloc_q_vectors;
4976         }
4977
4978         err = ixgbe_alloc_queues(adapter);
4979         if (err) {
4980                 e_dev_err("Unable to allocate memory for queues\n");
4981                 goto err_alloc_queues;
4982         }
4983
4984         e_dev_info("Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n",
4985                    (adapter->num_rx_queues > 1) ? "Enabled" : "Disabled",
4986                    adapter->num_rx_queues, adapter->num_tx_queues);
4987
4988         set_bit(__IXGBE_DOWN, &adapter->state);
4989
4990         return 0;
4991
4992 err_alloc_queues:
4993         ixgbe_free_q_vectors(adapter);
4994 err_alloc_q_vectors:
4995         ixgbe_reset_interrupt_capability(adapter);
4996 err_set_interrupt:
4997         return err;
4998 }
4999
5000 /**
5001  * ixgbe_clear_interrupt_scheme - Clear the current interrupt scheme settings
5002  * @adapter: board private structure to clear interrupt scheme on
5003  *
5004  * We go through and clear interrupt specific resources and reset the structure
5005  * to pre-load conditions
5006  **/
5007 void ixgbe_clear_interrupt_scheme(struct ixgbe_adapter *adapter)
5008 {
5009         int i;
5010
5011         for (i = 0; i < adapter->num_tx_queues; i++) {
5012                 kfree(adapter->tx_ring[i]);
5013                 adapter->tx_ring[i] = NULL;
5014         }
5015         for (i = 0; i < adapter->num_rx_queues; i++) {
5016                 struct ixgbe_ring *ring = adapter->rx_ring[i];
5017
5018                 /* ixgbe_get_stats64() might access this ring, we must wait
5019                  * a grace period before freeing it.
5020                  */
5021                 kfree_rcu(ring, rcu);
5022                 adapter->rx_ring[i] = NULL;
5023         }
5024
5025         adapter->num_tx_queues = 0;
5026         adapter->num_rx_queues = 0;
5027
5028         ixgbe_free_q_vectors(adapter);
5029         ixgbe_reset_interrupt_capability(adapter);
5030 }
5031
5032 /**
5033  * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)
5034  * @adapter: board private structure to initialize
5035  *
5036  * ixgbe_sw_init initializes the Adapter private data structure.
5037  * Fields are initialized based on PCI device information and
5038  * OS network device settings (MTU size).
5039  **/
5040 static int __devinit ixgbe_sw_init(struct ixgbe_adapter *adapter)
5041 {
5042         struct ixgbe_hw *hw = &adapter->hw;
5043         struct pci_dev *pdev = adapter->pdev;
5044         struct net_device *dev = adapter->netdev;
5045         unsigned int rss;
5046 #ifdef CONFIG_IXGBE_DCB
5047         int j;
5048         struct tc_configuration *tc;
5049 #endif
5050         int max_frame = dev->mtu + ETH_HLEN + ETH_FCS_LEN;
5051
5052         /* PCI config space info */
5053
5054         hw->vendor_id = pdev->vendor;
5055         hw->device_id = pdev->device;
5056         hw->revision_id = pdev->revision;
5057         hw->subsystem_vendor_id = pdev->subsystem_vendor;
5058         hw->subsystem_device_id = pdev->subsystem_device;
5059
5060         /* Set capability flags */
5061         rss = min(IXGBE_MAX_RSS_INDICES, (int)num_online_cpus());
5062         adapter->ring_feature[RING_F_RSS].indices = rss;
5063         adapter->flags |= IXGBE_FLAG_RSS_ENABLED;
5064         switch (hw->mac.type) {
5065         case ixgbe_mac_82598EB:
5066                 if (hw->device_id == IXGBE_DEV_ID_82598AT)
5067                         adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;
5068                 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82598;
5069                 break;
5070         case ixgbe_mac_82599EB:
5071         case ixgbe_mac_X540:
5072                 adapter->max_msix_q_vectors = MAX_MSIX_Q_VECTORS_82599;
5073                 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;
5074                 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;
5075                 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)
5076                         adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;
5077                 /* Flow Director hash filters enabled */
5078                 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
5079                 adapter->atr_sample_rate = 20;
5080                 adapter->ring_feature[RING_F_FDIR].indices =
5081                                                          IXGBE_MAX_FDIR_INDICES;
5082                 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;
5083 #ifdef IXGBE_FCOE
5084                 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;
5085                 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;
5086                 adapter->ring_feature[RING_F_FCOE].indices = 0;
5087 #ifdef CONFIG_IXGBE_DCB
5088                 /* Default traffic class to use for FCoE */
5089                 adapter->fcoe.up = IXGBE_FCOE_DEFTC;
5090 #endif
5091 #endif /* IXGBE_FCOE */
5092                 break;
5093         default:
5094                 break;
5095         }
5096
5097         /* n-tuple support exists, always init our spinlock */
5098         spin_lock_init(&adapter->fdir_perfect_lock);
5099
5100 #ifdef CONFIG_IXGBE_DCB
5101         /* Configure DCB traffic classes */
5102         for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {
5103                 tc = &adapter->dcb_cfg.tc_config[j];
5104                 tc->path[DCB_TX_CONFIG].bwg_id = 0;
5105                 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);
5106                 tc->path[DCB_RX_CONFIG].bwg_id = 0;
5107                 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);
5108                 tc->dcb_pfc = pfc_disabled;
5109         }
5110         adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;
5111         adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;
5112         adapter->dcb_cfg.pfc_mode_enable = false;
5113         adapter->dcb_set_bitmap = 0x00;
5114         adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;
5115         ixgbe_copy_dcb_cfg(&adapter->dcb_cfg, &adapter->temp_dcb_cfg,
5116                            MAX_TRAFFIC_CLASS);
5117
5118 #endif
5119
5120         /* default flow control settings */
5121         hw->fc.requested_mode = ixgbe_fc_full;
5122         hw->fc.current_mode = ixgbe_fc_full;    /* init for ethtool output */
5123 #ifdef CONFIG_DCB
5124         adapter->last_lfc_mode = hw->fc.current_mode;
5125 #endif
5126         hw->fc.high_water = FC_HIGH_WATER(max_frame);
5127         hw->fc.low_water = FC_LOW_WATER(max_frame);
5128         hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;
5129         hw->fc.send_xon = true;
5130         hw->fc.disable_fc_autoneg = false;
5131
5132         /* enable itr by default in dynamic mode */
5133         adapter->rx_itr_setting = 1;
5134         adapter->rx_eitr_param = 20000;
5135         adapter->tx_itr_setting = 1;
5136         adapter->tx_eitr_param = 10000;
5137
5138         /* set defaults for eitr in MegaBytes */
5139         adapter->eitr_low = 10;
5140         adapter->eitr_high = 20;
5141
5142         /* set default ring sizes */
5143         adapter->tx_ring_count = IXGBE_DEFAULT_TXD;
5144         adapter->rx_ring_count = IXGBE_DEFAULT_RXD;
5145
5146         /* set default work limits */
5147         adapter->tx_work_limit = adapter->tx_ring_count;
5148
5149         /* initialize eeprom parameters */
5150         if (ixgbe_init_eeprom_params_generic(hw)) {
5151                 e_dev_err("EEPROM initialization failed\n");
5152                 return -EIO;
5153         }
5154
5155         /* enable rx csum by default */
5156         adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
5157
5158         /* get assigned NUMA node */
5159         adapter->node = dev_to_node(&pdev->dev);
5160
5161         set_bit(__IXGBE_DOWN, &adapter->state);
5162
5163         return 0;
5164 }
5165
5166 /**
5167  * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)
5168  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
5169  *
5170  * Return 0 on success, negative on failure
5171  **/
5172 int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)
5173 {
5174         struct device *dev = tx_ring->dev;
5175         int size;
5176
5177         size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;
5178         tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
5179         if (!tx_ring->tx_buffer_info)
5180                 tx_ring->tx_buffer_info = vzalloc(size);
5181         if (!tx_ring->tx_buffer_info)
5182                 goto err;
5183
5184         /* round up to nearest 4K */
5185         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
5186         tx_ring->size = ALIGN(tx_ring->size, 4096);
5187
5188         tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
5189                                            &tx_ring->dma, GFP_KERNEL);
5190         if (!tx_ring->desc)
5191                 goto err;
5192
5193         tx_ring->next_to_use = 0;
5194         tx_ring->next_to_clean = 0;
5195         return 0;
5196
5197 err:
5198         vfree(tx_ring->tx_buffer_info);
5199         tx_ring->tx_buffer_info = NULL;
5200         dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
5201         return -ENOMEM;
5202 }
5203
5204 /**
5205  * ixgbe_setup_all_tx_resources - allocate all queues Tx resources
5206  * @adapter: board private structure
5207  *
5208  * If this function returns with an error, then it's possible one or
5209  * more of the rings is populated (while the rest are not).  It is the
5210  * callers duty to clean those orphaned rings.
5211  *
5212  * Return 0 on success, negative on failure
5213  **/
5214 static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)
5215 {
5216         int i, err = 0;
5217
5218         for (i = 0; i < adapter->num_tx_queues; i++) {
5219                 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);
5220                 if (!err)
5221                         continue;
5222                 e_err(probe, "Allocation for Tx Queue %u failed\n", i);
5223                 break;
5224         }
5225
5226         return err;
5227 }
5228
5229 /**
5230  * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)
5231  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
5232  *
5233  * Returns 0 on success, negative on failure
5234  **/
5235 int ixgbe_setup_rx_resources(struct ixgbe_ring *rx_ring)
5236 {
5237         struct device *dev = rx_ring->dev;
5238         int size;
5239
5240         size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;
5241         rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
5242         if (!rx_ring->rx_buffer_info)
5243                 rx_ring->rx_buffer_info = vzalloc(size);
5244         if (!rx_ring->rx_buffer_info)
5245                 goto err;
5246
5247         /* Round up to nearest 4K */
5248         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
5249         rx_ring->size = ALIGN(rx_ring->size, 4096);
5250
5251         rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
5252                                            &rx_ring->dma, GFP_KERNEL);
5253
5254         if (!rx_ring->desc)
5255                 goto err;
5256
5257         rx_ring->next_to_clean = 0;
5258         rx_ring->next_to_use = 0;
5259
5260         return 0;
5261 err:
5262         vfree(rx_ring->rx_buffer_info);
5263         rx_ring->rx_buffer_info = NULL;
5264         dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
5265         return -ENOMEM;
5266 }
5267
5268 /**
5269  * ixgbe_setup_all_rx_resources - allocate all queues Rx resources
5270  * @adapter: board private structure
5271  *
5272  * If this function returns with an error, then it's possible one or
5273  * more of the rings is populated (while the rest are not).  It is the
5274  * callers duty to clean those orphaned rings.
5275  *
5276  * Return 0 on success, negative on failure
5277  **/
5278 static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)
5279 {
5280         int i, err = 0;
5281
5282         for (i = 0; i < adapter->num_rx_queues; i++) {
5283                 err = ixgbe_setup_rx_resources(adapter->rx_ring[i]);
5284                 if (!err)
5285                         continue;
5286                 e_err(probe, "Allocation for Rx Queue %u failed\n", i);
5287                 break;
5288         }
5289
5290         return err;
5291 }
5292
5293 /**
5294  * ixgbe_free_tx_resources - Free Tx Resources per Queue
5295  * @tx_ring: Tx descriptor ring for a specific queue
5296  *
5297  * Free all transmit software resources
5298  **/
5299 void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)
5300 {
5301         ixgbe_clean_tx_ring(tx_ring);
5302
5303         vfree(tx_ring->tx_buffer_info);
5304         tx_ring->tx_buffer_info = NULL;
5305
5306         /* if not set, then don't free */
5307         if (!tx_ring->desc)
5308                 return;
5309
5310         dma_free_coherent(tx_ring->dev, tx_ring->size,
5311                           tx_ring->desc, tx_ring->dma);
5312
5313         tx_ring->desc = NULL;
5314 }
5315
5316 /**
5317  * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues
5318  * @adapter: board private structure
5319  *
5320  * Free all transmit software resources
5321  **/
5322 static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)
5323 {
5324         int i;
5325
5326         for (i = 0; i < adapter->num_tx_queues; i++)
5327                 if (adapter->tx_ring[i]->desc)
5328                         ixgbe_free_tx_resources(adapter->tx_ring[i]);
5329 }
5330
5331 /**
5332  * ixgbe_free_rx_resources - Free Rx Resources
5333  * @rx_ring: ring to clean the resources from
5334  *
5335  * Free all receive software resources
5336  **/
5337 void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)
5338 {
5339         ixgbe_clean_rx_ring(rx_ring);
5340
5341         vfree(rx_ring->rx_buffer_info);
5342         rx_ring->rx_buffer_info = NULL;
5343
5344         /* if not set, then don't free */
5345         if (!rx_ring->desc)
5346                 return;
5347
5348         dma_free_coherent(rx_ring->dev, rx_ring->size,
5349                           rx_ring->desc, rx_ring->dma);
5350
5351         rx_ring->desc = NULL;
5352 }
5353
5354 /**
5355  * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues
5356  * @adapter: board private structure
5357  *
5358  * Free all receive software resources
5359  **/
5360 static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)
5361 {
5362         int i;
5363
5364         for (i = 0; i < adapter->num_rx_queues; i++)
5365                 if (adapter->rx_ring[i]->desc)
5366                         ixgbe_free_rx_resources(adapter->rx_ring[i]);
5367 }
5368
5369 /**
5370  * ixgbe_change_mtu - Change the Maximum Transfer Unit
5371  * @netdev: network interface device structure
5372  * @new_mtu: new value for maximum frame size
5373  *
5374  * Returns 0 on success, negative on failure
5375  **/
5376 static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)
5377 {
5378         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5379         struct ixgbe_hw *hw = &adapter->hw;
5380         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5381
5382         /* MTU < 68 is an error and causes problems on some kernels */
5383         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED &&
5384             hw->mac.type != ixgbe_mac_X540) {
5385                 if ((new_mtu < 68) || (max_frame > MAXIMUM_ETHERNET_VLAN_SIZE))
5386                         return -EINVAL;
5387         } else {
5388                 if ((new_mtu < 68) || (max_frame > IXGBE_MAX_JUMBO_FRAME_SIZE))
5389                         return -EINVAL;
5390         }
5391
5392         e_info(probe, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5393         /* must set new MTU before calling down or up */
5394         netdev->mtu = new_mtu;
5395
5396         hw->fc.high_water = FC_HIGH_WATER(max_frame);
5397         hw->fc.low_water = FC_LOW_WATER(max_frame);
5398
5399         if (netif_running(netdev))
5400                 ixgbe_reinit_locked(adapter);
5401
5402         return 0;
5403 }
5404
5405 /**
5406  * ixgbe_open - Called when a network interface is made active
5407  * @netdev: network interface device structure
5408  *
5409  * Returns 0 on success, negative value on failure
5410  *
5411  * The open entry point is called when a network interface is made
5412  * active by the system (IFF_UP).  At this point all resources needed
5413  * for transmit and receive operations are allocated, the interrupt
5414  * handler is registered with the OS, the watchdog timer is started,
5415  * and the stack is notified that the interface is ready.
5416  **/
5417 static int ixgbe_open(struct net_device *netdev)
5418 {
5419         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5420         int err;
5421
5422         /* disallow open during test */
5423         if (test_bit(__IXGBE_TESTING, &adapter->state))
5424                 return -EBUSY;
5425
5426         netif_carrier_off(netdev);
5427
5428         /* allocate transmit descriptors */
5429         err = ixgbe_setup_all_tx_resources(adapter);
5430         if (err)
5431                 goto err_setup_tx;
5432
5433         /* allocate receive descriptors */
5434         err = ixgbe_setup_all_rx_resources(adapter);
5435         if (err)
5436                 goto err_setup_rx;
5437
5438         ixgbe_configure(adapter);
5439
5440         err = ixgbe_request_irq(adapter);
5441         if (err)
5442                 goto err_req_irq;
5443
5444         err = ixgbe_up_complete(adapter);
5445         if (err)
5446                 goto err_up;
5447
5448         netif_tx_start_all_queues(netdev);
5449
5450         return 0;
5451
5452 err_up:
5453         ixgbe_release_hw_control(adapter);
5454         ixgbe_free_irq(adapter);
5455 err_req_irq:
5456 err_setup_rx:
5457         ixgbe_free_all_rx_resources(adapter);
5458 err_setup_tx:
5459         ixgbe_free_all_tx_resources(adapter);
5460         ixgbe_reset(adapter);
5461
5462         return err;
5463 }
5464
5465 /**
5466  * ixgbe_close - Disables a network interface
5467  * @netdev: network interface device structure
5468  *
5469  * Returns 0, this is not allowed to fail
5470  *
5471  * The close entry point is called when an interface is de-activated
5472  * by the OS.  The hardware is still under the drivers control, but
5473  * needs to be disabled.  A global MAC reset is issued to stop the
5474  * hardware, and all transmit and receive resources are freed.
5475  **/
5476 static int ixgbe_close(struct net_device *netdev)
5477 {
5478         struct ixgbe_adapter *adapter = netdev_priv(netdev);
5479
5480         ixgbe_down(adapter);
5481         ixgbe_free_irq(adapter);
5482
5483         ixgbe_fdir_filter_exit(adapter);
5484
5485         ixgbe_free_all_tx_resources(adapter);
5486         ixgbe_free_all_rx_resources(adapter);
5487
5488         ixgbe_release_hw_control(adapter);
5489
5490         return 0;
5491 }
5492
5493 #ifdef CONFIG_PM
5494 static int ixgbe_resume(struct pci_dev *pdev)
5495 {
5496         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5497         struct net_device *netdev = adapter->netdev;
5498         u32 err;
5499
5500         pci_set_power_state(pdev, PCI_D0);
5501         pci_restore_state(pdev);
5502         /*
5503          * pci_restore_state clears dev->state_saved so call
5504          * pci_save_state to restore it.
5505          */
5506         pci_save_state(pdev);
5507
5508         err = pci_enable_device_mem(pdev);
5509         if (err) {
5510                 e_dev_err("Cannot enable PCI device from suspend\n");
5511                 return err;
5512         }
5513         pci_set_master(pdev);
5514
5515         pci_wake_from_d3(pdev, false);
5516
5517         err = ixgbe_init_interrupt_scheme(adapter);
5518         if (err) {
5519                 e_dev_err("Cannot initialize interrupts for device\n");
5520                 return err;
5521         }
5522
5523         ixgbe_reset(adapter);
5524
5525         IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
5526
5527         if (netif_running(netdev)) {
5528                 err = ixgbe_open(netdev);
5529                 if (err)
5530                         return err;
5531         }
5532
5533         netif_device_attach(netdev);
5534
5535         return 0;
5536 }
5537 #endif /* CONFIG_PM */
5538
5539 static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)
5540 {
5541         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
5542         struct net_device *netdev = adapter->netdev;
5543         struct ixgbe_hw *hw = &adapter->hw;
5544         u32 ctrl, fctrl;
5545         u32 wufc = adapter->wol;
5546 #ifdef CONFIG_PM
5547         int retval = 0;
5548 #endif
5549
5550         netif_device_detach(netdev);
5551
5552         if (netif_running(netdev)) {
5553                 ixgbe_down(adapter);
5554                 ixgbe_free_irq(adapter);
5555                 ixgbe_free_all_tx_resources(adapter);
5556                 ixgbe_free_all_rx_resources(adapter);
5557         }
5558
5559         ixgbe_clear_interrupt_scheme(adapter);
5560 #ifdef CONFIG_DCB
5561         kfree(adapter->ixgbe_ieee_pfc);
5562         kfree(adapter->ixgbe_ieee_ets);
5563 #endif
5564
5565 #ifdef CONFIG_PM
5566         retval = pci_save_state(pdev);
5567         if (retval)
5568                 return retval;
5569
5570 #endif
5571         if (wufc) {
5572                 ixgbe_set_rx_mode(netdev);
5573
5574                 /* turn on all-multi mode if wake on multicast is enabled */
5575                 if (wufc & IXGBE_WUFC_MC) {
5576                         fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5577                         fctrl |= IXGBE_FCTRL_MPE;
5578                         IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
5579                 }
5580
5581                 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);
5582                 ctrl |= IXGBE_CTRL_GIO_DIS;
5583                 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);
5584
5585                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);
5586         } else {
5587                 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);
5588                 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);
5589         }
5590
5591         switch (hw->mac.type) {
5592         case ixgbe_mac_82598EB:
5593                 pci_wake_from_d3(pdev, false);
5594                 break;
5595         case ixgbe_mac_82599EB:
5596         case ixgbe_mac_X540:
5597                 pci_wake_from_d3(pdev, !!wufc);
5598                 break;
5599         default:
5600                 break;
5601         }
5602
5603         *enable_wake = !!wufc;
5604
5605         ixgbe_release_hw_control(adapter);
5606
5607         pci_disable_device(pdev);
5608
5609         return 0;
5610 }
5611
5612 #ifdef CONFIG_PM
5613 static int ixgbe_suspend(struct pci_dev *pdev, pm_message_t state)
5614 {
5615         int retval;
5616         bool wake;
5617
5618         retval = __ixgbe_shutdown(pdev, &wake);
5619         if (retval)
5620                 return retval;
5621
5622         if (wake) {
5623                 pci_prepare_to_sleep(pdev);
5624         } else {
5625                 pci_wake_from_d3(pdev, false);
5626                 pci_set_power_state(pdev, PCI_D3hot);
5627         }
5628
5629         return 0;
5630 }
5631 #endif /* CONFIG_PM */
5632
5633 static void ixgbe_shutdown(struct pci_dev *pdev)
5634 {
5635         bool wake;
5636
5637         __ixgbe_shutdown(pdev, &wake);
5638
5639         if (system_state == SYSTEM_POWER_OFF) {
5640                 pci_wake_from_d3(pdev, wake);
5641                 pci_set_power_state(pdev, PCI_D3hot);
5642         }
5643 }
5644
5645 /**
5646  * ixgbe_update_stats - Update the board statistics counters.
5647  * @adapter: board private structure
5648  **/
5649 void ixgbe_update_stats(struct ixgbe_adapter *adapter)
5650 {
5651         struct net_device *netdev = adapter->netdev;
5652         struct ixgbe_hw *hw = &adapter->hw;
5653         struct ixgbe_hw_stats *hwstats = &adapter->stats;
5654         u64 total_mpc = 0;
5655         u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;
5656         u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;
5657         u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;
5658         u64 bytes = 0, packets = 0;
5659
5660         if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5661             test_bit(__IXGBE_RESETTING, &adapter->state))
5662                 return;
5663
5664         if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {
5665                 u64 rsc_count = 0;
5666                 u64 rsc_flush = 0;
5667                 for (i = 0; i < 16; i++)
5668                         adapter->hw_rx_no_dma_resources +=
5669                                 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
5670                 for (i = 0; i < adapter->num_rx_queues; i++) {
5671                         rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;
5672                         rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;
5673                 }
5674                 adapter->rsc_total_count = rsc_count;
5675                 adapter->rsc_total_flush = rsc_flush;
5676         }
5677
5678         for (i = 0; i < adapter->num_rx_queues; i++) {
5679                 struct ixgbe_ring *rx_ring = adapter->rx_ring[i];
5680                 non_eop_descs += rx_ring->rx_stats.non_eop_descs;
5681                 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;
5682                 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;
5683                 bytes += rx_ring->stats.bytes;
5684                 packets += rx_ring->stats.packets;
5685         }
5686         adapter->non_eop_descs = non_eop_descs;
5687         adapter->alloc_rx_page_failed = alloc_rx_page_failed;
5688         adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;
5689         netdev->stats.rx_bytes = bytes;
5690         netdev->stats.rx_packets = packets;
5691
5692         bytes = 0;
5693         packets = 0;
5694         /* gather some stats to the adapter struct that are per queue */
5695         for (i = 0; i < adapter->num_tx_queues; i++) {
5696                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
5697                 restart_queue += tx_ring->tx_stats.restart_queue;
5698                 tx_busy += tx_ring->tx_stats.tx_busy;
5699                 bytes += tx_ring->stats.bytes;
5700                 packets += tx_ring->stats.packets;
5701         }
5702         adapter->restart_queue = restart_queue;
5703         adapter->tx_busy = tx_busy;
5704         netdev->stats.tx_bytes = bytes;
5705         netdev->stats.tx_packets = packets;
5706
5707         hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
5708         for (i = 0; i < 8; i++) {
5709                 /* for packet buffers not used, the register should read 0 */
5710                 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));
5711                 missed_rx += mpc;
5712                 hwstats->mpc[i] += mpc;
5713                 total_mpc += hwstats->mpc[i];
5714                 if (hw->mac.type == ixgbe_mac_82598EB)
5715                         hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));
5716                 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
5717                 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));
5718                 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
5719                 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));
5720                 switch (hw->mac.type) {
5721                 case ixgbe_mac_82598EB:
5722                         hwstats->pxonrxc[i] +=
5723                                 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
5724                         break;
5725                 case ixgbe_mac_82599EB:
5726                 case ixgbe_mac_X540:
5727                         hwstats->pxonrxc[i] +=
5728                                 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
5729                         break;
5730                 default:
5731                         break;
5732                 }
5733                 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
5734                 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
5735         }
5736         hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);
5737         /* work around hardware counting issue */
5738         hwstats->gprc -= missed_rx;
5739
5740         ixgbe_update_xoff_received(adapter);
5741
5742         /* 82598 hardware only has a 32 bit counter in the high register */
5743         switch (hw->mac.type) {
5744         case ixgbe_mac_82598EB:
5745                 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
5746                 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
5747                 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
5748                 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
5749                 break;
5750         case ixgbe_mac_X540:
5751                 /* OS2BMC stats are X540 only*/
5752                 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);
5753                 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);
5754                 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);
5755                 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);
5756         case ixgbe_mac_82599EB:
5757                 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
5758                 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */
5759                 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
5760                 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */
5761                 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
5762                 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */
5763                 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
5764                 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
5765                 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
5766 #ifdef IXGBE_FCOE
5767                 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
5768                 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
5769                 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
5770                 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
5771                 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
5772                 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
5773 #endif /* IXGBE_FCOE */
5774                 break;
5775         default:
5776                 break;
5777         }
5778         bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
5779         hwstats->bprc += bprc;
5780         hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
5781         if (hw->mac.type == ixgbe_mac_82598EB)
5782                 hwstats->mprc -= bprc;
5783         hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
5784         hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
5785         hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
5786         hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
5787         hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
5788         hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
5789         hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
5790         hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
5791         lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
5792         hwstats->lxontxc += lxon;
5793         lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
5794         hwstats->lxofftxc += lxoff;
5795         hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5796         hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);
5797         hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
5798         /*
5799          * 82598 errata - tx of flow control packets is included in tx counters
5800          */
5801         xon_off_tot = lxon + lxoff;
5802         hwstats->gptc -= xon_off_tot;
5803         hwstats->mptc -= xon_off_tot;
5804         hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));
5805         hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
5806         hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
5807         hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
5808         hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
5809         hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
5810         hwstats->ptc64 -= xon_off_tot;
5811         hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
5812         hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
5813         hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
5814         hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
5815         hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
5816         hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
5817
5818         /* Fill out the OS statistics structure */
5819         netdev->stats.multicast = hwstats->mprc;
5820
5821         /* Rx Errors */
5822         netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;
5823         netdev->stats.rx_dropped = 0;
5824         netdev->stats.rx_length_errors = hwstats->rlec;
5825         netdev->stats.rx_crc_errors = hwstats->crcerrs;
5826         netdev->stats.rx_missed_errors = total_mpc;
5827 }
5828
5829 /**
5830  * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table
5831  * @adapter - pointer to the device adapter structure
5832  **/
5833 static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)
5834 {
5835         struct ixgbe_hw *hw = &adapter->hw;
5836         int i;
5837
5838         if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
5839                 return;
5840
5841         adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
5842
5843         /* if interface is down do nothing */
5844         if (test_bit(__IXGBE_DOWN, &adapter->state))
5845                 return;
5846
5847         /* do nothing if we are not using signature filters */
5848         if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))
5849                 return;
5850
5851         adapter->fdir_overflow++;
5852
5853         if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {
5854                 for (i = 0; i < adapter->num_tx_queues; i++)
5855                         set_bit(__IXGBE_TX_FDIR_INIT_DONE,
5856                                 &(adapter->tx_ring[i]->state));
5857                 /* re-enable flow director interrupts */
5858                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);
5859         } else {
5860                 e_err(probe, "failed to finish FDIR re-initialization, "
5861                       "ignored adding FDIR ATR filters\n");
5862         }
5863 }
5864
5865 /**
5866  * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts
5867  * @adapter - pointer to the device adapter structure
5868  *
5869  * This function serves two purposes.  First it strobes the interrupt lines
5870  * in order to make certain interrupts are occuring.  Secondly it sets the
5871  * bits needed to check for TX hangs.  As a result we should immediately
5872  * determine if a hang has occured.
5873  */
5874 static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)
5875 {
5876         struct ixgbe_hw *hw = &adapter->hw;
5877         u64 eics = 0;
5878         int i;
5879
5880         /* If we're down or resetting, just bail */
5881         if (test_bit(__IXGBE_DOWN, &adapter->state) ||
5882             test_bit(__IXGBE_RESETTING, &adapter->state))
5883                 return;
5884
5885         /* Force detection of hung controller */
5886         if (netif_carrier_ok(adapter->netdev)) {
5887                 for (i = 0; i < adapter->num_tx_queues; i++)
5888                         set_check_for_tx_hang(adapter->tx_ring[i]);
5889         }
5890
5891         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {
5892                 /*
5893                  * for legacy and MSI interrupts don't set any bits
5894                  * that are enabled for EIAM, because this operation
5895                  * would set *both* EIMS and EICS for any bit in EIAM
5896                  */
5897                 IXGBE_WRITE_REG(hw, IXGBE_EICS,
5898                         (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));
5899         } else {
5900                 /* get one bit for every active tx/rx interrupt vector */
5901                 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
5902                         struct ixgbe_q_vector *qv = adapter->q_vector[i];
5903                         if (qv->rx.count || qv->tx.count)
5904                                 eics |= ((u64)1 << i);
5905                 }
5906         }
5907
5908         /* Cause software interrupt to ensure rings are cleaned */
5909         ixgbe_irq_rearm_queues(adapter, eics);
5910
5911 }
5912
5913 /**
5914  * ixgbe_watchdog_update_link - update the link status
5915  * @adapter - pointer to the device adapter structure
5916  * @link_speed - pointer to a u32 to store the link_speed
5917  **/
5918 static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)
5919 {
5920         struct ixgbe_hw *hw = &adapter->hw;
5921         u32 link_speed = adapter->link_speed;
5922         bool link_up = adapter->link_up;
5923         int i;
5924
5925         if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))
5926                 return;
5927
5928         if (hw->mac.ops.check_link) {
5929                 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
5930         } else {
5931                 /* always assume link is up, if no check link function */
5932                 link_speed = IXGBE_LINK_SPEED_10GB_FULL;
5933                 link_up = true;
5934         }
5935         if (link_up) {
5936                 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
5937                         for (i = 0; i < MAX_TRAFFIC_CLASS; i++)
5938                                 hw->mac.ops.fc_enable(hw, i);
5939                 } else {
5940                         hw->mac.ops.fc_enable(hw, 0);
5941                 }
5942         }
5943
5944         if (link_up ||
5945             time_after(jiffies, (adapter->link_check_timeout +
5946                                  IXGBE_TRY_LINK_TIMEOUT))) {
5947                 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
5948                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);
5949                 IXGBE_WRITE_FLUSH(hw);
5950         }
5951
5952         adapter->link_up = link_up;
5953         adapter->link_speed = link_speed;
5954 }
5955
5956 /**
5957  * ixgbe_watchdog_link_is_up - update netif_carrier status and
5958  *                             print link up message
5959  * @adapter - pointer to the device adapter structure
5960  **/
5961 static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
5962 {
5963         struct net_device *netdev = adapter->netdev;
5964         struct ixgbe_hw *hw = &adapter->hw;
5965         u32 link_speed = adapter->link_speed;
5966         bool flow_rx, flow_tx;
5967
5968         /* only continue if link was previously down */
5969         if (netif_carrier_ok(netdev))
5970                 return;
5971
5972         adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
5973
5974         switch (hw->mac.type) {
5975         case ixgbe_mac_82598EB: {
5976                 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
5977                 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);
5978                 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);
5979                 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);
5980         }
5981                 break;
5982         case ixgbe_mac_X540:
5983         case ixgbe_mac_82599EB: {
5984                 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
5985                 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
5986                 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);
5987                 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);
5988         }
5989                 break;
5990         default:
5991                 flow_tx = false;
5992                 flow_rx = false;
5993                 break;
5994         }
5995         e_info(drv, "NIC Link is Up %s, Flow Control: %s\n",
5996                (link_speed == IXGBE_LINK_SPEED_10GB_FULL ?
5997                "10 Gbps" :
5998                (link_speed == IXGBE_LINK_SPEED_1GB_FULL ?
5999                "1 Gbps" :
6000                (link_speed == IXGBE_LINK_SPEED_100_FULL ?
6001                "100 Mbps" :
6002                "unknown speed"))),
6003                ((flow_rx && flow_tx) ? "RX/TX" :
6004                (flow_rx ? "RX" :
6005                (flow_tx ? "TX" : "None"))));
6006
6007         netif_carrier_on(netdev);
6008         ixgbe_check_vf_rate_limit(adapter);
6009 }
6010
6011 /**
6012  * ixgbe_watchdog_link_is_down - update netif_carrier status and
6013  *                               print link down message
6014  * @adapter - pointer to the adapter structure
6015  **/
6016 static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter* adapter)
6017 {
6018         struct net_device *netdev = adapter->netdev;
6019         struct ixgbe_hw *hw = &adapter->hw;
6020
6021         adapter->link_up = false;
6022         adapter->link_speed = 0;
6023
6024         /* only continue if link was up previously */
6025         if (!netif_carrier_ok(netdev))
6026                 return;
6027
6028         /* poll for SFP+ cable when link is down */
6029         if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)
6030                 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;
6031
6032         e_info(drv, "NIC Link is Down\n");
6033         netif_carrier_off(netdev);
6034 }
6035
6036 /**
6037  * ixgbe_watchdog_flush_tx - flush queues on link down
6038  * @adapter - pointer to the device adapter structure
6039  **/
6040 static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)
6041 {
6042         int i;
6043         int some_tx_pending = 0;
6044
6045         if (!netif_carrier_ok(adapter->netdev)) {
6046                 for (i = 0; i < adapter->num_tx_queues; i++) {
6047                         struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
6048                         if (tx_ring->next_to_use != tx_ring->next_to_clean) {
6049                                 some_tx_pending = 1;
6050                                 break;
6051                         }
6052                 }
6053
6054                 if (some_tx_pending) {
6055                         /* We've lost link, so the controller stops DMA,
6056                          * but we've got queued Tx work that's never going
6057                          * to get done, so reset controller to flush Tx.
6058                          * (Do the reset outside of interrupt context).
6059                          */
6060                         adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
6061                 }
6062         }
6063 }
6064
6065 static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)
6066 {
6067         u32 ssvpc;
6068
6069         /* Do not perform spoof check for 82598 */
6070         if (adapter->hw.mac.type == ixgbe_mac_82598EB)
6071                 return;
6072
6073         ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);
6074
6075         /*
6076          * ssvpc register is cleared on read, if zero then no
6077          * spoofed packets in the last interval.
6078          */
6079         if (!ssvpc)
6080                 return;
6081
6082         e_warn(drv, "%d Spoofed packets detected\n", ssvpc);
6083 }
6084
6085 /**
6086  * ixgbe_watchdog_subtask - check and bring link up
6087  * @adapter - pointer to the device adapter structure
6088  **/
6089 static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)
6090 {
6091         /* if interface is down do nothing */
6092         if (test_bit(__IXGBE_DOWN, &adapter->state))
6093                 return;
6094
6095         ixgbe_watchdog_update_link(adapter);
6096
6097         if (adapter->link_up)
6098                 ixgbe_watchdog_link_is_up(adapter);
6099         else
6100                 ixgbe_watchdog_link_is_down(adapter);
6101
6102         ixgbe_spoof_check(adapter);
6103         ixgbe_update_stats(adapter);
6104
6105         ixgbe_watchdog_flush_tx(adapter);
6106 }
6107
6108 /**
6109  * ixgbe_sfp_detection_subtask - poll for SFP+ cable
6110  * @adapter - the ixgbe adapter structure
6111  **/
6112 static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)
6113 {
6114         struct ixgbe_hw *hw = &adapter->hw;
6115         s32 err;
6116
6117         /* not searching for SFP so there is nothing to do here */
6118         if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&
6119             !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
6120                 return;
6121
6122         /* someone else is in init, wait until next service event */
6123         if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
6124                 return;
6125
6126         err = hw->phy.ops.identify_sfp(hw);
6127         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
6128                 goto sfp_out;
6129
6130         if (err == IXGBE_ERR_SFP_NOT_PRESENT) {
6131                 /* If no cable is present, then we need to reset
6132                  * the next time we find a good cable. */
6133                 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
6134         }
6135
6136         /* exit on error */
6137         if (err)
6138                 goto sfp_out;
6139
6140         /* exit if reset not needed */
6141         if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))
6142                 goto sfp_out;
6143
6144         adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;
6145
6146         /*
6147          * A module may be identified correctly, but the EEPROM may not have
6148          * support for that module.  setup_sfp() will fail in that case, so
6149          * we should not allow that module to load.
6150          */
6151         if (hw->mac.type == ixgbe_mac_82598EB)
6152                 err = hw->phy.ops.reset(hw);
6153         else
6154                 err = hw->mac.ops.setup_sfp(hw);
6155
6156         if (err == IXGBE_ERR_SFP_NOT_SUPPORTED)
6157                 goto sfp_out;
6158
6159         adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
6160         e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);
6161
6162 sfp_out:
6163         clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
6164
6165         if ((err == IXGBE_ERR_SFP_NOT_SUPPORTED) &&
6166             (adapter->netdev->reg_state == NETREG_REGISTERED)) {
6167                 e_dev_err("failed to initialize because an unsupported "
6168                           "SFP+ module type was detected.\n");
6169                 e_dev_err("Reload the driver after installing a "
6170                           "supported module.\n");
6171                 unregister_netdev(adapter->netdev);
6172         }
6173 }
6174
6175 /**
6176  * ixgbe_sfp_link_config_subtask - set up link SFP after module install
6177  * @adapter - the ixgbe adapter structure
6178  **/
6179 static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)
6180 {
6181         struct ixgbe_hw *hw = &adapter->hw;
6182         u32 autoneg;
6183         bool negotiation;
6184
6185         if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))
6186                 return;
6187
6188         /* someone else is in init, wait until next service event */
6189         if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))
6190                 return;
6191
6192         adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;
6193
6194         autoneg = hw->phy.autoneg_advertised;
6195         if ((!autoneg) && (hw->mac.ops.get_link_capabilities))
6196                 hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiation);
6197         hw->mac.autotry_restart = false;
6198         if (hw->mac.ops.setup_link)
6199                 hw->mac.ops.setup_link(hw, autoneg, negotiation, true);
6200
6201         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
6202         adapter->link_check_timeout = jiffies;
6203         clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);
6204 }
6205
6206 /**
6207  * ixgbe_service_timer - Timer Call-back
6208  * @data: pointer to adapter cast into an unsigned long
6209  **/
6210 static void ixgbe_service_timer(unsigned long data)
6211 {
6212         struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)data;
6213         unsigned long next_event_offset;
6214
6215         /* poll faster when waiting for link */
6216         if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
6217                 next_event_offset = HZ / 10;
6218         else
6219                 next_event_offset = HZ * 2;
6220
6221         /* Reset the timer */
6222         mod_timer(&adapter->service_timer, next_event_offset + jiffies);
6223
6224         ixgbe_service_event_schedule(adapter);
6225 }
6226
6227 static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)
6228 {
6229         if (!(adapter->flags2 & IXGBE_FLAG2_RESET_REQUESTED))
6230                 return;
6231
6232         adapter->flags2 &= ~IXGBE_FLAG2_RESET_REQUESTED;
6233
6234         /* If we're already down or resetting, just bail */
6235         if (test_bit(__IXGBE_DOWN, &adapter->state) ||
6236             test_bit(__IXGBE_RESETTING, &adapter->state))
6237                 return;
6238
6239         ixgbe_dump(adapter);
6240         netdev_err(adapter->netdev, "Reset adapter\n");
6241         adapter->tx_timeout_count++;
6242
6243         ixgbe_reinit_locked(adapter);
6244 }
6245
6246 /**
6247  * ixgbe_service_task - manages and runs subtasks
6248  * @work: pointer to work_struct containing our data
6249  **/
6250 static void ixgbe_service_task(struct work_struct *work)
6251 {
6252         struct ixgbe_adapter *adapter = container_of(work,
6253                                                      struct ixgbe_adapter,
6254                                                      service_task);
6255
6256         ixgbe_reset_subtask(adapter);
6257         ixgbe_sfp_detection_subtask(adapter);
6258         ixgbe_sfp_link_config_subtask(adapter);
6259         ixgbe_check_overtemp_subtask(adapter);
6260         ixgbe_watchdog_subtask(adapter);
6261         ixgbe_fdir_reinit_subtask(adapter);
6262         ixgbe_check_hang_subtask(adapter);
6263
6264         ixgbe_service_event_complete(adapter);
6265 }
6266
6267 void ixgbe_tx_ctxtdesc(struct ixgbe_ring *tx_ring, u32 vlan_macip_lens,
6268                        u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)
6269 {
6270         struct ixgbe_adv_tx_context_desc *context_desc;
6271         u16 i = tx_ring->next_to_use;
6272
6273         context_desc = IXGBE_TX_CTXTDESC_ADV(tx_ring, i);
6274
6275         i++;
6276         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
6277
6278         /* set bits to identify this as an advanced context descriptor */
6279         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
6280
6281         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
6282         context_desc->seqnum_seed       = cpu_to_le32(fcoe_sof_eof);
6283         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
6284         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
6285 }
6286
6287 static int ixgbe_tso(struct ixgbe_ring *tx_ring, struct sk_buff *skb,
6288                      u32 tx_flags, __be16 protocol, u8 *hdr_len)
6289 {
6290         int err;
6291         u32 vlan_macip_lens, type_tucmd;
6292         u32 mss_l4len_idx, l4len;
6293
6294         if (!skb_is_gso(skb))
6295                 return 0;
6296
6297         if (skb_header_cloned(skb)) {
6298                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
6299                 if (err)
6300                         return err;
6301         }
6302
6303         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6304         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
6305
6306         if (protocol == __constant_htons(ETH_P_IP)) {
6307                 struct iphdr *iph = ip_hdr(skb);
6308                 iph->tot_len = 0;
6309                 iph->check = 0;
6310                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
6311                                                          iph->daddr, 0,
6312                                                          IPPROTO_TCP,
6313                                                          0);
6314                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
6315         } else if (skb_is_gso_v6(skb)) {
6316                 ipv6_hdr(skb)->payload_len = 0;
6317                 tcp_hdr(skb)->check =
6318                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
6319                                      &ipv6_hdr(skb)->daddr,
6320                                      0, IPPROTO_TCP, 0);
6321         }
6322
6323         l4len = tcp_hdrlen(skb);
6324         *hdr_len = skb_transport_offset(skb) + l4len;
6325
6326         /* mss_l4len_id: use 1 as index for TSO */
6327         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
6328         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
6329         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
6330
6331         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
6332         vlan_macip_lens = skb_network_header_len(skb);
6333         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
6334         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
6335
6336         ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0, type_tucmd,
6337                           mss_l4len_idx);
6338
6339         return 1;
6340 }
6341
6342 static bool ixgbe_tx_csum(struct ixgbe_ring *tx_ring,
6343                           struct sk_buff *skb, u32 tx_flags,
6344                           __be16 protocol)
6345 {
6346         u32 vlan_macip_lens = 0;
6347         u32 mss_l4len_idx = 0;
6348         u32 type_tucmd = 0;
6349
6350         if (skb->ip_summed != CHECKSUM_PARTIAL) {
6351             if (!(tx_flags & IXGBE_TX_FLAGS_VLAN))
6352                         return false;
6353         } else {
6354                 u8 l4_hdr = 0;
6355                 switch (protocol) {
6356                 case __constant_htons(ETH_P_IP):
6357                         vlan_macip_lens |= skb_network_header_len(skb);
6358                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
6359                         l4_hdr = ip_hdr(skb)->protocol;
6360                         break;
6361                 case __constant_htons(ETH_P_IPV6):
6362                         vlan_macip_lens |= skb_network_header_len(skb);
6363                         l4_hdr = ipv6_hdr(skb)->nexthdr;
6364                         break;
6365                 default:
6366                         if (unlikely(net_ratelimit())) {
6367                                 dev_warn(tx_ring->dev,
6368                                  "partial checksum but proto=%x!\n",
6369                                  skb->protocol);
6370                         }
6371                         break;
6372                 }
6373
6374                 switch (l4_hdr) {
6375                 case IPPROTO_TCP:
6376                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
6377                         mss_l4len_idx = tcp_hdrlen(skb) <<
6378                                         IXGBE_ADVTXD_L4LEN_SHIFT;
6379                         break;
6380                 case IPPROTO_SCTP:
6381                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
6382                         mss_l4len_idx = sizeof(struct sctphdr) <<
6383                                         IXGBE_ADVTXD_L4LEN_SHIFT;
6384                         break;
6385                 case IPPROTO_UDP:
6386                         mss_l4len_idx = sizeof(struct udphdr) <<
6387                                         IXGBE_ADVTXD_L4LEN_SHIFT;
6388                         break;
6389                 default:
6390                         if (unlikely(net_ratelimit())) {
6391                                 dev_warn(tx_ring->dev,
6392                                  "partial checksum but l4 proto=%x!\n",
6393                                  skb->protocol);
6394                         }
6395                         break;
6396                 }
6397         }
6398
6399         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
6400         vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
6401
6402         ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, 0,
6403                           type_tucmd, mss_l4len_idx);
6404
6405         return (skb->ip_summed == CHECKSUM_PARTIAL);
6406 }
6407
6408 static int ixgbe_tx_map(struct ixgbe_adapter *adapter,
6409                         struct ixgbe_ring *tx_ring,
6410                         struct sk_buff *skb, u32 tx_flags,
6411                         unsigned int first, const u8 hdr_len)
6412 {
6413         struct device *dev = tx_ring->dev;
6414         struct ixgbe_tx_buffer *tx_buffer_info;
6415         unsigned int len;
6416         unsigned int total = skb->len;
6417         unsigned int offset = 0, size, count = 0;
6418         unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
6419         unsigned int f;
6420         unsigned int bytecount = skb->len;
6421         u16 gso_segs = 1;
6422         u16 i;
6423
6424         i = tx_ring->next_to_use;
6425
6426         if (tx_flags & IXGBE_TX_FLAGS_FCOE)
6427                 /* excluding fcoe_crc_eof for FCoE */
6428                 total -= sizeof(struct fcoe_crc_eof);
6429
6430         len = min(skb_headlen(skb), total);
6431         while (len) {
6432                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6433                 size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
6434
6435                 tx_buffer_info->length = size;
6436                 tx_buffer_info->mapped_as_page = false;
6437                 tx_buffer_info->dma = dma_map_single(dev,
6438                                                      skb->data + offset,
6439                                                      size, DMA_TO_DEVICE);
6440                 if (dma_mapping_error(dev, tx_buffer_info->dma))
6441                         goto dma_error;
6442                 tx_buffer_info->time_stamp = jiffies;
6443                 tx_buffer_info->next_to_watch = i;
6444
6445                 len -= size;
6446                 total -= size;
6447                 offset += size;
6448                 count++;
6449
6450                 if (len) {
6451                         i++;
6452                         if (i == tx_ring->count)
6453                                 i = 0;
6454                 }
6455         }
6456
6457         for (f = 0; f < nr_frags; f++) {
6458                 struct skb_frag_struct *frag;
6459
6460                 frag = &skb_shinfo(skb)->frags[f];
6461                 len = min((unsigned int)frag->size, total);
6462                 offset = frag->page_offset;
6463
6464                 while (len) {
6465                         i++;
6466                         if (i == tx_ring->count)
6467                                 i = 0;
6468
6469                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
6470                         size = min(len, (uint)IXGBE_MAX_DATA_PER_TXD);
6471
6472                         tx_buffer_info->length = size;
6473                         tx_buffer_info->dma = dma_map_page(dev,
6474                                                            frag->page,
6475                                                            offset, size,
6476                                                            DMA_TO_DEVICE);
6477                         tx_buffer_info->mapped_as_page = true;
6478                         if (dma_mapping_error(dev, tx_buffer_info->dma))
6479                                 goto dma_error;
6480                         tx_buffer_info->time_stamp = jiffies;
6481                         tx_buffer_info->next_to_watch = i;
6482
6483                         len -= size;
6484                         total -= size;
6485                         offset += size;
6486                         count++;
6487                 }
6488                 if (total == 0)
6489                         break;
6490         }
6491
6492         if (tx_flags & IXGBE_TX_FLAGS_TSO)
6493                 gso_segs = skb_shinfo(skb)->gso_segs;
6494 #ifdef IXGBE_FCOE
6495         /* adjust for FCoE Sequence Offload */
6496         else if (tx_flags & IXGBE_TX_FLAGS_FSO)
6497                 gso_segs = DIV_ROUND_UP(skb->len - hdr_len,
6498                                         skb_shinfo(skb)->gso_size);
6499 #endif /* IXGBE_FCOE */
6500         bytecount += (gso_segs - 1) * hdr_len;
6501
6502         /* multiply data chunks by size of headers */
6503         tx_ring->tx_buffer_info[i].bytecount = bytecount;
6504         tx_ring->tx_buffer_info[i].gso_segs = gso_segs;
6505         tx_ring->tx_buffer_info[i].skb = skb;
6506         tx_ring->tx_buffer_info[first].next_to_watch = i;
6507
6508         return count;
6509
6510 dma_error:
6511         e_dev_err("TX DMA map failed\n");
6512
6513         /* clear timestamp and dma mappings for failed tx_buffer_info map */
6514         tx_buffer_info->dma = 0;
6515         tx_buffer_info->time_stamp = 0;
6516         tx_buffer_info->next_to_watch = 0;
6517         if (count)
6518                 count--;
6519
6520         /* clear timestamp and dma mappings for remaining portion of packet */
6521         while (count--) {
6522                 if (i == 0)
6523                         i += tx_ring->count;
6524                 i--;
6525                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6526                 ixgbe_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
6527         }
6528
6529         return 0;
6530 }
6531
6532 static void ixgbe_tx_queue(struct ixgbe_ring *tx_ring,
6533                            int tx_flags, int count, u32 paylen, u8 hdr_len)
6534 {
6535         union ixgbe_adv_tx_desc *tx_desc = NULL;
6536         struct ixgbe_tx_buffer *tx_buffer_info;
6537         u32 olinfo_status = 0, cmd_type_len = 0;
6538         unsigned int i;
6539         u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS;
6540
6541         cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA;
6542
6543         cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT;
6544
6545         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
6546                 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
6547
6548         if (tx_flags & IXGBE_TX_FLAGS_TSO) {
6549                 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
6550
6551                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
6552                                  IXGBE_ADVTXD_POPTS_SHIFT;
6553
6554                 /* use index 1 context for tso */
6555                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
6556                 if (tx_flags & IXGBE_TX_FLAGS_IPV4)
6557                         olinfo_status |= IXGBE_TXD_POPTS_IXSM <<
6558                                          IXGBE_ADVTXD_POPTS_SHIFT;
6559
6560         } else if (tx_flags & IXGBE_TX_FLAGS_CSUM)
6561                 olinfo_status |= IXGBE_TXD_POPTS_TXSM <<
6562                                  IXGBE_ADVTXD_POPTS_SHIFT;
6563
6564         if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
6565                 olinfo_status |= IXGBE_ADVTXD_CC;
6566                 olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT);
6567                 if (tx_flags & IXGBE_TX_FLAGS_FSO)
6568                         cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
6569         }
6570
6571         olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT);
6572
6573         i = tx_ring->next_to_use;
6574         while (count--) {
6575                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
6576                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
6577                 tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma);
6578                 tx_desc->read.cmd_type_len =
6579                         cpu_to_le32(cmd_type_len | tx_buffer_info->length);
6580                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6581                 i++;
6582                 if (i == tx_ring->count)
6583                         i = 0;
6584         }
6585
6586         tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd);
6587
6588         /*
6589          * Force memory writes to complete before letting h/w
6590          * know there are new descriptors to fetch.  (Only
6591          * applicable for weak-ordered memory model archs,
6592          * such as IA-64).
6593          */
6594         wmb();
6595
6596         tx_ring->next_to_use = i;
6597         writel(i, tx_ring->tail);
6598 }
6599
6600 static void ixgbe_atr(struct ixgbe_ring *ring, struct sk_buff *skb,
6601                       u32 tx_flags, __be16 protocol)
6602 {
6603         struct ixgbe_q_vector *q_vector = ring->q_vector;
6604         union ixgbe_atr_hash_dword input = { .dword = 0 };
6605         union ixgbe_atr_hash_dword common = { .dword = 0 };
6606         union {
6607                 unsigned char *network;
6608                 struct iphdr *ipv4;
6609                 struct ipv6hdr *ipv6;
6610         } hdr;
6611         struct tcphdr *th;
6612         __be16 vlan_id;
6613
6614         /* if ring doesn't have a interrupt vector, cannot perform ATR */
6615         if (!q_vector)
6616                 return;
6617
6618         /* do nothing if sampling is disabled */
6619         if (!ring->atr_sample_rate)
6620                 return;
6621
6622         ring->atr_count++;
6623
6624         /* snag network header to get L4 type and address */
6625         hdr.network = skb_network_header(skb);
6626
6627         /* Currently only IPv4/IPv6 with TCP is supported */
6628         if ((protocol != __constant_htons(ETH_P_IPV6) ||
6629              hdr.ipv6->nexthdr != IPPROTO_TCP) &&
6630             (protocol != __constant_htons(ETH_P_IP) ||
6631              hdr.ipv4->protocol != IPPROTO_TCP))
6632                 return;
6633
6634         th = tcp_hdr(skb);
6635
6636         /* skip this packet since the socket is closing */
6637         if (th->fin)
6638                 return;
6639
6640         /* sample on all syn packets or once every atr sample count */
6641         if (!th->syn && (ring->atr_count < ring->atr_sample_rate))
6642                 return;
6643
6644         /* reset sample count */
6645         ring->atr_count = 0;
6646
6647         vlan_id = htons(tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);
6648
6649         /*
6650          * src and dst are inverted, think how the receiver sees them
6651          *
6652          * The input is broken into two sections, a non-compressed section
6653          * containing vm_pool, vlan_id, and flow_type.  The rest of the data
6654          * is XORed together and stored in the compressed dword.
6655          */
6656         input.formatted.vlan_id = vlan_id;
6657
6658         /*
6659          * since src port and flex bytes occupy the same word XOR them together
6660          * and write the value to source port portion of compressed dword
6661          */
6662         if (vlan_id)
6663                 common.port.src ^= th->dest ^ __constant_htons(ETH_P_8021Q);
6664         else
6665                 common.port.src ^= th->dest ^ protocol;
6666         common.port.dst ^= th->source;
6667
6668         if (protocol == __constant_htons(ETH_P_IP)) {
6669                 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;
6670                 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;
6671         } else {
6672                 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;
6673                 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^
6674                              hdr.ipv6->saddr.s6_addr32[1] ^
6675                              hdr.ipv6->saddr.s6_addr32[2] ^
6676                              hdr.ipv6->saddr.s6_addr32[3] ^
6677                              hdr.ipv6->daddr.s6_addr32[0] ^
6678                              hdr.ipv6->daddr.s6_addr32[1] ^
6679                              hdr.ipv6->daddr.s6_addr32[2] ^
6680                              hdr.ipv6->daddr.s6_addr32[3];
6681         }
6682
6683         /* This assumes the Rx queue and Tx queue are bound to the same CPU */
6684         ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,
6685                                               input, common, ring->queue_index);
6686 }
6687
6688 static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
6689 {
6690         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
6691         /* Herbert's original patch had:
6692          *  smp_mb__after_netif_stop_queue();
6693          * but since that doesn't exist yet, just open code it. */
6694         smp_mb();
6695
6696         /* We need to check again in a case another CPU has just
6697          * made room available. */
6698         if (likely(ixgbe_desc_unused(tx_ring) < size))
6699                 return -EBUSY;
6700
6701         /* A reprieve! - use start_queue because it doesn't call schedule */
6702         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
6703         ++tx_ring->tx_stats.restart_queue;
6704         return 0;
6705 }
6706
6707 static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)
6708 {
6709         if (likely(ixgbe_desc_unused(tx_ring) >= size))
6710                 return 0;
6711         return __ixgbe_maybe_stop_tx(tx_ring, size);
6712 }
6713
6714 static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb)
6715 {
6716         struct ixgbe_adapter *adapter = netdev_priv(dev);
6717         int txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :
6718                                                smp_processor_id();
6719 #ifdef IXGBE_FCOE
6720         __be16 protocol = vlan_get_protocol(skb);
6721
6722         if (((protocol == htons(ETH_P_FCOE)) ||
6723             (protocol == htons(ETH_P_FIP))) &&
6724             (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)) {
6725                 txq &= (adapter->ring_feature[RING_F_FCOE].indices - 1);
6726                 txq += adapter->ring_feature[RING_F_FCOE].mask;
6727                 return txq;
6728         }
6729 #endif
6730
6731         if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
6732                 while (unlikely(txq >= dev->real_num_tx_queues))
6733                         txq -= dev->real_num_tx_queues;
6734                 return txq;
6735         }
6736
6737         return skb_tx_hash(dev, skb);
6738 }
6739
6740 netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,
6741                           struct ixgbe_adapter *adapter,
6742                           struct ixgbe_ring *tx_ring)
6743 {
6744         int tso;
6745         u32  tx_flags = 0;
6746 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
6747         unsigned short f;
6748 #endif
6749         u16 first;
6750         u16 count = TXD_USE_COUNT(skb_headlen(skb));
6751         __be16 protocol;
6752         u8 hdr_len = 0;
6753
6754         /*
6755          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
6756          *       + 1 desc for skb_head_len/IXGBE_MAX_DATA_PER_TXD,
6757          *       + 2 desc gap to keep tail from touching head,
6758          *       + 1 desc for context descriptor,
6759          * otherwise try next time
6760          */
6761 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
6762         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6763                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
6764 #else
6765         count += skb_shinfo(skb)->nr_frags;
6766 #endif
6767         if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {
6768                 tx_ring->tx_stats.tx_busy++;
6769                 return NETDEV_TX_BUSY;
6770         }
6771
6772         protocol = vlan_get_protocol(skb);
6773
6774         if (vlan_tx_tag_present(skb)) {
6775                 tx_flags |= vlan_tx_tag_get(skb);
6776                 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED) {
6777                         tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;
6778                         tx_flags |= tx_ring->dcb_tc << 13;
6779                 }
6780                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
6781                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
6782         } else if (adapter->flags & IXGBE_FLAG_DCB_ENABLED &&
6783                    skb->priority != TC_PRIO_CONTROL) {
6784                 tx_flags |= tx_ring->dcb_tc << 13;
6785                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
6786                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
6787         }
6788
6789 #ifdef IXGBE_FCOE
6790         /* for FCoE with DCB, we force the priority to what
6791          * was specified by the switch */
6792         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED &&
6793             (protocol == htons(ETH_P_FCOE)))
6794                 tx_flags |= IXGBE_TX_FLAGS_FCOE;
6795
6796 #endif
6797         /* record the location of the first descriptor for this packet */
6798         first = tx_ring->next_to_use;
6799
6800         if (tx_flags & IXGBE_TX_FLAGS_FCOE) {
6801 #ifdef IXGBE_FCOE
6802                 /* setup tx offload for FCoE */
6803                 tso = ixgbe_fso(tx_ring, skb, tx_flags, &hdr_len);
6804                 if (tso < 0)
6805                         goto out_drop;
6806                 else if (tso)
6807                         tx_flags |= IXGBE_TX_FLAGS_FSO;
6808 #endif /* IXGBE_FCOE */
6809         } else {
6810                 if (protocol == htons(ETH_P_IP))
6811                         tx_flags |= IXGBE_TX_FLAGS_IPV4;
6812                 tso = ixgbe_tso(tx_ring, skb, tx_flags, protocol, &hdr_len);
6813                 if (tso < 0)
6814                         goto out_drop;
6815                 else if (tso)
6816                         tx_flags |= IXGBE_TX_FLAGS_TSO;
6817                 else if (ixgbe_tx_csum(tx_ring, skb, tx_flags, protocol))
6818                         tx_flags |= IXGBE_TX_FLAGS_CSUM;
6819         }
6820
6821         count = ixgbe_tx_map(adapter, tx_ring, skb, tx_flags, first, hdr_len);
6822         if (count) {
6823                 /* add the ATR filter if ATR is on */
6824                 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))
6825                         ixgbe_atr(tx_ring, skb, tx_flags, protocol);
6826                 ixgbe_tx_queue(tx_ring, tx_flags, count, skb->len, hdr_len);
6827                 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);
6828
6829         } else {
6830                 tx_ring->tx_buffer_info[first].time_stamp = 0;
6831                 tx_ring->next_to_use = first;
6832                 goto out_drop;
6833         }
6834
6835         return NETDEV_TX_OK;
6836
6837 out_drop:
6838         dev_kfree_skb_any(skb);
6839         return NETDEV_TX_OK;
6840 }
6841
6842 static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
6843 {
6844         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6845         struct ixgbe_ring *tx_ring;
6846
6847         tx_ring = adapter->tx_ring[skb->queue_mapping];
6848         return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);
6849 }
6850
6851 /**
6852  * ixgbe_set_mac - Change the Ethernet Address of the NIC
6853  * @netdev: network interface device structure
6854  * @p: pointer to an address structure
6855  *
6856  * Returns 0 on success, negative on failure
6857  **/
6858 static int ixgbe_set_mac(struct net_device *netdev, void *p)
6859 {
6860         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6861         struct ixgbe_hw *hw = &adapter->hw;
6862         struct sockaddr *addr = p;
6863
6864         if (!is_valid_ether_addr(addr->sa_data))
6865                 return -EADDRNOTAVAIL;
6866
6867         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
6868         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
6869
6870         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, adapter->num_vfs,
6871                             IXGBE_RAH_AV);
6872
6873         return 0;
6874 }
6875
6876 static int
6877 ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)
6878 {
6879         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6880         struct ixgbe_hw *hw = &adapter->hw;
6881         u16 value;
6882         int rc;
6883
6884         if (prtad != hw->phy.mdio.prtad)
6885                 return -EINVAL;
6886         rc = hw->phy.ops.read_reg(hw, addr, devad, &value);
6887         if (!rc)
6888                 rc = value;
6889         return rc;
6890 }
6891
6892 static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,
6893                             u16 addr, u16 value)
6894 {
6895         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6896         struct ixgbe_hw *hw = &adapter->hw;
6897
6898         if (prtad != hw->phy.mdio.prtad)
6899                 return -EINVAL;
6900         return hw->phy.ops.write_reg(hw, addr, devad, value);
6901 }
6902
6903 static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
6904 {
6905         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6906
6907         return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);
6908 }
6909
6910 /**
6911  * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding
6912  * netdev->dev_addrs
6913  * @netdev: network interface device structure
6914  *
6915  * Returns non-zero on failure
6916  **/
6917 static int ixgbe_add_sanmac_netdev(struct net_device *dev)
6918 {
6919         int err = 0;
6920         struct ixgbe_adapter *adapter = netdev_priv(dev);
6921         struct ixgbe_mac_info *mac = &adapter->hw.mac;
6922
6923         if (is_valid_ether_addr(mac->san_addr)) {
6924                 rtnl_lock();
6925                 err = dev_addr_add(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
6926                 rtnl_unlock();
6927         }
6928         return err;
6929 }
6930
6931 /**
6932  * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding
6933  * netdev->dev_addrs
6934  * @netdev: network interface device structure
6935  *
6936  * Returns non-zero on failure
6937  **/
6938 static int ixgbe_del_sanmac_netdev(struct net_device *dev)
6939 {
6940         int err = 0;
6941         struct ixgbe_adapter *adapter = netdev_priv(dev);
6942         struct ixgbe_mac_info *mac = &adapter->hw.mac;
6943
6944         if (is_valid_ether_addr(mac->san_addr)) {
6945                 rtnl_lock();
6946                 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);
6947                 rtnl_unlock();
6948         }
6949         return err;
6950 }
6951
6952 #ifdef CONFIG_NET_POLL_CONTROLLER
6953 /*
6954  * Polling 'interrupt' - used by things like netconsole to send skbs
6955  * without having to re-enable interrupts. It's not called while
6956  * the interrupt routine is executing.
6957  */
6958 static void ixgbe_netpoll(struct net_device *netdev)
6959 {
6960         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6961         int i;
6962
6963         /* if interface is down do nothing */
6964         if (test_bit(__IXGBE_DOWN, &adapter->state))
6965                 return;
6966
6967         adapter->flags |= IXGBE_FLAG_IN_NETPOLL;
6968         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
6969                 int num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
6970                 for (i = 0; i < num_q_vectors; i++) {
6971                         struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
6972                         ixgbe_msix_clean_many(0, q_vector);
6973                 }
6974         } else {
6975                 ixgbe_intr(adapter->pdev->irq, netdev);
6976         }
6977         adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL;
6978 }
6979 #endif
6980
6981 static struct rtnl_link_stats64 *ixgbe_get_stats64(struct net_device *netdev,
6982                                                    struct rtnl_link_stats64 *stats)
6983 {
6984         struct ixgbe_adapter *adapter = netdev_priv(netdev);
6985         int i;
6986
6987         rcu_read_lock();
6988         for (i = 0; i < adapter->num_rx_queues; i++) {
6989                 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->rx_ring[i]);
6990                 u64 bytes, packets;
6991                 unsigned int start;
6992
6993                 if (ring) {
6994                         do {
6995                                 start = u64_stats_fetch_begin_bh(&ring->syncp);
6996                                 packets = ring->stats.packets;
6997                                 bytes   = ring->stats.bytes;
6998                         } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
6999                         stats->rx_packets += packets;
7000                         stats->rx_bytes   += bytes;
7001                 }
7002         }
7003
7004         for (i = 0; i < adapter->num_tx_queues; i++) {
7005                 struct ixgbe_ring *ring = ACCESS_ONCE(adapter->tx_ring[i]);
7006                 u64 bytes, packets;
7007                 unsigned int start;
7008
7009                 if (ring) {
7010                         do {
7011                                 start = u64_stats_fetch_begin_bh(&ring->syncp);
7012                                 packets = ring->stats.packets;
7013                                 bytes   = ring->stats.bytes;
7014                         } while (u64_stats_fetch_retry_bh(&ring->syncp, start));
7015                         stats->tx_packets += packets;
7016                         stats->tx_bytes   += bytes;
7017                 }
7018         }
7019         rcu_read_unlock();
7020         /* following stats updated by ixgbe_watchdog_task() */
7021         stats->multicast        = netdev->stats.multicast;
7022         stats->rx_errors        = netdev->stats.rx_errors;
7023         stats->rx_length_errors = netdev->stats.rx_length_errors;
7024         stats->rx_crc_errors    = netdev->stats.rx_crc_errors;
7025         stats->rx_missed_errors = netdev->stats.rx_missed_errors;
7026         return stats;
7027 }
7028
7029 /* ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.
7030  * #adapter: pointer to ixgbe_adapter
7031  * @tc: number of traffic classes currently enabled
7032  *
7033  * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm
7034  * 802.1Q priority maps to a packet buffer that exists.
7035  */
7036 static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)
7037 {
7038         struct ixgbe_hw *hw = &adapter->hw;
7039         u32 reg, rsave;
7040         int i;
7041
7042         /* 82598 have a static priority to TC mapping that can not
7043          * be changed so no validation is needed.
7044          */
7045         if (hw->mac.type == ixgbe_mac_82598EB)
7046                 return;
7047
7048         reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);
7049         rsave = reg;
7050
7051         for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {
7052                 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);
7053
7054                 /* If up2tc is out of bounds default to zero */
7055                 if (up2tc > tc)
7056                         reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);
7057         }
7058
7059         if (reg != rsave)
7060                 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);
7061
7062         return;
7063 }
7064
7065
7066 /* ixgbe_setup_tc - routine to configure net_device for multiple traffic
7067  * classes.
7068  *
7069  * @netdev: net device to configure
7070  * @tc: number of traffic classes to enable
7071  */
7072 int ixgbe_setup_tc(struct net_device *dev, u8 tc)
7073 {
7074         struct ixgbe_adapter *adapter = netdev_priv(dev);
7075         struct ixgbe_hw *hw = &adapter->hw;
7076
7077         /* If DCB is anabled do not remove traffic classes, multiple
7078          * traffic classes are required to implement DCB
7079          */
7080         if (!tc && (adapter->flags & IXGBE_FLAG_DCB_ENABLED))
7081                 return 0;
7082
7083         /* Hardware supports up to 8 traffic classes */
7084         if (tc > MAX_TRAFFIC_CLASS ||
7085             (hw->mac.type == ixgbe_mac_82598EB && tc < MAX_TRAFFIC_CLASS))
7086                 return -EINVAL;
7087
7088         /* Hardware has to reinitialize queues and interrupts to
7089          * match packet buffer alignment. Unfortunantly, the
7090          * hardware is not flexible enough to do this dynamically.
7091          */
7092         if (netif_running(dev))
7093                 ixgbe_close(dev);
7094         ixgbe_clear_interrupt_scheme(adapter);
7095
7096         if (tc)
7097                 netdev_set_num_tc(dev, tc);
7098         else
7099                 netdev_reset_tc(dev);
7100
7101         ixgbe_init_interrupt_scheme(adapter);
7102         ixgbe_validate_rtr(adapter, tc);
7103         if (netif_running(dev))
7104                 ixgbe_open(dev);
7105
7106         return 0;
7107 }
7108
7109 void ixgbe_do_reset(struct net_device *netdev)
7110 {
7111         struct ixgbe_adapter *adapter = netdev_priv(netdev);
7112
7113         if (netif_running(netdev))
7114                 ixgbe_reinit_locked(adapter);
7115         else
7116                 ixgbe_reset(adapter);
7117 }
7118
7119 static u32 ixgbe_fix_features(struct net_device *netdev, u32 data)
7120 {
7121         struct ixgbe_adapter *adapter = netdev_priv(netdev);
7122
7123 #ifdef CONFIG_DCB
7124         if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)
7125                 data &= ~NETIF_F_HW_VLAN_RX;
7126 #endif
7127
7128         /* return error if RXHASH is being enabled when RSS is not supported */
7129         if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED))
7130                 data &= ~NETIF_F_RXHASH;
7131
7132         /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
7133         if (!(data & NETIF_F_RXCSUM))
7134                 data &= ~NETIF_F_LRO;
7135
7136         /* Turn off LRO if not RSC capable or invalid ITR settings */
7137         if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)) {
7138                 data &= ~NETIF_F_LRO;
7139         } else if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
7140                    (adapter->rx_itr_setting != 1 &&
7141                     adapter->rx_itr_setting > IXGBE_MAX_RSC_INT_RATE)) {
7142                 data &= ~NETIF_F_LRO;
7143                 e_info(probe, "rx-usecs set too low, not enabling RSC\n");
7144         }
7145
7146         return data;
7147 }
7148
7149 static int ixgbe_set_features(struct net_device *netdev, u32 data)
7150 {
7151         struct ixgbe_adapter *adapter = netdev_priv(netdev);
7152         bool need_reset = false;
7153
7154         /* If Rx checksum is disabled, then RSC/LRO should also be disabled */
7155         if (!(data & NETIF_F_RXCSUM))
7156                 adapter->flags &= ~IXGBE_FLAG_RX_CSUM_ENABLED;
7157         else
7158                 adapter->flags |= IXGBE_FLAG_RX_CSUM_ENABLED;
7159
7160         /* Make sure RSC matches LRO, reset if change */
7161         if (!!(data & NETIF_F_LRO) !=
7162              !!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {
7163                 adapter->flags2 ^= IXGBE_FLAG2_RSC_ENABLED;
7164                 switch (adapter->hw.mac.type) {
7165                 case ixgbe_mac_X540:
7166                 case ixgbe_mac_82599EB:
7167                         need_reset = true;
7168                         break;
7169                 default:
7170                         break;
7171                 }
7172         }
7173
7174         /*
7175          * Check if Flow Director n-tuple support was enabled or disabled.  If
7176          * the state changed, we need to reset.
7177          */
7178         if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)) {
7179                 /* turn off ATR, enable perfect filters and reset */
7180                 if (data & NETIF_F_NTUPLE) {
7181                         adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;
7182                         adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
7183                         need_reset = true;
7184                 }
7185         } else if (!(data & NETIF_F_NTUPLE)) {
7186                 /* turn off Flow Director, set ATR and reset */
7187                 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;
7188                 if ((adapter->flags &  IXGBE_FLAG_RSS_ENABLED) &&
7189                     !(adapter->flags &  IXGBE_FLAG_DCB_ENABLED))
7190                         adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;
7191                 need_reset = true;
7192         }
7193
7194         if (need_reset)
7195                 ixgbe_do_reset(netdev);
7196
7197         return 0;
7198
7199 }
7200
7201 static const struct net_device_ops ixgbe_netdev_ops = {
7202         .ndo_open               = ixgbe_open,
7203         .ndo_stop               = ixgbe_close,
7204         .ndo_start_xmit         = ixgbe_xmit_frame,
7205         .ndo_select_queue       = ixgbe_select_queue,
7206         .ndo_set_rx_mode        = ixgbe_set_rx_mode,
7207         .ndo_set_multicast_list = ixgbe_set_rx_mode,
7208         .ndo_validate_addr      = eth_validate_addr,
7209         .ndo_set_mac_address    = ixgbe_set_mac,
7210         .ndo_change_mtu         = ixgbe_change_mtu,
7211         .ndo_tx_timeout         = ixgbe_tx_timeout,
7212         .ndo_vlan_rx_add_vid    = ixgbe_vlan_rx_add_vid,
7213         .ndo_vlan_rx_kill_vid   = ixgbe_vlan_rx_kill_vid,
7214         .ndo_do_ioctl           = ixgbe_ioctl,
7215         .ndo_set_vf_mac         = ixgbe_ndo_set_vf_mac,
7216         .ndo_set_vf_vlan        = ixgbe_ndo_set_vf_vlan,
7217         .ndo_set_vf_tx_rate     = ixgbe_ndo_set_vf_bw,
7218         .ndo_get_vf_config      = ixgbe_ndo_get_vf_config,
7219         .ndo_get_stats64        = ixgbe_get_stats64,
7220         .ndo_setup_tc           = ixgbe_setup_tc,
7221 #ifdef CONFIG_NET_POLL_CONTROLLER
7222         .ndo_poll_controller    = ixgbe_netpoll,
7223 #endif
7224 #ifdef IXGBE_FCOE
7225         .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,
7226         .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,
7227         .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,
7228         .ndo_fcoe_enable = ixgbe_fcoe_enable,
7229         .ndo_fcoe_disable = ixgbe_fcoe_disable,
7230         .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,
7231 #endif /* IXGBE_FCOE */
7232         .ndo_set_features = ixgbe_set_features,
7233         .ndo_fix_features = ixgbe_fix_features,
7234 };
7235
7236 static void __devinit ixgbe_probe_vf(struct ixgbe_adapter *adapter,
7237                            const struct ixgbe_info *ii)
7238 {
7239 #ifdef CONFIG_PCI_IOV
7240         struct ixgbe_hw *hw = &adapter->hw;
7241         int err;
7242         int num_vf_macvlans, i;
7243         struct vf_macvlans *mv_list;
7244
7245         if (hw->mac.type == ixgbe_mac_82598EB || !max_vfs)
7246                 return;
7247
7248         /* The 82599 supports up to 64 VFs per physical function
7249          * but this implementation limits allocation to 63 so that
7250          * basic networking resources are still available to the
7251          * physical function
7252          */
7253         adapter->num_vfs = (max_vfs > 63) ? 63 : max_vfs;
7254         adapter->flags |= IXGBE_FLAG_SRIOV_ENABLED;
7255         err = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
7256         if (err) {
7257                 e_err(probe, "Failed to enable PCI sriov: %d\n", err);
7258                 goto err_novfs;
7259         }
7260
7261         num_vf_macvlans = hw->mac.num_rar_entries -
7262                 (IXGBE_MAX_PF_MACVLANS + 1 + adapter->num_vfs);
7263
7264         adapter->mv_list = mv_list = kcalloc(num_vf_macvlans,
7265                                              sizeof(struct vf_macvlans),
7266                                              GFP_KERNEL);
7267         if (mv_list) {
7268                 /* Initialize list of VF macvlans */
7269                 INIT_LIST_HEAD(&adapter->vf_mvs.l);
7270                 for (i = 0; i < num_vf_macvlans; i++) {
7271                         mv_list->vf = -1;
7272                         mv_list->free = true;
7273                         mv_list->rar_entry = hw->mac.num_rar_entries -
7274                                 (i + adapter->num_vfs + 1);
7275                         list_add(&mv_list->l, &adapter->vf_mvs.l);
7276                         mv_list++;
7277                 }
7278         }
7279
7280         /* If call to enable VFs succeeded then allocate memory
7281          * for per VF control structures.
7282          */
7283         adapter->vfinfo =
7284                 kcalloc(adapter->num_vfs,
7285                         sizeof(struct vf_data_storage), GFP_KERNEL);
7286         if (adapter->vfinfo) {
7287                 /* Now that we're sure SR-IOV is enabled
7288                  * and memory allocated set up the mailbox parameters
7289                  */
7290                 ixgbe_init_mbx_params_pf(hw);
7291                 memcpy(&hw->mbx.ops, ii->mbx_ops,
7292                        sizeof(hw->mbx.ops));
7293
7294                 /* Disable RSC when in SR-IOV mode */
7295                 adapter->flags2 &= ~(IXGBE_FLAG2_RSC_CAPABLE |
7296                                      IXGBE_FLAG2_RSC_ENABLED);
7297                 return;
7298         }
7299
7300         /* Oh oh */
7301         e_err(probe, "Unable to allocate memory for VF Data Storage - "
7302               "SRIOV disabled\n");
7303         pci_disable_sriov(adapter->pdev);
7304
7305 err_novfs:
7306         adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
7307         adapter->num_vfs = 0;
7308 #endif /* CONFIG_PCI_IOV */
7309 }
7310
7311 /**
7312  * ixgbe_probe - Device Initialization Routine
7313  * @pdev: PCI device information struct
7314  * @ent: entry in ixgbe_pci_tbl
7315  *
7316  * Returns 0 on success, negative on failure
7317  *
7318  * ixgbe_probe initializes an adapter identified by a pci_dev structure.
7319  * The OS initialization, configuring of the adapter private structure,
7320  * and a hardware reset occur.
7321  **/
7322 static int __devinit ixgbe_probe(struct pci_dev *pdev,
7323                                  const struct pci_device_id *ent)
7324 {
7325         struct net_device *netdev;
7326         struct ixgbe_adapter *adapter = NULL;
7327         struct ixgbe_hw *hw;
7328         const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];
7329         static int cards_found;
7330         int i, err, pci_using_dac;
7331         u8 part_str[IXGBE_PBANUM_LENGTH];
7332         unsigned int indices = num_possible_cpus();
7333 #ifdef IXGBE_FCOE
7334         u16 device_caps;
7335 #endif
7336         u32 eec;
7337
7338         /* Catch broken hardware that put the wrong VF device ID in
7339          * the PCIe SR-IOV capability.
7340          */
7341         if (pdev->is_virtfn) {
7342                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
7343                      pci_name(pdev), pdev->vendor, pdev->device);
7344                 return -EINVAL;
7345         }
7346
7347         err = pci_enable_device_mem(pdev);
7348         if (err)
7349                 return err;
7350
7351         if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) &&
7352             !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) {
7353                 pci_using_dac = 1;
7354         } else {
7355                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
7356                 if (err) {
7357                         err = dma_set_coherent_mask(&pdev->dev,
7358                                                     DMA_BIT_MASK(32));
7359                         if (err) {
7360                                 dev_err(&pdev->dev,
7361                                         "No usable DMA configuration, aborting\n");
7362                                 goto err_dma;
7363                         }
7364                 }
7365                 pci_using_dac = 0;
7366         }
7367
7368         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
7369                                            IORESOURCE_MEM), ixgbe_driver_name);
7370         if (err) {
7371                 dev_err(&pdev->dev,
7372                         "pci_request_selected_regions failed 0x%x\n", err);
7373                 goto err_pci_reg;
7374         }
7375
7376         pci_enable_pcie_error_reporting(pdev);
7377
7378         pci_set_master(pdev);
7379         pci_save_state(pdev);
7380
7381 #ifdef CONFIG_IXGBE_DCB
7382         indices *= MAX_TRAFFIC_CLASS;
7383 #endif
7384
7385         if (ii->mac == ixgbe_mac_82598EB)
7386                 indices = min_t(unsigned int, indices, IXGBE_MAX_RSS_INDICES);
7387         else
7388                 indices = min_t(unsigned int, indices, IXGBE_MAX_FDIR_INDICES);
7389
7390 #ifdef IXGBE_FCOE
7391         indices += min_t(unsigned int, num_possible_cpus(),
7392                          IXGBE_MAX_FCOE_INDICES);
7393 #endif
7394         netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);
7395         if (!netdev) {
7396                 err = -ENOMEM;
7397                 goto err_alloc_etherdev;
7398         }
7399
7400         SET_NETDEV_DEV(netdev, &pdev->dev);
7401
7402         adapter = netdev_priv(netdev);
7403         pci_set_drvdata(pdev, adapter);
7404
7405         adapter->netdev = netdev;
7406         adapter->pdev = pdev;
7407         hw = &adapter->hw;
7408         hw->back = adapter;
7409         adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
7410
7411         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
7412                               pci_resource_len(pdev, 0));
7413         if (!hw->hw_addr) {
7414                 err = -EIO;
7415                 goto err_ioremap;
7416         }
7417
7418         for (i = 1; i <= 5; i++) {
7419                 if (pci_resource_len(pdev, i) == 0)
7420                         continue;
7421         }
7422
7423         netdev->netdev_ops = &ixgbe_netdev_ops;
7424         ixgbe_set_ethtool_ops(netdev);
7425         netdev->watchdog_timeo = 5 * HZ;
7426         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
7427
7428         adapter->bd_number = cards_found;
7429
7430         /* Setup hw api */
7431         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
7432         hw->mac.type  = ii->mac;
7433
7434         /* EEPROM */
7435         memcpy(&hw->eeprom.ops, ii->eeprom_ops, sizeof(hw->eeprom.ops));
7436         eec = IXGBE_READ_REG(hw, IXGBE_EEC);
7437         /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */
7438         if (!(eec & (1 << 8)))
7439                 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;
7440
7441         /* PHY */
7442         memcpy(&hw->phy.ops, ii->phy_ops, sizeof(hw->phy.ops));
7443         hw->phy.sfp_type = ixgbe_sfp_type_unknown;
7444         /* ixgbe_identify_phy_generic will set prtad and mmds properly */
7445         hw->phy.mdio.prtad = MDIO_PRTAD_NONE;
7446         hw->phy.mdio.mmds = 0;
7447         hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
7448         hw->phy.mdio.dev = netdev;
7449         hw->phy.mdio.mdio_read = ixgbe_mdio_read;
7450         hw->phy.mdio.mdio_write = ixgbe_mdio_write;
7451
7452         ii->get_invariants(hw);
7453
7454         /* setup the private structure */
7455         err = ixgbe_sw_init(adapter);
7456         if (err)
7457                 goto err_sw_init;
7458
7459         /* Make it possible the adapter to be woken up via WOL */
7460         switch (adapter->hw.mac.type) {
7461         case ixgbe_mac_82599EB:
7462         case ixgbe_mac_X540:
7463                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7464                 break;
7465         default:
7466                 break;
7467         }
7468
7469         /*
7470          * If there is a fan on this device and it has failed log the
7471          * failure.
7472          */
7473         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {
7474                 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);
7475                 if (esdp & IXGBE_ESDP_SDP1)
7476                         e_crit(probe, "Fan has stopped, replace the adapter\n");
7477         }
7478
7479         /* reset_hw fills in the perm_addr as well */
7480         hw->phy.reset_if_overtemp = true;
7481         err = hw->mac.ops.reset_hw(hw);
7482         hw->phy.reset_if_overtemp = false;
7483         if (err == IXGBE_ERR_SFP_NOT_PRESENT &&
7484             hw->mac.type == ixgbe_mac_82598EB) {
7485                 err = 0;
7486         } else if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
7487                 e_dev_err("failed to load because an unsupported SFP+ "
7488                           "module type was detected.\n");
7489                 e_dev_err("Reload the driver after installing a supported "
7490                           "module.\n");
7491                 goto err_sw_init;
7492         } else if (err) {
7493                 e_dev_err("HW Init failed: %d\n", err);
7494                 goto err_sw_init;
7495         }
7496
7497         ixgbe_probe_vf(adapter, ii);
7498
7499         netdev->features = NETIF_F_SG |
7500                            NETIF_F_IP_CSUM |
7501                            NETIF_F_IPV6_CSUM |
7502                            NETIF_F_HW_VLAN_TX |
7503                            NETIF_F_HW_VLAN_RX |
7504                            NETIF_F_HW_VLAN_FILTER |
7505                            NETIF_F_TSO |
7506                            NETIF_F_TSO6 |
7507                            NETIF_F_GRO |
7508                            NETIF_F_RXHASH |
7509                            NETIF_F_RXCSUM;
7510
7511         netdev->hw_features = netdev->features;
7512
7513         switch (adapter->hw.mac.type) {
7514         case ixgbe_mac_82599EB:
7515         case ixgbe_mac_X540:
7516                 netdev->features |= NETIF_F_SCTP_CSUM;
7517                 netdev->hw_features |= NETIF_F_SCTP_CSUM |
7518                                        NETIF_F_NTUPLE;
7519                 break;
7520         default:
7521                 break;
7522         }
7523
7524         netdev->vlan_features |= NETIF_F_TSO;
7525         netdev->vlan_features |= NETIF_F_TSO6;
7526         netdev->vlan_features |= NETIF_F_IP_CSUM;
7527         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
7528         netdev->vlan_features |= NETIF_F_SG;
7529
7530         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7531                 adapter->flags &= ~(IXGBE_FLAG_RSS_ENABLED |
7532                                     IXGBE_FLAG_DCB_ENABLED);
7533
7534 #ifdef CONFIG_IXGBE_DCB
7535         netdev->dcbnl_ops = &dcbnl_ops;
7536 #endif
7537
7538 #ifdef IXGBE_FCOE
7539         if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7540                 if (hw->mac.ops.get_device_caps) {
7541                         hw->mac.ops.get_device_caps(hw, &device_caps);
7542                         if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)
7543                                 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;
7544                 }
7545         }
7546         if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {
7547                 netdev->vlan_features |= NETIF_F_FCOE_CRC;
7548                 netdev->vlan_features |= NETIF_F_FSO;
7549                 netdev->vlan_features |= NETIF_F_FCOE_MTU;
7550         }
7551 #endif /* IXGBE_FCOE */
7552         if (pci_using_dac) {
7553                 netdev->features |= NETIF_F_HIGHDMA;
7554                 netdev->vlan_features |= NETIF_F_HIGHDMA;
7555         }
7556
7557         if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)
7558                 netdev->hw_features |= NETIF_F_LRO;
7559         if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
7560                 netdev->features |= NETIF_F_LRO;
7561
7562         /* make sure the EEPROM is good */
7563         if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {
7564                 e_dev_err("The EEPROM Checksum Is Not Valid\n");
7565                 err = -EIO;
7566                 goto err_eeprom;
7567         }
7568
7569         memcpy(netdev->dev_addr, hw->mac.perm_addr, netdev->addr_len);
7570         memcpy(netdev->perm_addr, hw->mac.perm_addr, netdev->addr_len);
7571
7572         if (ixgbe_validate_mac_addr(netdev->perm_addr)) {
7573                 e_dev_err("invalid MAC address\n");
7574                 err = -EIO;
7575                 goto err_eeprom;
7576         }
7577
7578         /* power down the optics for multispeed fiber and 82599 SFP+ fiber */
7579         if (hw->mac.ops.disable_tx_laser &&
7580             ((hw->phy.multispeed_fiber) ||
7581              ((hw->mac.ops.get_media_type(hw) == ixgbe_media_type_fiber) &&
7582               (hw->mac.type == ixgbe_mac_82599EB))))
7583                 hw->mac.ops.disable_tx_laser(hw);
7584
7585         setup_timer(&adapter->service_timer, &ixgbe_service_timer,
7586                     (unsigned long) adapter);
7587
7588         INIT_WORK(&adapter->service_task, ixgbe_service_task);
7589         clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
7590
7591         err = ixgbe_init_interrupt_scheme(adapter);
7592         if (err)
7593                 goto err_sw_init;
7594
7595         if (!(adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
7596                 netdev->hw_features &= ~NETIF_F_RXHASH;
7597                 netdev->features &= ~NETIF_F_RXHASH;
7598         }
7599
7600         switch (pdev->device) {
7601         case IXGBE_DEV_ID_82599_SFP:
7602                 /* Only this subdevice supports WOL */
7603                 if (pdev->subsystem_device == IXGBE_SUBDEV_ID_82599_SFP)
7604                         adapter->wol = IXGBE_WUFC_MAG;
7605                 break;
7606         case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:
7607                 /* All except this subdevice support WOL */
7608                 if (pdev->subsystem_device != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)
7609                         adapter->wol = IXGBE_WUFC_MAG;
7610                 break;
7611         case IXGBE_DEV_ID_82599_KX4:
7612                 adapter->wol = IXGBE_WUFC_MAG;
7613                 break;
7614         default:
7615                 adapter->wol = 0;
7616                 break;
7617         }
7618         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
7619
7620         /* pick up the PCI bus settings for reporting later */
7621         hw->mac.ops.get_bus_info(hw);
7622
7623         /* print bus type/speed/width info */
7624         e_dev_info("(PCI Express:%s:%s) %pM\n",
7625                    (hw->bus.speed == ixgbe_bus_speed_5000 ? "5.0GT/s" :
7626                     hw->bus.speed == ixgbe_bus_speed_2500 ? "2.5GT/s" :
7627                     "Unknown"),
7628                    (hw->bus.width == ixgbe_bus_width_pcie_x8 ? "Width x8" :
7629                     hw->bus.width == ixgbe_bus_width_pcie_x4 ? "Width x4" :
7630                     hw->bus.width == ixgbe_bus_width_pcie_x1 ? "Width x1" :
7631                     "Unknown"),
7632                    netdev->dev_addr);
7633
7634         err = ixgbe_read_pba_string_generic(hw, part_str, IXGBE_PBANUM_LENGTH);
7635         if (err)
7636                 strncpy(part_str, "Unknown", IXGBE_PBANUM_LENGTH);
7637         if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
7638                 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
7639                            hw->mac.type, hw->phy.type, hw->phy.sfp_type,
7640                            part_str);
7641         else
7642                 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",
7643                            hw->mac.type, hw->phy.type, part_str);
7644
7645         if (hw->bus.width <= ixgbe_bus_width_pcie_x4) {
7646                 e_dev_warn("PCI-Express bandwidth available for this card is "
7647                            "not sufficient for optimal performance.\n");
7648                 e_dev_warn("For optimal performance a x8 PCI-Express slot "
7649                            "is required.\n");
7650         }
7651
7652         /* save off EEPROM version number */
7653         hw->eeprom.ops.read(hw, 0x29, &adapter->eeprom_version);
7654
7655         /* reset the hardware with the new settings */
7656         err = hw->mac.ops.start_hw(hw);
7657
7658         if (err == IXGBE_ERR_EEPROM_VERSION) {
7659                 /* We are running on a pre-production device, log a warning */
7660                 e_dev_warn("This device is a pre-production adapter/LOM. "
7661                            "Please be aware there may be issues associated "
7662                            "with your hardware.  If you are experiencing "
7663                            "problems please contact your Intel or hardware "
7664                            "representative who provided you with this "
7665                            "hardware.\n");
7666         }
7667         strcpy(netdev->name, "eth%d");
7668         err = register_netdev(netdev);
7669         if (err)
7670                 goto err_register;
7671
7672         /* carrier off reporting is important to ethtool even BEFORE open */
7673         netif_carrier_off(netdev);
7674
7675 #ifdef CONFIG_IXGBE_DCA
7676         if (dca_add_requester(&pdev->dev) == 0) {
7677                 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
7678                 ixgbe_setup_dca(adapter);
7679         }
7680 #endif
7681         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
7682                 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);
7683                 for (i = 0; i < adapter->num_vfs; i++)
7684                         ixgbe_vf_configuration(pdev, (i | 0x10000000));
7685         }
7686
7687         /* Inform firmware of driver version */
7688         if (hw->mac.ops.set_fw_drv_ver)
7689                 hw->mac.ops.set_fw_drv_ver(hw, MAJ, MIN, BUILD,
7690                                            FW_CEM_UNUSED_VER);
7691
7692         /* add san mac addr to netdev */
7693         ixgbe_add_sanmac_netdev(netdev);
7694
7695         e_dev_info("Intel(R) 10 Gigabit Network Connection\n");
7696         cards_found++;
7697         return 0;
7698
7699 err_register:
7700         ixgbe_release_hw_control(adapter);
7701         ixgbe_clear_interrupt_scheme(adapter);
7702 err_sw_init:
7703 err_eeprom:
7704         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7705                 ixgbe_disable_sriov(adapter);
7706         adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;
7707         iounmap(hw->hw_addr);
7708 err_ioremap:
7709         free_netdev(netdev);
7710 err_alloc_etherdev:
7711         pci_release_selected_regions(pdev,
7712                                      pci_select_bars(pdev, IORESOURCE_MEM));
7713 err_pci_reg:
7714 err_dma:
7715         pci_disable_device(pdev);
7716         return err;
7717 }
7718
7719 /**
7720  * ixgbe_remove - Device Removal Routine
7721  * @pdev: PCI device information struct
7722  *
7723  * ixgbe_remove is called by the PCI subsystem to alert the driver
7724  * that it should release a PCI device.  The could be caused by a
7725  * Hot-Plug event, or because the driver is going to be removed from
7726  * memory.
7727  **/
7728 static void __devexit ixgbe_remove(struct pci_dev *pdev)
7729 {
7730         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7731         struct net_device *netdev = adapter->netdev;
7732
7733         set_bit(__IXGBE_DOWN, &adapter->state);
7734         cancel_work_sync(&adapter->service_task);
7735
7736 #ifdef CONFIG_IXGBE_DCA
7737         if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
7738                 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
7739                 dca_remove_requester(&pdev->dev);
7740                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
7741         }
7742
7743 #endif
7744 #ifdef IXGBE_FCOE
7745         if (adapter->flags & IXGBE_FLAG_FCOE_ENABLED)
7746                 ixgbe_cleanup_fcoe(adapter);
7747
7748 #endif /* IXGBE_FCOE */
7749
7750         /* remove the added san mac */
7751         ixgbe_del_sanmac_netdev(netdev);
7752
7753         if (netdev->reg_state == NETREG_REGISTERED)
7754                 unregister_netdev(netdev);
7755
7756         if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)
7757                 ixgbe_disable_sriov(adapter);
7758
7759         ixgbe_clear_interrupt_scheme(adapter);
7760
7761         ixgbe_release_hw_control(adapter);
7762
7763         iounmap(adapter->hw.hw_addr);
7764         pci_release_selected_regions(pdev, pci_select_bars(pdev,
7765                                      IORESOURCE_MEM));
7766
7767         e_dev_info("complete\n");
7768
7769         free_netdev(netdev);
7770
7771         pci_disable_pcie_error_reporting(pdev);
7772
7773         pci_disable_device(pdev);
7774 }
7775
7776 /**
7777  * ixgbe_io_error_detected - called when PCI error is detected
7778  * @pdev: Pointer to PCI device
7779  * @state: The current pci connection state
7780  *
7781  * This function is called after a PCI bus error affecting
7782  * this device has been detected.
7783  */
7784 static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,
7785                                                 pci_channel_state_t state)
7786 {
7787         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7788         struct net_device *netdev = adapter->netdev;
7789
7790         netif_device_detach(netdev);
7791
7792         if (state == pci_channel_io_perm_failure)
7793                 return PCI_ERS_RESULT_DISCONNECT;
7794
7795         if (netif_running(netdev))
7796                 ixgbe_down(adapter);
7797         pci_disable_device(pdev);
7798
7799         /* Request a slot reset. */
7800         return PCI_ERS_RESULT_NEED_RESET;
7801 }
7802
7803 /**
7804  * ixgbe_io_slot_reset - called after the pci bus has been reset.
7805  * @pdev: Pointer to PCI device
7806  *
7807  * Restart the card from scratch, as if from a cold-boot.
7808  */
7809 static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)
7810 {
7811         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7812         pci_ers_result_t result;
7813         int err;
7814
7815         if (pci_enable_device_mem(pdev)) {
7816                 e_err(probe, "Cannot re-enable PCI device after reset.\n");
7817                 result = PCI_ERS_RESULT_DISCONNECT;
7818         } else {
7819                 pci_set_master(pdev);
7820                 pci_restore_state(pdev);
7821                 pci_save_state(pdev);
7822
7823                 pci_wake_from_d3(pdev, false);
7824
7825                 ixgbe_reset(adapter);
7826                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);
7827                 result = PCI_ERS_RESULT_RECOVERED;
7828         }
7829
7830         err = pci_cleanup_aer_uncorrect_error_status(pdev);
7831         if (err) {
7832                 e_dev_err("pci_cleanup_aer_uncorrect_error_status "
7833                           "failed 0x%0x\n", err);
7834                 /* non-fatal, continue */
7835         }
7836
7837         return result;
7838 }
7839
7840 /**
7841  * ixgbe_io_resume - called when traffic can start flowing again.
7842  * @pdev: Pointer to PCI device
7843  *
7844  * This callback is called when the error recovery driver tells us that
7845  * its OK to resume normal operation.
7846  */
7847 static void ixgbe_io_resume(struct pci_dev *pdev)
7848 {
7849         struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);
7850         struct net_device *netdev = adapter->netdev;
7851
7852         if (netif_running(netdev)) {
7853                 if (ixgbe_up(adapter)) {
7854                         e_info(probe, "ixgbe_up failed after reset\n");
7855                         return;
7856                 }
7857         }
7858
7859         netif_device_attach(netdev);
7860 }
7861
7862 static struct pci_error_handlers ixgbe_err_handler = {
7863         .error_detected = ixgbe_io_error_detected,
7864         .slot_reset = ixgbe_io_slot_reset,
7865         .resume = ixgbe_io_resume,
7866 };
7867
7868 static struct pci_driver ixgbe_driver = {
7869         .name     = ixgbe_driver_name,
7870         .id_table = ixgbe_pci_tbl,
7871         .probe    = ixgbe_probe,
7872         .remove   = __devexit_p(ixgbe_remove),
7873 #ifdef CONFIG_PM
7874         .suspend  = ixgbe_suspend,
7875         .resume   = ixgbe_resume,
7876 #endif
7877         .shutdown = ixgbe_shutdown,
7878         .err_handler = &ixgbe_err_handler
7879 };
7880
7881 /**
7882  * ixgbe_init_module - Driver Registration Routine
7883  *
7884  * ixgbe_init_module is the first routine called when the driver is
7885  * loaded. All it does is register with the PCI subsystem.
7886  **/
7887 static int __init ixgbe_init_module(void)
7888 {
7889         int ret;
7890         pr_info("%s - version %s\n", ixgbe_driver_string, ixgbe_driver_version);
7891         pr_info("%s\n", ixgbe_copyright);
7892
7893 #ifdef CONFIG_IXGBE_DCA
7894         dca_register_notify(&dca_notifier);
7895 #endif
7896
7897         ret = pci_register_driver(&ixgbe_driver);
7898         return ret;
7899 }
7900
7901 module_init(ixgbe_init_module);
7902
7903 /**
7904  * ixgbe_exit_module - Driver Exit Cleanup Routine
7905  *
7906  * ixgbe_exit_module is called just before the driver is removed
7907  * from memory.
7908  **/
7909 static void __exit ixgbe_exit_module(void)
7910 {
7911 #ifdef CONFIG_IXGBE_DCA
7912         dca_unregister_notify(&dca_notifier);
7913 #endif
7914         pci_unregister_driver(&ixgbe_driver);
7915         rcu_barrier(); /* Wait for completion of call_rcu()'s */
7916 }
7917
7918 #ifdef CONFIG_IXGBE_DCA
7919 static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,
7920                             void *p)
7921 {
7922         int ret_val;
7923
7924         ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,
7925                                          __ixgbe_notify_dca);
7926
7927         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7928 }
7929
7930 #endif /* CONFIG_IXGBE_DCA */
7931
7932 module_exit(ixgbe_exit_module);
7933
7934 /* ixgbe_main.c */