drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

   1 /*******************************************************************************
   2
   3   Intel 10 Gigabit PCI Express Linux driver
   4   Copyright(c) 1999 - 2011 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/types.h>
  29 #include <linux/module.h>
  30 #include <linux/pci.h>
  31 #include <linux/netdevice.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/string.h>
  34 #include <linux/in.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/ip.h>
  37 #include <linux/tcp.h>
  38 #include <linux/sctp.h>
  39 #include <linux/pkt_sched.h>
  40 #include <linux/ipv6.h>
  41 #include <linux/slab.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <linux/ethtool.h>
  45 #include <linux/if.h>
  46 #include <linux/if_vlan.h>
  47 #include <linux/prefetch.h>
  48 #include <scsi/fc/fc_fcoe.h>
  49
  50 #include "ixgbe.h"
  51 #include "ixgbe_common.h"
  52 #include "ixgbe_dcb_82599.h"
  53 #include "ixgbe_sriov.h"
  54
  55 char ixgbe_driver_name[] = "ixgbe";
  56 static const char ixgbe_driver_string[] =
  57                               "Intel(R) 10 Gigabit PCI Express Network Driver";
  58 #define MAJ 3
  59 #define MIN 6
  60 #define BUILD 7
  61 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
  62         __stringify(BUILD) "-k"
  63 const char ixgbe_driver_version[] = DRV_VERSION;
  64 static const char ixgbe_copyright[] =
  65                                 "Copyright (c) 1999-2011 Intel Corporation.";
  66
  67 static const struct ixgbe_info *ixgbe_info_tbl[] = {
  68         [board_82598] = &ixgbe_82598_info,
  69         [board_82599] = &ixgbe_82599_info,
  70         [board_X540] = &ixgbe_X540_info,
  71 };
  72
  73 /* ixgbe_pci_tbl - PCI Device ID Table
  74  *
  75  * Wildcard entries (PCI_ANY_ID) should come last
  76  * Last entry must be all 0s
  77  *
  78  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  79  *   Class, Class Mask, private data (not used) }
  80  */
  81 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
  82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
  83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
  84         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
  85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
  86         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
  87         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
  88         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
  89         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
  90         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
  91         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
  92         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
  93         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
  94         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
  95         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
  96         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
  97         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
  98         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
  99         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
 100         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
 101         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
 102         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
 103         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
 104         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
 105         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
 106         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
 107         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
 108         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
 109         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
 110         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
 111         /* required last entry */
 112         {0, }
 113 };
 114 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
 115
 116 #ifdef CONFIG_IXGBE_DCA
 117 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
 118                             void *p);
 119 static struct notifier_block dca_notifier = {
 120         .notifier_call = ixgbe_notify_dca,
 121         .next          = NULL,
 122         .priority      = 0
 123 };
 124 #endif
 125
 126 #ifdef CONFIG_PCI_IOV
 127 static unsigned int max_vfs;
 128 module_param(max_vfs, uint, 0);
 129 MODULE_PARM_DESC(max_vfs,
 130                  "Maximum number of virtual functions to allocate per physical function");
 131 #endif /* CONFIG_PCI_IOV */
 132
 133 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
 134 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
 135 MODULE_LICENSE("GPL");
 136 MODULE_VERSION(DRV_VERSION);
 137
 138 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
 139
 140 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
 141 {
 142         if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
 143             !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
 144                 schedule_work(&adapter->service_task);
 145 }
 146
 147 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
 148 {
 149         BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
 150
 151         /* flush memory to make sure state is correct before next watchog */
 152         smp_mb__before_clear_bit();
 153         clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
 154 }
 155
 156 struct ixgbe_reg_info {
 157         u32 ofs;
 158         char *name;
 159 };
 160
 161 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
 162
 163         /* General Registers */
 164         {IXGBE_CTRL, "CTRL"},
 165         {IXGBE_STATUS, "STATUS"},
 166         {IXGBE_CTRL_EXT, "CTRL_EXT"},
 167
 168         /* Interrupt Registers */
 169         {IXGBE_EICR, "EICR"},
 170
 171         /* RX Registers */
 172         {IXGBE_SRRCTL(0), "SRRCTL"},
 173         {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
 174         {IXGBE_RDLEN(0), "RDLEN"},
 175         {IXGBE_RDH(0), "RDH"},
 176         {IXGBE_RDT(0), "RDT"},
 177         {IXGBE_RXDCTL(0), "RXDCTL"},
 178         {IXGBE_RDBAL(0), "RDBAL"},
 179         {IXGBE_RDBAH(0), "RDBAH"},
 180
 181         /* TX Registers */
 182         {IXGBE_TDBAL(0), "TDBAL"},
 183         {IXGBE_TDBAH(0), "TDBAH"},
 184         {IXGBE_TDLEN(0), "TDLEN"},
 185         {IXGBE_TDH(0), "TDH"},
 186         {IXGBE_TDT(0), "TDT"},
 187         {IXGBE_TXDCTL(0), "TXDCTL"},
 188
 189         /* List Terminator */
 190         {}
 191 };
 192
 193
 194 /*
 195  * ixgbe_regdump - register printout routine
 196  */
 197 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
 198 {
 199         int i = 0, j = 0;
 200         char rname[16];
 201         u32 regs[64];
 202
 203         switch (reginfo->ofs) {
 204         case IXGBE_SRRCTL(0):
 205                 for (i = 0; i < 64; i++)
 206                         regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
 207                 break;
 208         case IXGBE_DCA_RXCTRL(0):
 209                 for (i = 0; i < 64; i++)
 210                         regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
 211                 break;
 212         case IXGBE_RDLEN(0):
 213                 for (i = 0; i < 64; i++)
 214                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
 215                 break;
 216         case IXGBE_RDH(0):
 217                 for (i = 0; i < 64; i++)
 218                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
 219                 break;
 220         case IXGBE_RDT(0):
 221                 for (i = 0; i < 64; i++)
 222                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
 223                 break;
 224         case IXGBE_RXDCTL(0):
 225                 for (i = 0; i < 64; i++)
 226                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
 227                 break;
 228         case IXGBE_RDBAL(0):
 229                 for (i = 0; i < 64; i++)
 230                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
 231                 break;
 232         case IXGBE_RDBAH(0):
 233                 for (i = 0; i < 64; i++)
 234                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
 235                 break;
 236         case IXGBE_TDBAL(0):
 237                 for (i = 0; i < 64; i++)
 238                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
 239                 break;
 240         case IXGBE_TDBAH(0):
 241                 for (i = 0; i < 64; i++)
 242                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
 243                 break;
 244         case IXGBE_TDLEN(0):
 245                 for (i = 0; i < 64; i++)
 246                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
 247                 break;
 248         case IXGBE_TDH(0):
 249                 for (i = 0; i < 64; i++)
 250                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
 251                 break;
 252         case IXGBE_TDT(0):
 253                 for (i = 0; i < 64; i++)
 254                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
 255                 break;
 256         case IXGBE_TXDCTL(0):
 257                 for (i = 0; i < 64; i++)
 258                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
 259                 break;
 260         default:
 261                 pr_info("%-15s %08x\n", reginfo->name,
 262                         IXGBE_READ_REG(hw, reginfo->ofs));
 263                 return;
 264         }
 265
 266         for (i = 0; i < 8; i++) {
 267                 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
 268                 pr_err("%-15s", rname);
 269                 for (j = 0; j < 8; j++)
 270                         pr_cont(" %08x", regs[i*8+j]);
 271                 pr_cont("\n");
 272         }
 273
 274 }
 275
 276 /*
 277  * ixgbe_dump - Print registers, tx-rings and rx-rings
 278  */
 279 static void ixgbe_dump(struct ixgbe_adapter *adapter)
 280 {
 281         struct net_device *netdev = adapter->netdev;
 282         struct ixgbe_hw *hw = &adapter->hw;
 283         struct ixgbe_reg_info *reginfo;
 284         int n = 0;
 285         struct ixgbe_ring *tx_ring;
 286         struct ixgbe_tx_buffer *tx_buffer_info;
 287         union ixgbe_adv_tx_desc *tx_desc;
 288         struct my_u0 { u64 a; u64 b; } *u0;
 289         struct ixgbe_ring *rx_ring;
 290         union ixgbe_adv_rx_desc *rx_desc;
 291         struct ixgbe_rx_buffer *rx_buffer_info;
 292         u32 staterr;
 293         int i = 0;
 294
 295         if (!netif_msg_hw(adapter))
 296                 return;
 297
 298         /* Print netdevice Info */
 299         if (netdev) {
 300                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 301                 pr_info("Device Name     state            "
 302                         "trans_start      last_rx\n");
 303                 pr_info("%-15s %016lX %016lX %016lX\n",
 304                         netdev->name,
 305                         netdev->state,
 306                         netdev->trans_start,
 307                         netdev->last_rx);
 308         }
 309
 310         /* Print Registers */
 311         dev_info(&adapter->pdev->dev, "Register Dump\n");
 312         pr_info(" Register Name   Value\n");
 313         for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
 314              reginfo->name; reginfo++) {
 315                 ixgbe_regdump(hw, reginfo);
 316         }
 317
 318         /* Print TX Ring Summary */
 319         if (!netdev || !netif_running(netdev))
 320                 goto exit;
 321
 322         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 323         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 324         for (n = 0; n < adapter->num_tx_queues; n++) {
 325                 tx_ring = adapter->tx_ring[n];
 326                 tx_buffer_info =
 327                         &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
 328                 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
 329                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
 330                            (u64)tx_buffer_info->dma,
 331                            tx_buffer_info->length,
 332                            tx_buffer_info->next_to_watch,
 333                            (u64)tx_buffer_info->time_stamp);
 334         }
 335
 336         /* Print TX Rings */
 337         if (!netif_msg_tx_done(adapter))
 338                 goto rx_ring_summary;
 339
 340         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 341
 342         /* Transmit Descriptor Formats
 343          *
 344          * Advanced Transmit Descriptor
 345          *   +--------------------------------------------------------------+
 346          * 0 |         Buffer Address [63:0]                                |
 347          *   +--------------------------------------------------------------+
 348          * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
 349          *   +--------------------------------------------------------------+
 350          *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
 351          */
 352
 353         for (n = 0; n < adapter->num_tx_queues; n++) {
 354                 tx_ring = adapter->tx_ring[n];
 355                 pr_info("------------------------------------\n");
 356                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 357                 pr_info("------------------------------------\n");
 358                 pr_info("T [desc]     [address 63:0  ] "
 359                         "[PlPOIdStDDt Ln] [bi->dma       ] "
 360                         "leng  ntw timestamp        bi->skb\n");
 361
 362                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 363                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 364                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
 365                         u0 = (struct my_u0 *)tx_desc;
 366                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
 367                                 " %04X  %p %016llX %p", i,
 368                                 le64_to_cpu(u0->a),
 369                                 le64_to_cpu(u0->b),
 370                                 (u64)tx_buffer_info->dma,
 371                                 tx_buffer_info->length,
 372                                 tx_buffer_info->next_to_watch,
 373                                 (u64)tx_buffer_info->time_stamp,
 374                                 tx_buffer_info->skb);
 375                         if (i == tx_ring->next_to_use &&
 376                                 i == tx_ring->next_to_clean)
 377                                 pr_cont(" NTC/U\n");
 378                         else if (i == tx_ring->next_to_use)
 379                                 pr_cont(" NTU\n");
 380                         else if (i == tx_ring->next_to_clean)
 381                                 pr_cont(" NTC\n");
 382                         else
 383                                 pr_cont("\n");
 384
 385                         if (netif_msg_pktdata(adapter) &&
 386                                 tx_buffer_info->dma != 0)
 387                                 print_hex_dump(KERN_INFO, "",
 388                                         DUMP_PREFIX_ADDRESS, 16, 1,
 389                                         phys_to_virt(tx_buffer_info->dma),
 390                                         tx_buffer_info->length, true);
 391                 }
 392         }
 393
 394         /* Print RX Rings Summary */
 395 rx_ring_summary:
 396         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 397         pr_info("Queue [NTU] [NTC]\n");
 398         for (n = 0; n < adapter->num_rx_queues; n++) {
 399                 rx_ring = adapter->rx_ring[n];
 400                 pr_info("%5d %5X %5X\n",
 401                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
 402         }
 403
 404         /* Print RX Rings */
 405         if (!netif_msg_rx_status(adapter))
 406                 goto exit;
 407
 408         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 409
 410         /* Advanced Receive Descriptor (Read) Format
 411          *    63                                           1        0
 412          *    +-----------------------------------------------------+
 413          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 414          *    +----------------------------------------------+------+
 415          *  8 |       Header Buffer Address [63:1]           |  DD  |
 416          *    +-----------------------------------------------------+
 417          *
 418          *
 419          * Advanced Receive Descriptor (Write-Back) Format
 420          *
 421          *   63       48 47    32 31  30      21 20 16 15   4 3     0
 422          *   +------------------------------------------------------+
 423          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 424          *   | Checksum   Ident  |   |           |    | Type | Type |
 425          *   +------------------------------------------------------+
 426          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 427          *   +------------------------------------------------------+
 428          *   63       48 47    32 31            20 19               0
 429          */
 430         for (n = 0; n < adapter->num_rx_queues; n++) {
 431                 rx_ring = adapter->rx_ring[n];
 432                 pr_info("------------------------------------\n");
 433                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 434                 pr_info("------------------------------------\n");
 435                 pr_info("R  [desc]      [ PktBuf     A0] "
 436                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
 437                         "<-- Adv Rx Read format\n");
 438                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
 439                         "[vl er S cks ln] ---------------- [bi->skb] "
 440                         "<-- Adv Rx Write-Back format\n");
 441
 442                 for (i = 0; i < rx_ring->count; i++) {
 443                         rx_buffer_info = &rx_ring->rx_buffer_info[i];
 444                         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
 445                         u0 = (struct my_u0 *)rx_desc;
 446                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 447                         if (staterr & IXGBE_RXD_STAT_DD) {
 448                                 /* Descriptor Done */
 449                                 pr_info("RWB[0x%03X]     %016llX "
 450                                         "%016llX ---------------- %p", i,
 451                                         le64_to_cpu(u0->a),
 452                                         le64_to_cpu(u0->b),
 453                                         rx_buffer_info->skb);
 454                         } else {
 455                                 pr_info("R  [0x%03X]     %016llX "
 456                                         "%016llX %016llX %p", i,
 457                                         le64_to_cpu(u0->a),
 458                                         le64_to_cpu(u0->b),
 459                                         (u64)rx_buffer_info->dma,
 460                                         rx_buffer_info->skb);
 461
 462                                 if (netif_msg_pktdata(adapter)) {
 463                                         print_hex_dump(KERN_INFO, "",
 464                                            DUMP_PREFIX_ADDRESS, 16, 1,
 465                                            phys_to_virt(rx_buffer_info->dma),
 466                                            rx_ring->rx_buf_len, true);
 467
 468                                         if (rx_ring->rx_buf_len
 469                                                 < IXGBE_RXBUFFER_2K)
 470                                                 print_hex_dump(KERN_INFO, "",
 471                                                   DUMP_PREFIX_ADDRESS, 16, 1,
 472                                                   phys_to_virt(
 473                                                     rx_buffer_info->page_dma +
 474                                                     rx_buffer_info->page_offset
 475                                                   ),
 476                                                   PAGE_SIZE/2, true);
 477                                 }
 478                         }
 479
 480                         if (i == rx_ring->next_to_use)
 481                                 pr_cont(" NTU\n");
 482                         else if (i == rx_ring->next_to_clean)
 483                                 pr_cont(" NTC\n");
 484                         else
 485                                 pr_cont("\n");
 486
 487                 }
 488         }
 489
 490 exit:
 491         return;
 492 }
 493
 494 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
 495 {
 496         u32 ctrl_ext;
 497
 498         /* Let firmware take over control of h/w */
 499         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 500         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 501                         ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
 502 }
 503
 504 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
 505 {
 506         u32 ctrl_ext;
 507
 508         /* Let firmware know the driver has taken over */
 509         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
 510         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
 511                         ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
 512 }
 513
 514 /*
 515  * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
 516  * @adapter: pointer to adapter struct
 517  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 518  * @queue: queue to map the corresponding interrupt to
 519  * @msix_vector: the vector to map to the corresponding queue
 520  *
 521  */
 522 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
 523                            u8 queue, u8 msix_vector)
 524 {
 525         u32 ivar, index;
 526         struct ixgbe_hw *hw = &adapter->hw;
 527         switch (hw->mac.type) {
 528         case ixgbe_mac_82598EB:
 529                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 530                 if (direction == -1)
 531                         direction = 0;
 532                 index = (((direction * 64) + queue) >> 2) & 0x1F;
 533                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
 534                 ivar &= ~(0xFF << (8 * (queue & 0x3)));
 535                 ivar |= (msix_vector << (8 * (queue & 0x3)));
 536                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
 537                 break;
 538         case ixgbe_mac_82599EB:
 539         case ixgbe_mac_X540:
 540                 if (direction == -1) {
 541                         /* other causes */
 542                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 543                         index = ((queue & 1) * 8);
 544                         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
 545                         ivar &= ~(0xFF << index);
 546                         ivar |= (msix_vector << index);
 547                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
 548                         break;
 549                 } else {
 550                         /* tx or rx causes */
 551                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 552                         index = ((16 * (queue & 1)) + (8 * direction));
 553                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
 554                         ivar &= ~(0xFF << index);
 555                         ivar |= (msix_vector << index);
 556                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
 557                         break;
 558                 }
 559         default:
 560                 break;
 561         }
 562 }
 563
 564 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
 565                                           u64 qmask)
 566 {
 567         u32 mask;
 568
 569         switch (adapter->hw.mac.type) {
 570         case ixgbe_mac_82598EB:
 571                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
 572                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
 573                 break;
 574         case ixgbe_mac_82599EB:
 575         case ixgbe_mac_X540:
 576                 mask = (qmask & 0xFFFFFFFF);
 577                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
 578                 mask = (qmask >> 32);
 579                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
 580                 break;
 581         default:
 582                 break;
 583         }
 584 }
 585
 586 static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
 587                                            struct ixgbe_tx_buffer *tx_buffer)
 588 {
 589         if (tx_buffer->dma) {
 590                 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
 591                         dma_unmap_page(ring->dev,
 592                                        tx_buffer->dma,
 593                                        tx_buffer->length,
 594                                        DMA_TO_DEVICE);
 595                 else
 596                         dma_unmap_single(ring->dev,
 597                                          tx_buffer->dma,
 598                                          tx_buffer->length,
 599                                          DMA_TO_DEVICE);
 600         }
 601         tx_buffer->dma = 0;
 602 }
 603
 604 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
 605                                       struct ixgbe_tx_buffer *tx_buffer_info)
 606 {
 607         ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
 608         if (tx_buffer_info->skb)
 609                 dev_kfree_skb_any(tx_buffer_info->skb);
 610         tx_buffer_info->skb = NULL;
 611         /* tx_buffer_info must be completely set up in the transmit path */
 612 }
 613
 614 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
 615 {
 616         struct ixgbe_hw *hw = &adapter->hw;
 617         struct ixgbe_hw_stats *hwstats = &adapter->stats;
 618         u32 data = 0;
 619         u32 xoff[8] = {0};
 620         int i;
 621
 622         if ((hw->fc.current_mode == ixgbe_fc_full) ||
 623             (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
 624                 switch (hw->mac.type) {
 625                 case ixgbe_mac_82598EB:
 626                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
 627                         break;
 628                 default:
 629                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
 630                 }
 631                 hwstats->lxoffrxc += data;
 632
 633                 /* refill credits (no tx hang) if we received xoff */
 634                 if (!data)
 635                         return;
 636
 637                 for (i = 0; i < adapter->num_tx_queues; i++)
 638                         clear_bit(__IXGBE_HANG_CHECK_ARMED,
 639                                   &adapter->tx_ring[i]->state);
 640                 return;
 641         } else if (!(adapter->dcb_cfg.pfc_mode_enable))
 642                 return;
 643
 644         /* update stats for each tc, only valid with PFC enabled */
 645         for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
 646                 switch (hw->mac.type) {
 647                 case ixgbe_mac_82598EB:
 648                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
 649                         break;
 650                 default:
 651                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
 652                 }
 653                 hwstats->pxoffrxc[i] += xoff[i];
 654         }
 655
 656         /* disarm tx queues that have received xoff frames */
 657         for (i = 0; i < adapter->num_tx_queues; i++) {
 658                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
 659                 u8 tc = tx_ring->dcb_tc;
 660
 661                 if (xoff[tc])
 662                         clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 663         }
 664 }
 665
 666 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
 667 {
 668         return ring->tx_stats.completed;
 669 }
 670
 671 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
 672 {
 673         struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
 674         struct ixgbe_hw *hw = &adapter->hw;
 675
 676         u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
 677         u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
 678
 679         if (head != tail)
 680                 return (head < tail) ?
 681                         tail - head : (tail + ring->count - head);
 682
 683         return 0;
 684 }
 685
 686 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
 687 {
 688         u32 tx_done = ixgbe_get_tx_completed(tx_ring);
 689         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
 690         u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
 691         bool ret = false;
 692
 693         clear_check_for_tx_hang(tx_ring);
 694
 695         /*
 696          * Check for a hung queue, but be thorough. This verifies
 697          * that a transmit has been completed since the previous
 698          * check AND there is at least one packet pending. The
 699          * ARMED bit is set to indicate a potential hang. The
 700          * bit is cleared if a pause frame is received to remove
 701          * false hang detection due to PFC or 802.3x frames. By
 702          * requiring this to fail twice we avoid races with
 703          * pfc clearing the ARMED bit and conditions where we
 704          * run the check_tx_hang logic with a transmit completion
 705          * pending but without time to complete it yet.
 706          */
 707         if ((tx_done_old == tx_done) && tx_pending) {
 708                 /* make sure it is true for two checks in a row */
 709                 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
 710                                        &tx_ring->state);
 711         } else {
 712                 /* update completed stats and continue */
 713                 tx_ring->tx_stats.tx_done_old = tx_done;
 714                 /* reset the countdown */
 715                 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
 716         }
 717
 718         return ret;
 719 }
 720
 721 /**
 722  * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
 723  * @adapter: driver private struct
 724  **/
 725 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
 726 {
 727
 728         /* Do the reset outside of interrupt context */
 729         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
 730                 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
 731                 ixgbe_service_event_schedule(adapter);
 732         }
 733 }
 734
 735 /**
 736  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
 737  * @q_vector: structure containing interrupt and ring information
 738  * @tx_ring: tx ring to clean
 739  **/
 740 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
 741                                struct ixgbe_ring *tx_ring)
 742 {
 743         struct ixgbe_adapter *adapter = q_vector->adapter;
 744         struct ixgbe_tx_buffer *tx_buffer;
 745         union ixgbe_adv_tx_desc *tx_desc;
 746         unsigned int total_bytes = 0, total_packets = 0;
 747         unsigned int budget = q_vector->tx.work_limit;
 748         u16 i = tx_ring->next_to_clean;
 749
 750         tx_buffer = &tx_ring->tx_buffer_info[i];
 751         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 752
 753         for (; budget; budget--) {
 754                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
 755
 756                 /* if next_to_watch is not set then there is no work pending */
 757                 if (!eop_desc)
 758                         break;
 759
 760                 /* if DD is not set pending work has not been completed */
 761                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 762                         break;
 763
 764                 /* count the packet as being completed */
 765                 tx_ring->tx_stats.completed++;
 766
 767                 /* clear next_to_watch to prevent false hangs */
 768                 tx_buffer->next_to_watch = NULL;
 769
 770                 /* prevent any other reads prior to eop_desc being verified */
 771                 rmb();
 772
 773                 do {
 774                         ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
 775                         tx_desc->wb.status = 0;
 776                         if (likely(tx_desc == eop_desc)) {
 777                                 eop_desc = NULL;
 778                                 dev_kfree_skb_any(tx_buffer->skb);
 779                                 tx_buffer->skb = NULL;
 780
 781                                 total_bytes += tx_buffer->bytecount;
 782                                 total_packets += tx_buffer->gso_segs;
 783                         }
 784
 785                         tx_buffer++;
 786                         tx_desc++;
 787                         i++;
 788                         if (unlikely(i == tx_ring->count)) {
 789                                 i = 0;
 790
 791                                 tx_buffer = tx_ring->tx_buffer_info;
 792                                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
 793                         }
 794
 795                 } while (eop_desc);
 796         }
 797
 798         tx_ring->next_to_clean = i;
 799         u64_stats_update_begin(&tx_ring->syncp);
 800         tx_ring->stats.bytes += total_bytes;
 801         tx_ring->stats.packets += total_packets;
 802         u64_stats_update_end(&tx_ring->syncp);
 803         q_vector->tx.total_bytes += total_bytes;
 804         q_vector->tx.total_packets += total_packets;
 805
 806         if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
 807                 /* schedule immediate reset if we believe we hung */
 808                 struct ixgbe_hw *hw = &adapter->hw;
 809                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
 810                 e_err(drv, "Detected Tx Unit Hang\n"
 811                         "  Tx Queue             <%d>\n"
 812                         "  TDH, TDT             <%x>, <%x>\n"
 813                         "  next_to_use          <%x>\n"
 814                         "  next_to_clean        <%x>\n"
 815                         "tx_buffer_info[next_to_clean]\n"
 816                         "  time_stamp           <%lx>\n"
 817                         "  jiffies              <%lx>\n",
 818                         tx_ring->queue_index,
 819                         IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
 820                         IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
 821                         tx_ring->next_to_use, i,
 822                         tx_ring->tx_buffer_info[i].time_stamp, jiffies);
 823
 824                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 825
 826                 e_info(probe,
 827                        "tx hang %d detected on queue %d, resetting adapter\n",
 828                         adapter->tx_timeout_count + 1, tx_ring->queue_index);
 829
 830                 /* schedule immediate reset if we believe we hung */
 831                 ixgbe_tx_timeout_reset(adapter);
 832
 833                 /* the adapter is about to reset, no point in enabling stuff */
 834                 return true;
 835         }
 836
 837 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 838         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 839                      (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 840                 /* Make sure that anybody stopping the queue after this
 841                  * sees the new next_to_clean.
 842                  */
 843                 smp_mb();
 844                 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
 845                     !test_bit(__IXGBE_DOWN, &adapter->state)) {
 846                         netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
 847                         ++tx_ring->tx_stats.restart_queue;
 848                 }
 849         }
 850
 851         return !!budget;
 852 }
 853
 854 #ifdef CONFIG_IXGBE_DCA
 855 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
 856                                 struct ixgbe_ring *rx_ring,
 857                                 int cpu)
 858 {
 859         struct ixgbe_hw *hw = &adapter->hw;
 860         u32 rxctrl;
 861         u8 reg_idx = rx_ring->reg_idx;
 862
 863         rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
 864         switch (hw->mac.type) {
 865         case ixgbe_mac_82598EB:
 866                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
 867                 rxctrl |= dca3_get_tag(rx_ring->dev, cpu);
 868                 break;
 869         case ixgbe_mac_82599EB:
 870         case ixgbe_mac_X540:
 871                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
 872                 rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) <<
 873                            IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
 874                 break;
 875         default:
 876                 break;
 877         }
 878         rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
 879         rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
 880         rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
 881         IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
 882 }
 883
 884 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
 885                                 struct ixgbe_ring *tx_ring,
 886                                 int cpu)
 887 {
 888         struct ixgbe_hw *hw = &adapter->hw;
 889         u32 txctrl;
 890         u8 reg_idx = tx_ring->reg_idx;
 891
 892         switch (hw->mac.type) {
 893         case ixgbe_mac_82598EB:
 894                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
 895                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
 896                 txctrl |= dca3_get_tag(tx_ring->dev, cpu);
 897                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 898                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
 899                 break;
 900         case ixgbe_mac_82599EB:
 901         case ixgbe_mac_X540:
 902                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
 903                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
 904                 txctrl |= (dca3_get_tag(tx_ring->dev, cpu) <<
 905                            IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
 906                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
 907                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
 908                 break;
 909         default:
 910                 break;
 911         }
 912 }
 913
 914 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
 915 {
 916         struct ixgbe_adapter *adapter = q_vector->adapter;
 917         struct ixgbe_ring *ring;
 918         int cpu = get_cpu();
 919
 920         if (q_vector->cpu == cpu)
 921                 goto out_no_update;
 922
 923         for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
 924                 ixgbe_update_tx_dca(adapter, ring, cpu);
 925
 926         for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
 927                 ixgbe_update_rx_dca(adapter, ring, cpu);
 928
 929         q_vector->cpu = cpu;
 930 out_no_update:
 931         put_cpu();
 932 }
 933
 934 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
 935 {
 936         int num_q_vectors;
 937         int i;
 938
 939         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
 940                 return;
 941
 942         /* always use CB2 mode, difference is masked in the CB driver */
 943         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
 944
 945         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
 946                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
 947         else
 948                 num_q_vectors = 1;
 949
 950         for (i = 0; i < num_q_vectors; i++) {
 951                 adapter->q_vector[i]->cpu = -1;
 952                 ixgbe_update_dca(adapter->q_vector[i]);
 953         }
 954 }
 955
 956 static int __ixgbe_notify_dca(struct device *dev, void *data)
 957 {
 958         struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
 959         unsigned long event = *(unsigned long *)data;
 960
 961         if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
 962                 return 0;
 963
 964         switch (event) {
 965         case DCA_PROVIDER_ADD:
 966                 /* if we're already enabled, don't do it again */
 967                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
 968                         break;
 969                 if (dca_add_requester(dev) == 0) {
 970                         adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
 971                         ixgbe_setup_dca(adapter);
 972                         break;
 973                 }
 974                 /* Fall Through since DCA is disabled. */
 975         case DCA_PROVIDER_REMOVE:
 976                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
 977                         dca_remove_requester(dev);
 978                         adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
 979                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
 980                 }
 981                 break;
 982         }
 983
 984         return 0;
 985 }
 986 #endif /* CONFIG_IXGBE_DCA */
 987
 988 static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
 989                                  struct sk_buff *skb)
 990 {
 991         skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
 992 }
 993
 994 /**
 995  * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
 996  * @adapter: address of board private structure
 997  * @rx_desc: advanced rx descriptor
 998  *
 999  * Returns : true if it is FCoE pkt
1000  */
1001 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
1002                                     union ixgbe_adv_rx_desc *rx_desc)
1003 {
1004         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1005
1006         return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
1007                ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1008                 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1009                              IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1010 }
1011
1012 /**
1013  * ixgbe_receive_skb - Send a completed packet up the stack
1014  * @adapter: board private structure
1015  * @skb: packet to send up
1016  * @status: hardware indication of status of receive
1017  * @rx_ring: rx descriptor ring (for a specific queue) to setup
1018  * @rx_desc: rx descriptor
1019  **/
1020 static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1021                               struct sk_buff *skb, u8 status,
1022                               struct ixgbe_ring *ring,
1023                               union ixgbe_adv_rx_desc *rx_desc)
1024 {
1025         struct ixgbe_adapter *adapter = q_vector->adapter;
1026         struct napi_struct *napi = &q_vector->napi;
1027         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1028         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1029
1030         if (is_vlan && (tag & VLAN_VID_MASK))
1031                 __vlan_hwaccel_put_tag(skb, tag);
1032
1033         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1034                 napi_gro_receive(napi, skb);
1035         else
1036                 netif_rx(skb);
1037 }
1038
1039 /**
1040  * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1041  * @adapter: address of board private structure
1042  * @status_err: hardware indication of status of receive
1043  * @skb: skb currently being received and modified
1044  * @status_err: status error value of last descriptor in packet
1045  **/
1046 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1047                                      union ixgbe_adv_rx_desc *rx_desc,
1048                                      struct sk_buff *skb,
1049                                      u32 status_err)
1050 {
1051         skb->ip_summed = CHECKSUM_NONE;
1052
1053         /* Rx csum disabled */
1054         if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1055                 return;
1056
1057         /* if IP and error */
1058         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1059             (status_err & IXGBE_RXDADV_ERR_IPE)) {
1060                 adapter->hw_csum_rx_error++;
1061                 return;
1062         }
1063
1064         if (!(status_err & IXGBE_RXD_STAT_L4CS))
1065                 return;
1066
1067         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1068                 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1069
1070                 /*
1071                  * 82599 errata, UDP frames with a 0 checksum can be marked as
1072                  * checksum errors.
1073                  */
1074                 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1075                     (adapter->hw.mac.type == ixgbe_mac_82599EB))
1076                         return;
1077
1078                 adapter->hw_csum_rx_error++;
1079                 return;
1080         }
1081
1082         /* It must be a TCP or UDP packet with a valid checksum */
1083         skb->ip_summed = CHECKSUM_UNNECESSARY;
1084 }
1085
1086 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1087 {
1088         /*
1089          * Force memory writes to complete before letting h/w
1090          * know there are new descriptors to fetch.  (Only
1091          * applicable for weak-ordered memory model archs,
1092          * such as IA-64).
1093          */
1094         wmb();
1095         writel(val, rx_ring->tail);
1096 }
1097
1098 /**
1099  * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1100  * @rx_ring: ring to place buffers on
1101  * @cleaned_count: number of buffers to replace
1102  **/
1103 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1104 {
1105         union ixgbe_adv_rx_desc *rx_desc;
1106         struct ixgbe_rx_buffer *bi;
1107         struct sk_buff *skb;
1108         u16 i = rx_ring->next_to_use;
1109
1110         /* do nothing if no valid netdev defined */
1111         if (!rx_ring->netdev)
1112                 return;
1113
1114         while (cleaned_count--) {
1115                 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1116                 bi = &rx_ring->rx_buffer_info[i];
1117                 skb = bi->skb;
1118
1119                 if (!skb) {
1120                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1121                                                         rx_ring->rx_buf_len);
1122                         if (!skb) {
1123                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1124                                 goto no_buffers;
1125                         }
1126                         /* initialize queue mapping */
1127                         skb_record_rx_queue(skb, rx_ring->queue_index);
1128                         bi->skb = skb;
1129                 }
1130
1131                 if (!bi->dma) {
1132                         bi->dma = dma_map_single(rx_ring->dev,
1133                                                  skb->data,
1134                                                  rx_ring->rx_buf_len,
1135                                                  DMA_FROM_DEVICE);
1136                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1137                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1138                                 bi->dma = 0;
1139                                 goto no_buffers;
1140                         }
1141                 }
1142
1143                 if (ring_is_ps_enabled(rx_ring)) {
1144                         if (!bi->page) {
1145                                 bi->page = netdev_alloc_page(rx_ring->netdev);
1146                                 if (!bi->page) {
1147                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1148                                         goto no_buffers;
1149                                 }
1150                         }
1151
1152                         if (!bi->page_dma) {
1153                                 /* use a half page if we're re-using */
1154                                 bi->page_offset ^= PAGE_SIZE / 2;
1155                                 bi->page_dma = dma_map_page(rx_ring->dev,
1156                                                             bi->page,
1157                                                             bi->page_offset,
1158                                                             PAGE_SIZE / 2,
1159                                                             DMA_FROM_DEVICE);
1160                                 if (dma_mapping_error(rx_ring->dev,
1161                                                       bi->page_dma)) {
1162                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1163                                         bi->page_dma = 0;
1164                                         goto no_buffers;
1165                                 }
1166                         }
1167
1168                         /* Refresh the desc even if buffer_addrs didn't change
1169                          * because each write-back erases this info. */
1170                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1171                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1172                 } else {
1173                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1174                         rx_desc->read.hdr_addr = 0;
1175                 }
1176
1177                 i++;
1178                 if (i == rx_ring->count)
1179                         i = 0;
1180         }
1181
1182 no_buffers:
1183         if (rx_ring->next_to_use != i) {
1184                 rx_ring->next_to_use = i;
1185                 ixgbe_release_rx_desc(rx_ring, i);
1186         }
1187 }
1188
1189 static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1190 {
1191         /* HW will not DMA in data larger than the given buffer, even if it
1192          * parses the (NFS, of course) header to be larger.  In that case, it
1193          * fills the header buffer and spills the rest into the page.
1194          */
1195         u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1196         u16 hlen = (hdr_info &  IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1197                     IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1198         if (hlen > IXGBE_RX_HDR_SIZE)
1199                 hlen = IXGBE_RX_HDR_SIZE;
1200         return hlen;
1201 }
1202
1203 /**
1204  * ixgbe_transform_rsc_queue - change rsc queue into a full packet
1205  * @skb: pointer to the last skb in the rsc queue
1206  *
1207  * This function changes a queue full of hw rsc buffers into a completed
1208  * packet.  It uses the ->prev pointers to find the first packet and then
1209  * turns it into the frag list owner.
1210  **/
1211 static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1212 {
1213         unsigned int frag_list_size = 0;
1214         unsigned int skb_cnt = 1;
1215
1216         while (skb->prev) {
1217                 struct sk_buff *prev = skb->prev;
1218                 frag_list_size += skb->len;
1219                 skb->prev = NULL;
1220                 skb = prev;
1221                 skb_cnt++;
1222         }
1223
1224         skb_shinfo(skb)->frag_list = skb->next;
1225         skb->next = NULL;
1226         skb->len += frag_list_size;
1227         skb->data_len += frag_list_size;
1228         skb->truesize += frag_list_size;
1229         IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
1230
1231         return skb;
1232 }
1233
1234 static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
1235 {
1236         return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
1237                 IXGBE_RXDADV_RSCCNT_MASK);
1238 }
1239
1240 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1241                                struct ixgbe_ring *rx_ring,
1242                                int budget)
1243 {
1244         struct ixgbe_adapter *adapter = q_vector->adapter;
1245         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
1246         struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
1247         struct sk_buff *skb;
1248         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1249         const int current_node = numa_node_id();
1250 #ifdef IXGBE_FCOE
1251         int ddp_bytes = 0;
1252 #endif /* IXGBE_FCOE */
1253         u32 staterr;
1254         u16 i;
1255         u16 cleaned_count = 0;
1256         bool pkt_is_rsc = false;
1257
1258         i = rx_ring->next_to_clean;
1259         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1260         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1261
1262         while (staterr & IXGBE_RXD_STAT_DD) {
1263                 u32 upper_len = 0;
1264
1265                 rmb(); /* read descriptor and rx_buffer_info after status DD */
1266
1267                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1268
1269                 skb = rx_buffer_info->skb;
1270                 rx_buffer_info->skb = NULL;
1271                 prefetch(skb->data);
1272
1273                 if (ring_is_rsc_enabled(rx_ring))
1274                         pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1275
1276                 /* linear means we are building an skb from multiple pages */
1277                 if (!skb_is_nonlinear(skb)) {
1278                         u16 hlen;
1279                         if (pkt_is_rsc &&
1280                             !(staterr & IXGBE_RXD_STAT_EOP) &&
1281                             !skb->prev) {
1282                                 /*
1283                                  * When HWRSC is enabled, delay unmapping
1284                                  * of the first packet. It carries the
1285                                  * header information, HW may still
1286                                  * access the header after the writeback.
1287                                  * Only unmap it when EOP is reached
1288                                  */
1289                                 IXGBE_RSC_CB(skb)->delay_unmap = true;
1290                                 IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1291                         } else {
1292                                 dma_unmap_single(rx_ring->dev,
1293                                                  rx_buffer_info->dma,
1294                                                  rx_ring->rx_buf_len,
1295                                                  DMA_FROM_DEVICE);
1296                         }
1297                         rx_buffer_info->dma = 0;
1298
1299                         if (ring_is_ps_enabled(rx_ring)) {
1300                                 hlen = ixgbe_get_hlen(rx_desc);
1301                                 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1302                         } else {
1303                                 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1304                         }
1305
1306                         skb_put(skb, hlen);
1307                 } else {
1308                         /* assume packet split since header is unmapped */
1309                         upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1310                 }
1311
1312                 if (upper_len) {
1313                         dma_unmap_page(rx_ring->dev,
1314                                        rx_buffer_info->page_dma,
1315                                        PAGE_SIZE / 2,
1316                                        DMA_FROM_DEVICE);
1317                         rx_buffer_info->page_dma = 0;
1318                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1319                                            rx_buffer_info->page,
1320                                            rx_buffer_info->page_offset,
1321                                            upper_len);
1322
1323                         if ((page_count(rx_buffer_info->page) == 1) &&
1324                             (page_to_nid(rx_buffer_info->page) == current_node))
1325                                 get_page(rx_buffer_info->page);
1326                         else
1327                                 rx_buffer_info->page = NULL;
1328
1329                         skb->len += upper_len;
1330                         skb->data_len += upper_len;
1331                         skb->truesize += PAGE_SIZE / 2;
1332                 }
1333
1334                 i++;
1335                 if (i == rx_ring->count)
1336                         i = 0;
1337
1338                 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1339                 prefetch(next_rxd);
1340                 cleaned_count++;
1341
1342                 if (pkt_is_rsc) {
1343                         u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1344                                      IXGBE_RXDADV_NEXTP_SHIFT;
1345                         next_buffer = &rx_ring->rx_buffer_info[nextp];
1346                 } else {
1347                         next_buffer = &rx_ring->rx_buffer_info[i];
1348                 }
1349
1350                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1351                         if (ring_is_ps_enabled(rx_ring)) {
1352                                 rx_buffer_info->skb = next_buffer->skb;
1353                                 rx_buffer_info->dma = next_buffer->dma;
1354                                 next_buffer->skb = skb;
1355                                 next_buffer->dma = 0;
1356                         } else {
1357                                 skb->next = next_buffer->skb;
1358                                 skb->next->prev = skb;
1359                         }
1360                         rx_ring->rx_stats.non_eop_descs++;
1361                         goto next_desc;
1362                 }
1363
1364                 if (skb->prev) {
1365                         skb = ixgbe_transform_rsc_queue(skb);
1366                         /* if we got here without RSC the packet is invalid */
1367                         if (!pkt_is_rsc) {
1368                                 __pskb_trim(skb, 0);
1369                                 rx_buffer_info->skb = skb;
1370                                 goto next_desc;
1371                         }
1372                 }
1373
1374                 if (ring_is_rsc_enabled(rx_ring)) {
1375                         if (IXGBE_RSC_CB(skb)->delay_unmap) {
1376                                 dma_unmap_single(rx_ring->dev,
1377                                                  IXGBE_RSC_CB(skb)->dma,
1378                                                  rx_ring->rx_buf_len,
1379                                                  DMA_FROM_DEVICE);
1380                                 IXGBE_RSC_CB(skb)->dma = 0;
1381                                 IXGBE_RSC_CB(skb)->delay_unmap = false;
1382                         }
1383                 }
1384                 if (pkt_is_rsc) {
1385                         if (ring_is_ps_enabled(rx_ring))
1386                                 rx_ring->rx_stats.rsc_count +=
1387                                         skb_shinfo(skb)->nr_frags;
1388                         else
1389                                 rx_ring->rx_stats.rsc_count +=
1390                                         IXGBE_RSC_CB(skb)->skb_cnt;
1391                         rx_ring->rx_stats.rsc_flush++;
1392                 }
1393
1394                 /* ERR_MASK will only have valid bits if EOP set */
1395                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
1396                         dev_kfree_skb_any(skb);
1397                         goto next_desc;
1398                 }
1399
1400                 ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
1401                 if (adapter->netdev->features & NETIF_F_RXHASH)
1402                         ixgbe_rx_hash(rx_desc, skb);
1403
1404                 /* probably a little skewed due to removing CRC */
1405                 total_rx_bytes += skb->len;
1406                 total_rx_packets++;
1407
1408                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1409 #ifdef IXGBE_FCOE
1410                 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1411                 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
1412                         ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
1413                                                    staterr);
1414                         if (!ddp_bytes) {
1415                                 dev_kfree_skb_any(skb);
1416                                 goto next_desc;
1417                         }
1418                 }
1419 #endif /* IXGBE_FCOE */
1420                 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1421
1422                 budget--;
1423 next_desc:
1424                 rx_desc->wb.upper.status_error = 0;
1425
1426                 if (!budget)
1427                         break;
1428
1429                 /* return some buffers to hardware, one at a time is too slow */
1430                 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1431                         ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1432                         cleaned_count = 0;
1433                 }
1434
1435                 /* use prefetched values */
1436                 rx_desc = next_rxd;
1437                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1438         }
1439
1440         rx_ring->next_to_clean = i;
1441         cleaned_count = ixgbe_desc_unused(rx_ring);
1442
1443         if (cleaned_count)
1444                 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1445
1446 #ifdef IXGBE_FCOE
1447         /* include DDPed FCoE data */
1448         if (ddp_bytes > 0) {
1449                 unsigned int mss;
1450
1451                 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1452                         sizeof(struct fc_frame_header) -
1453                         sizeof(struct fcoe_crc_eof);
1454                 if (mss > 512)
1455                         mss &= ~511;
1456                 total_rx_bytes += ddp_bytes;
1457                 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1458         }
1459 #endif /* IXGBE_FCOE */
1460
1461         u64_stats_update_begin(&rx_ring->syncp);
1462         rx_ring->stats.packets += total_rx_packets;
1463         rx_ring->stats.bytes += total_rx_bytes;
1464         u64_stats_update_end(&rx_ring->syncp);
1465         q_vector->rx.total_packets += total_rx_packets;
1466         q_vector->rx.total_bytes += total_rx_bytes;
1467
1468         return !!budget;
1469 }
1470
1471 /**
1472  * ixgbe_configure_msix - Configure MSI-X hardware
1473  * @adapter: board private structure
1474  *
1475  * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1476  * interrupts.
1477  **/
1478 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1479 {
1480         struct ixgbe_q_vector *q_vector;
1481         int q_vectors, v_idx;
1482         u32 mask;
1483
1484         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1485
1486         /* Populate MSIX to EITR Select */
1487         if (adapter->num_vfs > 32) {
1488                 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1489                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1490         }
1491
1492         /*
1493          * Populate the IVAR table and set the ITR values to the
1494          * corresponding register.
1495          */
1496         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1497                 struct ixgbe_ring *ring;
1498                 q_vector = adapter->q_vector[v_idx];
1499
1500                 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
1501                         ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1502
1503                 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
1504                         ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1505
1506                 if (q_vector->tx.ring && !q_vector->rx.ring) {
1507                         /* tx only vector */
1508                         if (adapter->tx_itr_setting == 1)
1509                                 q_vector->itr = IXGBE_10K_ITR;
1510                         else
1511                                 q_vector->itr = adapter->tx_itr_setting;
1512                 } else {
1513                         /* rx or rx/tx vector */
1514                         if (adapter->rx_itr_setting == 1)
1515                                 q_vector->itr = IXGBE_20K_ITR;
1516                         else
1517                                 q_vector->itr = adapter->rx_itr_setting;
1518                 }
1519
1520                 ixgbe_write_eitr(q_vector);
1521         }
1522
1523         switch (adapter->hw.mac.type) {
1524         case ixgbe_mac_82598EB:
1525                 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1526                                v_idx);
1527                 break;
1528         case ixgbe_mac_82599EB:
1529         case ixgbe_mac_X540:
1530                 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1531                 break;
1532         default:
1533                 break;
1534         }
1535         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1536
1537         /* set up to autoclear timer, and the vectors */
1538         mask = IXGBE_EIMS_ENABLE_MASK;
1539         mask &= ~(IXGBE_EIMS_OTHER |
1540                   IXGBE_EIMS_MAILBOX |
1541                   IXGBE_EIMS_LSC);
1542
1543         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1544 }
1545
1546 enum latency_range {
1547         lowest_latency = 0,
1548         low_latency = 1,
1549         bulk_latency = 2,
1550         latency_invalid = 255
1551 };
1552
1553 /**
1554  * ixgbe_update_itr - update the dynamic ITR value based on statistics
1555  * @q_vector: structure containing interrupt and ring information
1556  * @ring_container: structure containing ring performance data
1557  *
1558  *      Stores a new ITR value based on packets and byte
1559  *      counts during the last interrupt.  The advantage of per interrupt
1560  *      computation is faster updates and more accurate ITR for the current
1561  *      traffic pattern.  Constants in this function were computed
1562  *      based on theoretical maximum wire speed and thresholds were set based
1563  *      on testing data as well as attempting to minimize response time
1564  *      while increasing bulk throughput.
1565  *      this functionality is controlled by the InterruptThrottleRate module
1566  *      parameter (see ixgbe_param.c)
1567  **/
1568 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1569                              struct ixgbe_ring_container *ring_container)
1570 {
1571         u64 bytes_perint;
1572         struct ixgbe_adapter *adapter = q_vector->adapter;
1573         int bytes = ring_container->total_bytes;
1574         int packets = ring_container->total_packets;
1575         u32 timepassed_us;
1576         u8 itr_setting = ring_container->itr;
1577
1578         if (packets == 0)
1579                 return;
1580
1581         /* simple throttlerate management
1582          *    0-20MB/s lowest (100000 ints/s)
1583          *   20-100MB/s low   (20000 ints/s)
1584          *  100-1249MB/s bulk (8000 ints/s)
1585          */
1586         /* what was last interrupt timeslice? */
1587         timepassed_us = q_vector->itr >> 2;
1588         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1589
1590         switch (itr_setting) {
1591         case lowest_latency:
1592                 if (bytes_perint > adapter->eitr_low)
1593                         itr_setting = low_latency;
1594                 break;
1595         case low_latency:
1596                 if (bytes_perint > adapter->eitr_high)
1597                         itr_setting = bulk_latency;
1598                 else if (bytes_perint <= adapter->eitr_low)
1599                         itr_setting = lowest_latency;
1600                 break;
1601         case bulk_latency:
1602                 if (bytes_perint <= adapter->eitr_high)
1603                         itr_setting = low_latency;
1604                 break;
1605         }
1606
1607         /* clear work counters since we have the values we need */
1608         ring_container->total_bytes = 0;
1609         ring_container->total_packets = 0;
1610
1611         /* write updated itr to ring container */
1612         ring_container->itr = itr_setting;
1613 }
1614
1615 /**
1616  * ixgbe_write_eitr - write EITR register in hardware specific way
1617  * @q_vector: structure containing interrupt and ring information
1618  *
1619  * This function is made to be called by ethtool and by the driver
1620  * when it needs to update EITR registers at runtime.  Hardware
1621  * specific quirks/differences are taken care of here.
1622  */
1623 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1624 {
1625         struct ixgbe_adapter *adapter = q_vector->adapter;
1626         struct ixgbe_hw *hw = &adapter->hw;
1627         int v_idx = q_vector->v_idx;
1628         u32 itr_reg = q_vector->itr;
1629
1630         switch (adapter->hw.mac.type) {
1631         case ixgbe_mac_82598EB:
1632                 /* must write high and low 16 bits to reset counter */
1633                 itr_reg |= (itr_reg << 16);
1634                 break;
1635         case ixgbe_mac_82599EB:
1636         case ixgbe_mac_X540:
1637                 /*
1638                  * set the WDIS bit to not clear the timer bits and cause an
1639                  * immediate assertion of the interrupt
1640                  */
1641                 itr_reg |= IXGBE_EITR_CNT_WDIS;
1642                 break;
1643         default:
1644                 break;
1645         }
1646         IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1647 }
1648
1649 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
1650 {
1651         u32 new_itr = q_vector->itr;
1652         u8 current_itr;
1653
1654         ixgbe_update_itr(q_vector, &q_vector->tx);
1655         ixgbe_update_itr(q_vector, &q_vector->rx);
1656
1657         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1658
1659         switch (current_itr) {
1660         /* counts and packets in update_itr are dependent on these numbers */
1661         case lowest_latency:
1662                 new_itr = IXGBE_100K_ITR;
1663                 break;
1664         case low_latency:
1665                 new_itr = IXGBE_20K_ITR;
1666                 break;
1667         case bulk_latency:
1668                 new_itr = IXGBE_8K_ITR;
1669                 break;
1670         default:
1671                 break;
1672         }
1673
1674         if (new_itr != q_vector->itr) {
1675                 /* do an exponential smoothing */
1676                 new_itr = (10 * new_itr * q_vector->itr) /
1677                           ((9 * new_itr) + q_vector->itr);
1678
1679                 /* save the algorithm value here */
1680                 q_vector->itr = new_itr & IXGBE_MAX_EITR;
1681
1682                 ixgbe_write_eitr(q_vector);
1683         }
1684 }
1685
1686 /**
1687  * ixgbe_check_overtemp_subtask - check for over tempurature
1688  * @adapter: pointer to adapter
1689  **/
1690 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
1691 {
1692         struct ixgbe_hw *hw = &adapter->hw;
1693         u32 eicr = adapter->interrupt_event;
1694
1695         if (test_bit(__IXGBE_DOWN, &adapter->state))
1696                 return;
1697
1698         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1699             !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
1700                 return;
1701
1702         adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1703
1704         switch (hw->device_id) {
1705         case IXGBE_DEV_ID_82599_T3_LOM:
1706                 /*
1707                  * Since the warning interrupt is for both ports
1708                  * we don't have to check if:
1709                  *  - This interrupt wasn't for our port.
1710                  *  - We may have missed the interrupt so always have to
1711                  *    check if we  got a LSC
1712                  */
1713                 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
1714                     !(eicr & IXGBE_EICR_LSC))
1715                         return;
1716
1717                 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
1718                         u32 autoneg;
1719                         bool link_up = false;
1720
1721                         hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1722
1723                         if (link_up)
1724                                 return;
1725                 }
1726
1727                 /* Check if this is not due to overtemp */
1728                 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
1729                         return;
1730
1731                 break;
1732         default:
1733                 if (!(eicr & IXGBE_EICR_GPI_SDP0))
1734                         return;
1735                 break;
1736         }
1737         e_crit(drv,
1738                "Network adapter has been stopped because it has over heated. "
1739                "Restart the computer. If the problem persists, "
1740                "power off the system and replace the adapter\n");
1741
1742         adapter->interrupt_event = 0;
1743 }
1744
1745 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1746 {
1747         struct ixgbe_hw *hw = &adapter->hw;
1748
1749         if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1750             (eicr & IXGBE_EICR_GPI_SDP1)) {
1751                 e_crit(probe, "Fan has stopped, replace the adapter\n");
1752                 /* write to clear the interrupt */
1753                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1754         }
1755 }
1756
1757 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
1758 {
1759         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
1760                 return;
1761
1762         switch (adapter->hw.mac.type) {
1763         case ixgbe_mac_82599EB:
1764                 /*
1765                  * Need to check link state so complete overtemp check
1766                  * on service task
1767                  */
1768                 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
1769                     (!test_bit(__IXGBE_DOWN, &adapter->state))) {
1770                         adapter->interrupt_event = eicr;
1771                         adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1772                         ixgbe_service_event_schedule(adapter);
1773                         return;
1774                 }
1775                 return;
1776         case ixgbe_mac_X540:
1777                 if (!(eicr & IXGBE_EICR_TS))
1778                         return;
1779                 break;
1780         default:
1781                 return;
1782         }
1783
1784         e_crit(drv,
1785                "Network adapter has been stopped because it has over heated. "
1786                "Restart the computer. If the problem persists, "
1787                "power off the system and replace the adapter\n");
1788 }
1789
1790 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1791 {
1792         struct ixgbe_hw *hw = &adapter->hw;
1793
1794         if (eicr & IXGBE_EICR_GPI_SDP2) {
1795                 /* Clear the interrupt */
1796                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1797                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1798                         adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
1799                         ixgbe_service_event_schedule(adapter);
1800                 }
1801         }
1802
1803         if (eicr & IXGBE_EICR_GPI_SDP1) {
1804                 /* Clear the interrupt */
1805                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1806                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1807                         adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
1808                         ixgbe_service_event_schedule(adapter);
1809                 }
1810         }
1811 }
1812
1813 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1814 {
1815         struct ixgbe_hw *hw = &adapter->hw;
1816
1817         adapter->lsc_int++;
1818         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1819         adapter->link_check_timeout = jiffies;
1820         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1821                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1822                 IXGBE_WRITE_FLUSH(hw);
1823                 ixgbe_service_event_schedule(adapter);
1824         }
1825 }
1826
1827 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
1828                                            u64 qmask)
1829 {
1830         u32 mask;
1831         struct ixgbe_hw *hw = &adapter->hw;
1832
1833         switch (hw->mac.type) {
1834         case ixgbe_mac_82598EB:
1835                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1836                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1837                 break;
1838         case ixgbe_mac_82599EB:
1839         case ixgbe_mac_X540:
1840                 mask = (qmask & 0xFFFFFFFF);
1841                 if (mask)
1842                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1843                 mask = (qmask >> 32);
1844                 if (mask)
1845                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1846                 break;
1847         default:
1848                 break;
1849         }
1850         /* skip the flush */
1851 }
1852
1853 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1854                                             u64 qmask)
1855 {
1856         u32 mask;
1857         struct ixgbe_hw *hw = &adapter->hw;
1858
1859         switch (hw->mac.type) {
1860         case ixgbe_mac_82598EB:
1861                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1862                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1863                 break;
1864         case ixgbe_mac_82599EB:
1865         case ixgbe_mac_X540:
1866                 mask = (qmask & 0xFFFFFFFF);
1867                 if (mask)
1868                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1869                 mask = (qmask >> 32);
1870                 if (mask)
1871                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1872                 break;
1873         default:
1874                 break;
1875         }
1876         /* skip the flush */
1877 }
1878
1879 /**
1880  * ixgbe_irq_enable - Enable default interrupt generation settings
1881  * @adapter: board private structure
1882  **/
1883 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
1884                                     bool flush)
1885 {
1886         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1887
1888         /* don't reenable LSC while waiting for link */
1889         if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
1890                 mask &= ~IXGBE_EIMS_LSC;
1891
1892         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
1893                 switch (adapter->hw.mac.type) {
1894                 case ixgbe_mac_82599EB:
1895                         mask |= IXGBE_EIMS_GPI_SDP0;
1896                         break;
1897                 case ixgbe_mac_X540:
1898                         mask |= IXGBE_EIMS_TS;
1899                         break;
1900                 default:
1901                         break;
1902                 }
1903         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
1904                 mask |= IXGBE_EIMS_GPI_SDP1;
1905         switch (adapter->hw.mac.type) {
1906         case ixgbe_mac_82599EB:
1907                 mask |= IXGBE_EIMS_GPI_SDP1;
1908                 mask |= IXGBE_EIMS_GPI_SDP2;
1909         case ixgbe_mac_X540:
1910                 mask |= IXGBE_EIMS_ECC;
1911                 mask |= IXGBE_EIMS_MAILBOX;
1912                 break;
1913         default:
1914                 break;
1915         }
1916         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
1917             !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
1918                 mask |= IXGBE_EIMS_FLOW_DIR;
1919
1920         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
1921         if (queues)
1922                 ixgbe_irq_enable_queues(adapter, ~0);
1923         if (flush)
1924                 IXGBE_WRITE_FLUSH(&adapter->hw);
1925 }
1926
1927 static irqreturn_t ixgbe_msix_other(int irq, void *data)
1928 {
1929         struct ixgbe_adapter *adapter = data;
1930         struct ixgbe_hw *hw = &adapter->hw;
1931         u32 eicr;
1932
1933         /*
1934          * Workaround for Silicon errata.  Use clear-by-write instead
1935          * of clear-by-read.  Reading with EICS will return the
1936          * interrupt causes without clearing, which later be done
1937          * with the write to EICR.
1938          */
1939         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1940         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1941
1942         if (eicr & IXGBE_EICR_LSC)
1943                 ixgbe_check_lsc(adapter);
1944
1945         if (eicr & IXGBE_EICR_MAILBOX)
1946                 ixgbe_msg_task(adapter);
1947
1948         switch (hw->mac.type) {
1949         case ixgbe_mac_82599EB:
1950         case ixgbe_mac_X540:
1951                 if (eicr & IXGBE_EICR_ECC)
1952                         e_info(link, "Received unrecoverable ECC Err, please "
1953                                "reboot\n");
1954                 /* Handle Flow Director Full threshold interrupt */
1955                 if (eicr & IXGBE_EICR_FLOW_DIR) {
1956                         int reinit_count = 0;
1957                         int i;
1958                         for (i = 0; i < adapter->num_tx_queues; i++) {
1959                                 struct ixgbe_ring *ring = adapter->tx_ring[i];
1960                                 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1961                                                        &ring->state))
1962                                         reinit_count++;
1963                         }
1964                         if (reinit_count) {
1965                                 /* no more flow director interrupts until after init */
1966                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
1967                                 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
1968                                 ixgbe_service_event_schedule(adapter);
1969                         }
1970                 }
1971                 ixgbe_check_sfp_event(adapter, eicr);
1972                 ixgbe_check_overtemp_event(adapter, eicr);
1973                 break;
1974         default:
1975                 break;
1976         }
1977
1978         ixgbe_check_fan_failure(adapter, eicr);
1979
1980         /* re-enable the original interrupt state, no lsc, no queues */
1981         if (!test_bit(__IXGBE_DOWN, &adapter->state))
1982                 ixgbe_irq_enable(adapter, false, false);
1983
1984         return IRQ_HANDLED;
1985 }
1986
1987 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
1988 {
1989         struct ixgbe_q_vector *q_vector = data;
1990
1991         /* EIAM disabled interrupts (on this vector) for us */
1992
1993         if (q_vector->rx.ring || q_vector->tx.ring)
1994                 napi_schedule(&q_vector->napi);
1995
1996         return IRQ_HANDLED;
1997 }
1998
1999 static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
2000                                      int r_idx)
2001 {
2002         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2003         struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
2004
2005         rx_ring->q_vector = q_vector;
2006         rx_ring->next = q_vector->rx.ring;
2007         q_vector->rx.ring = rx_ring;
2008         q_vector->rx.count++;
2009 }
2010
2011 static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
2012                                      int t_idx)
2013 {
2014         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2015         struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
2016
2017         tx_ring->q_vector = q_vector;
2018         tx_ring->next = q_vector->tx.ring;
2019         q_vector->tx.ring = tx_ring;
2020         q_vector->tx.count++;
2021         q_vector->tx.work_limit = a->tx_work_limit;
2022 }
2023
2024 /**
2025  * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
2026  * @adapter: board private structure to initialize
2027  *
2028  * This function maps descriptor rings to the queue-specific vectors
2029  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
2030  * one vector per ring/queue, but on a constrained vector budget, we
2031  * group the rings as "efficiently" as possible.  You would add new
2032  * mapping configurations in here.
2033  **/
2034 static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2035 {
2036         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2037         int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0;
2038         int txr_remaining = adapter->num_tx_queues, txr_idx = 0;
2039         int v_start = 0;
2040
2041         /* only one q_vector if MSI-X is disabled. */
2042         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2043                 q_vectors = 1;
2044
2045         /*
2046          * If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2047          * group them so there are multiple queues per vector.
2048          *
2049          * Re-adjusting *qpv takes care of the remainder.
2050          */
2051         for (; v_start < q_vectors && rxr_remaining; v_start++) {
2052                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start);
2053                 for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--)
2054                         map_vector_to_rxq(adapter, v_start, rxr_idx);
2055         }
2056
2057         /*
2058          * If there are not enough q_vectors for each ring to have it's own
2059          * vector then we must pair up Rx/Tx on a each vector
2060          */
2061         if ((v_start + txr_remaining) > q_vectors)
2062                 v_start = 0;
2063
2064         for (; v_start < q_vectors && txr_remaining; v_start++) {
2065                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start);
2066                 for (; tqpv; tqpv--, txr_idx++, txr_remaining--)
2067                         map_vector_to_txq(adapter, v_start, txr_idx);
2068         }
2069 }
2070
2071 /**
2072  * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2073  * @adapter: board private structure
2074  *
2075  * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2076  * interrupts from the kernel.
2077  **/
2078 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2079 {
2080         struct net_device *netdev = adapter->netdev;
2081         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2082         int vector, err;
2083         int ri = 0, ti = 0;
2084
2085         for (vector = 0; vector < q_vectors; vector++) {
2086                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2087                 struct msix_entry *entry = &adapter->msix_entries[vector];
2088
2089                 if (q_vector->tx.ring && q_vector->rx.ring) {
2090                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2091                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
2092                         ti++;
2093                 } else if (q_vector->rx.ring) {
2094                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2095                                  "%s-%s-%d", netdev->name, "rx", ri++);
2096                 } else if (q_vector->tx.ring) {
2097                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2098                                  "%s-%s-%d", netdev->name, "tx", ti++);
2099                 } else {
2100                         /* skip this unused q_vector */
2101                         continue;
2102                 }
2103                 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2104                                   q_vector->name, q_vector);
2105                 if (err) {
2106                         e_err(probe, "request_irq failed for MSIX interrupt "
2107                               "Error: %d\n", err);
2108                         goto free_queue_irqs;
2109                 }
2110                 /* If Flow Director is enabled, set interrupt affinity */
2111                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2112                         /* assign the mask for this irq */
2113                         irq_set_affinity_hint(entry->vector,
2114                                               q_vector->affinity_mask);
2115                 }
2116         }
2117
2118         err = request_irq(adapter->msix_entries[vector].vector,
2119                           ixgbe_msix_other, 0, netdev->name, adapter);
2120         if (err) {
2121                 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2122                 goto free_queue_irqs;
2123         }
2124
2125         return 0;
2126
2127 free_queue_irqs:
2128         while (vector) {
2129                 vector--;
2130                 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2131                                       NULL);
2132                 free_irq(adapter->msix_entries[vector].vector,
2133                          adapter->q_vector[vector]);
2134         }
2135         adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2136         pci_disable_msix(adapter->pdev);
2137         kfree(adapter->msix_entries);
2138         adapter->msix_entries = NULL;
2139         return err;
2140 }
2141
2142 /**
2143  * ixgbe_intr - legacy mode Interrupt Handler
2144  * @irq: interrupt number
2145  * @data: pointer to a network interface device structure
2146  **/
2147 static irqreturn_t ixgbe_intr(int irq, void *data)
2148 {
2149         struct ixgbe_adapter *adapter = data;
2150         struct ixgbe_hw *hw = &adapter->hw;
2151         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2152         u32 eicr;
2153
2154         /*
2155          * Workaround for silicon errata on 82598.  Mask the interrupts
2156          * before the read of EICR.
2157          */
2158         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2159
2160         /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2161          * therefore no explict interrupt disable is necessary */
2162         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2163         if (!eicr) {
2164                 /*
2165                  * shared interrupt alert!
2166                  * make sure interrupts are enabled because the read will
2167                  * have disabled interrupts due to EIAM
2168                  * finish the workaround of silicon errata on 82598.  Unmask
2169                  * the interrupt that we masked before the EICR read.
2170                  */
2171                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2172                         ixgbe_irq_enable(adapter, true, true);
2173                 return IRQ_NONE;        /* Not our interrupt */
2174         }
2175
2176         if (eicr & IXGBE_EICR_LSC)
2177                 ixgbe_check_lsc(adapter);
2178
2179         switch (hw->mac.type) {
2180         case ixgbe_mac_82599EB:
2181                 ixgbe_check_sfp_event(adapter, eicr);
2182                 /* Fall through */
2183         case ixgbe_mac_X540:
2184                 if (eicr & IXGBE_EICR_ECC)
2185                         e_info(link, "Received unrecoverable ECC err, please "
2186                                      "reboot\n");
2187                 ixgbe_check_overtemp_event(adapter, eicr);
2188                 break;
2189         default:
2190                 break;
2191         }
2192
2193         ixgbe_check_fan_failure(adapter, eicr);
2194
2195         if (napi_schedule_prep(&(q_vector->napi))) {
2196                 /* would disable interrupts here but EIAM disabled it */
2197                 __napi_schedule(&(q_vector->napi));
2198         }
2199
2200         /*
2201          * re-enable link(maybe) and non-queue interrupts, no flush.
2202          * ixgbe_poll will re-enable the queue interrupts
2203          */
2204
2205         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2206                 ixgbe_irq_enable(adapter, false, false);
2207
2208         return IRQ_HANDLED;
2209 }
2210
2211 static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2212 {
2213         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2214         int i;
2215
2216         /* legacy and MSI only use one vector */
2217         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2218                 q_vectors = 1;
2219
2220         for (i = 0; i < adapter->num_rx_queues; i++) {
2221                 adapter->rx_ring[i]->q_vector = NULL;
2222                 adapter->rx_ring[i]->next = NULL;
2223         }
2224         for (i = 0; i < adapter->num_tx_queues; i++) {
2225                 adapter->tx_ring[i]->q_vector = NULL;
2226                 adapter->tx_ring[i]->next = NULL;
2227         }
2228
2229         for (i = 0; i < q_vectors; i++) {
2230                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2231                 memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container));
2232                 memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container));
2233         }
2234 }
2235
2236 /**
2237  * ixgbe_request_irq - initialize interrupts
2238  * @adapter: board private structure
2239  *
2240  * Attempts to configure interrupts using the best available
2241  * capabilities of the hardware and kernel.
2242  **/
2243 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2244 {
2245         struct net_device *netdev = adapter->netdev;
2246         int err;
2247
2248         /* map all of the rings to the q_vectors */
2249         ixgbe_map_rings_to_vectors(adapter);
2250
2251         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
2252                 err = ixgbe_request_msix_irqs(adapter);
2253         else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
2254                 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2255                                   netdev->name, adapter);
2256         else
2257                 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2258                                   netdev->name, adapter);
2259
2260         if (err) {
2261                 e_err(probe, "request_irq failed, Error %d\n", err);
2262
2263                 /* place q_vectors and rings back into a known good state */
2264                 ixgbe_reset_q_vectors(adapter);
2265         }
2266
2267         return err;
2268 }
2269
2270 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2271 {
2272         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2273                 int i, q_vectors;
2274
2275                 q_vectors = adapter->num_msix_vectors;
2276                 i = q_vectors - 1;
2277                 free_irq(adapter->msix_entries[i].vector, adapter);
2278                 i--;
2279
2280                 for (; i >= 0; i--) {
2281                         /* free only the irqs that were actually requested */
2282                         if (!adapter->q_vector[i]->rx.ring &&
2283                             !adapter->q_vector[i]->tx.ring)
2284                                 continue;
2285
2286                         /* clear the affinity_mask in the IRQ descriptor */
2287                         irq_set_affinity_hint(adapter->msix_entries[i].vector,
2288                                               NULL);
2289
2290                         free_irq(adapter->msix_entries[i].vector,
2291                                  adapter->q_vector[i]);
2292                 }
2293         } else {
2294                 free_irq(adapter->pdev->irq, adapter);
2295         }
2296
2297         /* clear q_vector state information */
2298         ixgbe_reset_q_vectors(adapter);
2299 }
2300
2301 /**
2302  * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2303  * @adapter: board private structure
2304  **/
2305 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2306 {
2307         switch (adapter->hw.mac.type) {
2308         case ixgbe_mac_82598EB:
2309                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2310                 break;
2311         case ixgbe_mac_82599EB:
2312         case ixgbe_mac_X540:
2313                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2314                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2315                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2316                 break;
2317         default:
2318                 break;
2319         }
2320         IXGBE_WRITE_FLUSH(&adapter->hw);
2321         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2322                 int i;
2323                 for (i = 0; i < adapter->num_msix_vectors; i++)
2324                         synchronize_irq(adapter->msix_entries[i].vector);
2325         } else {
2326                 synchronize_irq(adapter->pdev->irq);
2327         }
2328 }
2329
2330 /**
2331  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2332  *
2333  **/
2334 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2335 {
2336         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2337
2338         /* rx/tx vector */
2339         if (adapter->rx_itr_setting == 1)
2340                 q_vector->itr = IXGBE_20K_ITR;
2341         else
2342                 q_vector->itr = adapter->rx_itr_setting;
2343
2344         ixgbe_write_eitr(q_vector);
2345
2346         ixgbe_set_ivar(adapter, 0, 0, 0);
2347         ixgbe_set_ivar(adapter, 1, 0, 0);
2348
2349         e_info(hw, "Legacy interrupt IVAR setup done\n");
2350 }
2351
2352 /**
2353  * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2354  * @adapter: board private structure
2355  * @ring: structure containing ring specific data
2356  *
2357  * Configure the Tx descriptor ring after a reset.
2358  **/
2359 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2360                              struct ixgbe_ring *ring)
2361 {
2362         struct ixgbe_hw *hw = &adapter->hw;
2363         u64 tdba = ring->dma;
2364         int wait_loop = 10;
2365         u32 txdctl = IXGBE_TXDCTL_ENABLE;
2366         u8 reg_idx = ring->reg_idx;
2367
2368         /* disable queue to avoid issues while updating state */
2369         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2370         IXGBE_WRITE_FLUSH(hw);
2371
2372         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2373                         (tdba & DMA_BIT_MASK(32)));
2374         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2375         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2376                         ring->count * sizeof(union ixgbe_adv_tx_desc));
2377         IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2378         IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2379         ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2380
2381         /*
2382          * set WTHRESH to encourage burst writeback, it should not be set
2383          * higher than 1 when ITR is 0 as it could cause false TX hangs
2384          *
2385          * In order to avoid issues WTHRESH + PTHRESH should always be equal
2386          * to or less than the number of on chip descriptors, which is
2387          * currently 40.
2388          */
2389         if (!adapter->tx_itr_setting || !adapter->rx_itr_setting)
2390                 txdctl |= (1 << 16);    /* WTHRESH = 1 */
2391         else
2392                 txdctl |= (8 << 16);    /* WTHRESH = 8 */
2393
2394         /* PTHRESH=32 is needed to avoid a Tx hang with DFP enabled. */
2395         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
2396                    32;          /* PTHRESH = 32 */
2397
2398         /* reinitialize flowdirector state */
2399         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2400             adapter->atr_sample_rate) {
2401                 ring->atr_sample_rate = adapter->atr_sample_rate;
2402                 ring->atr_count = 0;
2403                 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2404         } else {
2405                 ring->atr_sample_rate = 0;
2406         }
2407
2408         clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2409
2410         /* enable queue */
2411         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2412
2413         /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2414         if (hw->mac.type == ixgbe_mac_82598EB &&
2415             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2416                 return;
2417
2418         /* poll to verify queue is enabled */
2419         do {
2420                 usleep_range(1000, 2000);
2421                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2422         } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2423         if (!wait_loop)
2424                 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2425 }
2426
2427 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2428 {
2429         struct ixgbe_hw *hw = &adapter->hw;
2430         u32 rttdcs;
2431         u32 reg;
2432         u8 tcs = netdev_get_num_tc(adapter->netdev);
2433
2434         if (hw->mac.type == ixgbe_mac_82598EB)
2435                 return;
2436
2437         /* disable the arbiter while setting MTQC */
2438         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2439         rttdcs |= IXGBE_RTTDCS_ARBDIS;
2440         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2441
2442         /* set transmit pool layout */
2443         switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2444         case (IXGBE_FLAG_SRIOV_ENABLED):
2445                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2446                                 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2447                 break;
2448         default:
2449                 if (!tcs)
2450                         reg = IXGBE_MTQC_64Q_1PB;
2451                 else if (tcs <= 4)
2452                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2453                 else
2454                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2455
2456                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2457
2458                 /* Enable Security TX Buffer IFG for multiple pb */
2459                 if (tcs) {
2460                         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2461                         reg |= IXGBE_SECTX_DCB;
2462                         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2463                 }
2464                 break;
2465         }
2466
2467         /* re-enable the arbiter */
2468         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2469         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2470 }
2471
2472 /**
2473  * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2474  * @adapter: board private structure
2475  *
2476  * Configure the Tx unit of the MAC after a reset.
2477  **/
2478 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2479 {
2480         struct ixgbe_hw *hw = &adapter->hw;
2481         u32 dmatxctl;
2482         u32 i;
2483
2484         ixgbe_setup_mtqc(adapter);
2485
2486         if (hw->mac.type != ixgbe_mac_82598EB) {
2487                 /* DMATXCTL.EN must be before Tx queues are enabled */
2488                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2489                 dmatxctl |= IXGBE_DMATXCTL_TE;
2490                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2491         }
2492
2493         /* Setup the HW Tx Head and Tail descriptor pointers */
2494         for (i = 0; i < adapter->num_tx_queues; i++)
2495                 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2496 }
2497
2498 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2499
2500 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2501                                    struct ixgbe_ring *rx_ring)
2502 {
2503         u32 srrctl;
2504         u8 reg_idx = rx_ring->reg_idx;
2505
2506         switch (adapter->hw.mac.type) {
2507         case ixgbe_mac_82598EB: {
2508                 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2509                 const int mask = feature[RING_F_RSS].mask;
2510                 reg_idx = reg_idx & mask;
2511         }
2512                 break;
2513         case ixgbe_mac_82599EB:
2514         case ixgbe_mac_X540:
2515         default:
2516                 break;
2517         }
2518
2519         srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2520
2521         srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2522         srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2523         if (adapter->num_vfs)
2524                 srrctl |= IXGBE_SRRCTL_DROP_EN;
2525
2526         srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2527                   IXGBE_SRRCTL_BSIZEHDR_MASK;
2528
2529         if (ring_is_ps_enabled(rx_ring)) {
2530 #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2531                 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2532 #else
2533                 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2534 #endif
2535                 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2536         } else {
2537                 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2538                           IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2539                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2540         }
2541
2542         IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2543 }
2544
2545 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2546 {
2547         struct ixgbe_hw *hw = &adapter->hw;
2548         static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2549                           0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2550                           0x6A3E67EA, 0x14364D17, 0x3BED200D};
2551         u32 mrqc = 0, reta = 0;
2552         u32 rxcsum;
2553         int i, j;
2554         u8 tcs = netdev_get_num_tc(adapter->netdev);
2555         int maxq = adapter->ring_feature[RING_F_RSS].indices;
2556
2557         if (tcs)
2558                 maxq = min(maxq, adapter->num_tx_queues / tcs);
2559
2560         /* Fill out hash function seeds */
2561         for (i = 0; i < 10; i++)
2562                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2563
2564         /* Fill out redirection table */
2565         for (i = 0, j = 0; i < 128; i++, j++) {
2566                 if (j == maxq)
2567                         j = 0;
2568                 /* reta = 4-byte sliding window of
2569                  * 0x00..(indices-1)(indices-1)00..etc. */
2570                 reta = (reta << 8) | (j * 0x11);
2571                 if ((i & 3) == 3)
2572                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2573         }
2574
2575         /* Disable indicating checksum in descriptor, enables RSS hash */
2576         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2577         rxcsum |= IXGBE_RXCSUM_PCSD;
2578         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2579
2580         if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
2581             (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
2582                 mrqc = IXGBE_MRQC_RSSEN;
2583         } else {
2584                 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2585                                              | IXGBE_FLAG_SRIOV_ENABLED);
2586
2587                 switch (mask) {
2588                 case (IXGBE_FLAG_RSS_ENABLED):
2589                         if (!tcs)
2590                                 mrqc = IXGBE_MRQC_RSSEN;
2591                         else if (tcs <= 4)
2592                                 mrqc = IXGBE_MRQC_RTRSS4TCEN;
2593                         else
2594                                 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2595                         break;
2596                 case (IXGBE_FLAG_SRIOV_ENABLED):
2597                         mrqc = IXGBE_MRQC_VMDQEN;
2598                         break;
2599                 default:
2600                         break;
2601                 }
2602         }
2603
2604         /* Perform hash on these packet types */
2605         mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2606               | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2607               | IXGBE_MRQC_RSS_FIELD_IPV6
2608               | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2609
2610         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2611 }
2612
2613 /**
2614  * ixgbe_configure_rscctl - enable RSC for the indicated ring
2615  * @adapter:    address of board private structure
2616  * @index:      index of ring to set
2617  **/
2618 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2619                                    struct ixgbe_ring *ring)
2620 {
2621         struct ixgbe_hw *hw = &adapter->hw;
2622         u32 rscctrl;
2623         int rx_buf_len;
2624         u8 reg_idx = ring->reg_idx;
2625
2626         if (!ring_is_rsc_enabled(ring))
2627                 return;
2628
2629         rx_buf_len = ring->rx_buf_len;
2630         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2631         rscctrl |= IXGBE_RSCCTL_RSCEN;
2632         /*
2633          * we must limit the number of descriptors so that the
2634          * total size of max desc * buf_len is not greater
2635          * than 65535
2636          */
2637         if (ring_is_ps_enabled(ring)) {
2638 #if (MAX_SKB_FRAGS > 16)
2639                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2640 #elif (MAX_SKB_FRAGS > 8)
2641                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2642 #elif (MAX_SKB_FRAGS > 4)
2643                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2644 #else
2645                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2646 #endif
2647         } else {
2648                 if (rx_buf_len < IXGBE_RXBUFFER_4K)
2649                         rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2650                 else if (rx_buf_len < IXGBE_RXBUFFER_8K)
2651                         rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2652                 else
2653                         rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2654         }
2655         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2656 }
2657
2658 /**
2659  *  ixgbe_set_uta - Set unicast filter table address
2660  *  @adapter: board private structure
2661  *
2662  *  The unicast table address is a register array of 32-bit registers.
2663  *  The table is meant to be used in a way similar to how the MTA is used
2664  *  however due to certain limitations in the hardware it is necessary to
2665  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2666  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
2667  **/
2668 static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2669 {
2670         struct ixgbe_hw *hw = &adapter->hw;
2671         int i;
2672
2673         /* The UTA table only exists on 82599 hardware and newer */
2674         if (hw->mac.type < ixgbe_mac_82599EB)
2675                 return;
2676
2677         /* we only need to do this if VMDq is enabled */
2678         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2679                 return;
2680
2681         for (i = 0; i < 128; i++)
2682                 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2683 }
2684
2685 #define IXGBE_MAX_RX_DESC_POLL 10
2686 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
2687                                        struct ixgbe_ring *ring)
2688 {
2689         struct ixgbe_hw *hw = &adapter->hw;
2690         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2691         u32 rxdctl;
2692         u8 reg_idx = ring->reg_idx;
2693
2694         /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2695         if (hw->mac.type == ixgbe_mac_82598EB &&
2696             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2697                 return;
2698
2699         do {
2700                 usleep_range(1000, 2000);
2701                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2702         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
2703
2704         if (!wait_loop) {
2705                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
2706                       "the polling period\n", reg_idx);
2707         }
2708 }
2709
2710 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
2711                             struct ixgbe_ring *ring)
2712 {
2713         struct ixgbe_hw *hw = &adapter->hw;
2714         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2715         u32 rxdctl;
2716         u8 reg_idx = ring->reg_idx;
2717
2718         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2719         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
2720
2721         /* write value back with RXDCTL.ENABLE bit cleared */
2722         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2723
2724         if (hw->mac.type == ixgbe_mac_82598EB &&
2725             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2726                 return;
2727
2728         /* the hardware may take up to 100us to really disable the rx queue */
2729         do {
2730                 udelay(10);
2731                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2732         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
2733
2734         if (!wait_loop) {
2735                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
2736                       "the polling period\n", reg_idx);
2737         }
2738 }
2739
2740 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
2741                              struct ixgbe_ring *ring)
2742 {
2743         struct ixgbe_hw *hw = &adapter->hw;
2744         u64 rdba = ring->dma;
2745         u32 rxdctl;
2746         u8 reg_idx = ring->reg_idx;
2747
2748         /* disable queue to avoid issues while updating state */
2749         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2750         ixgbe_disable_rx_queue(adapter, ring);
2751
2752         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
2753         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
2754         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
2755                         ring->count * sizeof(union ixgbe_adv_rx_desc));
2756         IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
2757         IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
2758         ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
2759
2760         ixgbe_configure_srrctl(adapter, ring);
2761         ixgbe_configure_rscctl(adapter, ring);
2762
2763         /* If operating in IOV mode set RLPML for X540 */
2764         if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
2765             hw->mac.type == ixgbe_mac_X540) {
2766                 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
2767                 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
2768                             ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
2769         }
2770
2771         if (hw->mac.type == ixgbe_mac_82598EB) {
2772                 /*
2773                  * enable cache line friendly hardware writes:
2774                  * PTHRESH=32 descriptors (half the internal cache),
2775                  * this also removes ugly rx_no_buffer_count increment
2776                  * HTHRESH=4 descriptors (to minimize latency on fetch)
2777                  * WTHRESH=8 burst writeback up to two cache lines
2778                  */
2779                 rxdctl &= ~0x3FFFFF;
2780                 rxdctl |=  0x080420;
2781         }
2782
2783         /* enable receive descriptor ring */
2784         rxdctl |= IXGBE_RXDCTL_ENABLE;
2785         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2786
2787         ixgbe_rx_desc_queue_enable(adapter, ring);
2788         ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
2789 }
2790
2791 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
2792 {
2793         struct ixgbe_hw *hw = &adapter->hw;
2794         int p;
2795
2796         /* PSRTYPE must be initialized in non 82598 adapters */
2797         u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
2798                       IXGBE_PSRTYPE_UDPHDR |
2799                       IXGBE_PSRTYPE_IPV4HDR |
2800                       IXGBE_PSRTYPE_L2HDR |
2801                       IXGBE_PSRTYPE_IPV6HDR;
2802
2803         if (hw->mac.type == ixgbe_mac_82598EB)
2804                 return;
2805
2806         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
2807                 psrtype |= (adapter->num_rx_queues_per_pool << 29);
2808
2809         for (p = 0; p < adapter->num_rx_pools; p++)
2810                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
2811                                 psrtype);
2812 }
2813
2814 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
2815 {
2816         struct ixgbe_hw *hw = &adapter->hw;
2817         u32 gcr_ext;
2818         u32 vt_reg_bits;
2819         u32 reg_offset, vf_shift;
2820         u32 vmdctl;
2821         int i;
2822
2823         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2824                 return;
2825
2826         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2827         vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
2828         vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
2829         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
2830
2831         vf_shift = adapter->num_vfs % 32;
2832         reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
2833
2834         /* Enable only the PF's pool for Tx/Rx */
2835         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
2836         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
2837         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
2838         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
2839         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2840
2841         /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
2842         hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
2843
2844         /*
2845          * Set up VF register offsets for selected VT Mode,
2846          * i.e. 32 or 64 VFs for SR-IOV
2847          */
2848         gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
2849         gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
2850         gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
2851         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
2852
2853         /* enable Tx loopback for VF/PF communication */
2854         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2855         /* Enable MAC Anti-Spoofing */
2856         hw->mac.ops.set_mac_anti_spoofing(hw,
2857                                            (adapter->num_vfs != 0),
2858                                           adapter->num_vfs);
2859         /* For VFs that have spoof checking turned off */
2860         for (i = 0; i < adapter->num_vfs; i++) {
2861                 if (!adapter->vfinfo[i].spoofchk_enabled)
2862                         ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);
2863         }
2864 }
2865