drivers/net/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2009 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/ipv6.h>
  35 #include <linux/slab.h>
  36 #include <net/checksum.h>
  37 #include <net/ip6_checksum.h>
  38 #include <linux/net_tstamp.h>
  39 #include <linux/mii.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/pci.h>
  43 #include <linux/pci-aspm.h>
  44 #include <linux/delay.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/if_ether.h>
  47 #include <linux/aer.h>
  48 #ifdef CONFIG_IGB_DCA
  49 #include <linux/dca.h>
  50 #endif
  51 #include "igb.h"
  52
  53 #define DRV_VERSION "2.4.13-k2"
  54 char igb_driver_name[] = "igb";
  55 char igb_driver_version[] = DRV_VERSION;
  56 static const char igb_driver_string[] =
  57                                 "Intel(R) Gigabit Ethernet Network Driver";
  58 static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
  59
  60 static const struct e1000_info *igb_info_tbl[] = {
  61         [board_82575] = &e1000_82575_info,
  62 };
  63
  64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  85         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
  86         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  87         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  88         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  89         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  90         /* required last entry */
  91         {0, }
  92 };
  93
  94 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  95
  96 void igb_reset(struct igb_adapter *);
  97 static int igb_setup_all_tx_resources(struct igb_adapter *);
  98 static int igb_setup_all_rx_resources(struct igb_adapter *);
  99 static void igb_free_all_tx_resources(struct igb_adapter *);
 100 static void igb_free_all_rx_resources(struct igb_adapter *);
 101 static void igb_setup_mrqc(struct igb_adapter *);
 102 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
 103 static void __devexit igb_remove(struct pci_dev *pdev);
 104 static int igb_sw_init(struct igb_adapter *);
 105 static int igb_open(struct net_device *);
 106 static int igb_close(struct net_device *);
 107 static void igb_configure_tx(struct igb_adapter *);
 108 static void igb_configure_rx(struct igb_adapter *);
 109 static void igb_clean_all_tx_rings(struct igb_adapter *);
 110 static void igb_clean_all_rx_rings(struct igb_adapter *);
 111 static void igb_clean_tx_ring(struct igb_ring *);
 112 static void igb_clean_rx_ring(struct igb_ring *);
 113 static void igb_set_rx_mode(struct net_device *);
 114 static void igb_update_phy_info(unsigned long);
 115 static void igb_watchdog(unsigned long);
 116 static void igb_watchdog_task(struct work_struct *);
 117 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
 118 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
 119                                                  struct rtnl_link_stats64 *stats);
 120 static int igb_change_mtu(struct net_device *, int);
 121 static int igb_set_mac(struct net_device *, void *);
 122 static void igb_set_uta(struct igb_adapter *adapter);
 123 static irqreturn_t igb_intr(int irq, void *);
 124 static irqreturn_t igb_intr_msi(int irq, void *);
 125 static irqreturn_t igb_msix_other(int irq, void *);
 126 static irqreturn_t igb_msix_ring(int irq, void *);
 127 #ifdef CONFIG_IGB_DCA
 128 static void igb_update_dca(struct igb_q_vector *);
 129 static void igb_setup_dca(struct igb_adapter *);
 130 #endif /* CONFIG_IGB_DCA */
 131 static bool igb_clean_tx_irq(struct igb_q_vector *);
 132 static int igb_poll(struct napi_struct *, int);
 133 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
 134 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 135 static void igb_tx_timeout(struct net_device *);
 136 static void igb_reset_task(struct work_struct *);
 137 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
 138 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 139 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 140 static void igb_restore_vlan(struct igb_adapter *);
 141 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 142 static void igb_ping_all_vfs(struct igb_adapter *);
 143 static void igb_msg_task(struct igb_adapter *);
 144 static void igb_vmm_control(struct igb_adapter *);
 145 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 146 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 147 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 148 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 149                                int vf, u16 vlan, u8 qos);
 150 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 151 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 152                                  struct ifla_vf_info *ivi);
 153 static void igb_check_vf_rate_limit(struct igb_adapter *);
 154
 155 #ifdef CONFIG_PM
 156 static int igb_suspend(struct pci_dev *, pm_message_t);
 157 static int igb_resume(struct pci_dev *);
 158 #endif
 159 static void igb_shutdown(struct pci_dev *);
 160 #ifdef CONFIG_IGB_DCA
 161 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 162 static struct notifier_block dca_notifier = {
 163         .notifier_call  = igb_notify_dca,
 164         .next           = NULL,
 165         .priority       = 0
 166 };
 167 #endif
 168 #ifdef CONFIG_NET_POLL_CONTROLLER
 169 /* for netdump / net console */
 170 static void igb_netpoll(struct net_device *);
 171 #endif
 172 #ifdef CONFIG_PCI_IOV
 173 static unsigned int max_vfs = 0;
 174 module_param(max_vfs, uint, 0);
 175 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 176                  "per physical function");
 177 #endif /* CONFIG_PCI_IOV */
 178
 179 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 180                      pci_channel_state_t);
 181 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 182 static void igb_io_resume(struct pci_dev *);
 183
 184 static struct pci_error_handlers igb_err_handler = {
 185         .error_detected = igb_io_error_detected,
 186         .slot_reset = igb_io_slot_reset,
 187         .resume = igb_io_resume,
 188 };
 189
 190
 191 static struct pci_driver igb_driver = {
 192         .name     = igb_driver_name,
 193         .id_table = igb_pci_tbl,
 194         .probe    = igb_probe,
 195         .remove   = __devexit_p(igb_remove),
 196 #ifdef CONFIG_PM
 197         /* Power Managment Hooks */
 198         .suspend  = igb_suspend,
 199         .resume   = igb_resume,
 200 #endif
 201         .shutdown = igb_shutdown,
 202         .err_handler = &igb_err_handler
 203 };
 204
 205 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 206 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 207 MODULE_LICENSE("GPL");
 208 MODULE_VERSION(DRV_VERSION);
 209
 210 struct igb_reg_info {
 211         u32 ofs;
 212         char *name;
 213 };
 214
 215 static const struct igb_reg_info igb_reg_info_tbl[] = {
 216
 217         /* General Registers */
 218         {E1000_CTRL, "CTRL"},
 219         {E1000_STATUS, "STATUS"},
 220         {E1000_CTRL_EXT, "CTRL_EXT"},
 221
 222         /* Interrupt Registers */
 223         {E1000_ICR, "ICR"},
 224
 225         /* RX Registers */
 226         {E1000_RCTL, "RCTL"},
 227         {E1000_RDLEN(0), "RDLEN"},
 228         {E1000_RDH(0), "RDH"},
 229         {E1000_RDT(0), "RDT"},
 230         {E1000_RXDCTL(0), "RXDCTL"},
 231         {E1000_RDBAL(0), "RDBAL"},
 232         {E1000_RDBAH(0), "RDBAH"},
 233
 234         /* TX Registers */
 235         {E1000_TCTL, "TCTL"},
 236         {E1000_TDBAL(0), "TDBAL"},
 237         {E1000_TDBAH(0), "TDBAH"},
 238         {E1000_TDLEN(0), "TDLEN"},
 239         {E1000_TDH(0), "TDH"},
 240         {E1000_TDT(0), "TDT"},
 241         {E1000_TXDCTL(0), "TXDCTL"},
 242         {E1000_TDFH, "TDFH"},
 243         {E1000_TDFT, "TDFT"},
 244         {E1000_TDFHS, "TDFHS"},
 245         {E1000_TDFPC, "TDFPC"},
 246
 247         /* List Terminator */
 248         {}
 249 };
 250
 251 /*
 252  * igb_regdump - register printout routine
 253  */
 254 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
 255 {
 256         int n = 0;
 257         char rname[16];
 258         u32 regs[8];
 259
 260         switch (reginfo->ofs) {
 261         case E1000_RDLEN(0):
 262                 for (n = 0; n < 4; n++)
 263                         regs[n] = rd32(E1000_RDLEN(n));
 264                 break;
 265         case E1000_RDH(0):
 266                 for (n = 0; n < 4; n++)
 267                         regs[n] = rd32(E1000_RDH(n));
 268                 break;
 269         case E1000_RDT(0):
 270                 for (n = 0; n < 4; n++)
 271                         regs[n] = rd32(E1000_RDT(n));
 272                 break;
 273         case E1000_RXDCTL(0):
 274                 for (n = 0; n < 4; n++)
 275                         regs[n] = rd32(E1000_RXDCTL(n));
 276                 break;
 277         case E1000_RDBAL(0):
 278                 for (n = 0; n < 4; n++)
 279                         regs[n] = rd32(E1000_RDBAL(n));
 280                 break;
 281         case E1000_RDBAH(0):
 282                 for (n = 0; n < 4; n++)
 283                         regs[n] = rd32(E1000_RDBAH(n));
 284                 break;
 285         case E1000_TDBAL(0):
 286                 for (n = 0; n < 4; n++)
 287                         regs[n] = rd32(E1000_RDBAL(n));
 288                 break;
 289         case E1000_TDBAH(0):
 290                 for (n = 0; n < 4; n++)
 291                         regs[n] = rd32(E1000_TDBAH(n));
 292                 break;
 293         case E1000_TDLEN(0):
 294                 for (n = 0; n < 4; n++)
 295                         regs[n] = rd32(E1000_TDLEN(n));
 296                 break;
 297         case E1000_TDH(0):
 298                 for (n = 0; n < 4; n++)
 299                         regs[n] = rd32(E1000_TDH(n));
 300                 break;
 301         case E1000_TDT(0):
 302                 for (n = 0; n < 4; n++)
 303                         regs[n] = rd32(E1000_TDT(n));
 304                 break;
 305         case E1000_TXDCTL(0):
 306                 for (n = 0; n < 4; n++)
 307                         regs[n] = rd32(E1000_TXDCTL(n));
 308                 break;
 309         default:
 310                 printk(KERN_INFO "%-15s %08x\n",
 311                         reginfo->name, rd32(reginfo->ofs));
 312                 return;
 313         }
 314
 315         snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
 316         printk(KERN_INFO "%-15s ", rname);
 317         for (n = 0; n < 4; n++)
 318                 printk(KERN_CONT "%08x ", regs[n]);
 319         printk(KERN_CONT "\n");
 320 }
 321
 322 /*
 323  * igb_dump - Print registers, tx-rings and rx-rings
 324  */
 325 static void igb_dump(struct igb_adapter *adapter)
 326 {
 327         struct net_device *netdev = adapter->netdev;
 328         struct e1000_hw *hw = &adapter->hw;
 329         struct igb_reg_info *reginfo;
 330         int n = 0;
 331         struct igb_ring *tx_ring;
 332         union e1000_adv_tx_desc *tx_desc;
 333         struct my_u0 { u64 a; u64 b; } *u0;
 334         struct igb_buffer *buffer_info;
 335         struct igb_ring *rx_ring;
 336         union e1000_adv_rx_desc *rx_desc;
 337         u32 staterr;
 338         int i = 0;
 339
 340         if (!netif_msg_hw(adapter))
 341                 return;
 342
 343         /* Print netdevice Info */
 344         if (netdev) {
 345                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 346                 printk(KERN_INFO "Device Name     state            "
 347                         "trans_start      last_rx\n");
 348                 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
 349                 netdev->name,
 350                 netdev->state,
 351                 netdev->trans_start,
 352                 netdev->last_rx);
 353         }
 354
 355         /* Print Registers */
 356         dev_info(&adapter->pdev->dev, "Register Dump\n");
 357         printk(KERN_INFO " Register Name   Value\n");
 358         for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
 359              reginfo->name; reginfo++) {
 360                 igb_regdump(hw, reginfo);
 361         }
 362
 363         /* Print TX Ring Summary */
 364         if (!netdev || !netif_running(netdev))
 365                 goto exit;
 366
 367         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
 368         printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma  ]"
 369                 " leng ntw timestamp\n");
 370         for (n = 0; n < adapter->num_tx_queues; n++) {
 371                 tx_ring = adapter->tx_ring[n];
 372                 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
 373                 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
 374                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
 375                            (u64)buffer_info->dma,
 376                            buffer_info->length,
 377                            buffer_info->next_to_watch,
 378                            (u64)buffer_info->time_stamp);
 379         }
 380
 381         /* Print TX Rings */
 382         if (!netif_msg_tx_done(adapter))
 383                 goto rx_ring_summary;
 384
 385         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
 386
 387         /* Transmit Descriptor Formats
 388          *
 389          * Advanced Transmit Descriptor
 390          *   +--------------------------------------------------------------+
 391          * 0 |         Buffer Address [63:0]                                |
 392          *   +--------------------------------------------------------------+
 393          * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
 394          *   +--------------------------------------------------------------+
 395          *   63      46 45    40 39 38 36 35 32 31   24             15       0
 396          */
 397
 398         for (n = 0; n < adapter->num_tx_queues; n++) {
 399                 tx_ring = adapter->tx_ring[n];
 400                 printk(KERN_INFO "------------------------------------\n");
 401                 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
 402                 printk(KERN_INFO "------------------------------------\n");
 403                 printk(KERN_INFO "T [desc]     [address 63:0  ] "
 404                         "[PlPOCIStDDM Ln] [bi->dma       ] "
 405                         "leng  ntw timestamp        bi->skb\n");
 406
 407                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 408                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
 409                         buffer_info = &tx_ring->buffer_info[i];
 410                         u0 = (struct my_u0 *)tx_desc;
 411                         printk(KERN_INFO "T [0x%03X]    %016llX %016llX %016llX"
 412                                 " %04X  %3X %016llX %p", i,
 413                                 le64_to_cpu(u0->a),
 414                                 le64_to_cpu(u0->b),
 415                                 (u64)buffer_info->dma,
 416                                 buffer_info->length,
 417                                 buffer_info->next_to_watch,
 418                                 (u64)buffer_info->time_stamp,
 419                                 buffer_info->skb);
 420                         if (i == tx_ring->next_to_use &&
 421                                 i == tx_ring->next_to_clean)
 422                                 printk(KERN_CONT " NTC/U\n");
 423                         else if (i == tx_ring->next_to_use)
 424                                 printk(KERN_CONT " NTU\n");
 425                         else if (i == tx_ring->next_to_clean)
 426                                 printk(KERN_CONT " NTC\n");
 427                         else
 428                                 printk(KERN_CONT "\n");
 429
 430                         if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
 431                                 print_hex_dump(KERN_INFO, "",
 432                                         DUMP_PREFIX_ADDRESS,
 433                                         16, 1, phys_to_virt(buffer_info->dma),
 434                                         buffer_info->length, true);
 435                 }
 436         }
 437
 438         /* Print RX Rings Summary */
 439 rx_ring_summary:
 440         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
 441         printk(KERN_INFO "Queue [NTU] [NTC]\n");
 442         for (n = 0; n < adapter->num_rx_queues; n++) {
 443                 rx_ring = adapter->rx_ring[n];
 444                 printk(KERN_INFO " %5d %5X %5X\n", n,
 445                            rx_ring->next_to_use, rx_ring->next_to_clean);
 446         }
 447
 448         /* Print RX Rings */
 449         if (!netif_msg_rx_status(adapter))
 450                 goto exit;
 451
 452         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
 453
 454         /* Advanced Receive Descriptor (Read) Format
 455          *    63                                           1        0
 456          *    +-----------------------------------------------------+
 457          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
 458          *    +----------------------------------------------+------+
 459          *  8 |       Header Buffer Address [63:1]           |  DD  |
 460          *    +-----------------------------------------------------+
 461          *
 462          *
 463          * Advanced Receive Descriptor (Write-Back) Format
 464          *
 465          *   63       48 47    32 31  30      21 20 17 16   4 3     0
 466          *   +------------------------------------------------------+
 467          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
 468          *   | Checksum   Ident  |   |           |    | Type | Type |
 469          *   +------------------------------------------------------+
 470          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 471          *   +------------------------------------------------------+
 472          *   63       48 47    32 31            20 19               0
 473          */
 474
 475         for (n = 0; n < adapter->num_rx_queues; n++) {
 476                 rx_ring = adapter->rx_ring[n];
 477                 printk(KERN_INFO "------------------------------------\n");
 478                 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
 479                 printk(KERN_INFO "------------------------------------\n");
 480                 printk(KERN_INFO "R  [desc]      [ PktBuf     A0] "
 481                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
 482                         "<-- Adv Rx Read format\n");
 483                 printk(KERN_INFO "RWB[desc]      [PcsmIpSHl PtRs] "
 484                         "[vl er S cks ln] ---------------- [bi->skb] "
 485                         "<-- Adv Rx Write-Back format\n");
 486
 487                 for (i = 0; i < rx_ring->count; i++) {
 488                         buffer_info = &rx_ring->buffer_info[i];
 489                         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
 490                         u0 = (struct my_u0 *)rx_desc;
 491                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 492                         if (staterr & E1000_RXD_STAT_DD) {
 493                                 /* Descriptor Done */
 494                                 printk(KERN_INFO "RWB[0x%03X]     %016llX "
 495                                         "%016llX ---------------- %p", i,
 496                                         le64_to_cpu(u0->a),
 497                                         le64_to_cpu(u0->b),
 498                                         buffer_info->skb);
 499                         } else {
 500                                 printk(KERN_INFO "R  [0x%03X]     %016llX "
 501                                         "%016llX %016llX %p", i,
 502                                         le64_to_cpu(u0->a),
 503                                         le64_to_cpu(u0->b),
 504                                         (u64)buffer_info->dma,
 505                                         buffer_info->skb);
 506
 507                                 if (netif_msg_pktdata(adapter)) {
 508                                         print_hex_dump(KERN_INFO, "",
 509                                                 DUMP_PREFIX_ADDRESS,
 510                                                 16, 1,
 511                                                 phys_to_virt(buffer_info->dma),
 512                                                 rx_ring->rx_buffer_len, true);
 513                                         if (rx_ring->rx_buffer_len
 514                                                 < IGB_RXBUFFER_1024)
 515                                                 print_hex_dump(KERN_INFO, "",
 516                                                   DUMP_PREFIX_ADDRESS,
 517                                                   16, 1,
 518                                                   phys_to_virt(
 519                                                     buffer_info->page_dma +
 520                                                     buffer_info->page_offset),
 521                                                   PAGE_SIZE/2, true);
 522                                 }
 523                         }
 524
 525                         if (i == rx_ring->next_to_use)
 526                                 printk(KERN_CONT " NTU\n");
 527                         else if (i == rx_ring->next_to_clean)
 528                                 printk(KERN_CONT " NTC\n");
 529                         else
 530                                 printk(KERN_CONT "\n");
 531
 532                 }
 533         }
 534
 535 exit:
 536         return;
 537 }
 538
 539
 540 /**
 541  * igb_read_clock - read raw cycle counter (to be used by time counter)
 542  */
 543 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 544 {
 545         struct igb_adapter *adapter =
 546                 container_of(tc, struct igb_adapter, cycles);
 547         struct e1000_hw *hw = &adapter->hw;
 548         u64 stamp = 0;
 549         int shift = 0;
 550
 551         /*
 552          * The timestamp latches on lowest register read. For the 82580
 553          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 554          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 555          */
 556         if (hw->mac.type == e1000_82580) {
 557                 stamp = rd32(E1000_SYSTIMR) >> 8;
 558                 shift = IGB_82580_TSYNC_SHIFT;
 559         }
 560
 561         stamp |= (u64)rd32(E1000_SYSTIML) << shift;
 562         stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
 563         return stamp;
 564 }
 565
 566 /**
 567  * igb_get_hw_dev - return device
 568  * used by hardware layer to print debugging information
 569  **/
 570 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 571 {
 572         struct igb_adapter *adapter = hw->back;
 573         return adapter->netdev;
 574 }
 575
 576 /**
 577  * igb_init_module - Driver Registration Routine
 578  *
 579  * igb_init_module is the first routine called when the driver is
 580  * loaded. All it does is register with the PCI subsystem.
 581  **/
 582 static int __init igb_init_module(void)
 583 {
 584         int ret;
 585         printk(KERN_INFO "%s - version %s\n",
 586                igb_driver_string, igb_driver_version);
 587
 588         printk(KERN_INFO "%s\n", igb_copyright);
 589
 590 #ifdef CONFIG_IGB_DCA
 591         dca_register_notify(&dca_notifier);
 592 #endif
 593         ret = pci_register_driver(&igb_driver);
 594         return ret;
 595 }
 596
 597 module_init(igb_init_module);
 598
 599 /**
 600  * igb_exit_module - Driver Exit Cleanup Routine
 601  *
 602  * igb_exit_module is called just before the driver is removed
 603  * from memory.
 604  **/
 605 static void __exit igb_exit_module(void)
 606 {
 607 #ifdef CONFIG_IGB_DCA
 608         dca_unregister_notify(&dca_notifier);
 609 #endif
 610         pci_unregister_driver(&igb_driver);
 611 }
 612
 613 module_exit(igb_exit_module);
 614
 615 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 616 /**
 617  * igb_cache_ring_register - Descriptor ring to register mapping
 618  * @adapter: board private structure to initialize
 619  *
 620  * Once we know the feature-set enabled for the device, we'll cache
 621  * the register offset the descriptor ring is assigned to.
 622  **/
 623 static void igb_cache_ring_register(struct igb_adapter *adapter)
 624 {
 625         int i = 0, j = 0;
 626         u32 rbase_offset = adapter->vfs_allocated_count;
 627
 628         switch (adapter->hw.mac.type) {
 629         case e1000_82576:
 630                 /* The queues are allocated for virtualization such that VF 0
 631                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 632                  * In order to avoid collision we start at the first free queue
 633                  * and continue consuming queues in the same sequence
 634                  */
 635                 if (adapter->vfs_allocated_count) {
 636                         for (; i < adapter->rss_queues; i++)
 637                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 638                                                                Q_IDX_82576(i);
 639                 }
 640         case e1000_82575:
 641         case e1000_82580:
 642         case e1000_i350:
 643         default:
 644                 for (; i < adapter->num_rx_queues; i++)
 645                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 646                 for (; j < adapter->num_tx_queues; j++)
 647                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 648                 break;
 649         }
 650 }
 651
 652 static void igb_free_queues(struct igb_adapter *adapter)
 653 {
 654         int i;
 655
 656         for (i = 0; i < adapter->num_tx_queues; i++) {
 657                 kfree(adapter->tx_ring[i]);
 658                 adapter->tx_ring[i] = NULL;
 659         }
 660         for (i = 0; i < adapter->num_rx_queues; i++) {
 661                 kfree(adapter->rx_ring[i]);
 662                 adapter->rx_ring[i] = NULL;
 663         }
 664         adapter->num_rx_queues = 0;
 665         adapter->num_tx_queues = 0;
 666 }
 667
 668 /**
 669  * igb_alloc_queues - Allocate memory for all rings
 670  * @adapter: board private structure to initialize
 671  *
 672  * We allocate one ring per queue at run-time since we don't know the
 673  * number of queues at compile-time.
 674  **/
 675 static int igb_alloc_queues(struct igb_adapter *adapter)
 676 {
 677         struct igb_ring *ring;
 678         int i;
 679
 680         for (i = 0; i < adapter->num_tx_queues; i++) {
 681                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 682                 if (!ring)
 683                         goto err;
 684                 ring->count = adapter->tx_ring_count;
 685                 ring->queue_index = i;
 686                 ring->dev = &adapter->pdev->dev;
 687                 ring->netdev = adapter->netdev;
 688                 /* For 82575, context index must be unique per ring. */
 689                 if (adapter->hw.mac.type == e1000_82575)
 690                         ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
 691                 adapter->tx_ring[i] = ring;
 692         }
 693
 694         for (i = 0; i < adapter->num_rx_queues; i++) {
 695                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 696                 if (!ring)
 697                         goto err;
 698                 ring->count = adapter->rx_ring_count;
 699                 ring->queue_index = i;
 700                 ring->dev = &adapter->pdev->dev;
 701                 ring->netdev = adapter->netdev;
 702                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 703                 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
 704                 /* set flag indicating ring supports SCTP checksum offload */
 705                 if (adapter->hw.mac.type >= e1000_82576)
 706                         ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
 707                 adapter->rx_ring[i] = ring;
 708         }
 709
 710         igb_cache_ring_register(adapter);
 711
 712         return 0;
 713
 714 err:
 715         igb_free_queues(adapter);
 716
 717         return -ENOMEM;
 718 }
 719
 720 #define IGB_N0_QUEUE -1
 721 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 722 {
 723         u32 msixbm = 0;
 724         struct igb_adapter *adapter = q_vector->adapter;
 725         struct e1000_hw *hw = &adapter->hw;
 726         u32 ivar, index;
 727         int rx_queue = IGB_N0_QUEUE;
 728         int tx_queue = IGB_N0_QUEUE;
 729
 730         if (q_vector->rx_ring)
 731                 rx_queue = q_vector->rx_ring->reg_idx;
 732         if (q_vector->tx_ring)
 733                 tx_queue = q_vector->tx_ring->reg_idx;
 734
 735         switch (hw->mac.type) {
 736         case e1000_82575:
 737                 /* The 82575 assigns vectors using a bitmask, which matches the
 738                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 739                    or more queues to a vector, we write the appropriate bits
 740                    into the MSIXBM register for that vector. */
 741                 if (rx_queue > IGB_N0_QUEUE)
 742                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 743                 if (tx_queue > IGB_N0_QUEUE)
 744                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 745                 if (!adapter->msix_entries && msix_vector == 0)
 746                         msixbm |= E1000_EIMS_OTHER;
 747                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 748                 q_vector->eims_value = msixbm;
 749                 break;
 750         case e1000_82576:
 751                 /* 82576 uses a table-based method for assigning vectors.
 752                    Each queue has a single entry in the table to which we write
 753                    a vector number along with a "valid" bit.  Sadly, the layout
 754                    of the table is somewhat counterintuitive. */
 755                 if (rx_queue > IGB_N0_QUEUE) {
 756                         index = (rx_queue & 0x7);
 757                         ivar = array_rd32(E1000_IVAR0, index);
 758                         if (rx_queue < 8) {
 759                                 /* vector goes into low byte of register */
 760                                 ivar = ivar & 0xFFFFFF00;
 761                                 ivar |= msix_vector | E1000_IVAR_VALID;
 762                         } else {
 763                                 /* vector goes into third byte of register */
 764                                 ivar = ivar & 0xFF00FFFF;
 765                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 766                         }
 767                         array_wr32(E1000_IVAR0, index, ivar);
 768                 }
 769                 if (tx_queue > IGB_N0_QUEUE) {
 770                         index = (tx_queue & 0x7);
 771                         ivar = array_rd32(E1000_IVAR0, index);
 772                         if (tx_queue < 8) {
 773                                 /* vector goes into second byte of register */
 774                                 ivar = ivar & 0xFFFF00FF;
 775                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 776                         } else {
 777                                 /* vector goes into high byte of register */
 778                                 ivar = ivar & 0x00FFFFFF;
 779                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 780                         }
 781                         array_wr32(E1000_IVAR0, index, ivar);
 782                 }
 783                 q_vector->eims_value = 1 << msix_vector;
 784                 break;
 785         case e1000_82580:
 786         case e1000_i350:
 787                 /* 82580 uses the same table-based approach as 82576 but has fewer
 788                    entries as a result we carry over for queues greater than 4. */
 789                 if (rx_queue > IGB_N0_QUEUE) {
 790                         index = (rx_queue >> 1);
 791                         ivar = array_rd32(E1000_IVAR0, index);
 792                         if (rx_queue & 0x1) {
 793                                 /* vector goes into third byte of register */
 794                                 ivar = ivar & 0xFF00FFFF;
 795                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 796                         } else {
 797                                 /* vector goes into low byte of register */
 798                                 ivar = ivar & 0xFFFFFF00;
 799                                 ivar |= msix_vector | E1000_IVAR_VALID;
 800                         }
 801                         array_wr32(E1000_IVAR0, index, ivar);
 802                 }
 803                 if (tx_queue > IGB_N0_QUEUE) {
 804                         index = (tx_queue >> 1);
 805                         ivar = array_rd32(E1000_IVAR0, index);
 806                         if (tx_queue & 0x1) {
 807                                 /* vector goes into high byte of register */
 808                                 ivar = ivar & 0x00FFFFFF;
 809                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 810                         } else {
 811                                 /* vector goes into second byte of register */
 812                                 ivar = ivar & 0xFFFF00FF;
 813                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 814                         }
 815                         array_wr32(E1000_IVAR0, index, ivar);
 816                 }
 817                 q_vector->eims_value = 1 << msix_vector;
 818                 break;
 819         default:
 820                 BUG();
 821                 break;
 822         }
 823
 824         /* add q_vector eims value to global eims_enable_mask */
 825         adapter->eims_enable_mask |= q_vector->eims_value;
 826
 827         /* configure q_vector to set itr on first interrupt */
 828         q_vector->set_itr = 1;
 829 }
 830
 831 /**
 832  * igb_configure_msix - Configure MSI-X hardware
 833  *
 834  * igb_configure_msix sets up the hardware to properly
 835  * generate MSI-X interrupts.
 836  **/
 837 static void igb_configure_msix(struct igb_adapter *adapter)
 838 {
 839         u32 tmp;
 840         int i, vector = 0;
 841         struct e1000_hw *hw = &adapter->hw;
 842
 843         adapter->eims_enable_mask = 0;
 844
 845         /* set vector for other causes, i.e. link changes */
 846         switch (hw->mac.type) {
 847         case e1000_82575:
 848                 tmp = rd32(E1000_CTRL_EXT);
 849                 /* enable MSI-X PBA support*/
 850                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 851
 852                 /* Auto-Mask interrupts upon ICR read. */
 853                 tmp |= E1000_CTRL_EXT_EIAME;
 854                 tmp |= E1000_CTRL_EXT_IRCA;
 855
 856                 wr32(E1000_CTRL_EXT, tmp);
 857
 858                 /* enable msix_other interrupt */
 859                 array_wr32(E1000_MSIXBM(0), vector++,
 860                                       E1000_EIMS_OTHER);
 861                 adapter->eims_other = E1000_EIMS_OTHER;
 862
 863                 break;
 864
 865         case e1000_82576:
 866         case e1000_82580:
 867         case e1000_i350:
 868                 /* Turn on MSI-X capability first, or our settings
 869                  * won't stick.  And it will take days to debug. */
 870                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 871                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 872                                 E1000_GPIE_NSICR);
 873
 874                 /* enable msix_other interrupt */
 875                 adapter->eims_other = 1 << vector;
 876                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 877
 878                 wr32(E1000_IVAR_MISC, tmp);
 879                 break;
 880         default:
 881                 /* do nothing, since nothing else supports MSI-X */
 882                 break;
 883         } /* switch (hw->mac.type) */
 884
 885         adapter->eims_enable_mask |= adapter->eims_other;
 886
 887         for (i = 0; i < adapter->num_q_vectors; i++)
 888                 igb_assign_vector(adapter->q_vector[i], vector++);
 889
 890         wrfl();
 891 }
 892
 893 /**
 894  * igb_request_msix - Initialize MSI-X interrupts
 895  *
 896  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 897  * kernel.
 898  **/
 899 static int igb_request_msix(struct igb_adapter *adapter)
 900 {
 901         struct net_device *netdev = adapter->netdev;
 902         struct e1000_hw *hw = &adapter->hw;
 903         int i, err = 0, vector = 0;
 904
 905         err = request_irq(adapter->msix_entries[vector].vector,
 906                           igb_msix_other, 0, netdev->name, adapter);
 907         if (err)
 908                 goto out;
 909         vector++;
 910
 911         for (i = 0; i < adapter->num_q_vectors; i++) {
 912                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 913
 914                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 915
 916                 if (q_vector->rx_ring && q_vector->tx_ring)
 917                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 918                                 q_vector->rx_ring->queue_index);
 919                 else if (q_vector->tx_ring)
 920                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 921                                 q_vector->tx_ring->queue_index);
 922                 else if (q_vector->rx_ring)
 923                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 924                                 q_vector->rx_ring->queue_index);
 925                 else
 926                         sprintf(q_vector->name, "%s-unused", netdev->name);
 927
 928                 err = request_irq(adapter->msix_entries[vector].vector,
 929                                   igb_msix_ring, 0, q_vector->name,
 930                                   q_vector);
 931                 if (err)
 932                         goto out;
 933                 vector++;
 934         }
 935
 936         igb_configure_msix(adapter);
 937         return 0;
 938 out:
 939         return err;
 940 }
 941
 942 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 943 {
 944         if (adapter->msix_entries) {
 945                 pci_disable_msix(adapter->pdev);
 946                 kfree(adapter->msix_entries);
 947                 adapter->msix_entries = NULL;
 948         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 949                 pci_disable_msi(adapter->pdev);
 950         }
 951 }
 952
 953 /**
 954  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 955  * @adapter: board private structure to initialize
 956  *
 957  * This function frees the memory allocated to the q_vectors.  In addition if
 958  * NAPI is enabled it will delete any references to the NAPI struct prior
 959  * to freeing the q_vector.
 960  **/
 961 static void igb_free_q_vectors(struct igb_adapter *adapter)
 962 {
 963         int v_idx;
 964
 965         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 966                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 967                 adapter->q_vector[v_idx] = NULL;
 968                 if (!q_vector)
 969                         continue;
 970                 netif_napi_del(&q_vector->napi);
 971                 kfree(q_vector);
 972         }
 973         adapter->num_q_vectors = 0;
 974 }
 975
 976 /**
 977  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 978  *
 979  * This function resets the device so that it has 0 rx queues, tx queues, and
 980  * MSI-X interrupts allocated.
 981  */
 982 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 983 {
 984         igb_free_queues(adapter);
 985         igb_free_q_vectors(adapter);
 986         igb_reset_interrupt_capability(adapter);
 987 }
 988
 989 /**
 990  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 991  *
 992  * Attempt to configure interrupts using the best available
 993  * capabilities of the hardware and kernel.
 994  **/
 995 static int igb_set_interrupt_capability(struct igb_adapter *adapter)
 996 {
 997         int err;
 998         int numvecs, i;
 999
1000         /* Number of supported queues. */
1001         adapter->num_rx_queues = adapter->rss_queues;
1002         if (adapter->vfs_allocated_count)
1003                 adapter->num_tx_queues = 1;
1004         else
1005                 adapter->num_tx_queues = adapter->rss_queues;
1006
1007         /* start with one vector for every rx queue */
1008         numvecs = adapter->num_rx_queues;
1009
1010         /* if tx handler is separate add 1 for every tx queue */
1011         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1012                 numvecs += adapter->num_tx_queues;
1013
1014         /* store the number of vectors reserved for queues */
1015         adapter->num_q_vectors = numvecs;
1016
1017         /* add 1 vector for link status interrupts */
1018         numvecs++;
1019         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1020                                         GFP_KERNEL);
1021         if (!adapter->msix_entries)
1022                 goto msi_only;
1023
1024         for (i = 0; i < numvecs; i++)
1025                 adapter->msix_entries[i].entry = i;
1026
1027         err = pci_enable_msix(adapter->pdev,
1028                               adapter->msix_entries,
1029                               numvecs);
1030         if (err == 0)
1031                 goto out;
1032
1033         igb_reset_interrupt_capability(adapter);
1034
1035         /* If we can't do MSI-X, try MSI */
1036 msi_only:
1037 #ifdef CONFIG_PCI_IOV
1038         /* disable SR-IOV for non MSI-X configurations */
1039         if (adapter->vf_data) {
1040                 struct e1000_hw *hw = &adapter->hw;
1041                 /* disable iov and allow time for transactions to clear */
1042                 pci_disable_sriov(adapter->pdev);
1043                 msleep(500);
1044
1045                 kfree(adapter->vf_data);
1046                 adapter->vf_data = NULL;
1047                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1048                 msleep(100);
1049                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1050         }
1051 #endif
1052         adapter->vfs_allocated_count = 0;
1053         adapter->rss_queues = 1;
1054         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1055         adapter->num_rx_queues = 1;
1056         adapter->num_tx_queues = 1;
1057         adapter->num_q_vectors = 1;
1058         if (!pci_enable_msi(adapter->pdev))
1059                 adapter->flags |= IGB_FLAG_HAS_MSI;
1060 out:
1061         /* Notify the stack of the (possibly) reduced queue counts. */
1062         netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1063         return netif_set_real_num_rx_queues(adapter->netdev,
1064                                             adapter->num_rx_queues);
1065 }
1066
1067 /**
1068  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1069  * @adapter: board private structure to initialize
1070  *
1071  * We allocate one q_vector per queue interrupt.  If allocation fails we
1072  * return -ENOMEM.
1073  **/
1074 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1075 {
1076         struct igb_q_vector *q_vector;
1077         struct e1000_hw *hw = &adapter->hw;
1078         int v_idx;
1079
1080         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1081                 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1082                 if (!q_vector)
1083                         goto err_out;
1084                 q_vector->adapter = adapter;
1085                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1086                 q_vector->itr_val = IGB_START_ITR;
1087                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1088                 adapter->q_vector[v_idx] = q_vector;
1089         }
1090         return 0;
1091
1092 err_out:
1093         igb_free_q_vectors(adapter);
1094         return -ENOMEM;
1095 }
1096
1097 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1098                                       int ring_idx, int v_idx)
1099 {
1100         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1101
1102         q_vector->rx_ring = adapter->rx_ring[ring_idx];
1103         q_vector->rx_ring->q_vector = q_vector;
1104         q_vector->itr_val = adapter->rx_itr_setting;
1105         if (q_vector->itr_val && q_vector->itr_val <= 3)
1106                 q_vector->itr_val = IGB_START_ITR;
1107 }
1108
1109 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1110                                       int ring_idx, int v_idx)
1111 {
1112         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1113
1114         q_vector->tx_ring = adapter->tx_ring[ring_idx];
1115         q_vector->tx_ring->q_vector = q_vector;
1116         q_vector->itr_val = adapter->tx_itr_setting;
1117         if (q_vector->itr_val && q_vector->itr_val <= 3)
1118                 q_vector->itr_val = IGB_START_ITR;
1119 }
1120
1121 /**
1122  * igb_map_ring_to_vector - maps allocated queues to vectors
1123  *
1124  * This function maps the recently allocated queues to vectors.
1125  **/
1126 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1127 {
1128         int i;
1129         int v_idx = 0;
1130
1131         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1132             (adapter->num_q_vectors < adapter->num_tx_queues))
1133                 return -ENOMEM;
1134
1135         if (adapter->num_q_vectors >=
1136             (adapter->num_rx_queues + adapter->num_tx_queues)) {
1137                 for (i = 0; i < adapter->num_rx_queues; i++)
1138                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1139                 for (i = 0; i < adapter->num_tx_queues; i++)
1140                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1141         } else {
1142                 for (i = 0; i < adapter->num_rx_queues; i++) {
1143                         if (i < adapter->num_tx_queues)
1144                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1145                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1146                 }
1147                 for (; i < adapter->num_tx_queues; i++)
1148                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1149         }
1150         return 0;
1151 }
1152
1153 /**
1154  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1155  *
1156  * This function initializes the interrupts and allocates all of the queues.
1157  **/
1158 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1159 {
1160         struct pci_dev *pdev = adapter->pdev;
1161         int err;
1162
1163         err = igb_set_interrupt_capability(adapter);
1164         if (err)
1165                 return err;
1166
1167         err = igb_alloc_q_vectors(adapter);
1168         if (err) {
1169                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1170                 goto err_alloc_q_vectors;
1171         }
1172
1173         err = igb_alloc_queues(adapter);
1174         if (err) {
1175                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1176                 goto err_alloc_queues;
1177         }
1178
1179         err = igb_map_ring_to_vector(adapter);
1180         if (err) {
1181                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1182                 goto err_map_queues;
1183         }
1184
1185
1186         return 0;
1187 err_map_queues:
1188         igb_free_queues(adapter);
1189 err_alloc_queues:
1190         igb_free_q_vectors(adapter);
1191 err_alloc_q_vectors:
1192         igb_reset_interrupt_capability(adapter);
1193         return err;
1194 }
1195
1196 /**
1197  * igb_request_irq - initialize interrupts
1198  *
1199  * Attempts to configure interrupts using the best available
1200  * capabilities of the hardware and kernel.
1201  **/
1202 static int igb_request_irq(struct igb_adapter *adapter)
1203 {
1204         struct net_device *netdev = adapter->netdev;
1205         struct pci_dev *pdev = adapter->pdev;
1206         int err = 0;
1207
1208         if (adapter->msix_entries) {
1209                 err = igb_request_msix(adapter);
1210                 if (!err)
1211                         goto request_done;
1212                 /* fall back to MSI */
1213                 igb_clear_interrupt_scheme(adapter);
1214                 if (!pci_enable_msi(adapter->pdev))
1215                         adapter->flags |= IGB_FLAG_HAS_MSI;
1216                 igb_free_all_tx_resources(adapter);
1217                 igb_free_all_rx_resources(adapter);
1218                 adapter->num_tx_queues = 1;
1219                 adapter->num_rx_queues = 1;
1220                 adapter->num_q_vectors = 1;
1221                 err = igb_alloc_q_vectors(adapter);
1222                 if (err) {
1223                         dev_err(&pdev->dev,
1224                                 "Unable to allocate memory for vectors\n");
1225                         goto request_done;
1226                 }
1227                 err = igb_alloc_queues(adapter);
1228                 if (err) {
1229                         dev_err(&pdev->dev,
1230                                 "Unable to allocate memory for queues\n");
1231                         igb_free_q_vectors(adapter);
1232                         goto request_done;
1233                 }
1234                 igb_setup_all_tx_resources(adapter);
1235                 igb_setup_all_rx_resources(adapter);
1236         } else {
1237                 igb_assign_vector(adapter->q_vector[0], 0);
1238         }
1239
1240         if (adapter->flags & IGB_FLAG_HAS_MSI) {
1241                 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1242                                   netdev->name, adapter);
1243                 if (!err)
1244                         goto request_done;
1245
1246                 /* fall back to legacy interrupts */
1247                 igb_reset_interrupt_capability(adapter);
1248                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1249         }
1250
1251         err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1252                           netdev->name, adapter);
1253
1254         if (err)
1255                 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1256                         err);
1257
1258 request_done:
1259         return err;
1260 }
1261
1262 static void igb_free_irq(struct igb_adapter *adapter)
1263 {
1264         if (adapter->msix_entries) {
1265                 int vector = 0, i;
1266
1267                 free_irq(adapter->msix_entries[vector++].vector, adapter);
1268
1269                 for (i = 0; i < adapter->num_q_vectors; i++) {
1270                         struct igb_q_vector *q_vector = adapter->q_vector[i];
1271                         free_irq(adapter->msix_entries[vector++].vector,
1272                                  q_vector);
1273                 }
1274         } else {
1275                 free_irq(adapter->pdev->irq, adapter);
1276         }
1277 }
1278
1279 /**
1280  * igb_irq_disable - Mask off interrupt generation on the NIC
1281  * @adapter: board private structure
1282  **/
1283 static void igb_irq_disable(struct igb_adapter *adapter)
1284 {
1285         struct e1000_hw *hw = &adapter->hw;
1286
1287         /*
1288          * we need to be careful when disabling interrupts.  The VFs are also
1289          * mapped into these registers and so clearing the bits can cause
1290          * issues on the VF drivers so we only need to clear what we set
1291          */
1292         if (adapter->msix_entries) {
1293                 u32 regval = rd32(E1000_EIAM);
1294                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1295                 wr32(E1000_EIMC, adapter->eims_enable_mask);
1296                 regval = rd32(E1000_EIAC);
1297                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1298         }
1299
1300         wr32(E1000_IAM, 0);
1301         wr32(E1000_IMC, ~0);
1302         wrfl();
1303         if (adapter->msix_entries) {
1304                 int i;
1305                 for (i = 0; i < adapter->num_q_vectors; i++)
1306                         synchronize_irq(adapter->msix_entries[i].vector);
1307         } else {
1308                 synchronize_irq(adapter->pdev->irq);
1309         }
1310 }
1311
1312 /**
1313  * igb_irq_enable - Enable default interrupt generation settings
1314  * @adapter: board private structure
1315  **/
1316 static void igb_irq_enable(struct igb_adapter *adapter)
1317 {
1318         struct e1000_hw *hw = &adapter->hw;
1319
1320         if (adapter->msix_entries) {
1321                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1322                 u32 regval = rd32(E1000_EIAC);
1323                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1324                 regval = rd32(E1000_EIAM);
1325                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1326                 wr32(E1000_EIMS, adapter->eims_enable_mask);
1327                 if (adapter->vfs_allocated_count) {
1328                         wr32(E1000_MBVFIMR, 0xFF);
1329                         ims |= E1000_IMS_VMMB;
1330                 }
1331                 if (adapter->hw.mac.type == e1000_82580)
1332                         ims |= E1000_IMS_DRSTA;
1333
1334                 wr32(E1000_IMS, ims);
1335         } else {
1336                 wr32(E1000_IMS, IMS_ENABLE_MASK |
1337                                 E1000_IMS_DRSTA);
1338                 wr32(E1000_IAM, IMS_ENABLE_MASK |
1339                                 E1000_IMS_DRSTA);
1340         }
1341 }
1342
1343 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1344 {
1345         struct e1000_hw *hw = &adapter->hw;
1346         u16 vid = adapter->hw.mng_cookie.vlan_id;
1347         u16 old_vid = adapter->mng_vlan_id;
1348
1349         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1350                 /* add VID to filter table */
1351                 igb_vfta_set(hw, vid, true);
1352                 adapter->mng_vlan_id = vid;
1353         } else {
1354                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1355         }
1356
1357         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1358             (vid != old_vid) &&
1359             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1360                 /* remove VID from filter table */
1361                 igb_vfta_set(hw, old_vid, false);
1362         }
1363 }
1364
1365 /**
1366  * igb_release_hw_control - release control of the h/w to f/w
1367  * @adapter: address of board private structure
1368  *
1369  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1370  * For ASF and Pass Through versions of f/w this means that the
1371  * driver is no longer loaded.
1372  *
1373  **/
1374 static void igb_release_hw_control(struct igb_adapter *adapter)
1375 {
1376         struct e1000_hw *hw = &adapter->hw;
1377         u32 ctrl_ext;
1378
1379         /* Let firmware take over control of h/w */
1380         ctrl_ext = rd32(E1000_CTRL_EXT);
1381         wr32(E1000_CTRL_EXT,
1382                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1383 }
1384
1385 /**
1386  * igb_get_hw_control - get control of the h/w from f/w
1387  * @adapter: address of board private structure
1388  *
1389  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1390  * For ASF and Pass Through versions of f/w this means that
1391  * the driver is loaded.
1392  *
1393  **/
1394 static void igb_get_hw_control(struct igb_adapter *adapter)
1395 {
1396         struct e1000_hw *hw = &adapter->hw;
1397         u32 ctrl_ext;
1398
1399         /* Let firmware know the driver has taken over */
1400         ctrl_ext = rd32(E1000_CTRL_EXT);
1401         wr32(E1000_CTRL_EXT,
1402                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1403 }
1404
1405 /**
1406  * igb_configure - configure the hardware for RX and TX
1407  * @adapter: private board structure
1408  **/
1409 static void igb_configure(struct igb_adapter *adapter)
1410 {
1411         struct net_device *netdev = adapter->netdev;
1412         int i;
1413
1414         igb_get_hw_control(adapter);
1415         igb_set_rx_mode(netdev);
1416
1417         igb_restore_vlan(adapter);
1418
1419         igb_setup_tctl(adapter);
1420         igb_setup_mrqc(adapter);
1421         igb_setup_rctl(adapter);
1422
1423         igb_configure_tx(adapter);
1424         igb_configure_rx(adapter);
1425
1426         igb_rx_fifo_flush_82575(&adapter->hw);
1427
1428         /* call igb_desc_unused which always leaves
1429          * at least 1 descriptor unused to make sure
1430          * next_to_use != next_to_clean */
1431         for (i = 0; i < adapter->num_rx_queues; i++) {
1432                 struct igb_ring *ring = adapter->rx_ring[i];
1433                 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1434         }
1435 }
1436
1437 /**
1438  * igb_power_up_link - Power up the phy/serdes link
1439  * @adapter: address of board private structure
1440  **/
1441 void igb_power_up_link(struct igb_adapter *adapter)
1442 {
1443         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1444                 igb_power_up_phy_copper(&adapter->hw);
1445         else
1446                 igb_power_up_serdes_link_82575(&adapter->hw);
1447 }
1448
1449 /**
1450  * igb_power_down_link - Power down the phy/serdes link
1451  * @adapter: address of board private structure
1452  */
1453 static void igb_power_down_link(struct igb_adapter *adapter)
1454 {
1455         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1456                 igb_power_down_phy_copper_82575(&adapter->hw);
1457         else
1458                 igb_shutdown_serdes_link_82575(&adapter->hw);
1459 }
1460
1461 /**
1462  * igb_up - Open the interface and prepare it to handle traffic
1463  * @adapter: board private structure
1464  **/
1465 int igb_up(struct igb_adapter *adapter)
1466 {
1467         struct e1000_hw *hw = &adapter->hw;
1468         int i;
1469
1470         /* hardware has been reset, we need to reload some things */
1471         igb_configure(adapter);
1472
1473         clear_bit(__IGB_DOWN, &adapter->state);
1474
1475         for (i = 0; i < adapter->num_q_vectors; i++) {
1476                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1477                 napi_enable(&q_vector->napi);
1478         }
1479         if (adapter->msix_entries)
1480                 igb_configure_msix(adapter);
1481         else
1482                 igb_assign_vector(adapter->q_vector[0], 0);
1483
1484         /* Clear any pending interrupts. */
1485         rd32(E1000_ICR);
1486         igb_irq_enable(adapter);
1487
1488         /* notify VFs that reset has been completed */
1489         if (adapter->vfs_allocated_count) {
1490                 u32 reg_data = rd32(E1000_CTRL_EXT);
1491                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1492                 wr32(E1000_CTRL_EXT, reg_data);
1493         }
1494
1495         netif_tx_start_all_queues(adapter->netdev);
1496
1497         /* start the watchdog. */
1498         hw->mac.get_link_status = 1;
1499         schedule_work(&adapter->watchdog_task);
1500
1501         return 0;
1502 }
1503
1504 void igb_down(struct igb_adapter *adapter)
1505 {
1506         struct net_device *netdev = adapter->netdev;
1507         struct e1000_hw *hw = &adapter->hw;
1508         u32 tctl, rctl;
1509         int i;
1510
1511         /* signal that we're down so the interrupt handler does not
1512          * reschedule our watchdog timer */
1513         set_bit(__IGB_DOWN, &adapter->state);
1514
1515         /* disable receives in the hardware */
1516         rctl = rd32(E1000_RCTL);
1517         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1518         /* flush and sleep below */
1519
1520         netif_tx_stop_all_queues(netdev);
1521
1522         /* disable transmits in the hardware */
1523         tctl = rd32(E1000_TCTL);
1524         tctl &= ~E1000_TCTL_EN;
1525         wr32(E1000_TCTL, tctl);
1526         /* flush both disables and wait for them to finish */
1527         wrfl();
1528         msleep(10);
1529
1530         for (i = 0; i < adapter->num_q_vectors; i++) {
1531                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1532                 napi_disable(&q_vector->napi);
1533         }
1534
1535         igb_irq_disable(adapter);
1536
1537         del_timer_sync(&adapter->watchdog_timer);
1538         del_timer_sync(&adapter->phy_info_timer);
1539
1540         netif_carrier_off(netdev);
1541
1542         /* record the stats before reset*/
1543         spin_lock(&adapter->stats64_lock);
1544         igb_update_stats(adapter, &adapter->stats64);
1545         spin_unlock(&adapter->stats64_lock);
1546
1547         adapter->link_speed = 0;
1548         adapter->link_duplex = 0;
1549
1550         if (!pci_channel_offline(adapter->pdev))
1551                 igb_reset(adapter);
1552         igb_clean_all_tx_rings(adapter);
1553         igb_clean_all_rx_rings(adapter);
1554 #ifdef CONFIG_IGB_DCA
1555
1556         /* since we reset the hardware DCA settings were cleared */
1557         igb_setup_dca(adapter);
1558 #endif
1559 }
1560
1561 void igb_reinit_locked(struct igb_adapter *adapter)
1562 {
1563         WARN_ON(in_interrupt());
1564         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1565                 msleep(1);
1566         igb_down(adapter);
1567         igb_up(adapter);
1568         clear_bit(__IGB_RESETTING, &adapter->state);
1569 }
1570
1571 void igb_reset(struct igb_adapter *adapter)
1572 {
1573         struct pci_dev *pdev = adapter->pdev;
1574         struct e1000_hw *hw = &adapter->hw;
1575         struct e1000_mac_info *mac = &hw->mac;
1576         struct e1000_fc_info *fc = &hw->fc;
1577         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1578         u16 hwm;
1579
1580         /* Repartition Pba for greater than 9k mtu
1581          * To take effect CTRL.RST is required.
1582          */
1583         switch (mac->type) {
1584         case e1000_i350:
1585         case e1000_82580:
1586                 pba = rd32(E1000_RXPBS);
1587                 pba = igb_rxpbs_adjust_82580(pba);
1588                 break;
1589         case e1000_82576:
1590                 pba = rd32(E1000_RXPBS);
1591                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1592                 break;
1593         case e1000_82575:
1594         default:
1595                 pba = E1000_PBA_34K;
1596                 break;
1597         }
1598
1599         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1600             (mac->type < e1000_82576)) {
1601                 /* adjust PBA for jumbo frames */
1602                 wr32(E1000_PBA, pba);
1603
1604                 /* To maintain wire speed transmits, the Tx FIFO should be
1605                  * large enough to accommodate two full transmit packets,
1606                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1607                  * the Rx FIFO should be large enough to accommodate at least
1608                  * one full receive packet and is similarly rounded up and
1609                  * expressed in KB. */
1610                 pba = rd32(E1000_PBA);
1611                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1612                 tx_space = pba >> 16;
1613                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1614                 pba &= 0xffff;
1615                 /* the tx fifo also stores 16 bytes of information about the tx
1616                  * but don't include ethernet FCS because hardware appends it */
1617                 min_tx_space = (adapter->max_frame_size +
1618                                 sizeof(union e1000_adv_tx_desc) -
1619                                 ETH_FCS_LEN) * 2;
1620                 min_tx_space = ALIGN(min_tx_space, 1024);
1621                 min_tx_space >>= 10;
1622                 /* software strips receive CRC, so leave room for it */
1623                 min_rx_space = adapter->max_frame_size;
1624                 min_rx_space = ALIGN(min_rx_space, 1024);
1625                 min_rx_space >>= 10;
1626
1627                 /* If current Tx allocation is less than the min Tx FIFO size,
1628                  * and the min Tx FIFO size is less than the current Rx FIFO
1629                  * allocation, take space away from current Rx allocation */
1630                 if (tx_space < min_tx_space &&
1631                     ((min_tx_space - tx_space) < pba)) {
1632                         pba = pba - (min_tx_space - tx_space);
1633
1634                         /* if short on rx space, rx wins and must trump tx
1635                          * adjustment */
1636                         if (pba < min_rx_space)
1637                                 pba = min_rx_space;
1638                 }
1639                 wr32(E1000_PBA, pba);
1640         }
1641
1642         /* flow control settings */
1643         /* The high water mark must be low enough to fit one full frame
1644          * (or the size used for early receive) above it in the Rx FIFO.
1645          * Set it to the lower of:
1646          * - 90% of the Rx FIFO size, or
1647          * - the full Rx FIFO size minus one full frame */
1648         hwm = min(((pba << 10) * 9 / 10),
1649                         ((pba << 10) - 2 * adapter->max_frame_size));
1650
1651         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1652         fc->low_water = fc->high_water - 16;
1653         fc->pause_time = 0xFFFF;
1654         fc->send_xon = 1;
1655         fc->current_mode = fc->requested_mode;
1656
1657         /* disable receive for all VFs and wait one second */
1658         if (adapter->vfs_allocated_count) {
1659                 int i;
1660                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1661                         adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1662
1663                 /* ping all the active vfs to let them know we are going down */
1664                 igb_ping_all_vfs(adapter);
1665
1666                 /* disable transmits and receives */
1667                 wr32(E1000_VFRE, 0);
1668                 wr32(E1000_VFTE, 0);
1669         }
1670
1671         /* Allow time for pending master requests to run */
1672         hw->mac.ops.reset_hw(hw);
1673         wr32(E1000_WUC, 0);
1674
1675         if (hw->mac.ops.init_hw(hw))
1676                 dev_err(&pdev->dev, "Hardware Error\n");
1677
1678         if (hw->mac.type == e1000_82580) {
1679                 u32 reg = rd32(E1000_PCIEMISC);
1680                 wr32(E1000_PCIEMISC,
1681                                 reg & ~E1000_PCIEMISC_LX_DECISION);
1682         }
1683         if (!netif_running(adapter->netdev))
1684                 igb_power_down_link(adapter);
1685
1686         igb_update_mng_vlan(adapter);
1687
1688         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1689         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1690
1691         igb_get_phy_info(hw);
1692 }
1693
1694 static const struct net_device_ops igb_netdev_ops = {
1695         .ndo_open               = igb_open,
1696         .ndo_stop               = igb_close,
1697         .ndo_start_xmit         = igb_xmit_frame_adv,
1698         .ndo_get_stats64        = igb_get_stats64,
1699         .ndo_set_rx_mode        = igb_set_rx_mode,
1700         .ndo_set_multicast_list = igb_set_rx_mode,
1701         .ndo_set_mac_address    = igb_set_mac,
1702         .ndo_change_mtu         = igb_change_mtu,
1703         .ndo_do_ioctl           = igb_ioctl,
1704         .ndo_tx_timeout         = igb_tx_timeout,
1705         .ndo_validate_addr      = eth_validate_addr,
1706         .ndo_vlan_rx_register   = igb_vlan_rx_register,
1707         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1708         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1709         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1710         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1711         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1712         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1713 #ifdef CONFIG_NET_POLL_CONTROLLER
1714         .ndo_poll_controller    = igb_netpoll,
1715 #endif
1716 };
1717
1718 /**
1719  * igb_probe - Device Initialization Routine
1720  * @pdev: PCI device information struct
1721  * @ent: entry in igb_pci_tbl
1722  *
1723  * Returns 0 on success, negative on failure
1724  *
1725  * igb_probe initializes an adapter identified by a pci_dev structure.
1726  * The OS initialization, configuring of the adapter private structure,
1727  * and a hardware reset occur.
1728  **/
1729 static int __devinit igb_probe(struct pci_dev *pdev,
1730                                const struct pci_device_id *ent)
1731 {
1732         struct net_device *netdev;
1733         struct igb_adapter *adapter;
1734         struct e1000_hw *hw;
1735         u16 eeprom_data = 0;
1736         s32 ret_val;
1737         static int global_quad_port_a; /* global quad port a indication */
1738         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1739         unsigned long mmio_start, mmio_len;
1740         int err, pci_using_dac;
1741         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1742         u8 part_str[E1000_PBANUM_LENGTH];
1743
1744         /* Catch broken hardware that put the wrong VF device ID in
1745          * the PCIe SR-IOV capability.
1746          */
1747         if (pdev->is_virtfn) {
1748                 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1749                      pci_name(pdev), pdev->vendor, pdev->device);
1750                 return -EINVAL;
1751         }
1752
1753         err = pci_enable_device_mem(pdev);
1754         if (err)
1755                 return err;
1756
1757         pci_using_dac = 0;
1758         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1759         if (!err) {
1760                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1761                 if (!err)
1762                         pci_using_dac = 1;
1763         } else {
1764                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1765                 if (err) {
1766                         err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1767                         if (err) {
1768                                 dev_err(&pdev->dev, "No usable DMA "
1769                                         "configuration, aborting\n");
1770                                 goto err_dma;
1771                         }
1772                 }
1773         }
1774
1775         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1776                                            IORESOURCE_MEM),
1777                                            igb_driver_name);
1778         if (err)
1779                 goto err_pci_reg;
1780
1781         pci_enable_pcie_error_reporting(pdev);
1782
1783         pci_set_master(pdev);
1784         pci_save_state(pdev);
1785
1786         err = -ENOMEM;
1787         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1788                                    IGB_ABS_MAX_TX_QUEUES);
1789         if (!netdev)
1790                 goto err_alloc_etherdev;
1791
1792         SET_NETDEV_DEV(netdev, &pdev->dev);
1793
1794         pci_set_drvdata(pdev, netdev);
1795         adapter = netdev_priv(netdev);
1796         adapter->netdev = netdev;
1797         adapter->pdev = pdev;
1798         hw = &adapter->hw;
1799         hw->back = adapter;
1800         adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1801
1802         mmio_start = pci_resource_start(pdev, 0);
1803         mmio_len = pci_resource_len(pdev, 0);
1804
1805         err = -EIO;
1806         hw->hw_addr = ioremap(mmio_start, mmio_len);
1807         if (!hw->hw_addr)
1808                 goto err_ioremap;
1809
1810         netdev->netdev_ops = &igb_netdev_ops;
1811         igb_set_ethtool_ops(netdev);
1812         netdev->watchdog_timeo = 5 * HZ;
1813
1814         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1815
1816         netdev->mem_start = mmio_start;
1817         netdev->mem_end = mmio_start + mmio_len;
1818
1819         /* PCI config space info */
1820         hw->vendor_id = pdev->vendor;
1821         hw->device_id = pdev->device;
1822         hw->revision_id = pdev->revision;
1823         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1824         hw->subsystem_device_id = pdev->subsystem_device;
1825
1826         /* Copy the default MAC, PHY and NVM function pointers */
1827         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1828         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1829         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1830         /* Initialize skew-specific constants */
1831         err = ei->get_invariants(hw);
1832         if (err)
1833                 goto err_sw_init;
1834
1835         /* setup the private structure */
1836         err = igb_sw_init(adapter);
1837         if (err)
1838                 goto err_sw_init;
1839
1840         igb_get_bus_info_pcie(hw);
1841
1842         hw->phy.autoneg_wait_to_complete = false;
1843
1844         /* Copper options */
1845         if (hw->phy.media_type == e1000_media_type_copper) {
1846                 hw->phy.mdix = AUTO_ALL_MODES;
1847                 hw->phy.disable_polarity_correction = false;
1848                 hw->phy.ms_type = e1000_ms_hw_default;
1849         }
1850
1851         if (igb_check_reset_block(hw))
1852                 dev_info(&pdev->dev,
1853                         "PHY reset is blocked due to SOL/IDER session.\n");
1854
1855         netdev->features = NETIF_F_SG |
1856                            NETIF_F_IP_CSUM |
1857                            NETIF_F_HW_VLAN_TX |
1858                            NETIF_F_HW_VLAN_RX |
1859                            NETIF_F_HW_VLAN_FILTER;
1860
1861         netdev->features |= NETIF_F_IPV6_CSUM;
1862         netdev->features |= NETIF_F_TSO;
1863         netdev->features |= NETIF_F_TSO6;
1864         netdev->features |= NETIF_F_GRO;
1865
1866         netdev->vlan_features |= NETIF_F_TSO;
1867         netdev->vlan_features |= NETIF_F_TSO6;
1868         netdev->vlan_features |= NETIF_F_IP_CSUM;
1869         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1870         netdev->vlan_features |= NETIF_F_SG;
1871
1872         if (pci_using_dac) {
1873                 netdev->features |= NETIF_F_HIGHDMA;
1874                 netdev->vlan_features |= NETIF_F_HIGHDMA;
1875         }
1876
1877         if (hw->mac.type >= e1000_82576)
1878                 netdev->features |= NETIF_F_SCTP_CSUM;
1879
1880         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1881
1882         /* before reading the NVM, reset the controller to put the device in a
1883          * known good starting state */
1884         hw->mac.ops.reset_hw(hw);
1885
1886         /* make sure the NVM is good */
1887         if (igb_validate_nvm_checksum(hw) < 0) {
1888                 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1889                 err = -EIO;
1890                 goto err_eeprom;
1891         }
1892
1893         /* copy the MAC address out of the NVM */
1894         if (hw->mac.ops.read_mac_addr(hw))
1895                 dev_err(&pdev->dev, "NVM Read Error\n");
1896
1897         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1898         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1899
1900         if (!is_valid_ether_addr(netdev->perm_addr)) {
1901                 dev_err(&pdev->dev, "Invalid MAC Address\n");
1902                 err = -EIO;
1903                 goto err_eeprom;
1904         }
1905
1906         setup_timer(&adapter->watchdog_timer, igb_watchdog,
1907                     (unsigned long) adapter);
1908         setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
1909                     (unsigned long) adapter);
1910
1911         INIT_WORK(&adapter->reset_task, igb_reset_task);
1912         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1913
1914         /* Initialize link properties that are user-changeable */
1915         adapter->fc_autoneg = true;
1916         hw->mac.autoneg = true;
1917         hw->phy.autoneg_advertised = 0x2f;
1918
1919         hw->fc.requested_mode = e1000_fc_default;
1920         hw->fc.current_mode = e1000_fc_default;
1921
1922         igb_validate_mdi_setting(hw);
1923
1924         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1925          * enable the ACPI Magic Packet filter
1926          */
1927
1928         if (hw->bus.func == 0)
1929                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1930         else if (hw->mac.type == e1000_82580)
1931                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1932                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1933                                  &eeprom_data);
1934         else if (hw->bus.func == 1)
1935                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1936
1937         if (eeprom_data & eeprom_apme_mask)
1938                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1939
1940         /* now that we have the eeprom settings, apply the special cases where
1941          * the eeprom may be wrong or the board simply won't support wake on
1942          * lan on a particular port */
1943         switch (pdev->device) {
1944         case E1000_DEV_ID_82575GB_QUAD_COPPER:
1945                 adapter->eeprom_wol = 0;
1946                 break;
1947         case E1000_DEV_ID_82575EB_FIBER_SERDES:
1948         case E1000_DEV_ID_82576_FIBER:
1949         case E1000_DEV_ID_82576_SERDES:
1950                 /* Wake events only supported on port A for dual fiber
1951                  * regardless of eeprom setting */
1952                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1953                         adapter->eeprom_wol = 0;
1954                 break;
1955         case E1000_DEV_ID_82576_QUAD_COPPER:
1956         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1957                 /* if quad port adapter, disable WoL on all but port A */
1958                 if (global_quad_port_a != 0)
1959                         adapter->eeprom_wol = 0;
1960                 else
1961                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1962                 /* Reset for multiple quad port adapters */
1963                 if (++global_quad_port_a == 4)
1964                         global_quad_port_a = 0;
1965                 break;
1966         }
1967
1968         /* initialize the wol settings based on the eeprom settings */
1969         adapter->wol = adapter->eeprom_wol;
1970         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1971
1972         /* reset the hardware with the new settings */
1973         igb_reset(adapter);
1974
1975         /* let the f/w know that the h/w is now under the control of the
1976          * driver. */
1977         igb_get_hw_control(adapter);
1978
1979         strcpy(netdev->name, "eth%d");
1980         err = register_netdev(netdev);
1981         if (err)
1982                 goto err_register;
1983
1984         /* carrier off reporting is important to ethtool even BEFORE open */
1985         netif_carrier_off(netdev);
1986
1987 #ifdef CONFIG_IGB_DCA
1988         if (dca_add_requester(&pdev->dev) == 0) {
1989                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1990                 dev_info(&pdev->dev, "DCA enabled\n");
1991                 igb_setup_dca(adapter);
1992         }
1993
1994 #endif
1995         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1996         /* print bus type/speed/width info */
1997         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1998                  netdev->name,
1999                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2000                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2001                                                             "unknown"),
2002                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2003                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2004                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2005                    "unknown"),
2006                  netdev->dev_addr);
2007
2008         ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2009         if (ret_val)
2010                 strcpy(part_str, "Unknown");
2011         dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2012         dev_info(&pdev->dev,
2013                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2014                 adapter->msix_entries ? "MSI-X" :
2015                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2016                 adapter->num_rx_queues, adapter->num_tx_queues);
2017
2018         return 0;
2019
2020 err_register:
2021         igb_release_hw_control(adapter);
2022 err_eeprom:
2023         if (!igb_check_reset_block(hw))
2024                 igb_reset_phy(hw);
2025
2026         if (hw->flash_address)
2027                 iounmap(hw->flash_address);
2028 err_sw_init:
2029         igb_clear_interrupt_scheme(adapter);
2030         iounmap(hw->hw_addr);
2031 err_ioremap:
2032         free_netdev(netdev);
2033 err_alloc_etherdev:
2034         pci_release_selected_regions(pdev,
2035                                      pci_select_bars(pdev, IORESOURCE_MEM));
2036 err_pci_reg:
2037 err_dma:
2038         pci_disable_device(pdev);
2039         return err;
2040 }
2041
2042 /**
2043  * igb_remove - Device Removal Routine
2044  * @pdev: PCI device information struct
2045  *
2046  * igb_remove is called by the PCI subsystem to alert the driver
2047  * that it should release a PCI device.  The could be caused by a
2048  * Hot-Plug event, or because the driver is going to be removed from
2049  * memory.
2050  **/
2051 static void __devexit igb_remove(struct pci_dev *pdev)
2052 {
2053         struct net_device *netdev = pci_get_drvdata(pdev);
2054         struct igb_adapter *adapter = netdev_priv(netdev);
2055         struct e1000_hw *hw = &adapter->hw;
2056
2057         /*
2058          * The watchdog timer may be rescheduled, so explicitly
2059          * disable watchdog from being rescheduled.
2060          */
2061         set_bit(__IGB_DOWN, &adapter->state);
2062         del_timer_sync(&adapter->watchdog_timer);
2063         del_timer_sync(&adapter->phy_info_timer);
2064
2065         cancel_work_sync(&adapter->reset_task);
2066         cancel_work_sync(&adapter->watchdog_task);
2067
2068 #ifdef CONFIG_IGB_DCA
2069         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2070                 dev_info(&pdev->dev, "DCA disabled\n");
2071                 dca_remove_requester(&pdev->dev);
2072                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2073                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2074         }
2075 #endif
2076
2077         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
2078          * would have already happened in close and is redundant. */
2079         igb_release_hw_control(adapter);
2080
2081         unregister_netdev(netdev);
2082
2083         igb_clear_interrupt_scheme(adapter);
2084
2085 #ifdef CONFIG_PCI_IOV
2086         /* reclaim resources allocated to VFs */
2087         if (adapter->vf_data) {
2088                 /* disable iov and allow time for transactions to clear */
2089                 pci_disable_sriov(pdev);
2090                 msleep(500);
2091
2092                 kfree(adapter->vf_data);
2093                 adapter->vf_data = NULL;
2094                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2095                 msleep(100);
2096                 dev_info(&pdev->dev, "IOV Disabled\n");
2097         }
2098 #endif
2099
2100         iounmap(hw->hw_addr);
2101         if (hw->flash_address)
2102                 iounmap(hw->flash_address);
2103         pci_release_selected_regions(pdev,
2104                                      pci_select_bars(pdev, IORESOURCE_MEM));
2105
2106         free_netdev(netdev);
2107
2108         pci_disable_pcie_error_reporting(pdev);
2109
2110         pci_disable_device(pdev);
2111 }
2112
2113 /**
2114  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2115  * @adapter: board private structure to initialize
2116  *
2117  * This function initializes the vf specific data storage and then attempts to
2118  * allocate the VFs.  The reason for ordering it this way is because it is much
2119  * mor expensive time wise to disable SR-IOV than it is to allocate and free
2120  * the memory for the VFs.
2121  **/
2122 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2123 {
2124 #ifdef CONFIG_PCI_IOV
2125         struct pci_dev *pdev = adapter->pdev;
2126
2127         if (adapter->vfs_allocated_count) {
2128                 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2129                                            sizeof(struct vf_data_storage),
2130                                            GFP_KERNEL);
2131                 /* if allocation failed then we do not support SR-IOV */
2132                 if (!adapter->vf_data) {
2133                         adapter->vfs_allocated_count = 0;
2134                         dev_err(&pdev->dev, "Unable to allocate memory for VF "
2135                                 "Data Storage\n");
2136                 }
2137         }
2138
2139         if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2140                 kfree(adapter->vf_data);
2141                 adapter->vf_data = NULL;
2142 #endif /* CONFIG_PCI_IOV */
2143                 adapter->vfs_allocated_count = 0;
2144 #ifdef CONFIG_PCI_IOV
2145         } else {
2146                 unsigned char mac_addr[ETH_ALEN];
2147                 int i;
2148                 dev_info(&pdev->dev, "%d vfs allocated\n",
2149                          adapter->vfs_allocated_count);
2150                 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2151                         random_ether_addr(mac_addr);
2152                         igb_set_vf_mac(adapter, i, mac_addr);
2153                 }
2154         }
2155 #endif /* CONFIG_PCI_IOV */
2156 }
2157
2158
2159 /**
2160  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2161  * @adapter: board private structure to initialize
2162  *
2163  * igb_init_hw_timer initializes the function pointer and values for the hw
2164  * timer found in hardware.
2165  **/
2166 static void igb_init_hw_timer(struct igb_adapter *adapter)
2167 {
2168         struct e1000_hw *hw = &adapter->hw;
2169
2170         switch (hw->mac.type) {
2171         case e1000_i350:
2172         case e1000_82580:
2173                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2174                 adapter->cycles.read = igb_read_clock;
2175                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2176                 adapter->cycles.mult = 1;
2177                 /*
2178                  * The 82580 timesync updates the system timer every 8ns by 8ns
2179                  * and the value cannot be shifted.  Instead we need to shift
2180                  * the registers to generate a 64bit timer value.  As a result
2181                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2182                  * 24 in order to generate a larger value for synchronization.
2183                  */
2184                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2185                 /* disable system timer temporarily by setting bit 31 */
2186                 wr32(E1000_TSAUXC, 0x80000000);
2187                 wrfl();
2188
2189                 /* Set registers so that rollover occurs soon to test this. */
2190                 wr32(E1000_SYSTIMR, 0x00000000);
2191                 wr32(E1000_SYSTIML, 0x80000000);
2192                 wr32(E1000_SYSTIMH, 0x000000FF);
2193                 wrfl();
2194
2195                 /* enable system timer by clearing bit 31 */
2196                 wr32(E1000_TSAUXC, 0x0);
2197                 wrfl();
2198
2199                 timecounter_init(&adapter->clock,
2200                                  &adapter->cycles,
2201                                  ktime_to_ns(ktime_get_real()));
2202                 /*
2203                  * Synchronize our NIC clock against system wall clock. NIC
2204                  * time stamp reading requires ~3us per sample, each sample
2205                  * was pretty stable even under load => only require 10
2206                  * samples for each offset comparison.
2207                  */
2208                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2209                 adapter->compare.source = &adapter->clock;
2210                 adapter->compare.target = ktime_get_real;
2211                 adapter->compare.num_samples = 10;
2212                 timecompare_update(&adapter->compare, 0);
2213                 break;
2214         case e1000_82576:
2215                 /*
2216                  * Initialize hardware timer: we keep it running just in case
2217                  * that some program needs it later on.
2218                  */
2219                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2220                 adapter->cycles.read = igb_read_clock;
2221                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2222                 adapter->cycles.mult = 1;
2223                 /**
2224                  * Scale the NIC clock cycle by a large factor so that
2225                  * relatively small clock corrections can be added or
2226                  * substracted at each clock tick. The drawbacks of a large
2227                  * factor are a) that the clock register overflows more quickly
2228                  * (not such a big deal) and b) that the increment per tick has
2229                  * to fit into 24 bits.  As a result we need to use a shift of
2230                  * 19 so we can fit a value of 16 into the TIMINCA register.
2231                  */
2232                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2233                 wr32(E1000_TIMINCA,
2234                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
2235                                 (16 << IGB_82576_TSYNC_SHIFT));
2236
2237                 /* Set registers so that rollover occurs soon to test this. */
2238                 wr32(E1000_SYSTIML, 0x00000000);
2239                 wr32(E1000_SYSTIMH, 0xFF800000);
2240                 wrfl();
2241
2242                 timecounter_init(&adapter->clock,
2243                                  &adapter->cycles,
2244                                  ktime_to_ns(ktime_get_real()));
2245                 /*
2246                  * Synchronize our NIC clock against system wall clock. NIC
2247                  * time stamp reading requires ~3us per sample, each sample
2248                  * was pretty stable even under load => only require 10
2249                  * samples for each offset comparison.
2250                  */
2251                 memset(&adapter->compare, 0, sizeof(adapter->compare));
2252                 adapter->compare.source = &adapter->clock;
2253                 adapter->compare.target = ktime_get_real;
2254                 adapter->compare.num_samples = 10;
2255                 timecompare_update(&adapter->compare, 0);
2256                 break;
2257         case e1000_82575:
2258                 /* 82575 does not support timesync */
2259         default:
2260                 break;
2261         }
2262
2263 }
2264
2265 /**
2266  * igb_sw_init - Initialize general software structures (struct igb_adapter)
2267  * @adapter: board private structure to initialize
2268  *
2269  * igb_sw_init initializes the Adapter private data structure.
2270  * Fields are initialized based on PCI device information and
2271  * OS network device settings (MTU size).
2272  **/
2273 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2274 {
2275         struct e1000_hw *hw = &adapter->hw;
2276         struct net_device *netdev = adapter->netdev;
2277         struct pci_dev *pdev = adapter->pdev;
2278
2279         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2280
2281         adapter->tx_ring_count = IGB_DEFAULT_TXD;
2282         adapter->rx_ring_count = IGB_DEFAULT_RXD;
2283         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2284         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2285
2286         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2287         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2288
2289         spin_lock_init(&adapter->stats64_lock);
2290 #ifdef CONFIG_PCI_IOV
2291         switch (hw->mac.type) {
2292         case e1000_82576:
2293         case e1000_i350:
2294                 if (max_vfs > 7) {
2295                         dev_warn(&pdev->dev,
2296                                  "Maximum of 7 VFs per PF, using max\n");
2297                         adapter->vfs_allocated_count = 7;
2298                 } else
2299                         adapter->vfs_allocated_count = max_vfs;
2300                 break;
2301         default:
2302                 break;
2303         }
2304 #endif /* CONFIG_PCI_IOV */
2305         adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2306
2307         /*
2308          * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2309          * then we should combine the queues into a queue pair in order to
2310          * conserve interrupts due to limited supply
2311          */
2312         if ((adapter->rss_queues > 4) ||
2313             ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2314                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2315
2316         /* This call may decrease the number of queues */
2317         if (igb_init_interrupt_scheme(adapter)) {
2318                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2319                 return -ENOMEM;
2320         }
2321
2322         igb_init_hw_timer(adapter);
2323         igb_probe_vfs(adapter);
2324
2325         /* Explicitly disable IRQ since the NIC can be in any state. */
2326         igb_irq_disable(adapter);
2327
2328         set_bit(__IGB_DOWN, &adapter->state);
2329         return 0;
2330 }
2331
2332 /**
2333  * igb_open - Called when a network interface is made active
2334  * @netdev: network interface device structure
2335  *
2336  * Returns 0 on success, negative value on failure
2337  *
2338  * The open entry point is called when a network interface is made
2339  * active by the system (IFF_UP).  At this point all resources needed
2340  * for transmit and receive operations are allocated, the interrupt
2341  * handler is registered with the OS, the watchdog timer is started,
2342  * and the stack is notified that the interface is ready.
2343  **/
2344 static int igb_open(struct net_device *netdev)
2345 {
2346         struct igb_adapter *adapter = netdev_priv(netdev);
2347         struct e1000_hw *hw = &adapter->hw;
2348         int err;
2349         int i;
2350
2351         /* disallow open during test */
2352         if (test_bit(__IGB_TESTING, &adapter->state))
2353                 return -EBUSY;
2354
2355         netif_carrier_off(netdev);
2356
2357         /* allocate transmit descriptors */
2358         err = igb_setup_all_tx_resources(adapter);
2359         if (err)
2360                 goto err_setup_tx;
2361
2362         /* allocate receive descriptors */
2363         err = igb_setup_all_rx_resources(adapter);
2364         if (err)
2365                 goto err_setup_rx;
2366
2367         igb_power_up_link(adapter);
2368
2369         /* before we allocate an interrupt, we must be ready to handle it.
2370          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2371          * as soon as we call pci_request_irq, so we have to setup our
2372          * clean_rx handler before we do so.  */
2373         igb_configure(adapter);
2374
2375         err = igb_request_irq(adapter);
2376         if (err)
2377                 goto err_req_irq;
2378
2379         /* From here on the code is the same as igb_up() */
2380         clear_bit(__IGB_DOWN, &adapter->state);
2381
2382         for (i = 0; i < adapter->num_q_vectors; i++) {
2383                 struct igb_q_vector *q_vector = adapter->q_vector[i];
2384                 napi_enable(&q_vector->napi);
2385         }
2386
2387         /* Clear any pending interrupts. */
2388         rd32(E1000_ICR);
2389
2390         igb_irq_enable(adapter);
2391
2392         /* notify VFs that reset has been completed */
2393         if (adapter->vfs_allocated_count) {
2394                 u32 reg_data = rd32(E1000_CTRL_EXT);
2395                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2396                 wr32(E1000_CTRL_EXT, reg_data);
2397         }
2398
2399         netif_tx_start_all_queues(netdev);
2400
2401         /* start the watchdog. */
2402         hw->mac.get_link_status = 1;
2403         schedule_work(&adapter->watchdog_task);
2404
2405         return 0;
2406
2407 err_req_irq:
2408         igb_release_hw_control(adapter);
2409         igb_power_down_link(adapter);
2410         igb_free_all_rx_resources(adapter);
2411 err_setup_rx:
2412         igb_free_all_tx_resources(adapter);
2413 err_setup_tx:
2414         igb_reset(adapter);
2415
2416         return err;
2417 }
2418
2419 /**
2420  * igb_close - Disables a network interface
2421  * @netdev: network interface device structure
2422  *
2423  * Returns 0, this is not allowed to fail
2424  *
2425  * The close entry point is called when an interface is de-activated
2426  * by the OS.  The hardware is still under the driver's control, but
2427  * needs to be disabled.  A global MAC reset is issued to stop the
2428  * hardware, and all transmit and receive resources are freed.
2429  **/
2430 static int igb_close(struct net_device *netdev)
2431 {
2432         struct igb_adapter *adapter = netdev_priv(netdev);
2433
2434         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2435         igb_down(adapter);
2436
2437         igb_free_irq(adapter);
2438
2439         igb_free_all_tx_resources(adapter);
2440         igb_free_all_rx_resources(adapter);
2441
2442         return 0;
2443 }
2444
2445 /**
2446  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2447  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2448  *
2449  * Return 0 on success, negative on failure
2450  **/
2451 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2452 {
2453         struct device *dev = tx_ring->dev;
2454         int size;
2455
2456         size = sizeof(struct igb_buffer) * tx_ring->count;
2457         tx_ring->buffer_info = vzalloc(size);
2458         if (!tx_ring->buffer_info)
2459                 goto err;
2460
2461         /* round up to nearest 4K */
2462         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2463         tx_ring->size = ALIGN(tx_ring->size, 4096);
2464
2465         tx_ring->desc = dma_alloc_coherent(dev,
2466                                            tx_ring->size,
2467                                            &tx_ring->dma,
2468                                            GFP_KERNEL);
2469
2470         if (!tx_ring->desc)
2471                 goto err;
2472
2473         tx_ring->next_to_use = 0;
2474         tx_ring->next_to_clean = 0;
2475         return 0;
2476
2477 err:
2478         vfree(tx_ring->buffer_info);
2479         dev_err(dev,
2480                 "Unable to allocate memory for the transmit descriptor ring\n");
2481         return -ENOMEM;
2482 }
2483
2484 /**
2485  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2486  *                                (Descriptors) for all queues
2487  * @adapter: board private structure
2488  *
2489  * Return 0 on success, negative on failure
2490  **/
2491 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2492 {
2493         struct pci_dev *pdev = adapter->pdev;
2494         int i, err = 0;
2495
2496         for (i = 0; i < adapter->num_tx_queues; i++) {
2497                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2498                 if (err) {
2499                         dev_err(&pdev->dev,
2500                                 "Allocation for Tx Queue %u failed\n", i);
2501                         for (i--; i >= 0; i--)
2502                                 igb_free_tx_resources(adapter->tx_ring[i]);
2503                         break;
2504                 }
2505         }
2506
2507         for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2508                 int r_idx = i % adapter->num_tx_queues;
2509                 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2510         }
2511         return err;
2512 }
2513
2514 /**
2515  * igb_setup_tctl - configure the transmit control registers
2516  * @adapter: Board private structure
2517  **/
2518 void igb_setup_tctl(struct igb_adapter *adapter)
2519 {
2520         struct e1000_hw *hw = &adapter->hw;
2521         u32 tctl;
2522
2523         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2524         wr32(E1000_TXDCTL(0), 0);
2525
2526         /* Program the Transmit Control Register */
2527         tctl = rd32(E1000_TCTL);
2528         tctl &= ~E1000_TCTL_CT;
2529         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2530                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2531
2532         igb_config_collision_dist(hw);
2533
2534         /* Enable transmits */
2535         tctl |= E1000_TCTL_EN;
2536
2537         wr32(E1000_TCTL, tctl);
2538 }
2539
2540 /**
2541  * igb_configure_tx_ring - Configure transmit ring after Reset
2542  * @adapter: board private structure
2543  * @ring: tx ring to configure
2544  *
2545  * Configure a transmit ring after a reset.
2546  **/
2547 void igb_configure_tx_ring(struct igb_adapter *adapter,
2548                            struct igb_ring *ring)
2549 {
2550         struct e1000_hw *hw = &adapter->hw;
2551         u32 txdctl;
2552         u64 tdba = ring->dma;
2553         int reg_idx = ring->reg_idx;
2554
2555         /* disable the queue */
2556         txdctl = rd32(E1000_TXDCTL(reg_idx));
2557         wr32(E1000_TXDCTL(reg_idx),
2558                         txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2559         wrfl();
2560         mdelay(10);
2561
2562         wr32(E1000_TDLEN(reg_idx),
2563                         ring->count * sizeof(union e1000_adv_tx_desc));
2564         wr32(E1000_TDBAL(reg_idx),
2565                         tdba & 0x00000000ffffffffULL);
2566         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2567
2568         ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2569         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2570         writel(0, ring->head);
2571         writel(0, ring->tail);
2572
2573         txdctl |= IGB_TX_PTHRESH;
2574         txdctl |= IGB_TX_HTHRESH << 8;
2575         txdctl |= IGB_TX_WTHRESH << 16;
2576
2577         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2578         wr32(E1000_TXDCTL(reg_idx), txdctl);
2579 }
2580
2581 /**
2582  * igb_configure_tx - Configure transmit Unit after Reset
2583  * @adapter: board private structure
2584  *
2585  * Configure the Tx unit of the MAC after a reset.
2586  **/
2587 static void igb_configure_tx(struct igb_adapter *adapter)
2588 {
2589         int i;
2590
2591         for (i = 0; i < adapter->num_tx_queues; i++)
2592                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2593 }
2594
2595 /**
2596  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2597  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2598  *
2599  * Returns 0 on success, negative on failure
2600  **/
2601 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2602 {
2603         struct device *dev = rx_ring->dev;
2604         int size, desc_len;
2605
2606         size = sizeof(struct igb_buffer) * rx_ring->count;
2607         rx_ring->buffer_info = vzalloc(size);
2608         if (!rx_ring->buffer_info)
2609                 goto err;
2610
2611         desc_len = sizeof(union e1000_adv_rx_desc);
2612
2613         /* Round up to nearest 4K */
2614         rx_ring->size = rx_ring->count * desc_len;
2615         rx_ring->size = ALIGN(rx_ring->size, 4096);
2616
2617         rx_ring->desc = dma_alloc_coherent(dev,
2618                                            rx_ring->size,
2619                                            &rx_ring->dma,
2620                                            GFP_KERNEL);
2621
2622         if (!rx_ring->desc)
2623                 goto err;
2624
2625         rx_ring->next_to_clean = 0;
2626         rx_ring->next_to_use = 0;
2627
2628         return 0;
2629
2630 err:
2631         vfree(rx_ring->buffer_info);
2632         rx_ring->buffer_info = NULL;
2633         dev_err(dev, "Unable to allocate memory for the receive descriptor"
2634                 " ring\n");
2635         return -ENOMEM;
2636 }
2637
2638 /**
2639  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2640  *                                (Descriptors) for all queues
2641  * @adapter: board private structure
2642  *
2643  * Return 0 on success, negative on failure
2644  **/
2645 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2646 {
2647         struct pci_dev *pdev = adapter->pdev;
2648         int i, err = 0;
2649
2650         for (i = 0; i < adapter->num_rx_queues; i++) {
2651                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2652                 if (err) {
2653                         dev_err(&pdev->dev,
2654                                 "Allocation for Rx Queue %u failed\n", i);
2655                         for (i--; i >= 0; i--)
2656                                 igb_free_rx_resources(adapter->rx_ring[i]);
2657                         break;
2658                 }
2659         }
2660
2661         return err;
2662 }
2663
2664 /**
2665  * igb_setup_mrqc - configure the multiple receive queue control registers
2666  * @adapter: Board private structure
2667  **/
2668 static void igb_setup_mrqc(struct igb_adapter *adapter)
2669 {
2670         struct e1000_hw *hw = &adapter->hw;
2671         u32 mrqc, rxcsum;
2672         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2673         union e1000_reta {
2674                 u32 dword;
2675                 u8  bytes[4];
2676         } reta;
2677         static const u8 rsshash[40] = {
2678                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2679                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2680                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2681                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2682
2683         /* Fill out hash function seeds */
2684         for (j = 0; j < 10; j++) {
2685                 u32 rsskey = rsshash[(j * 4)];
2686                 rsskey |= rsshash[(j * 4) + 1] << 8;
2687                 rsskey |= rsshash[(j * 4) + 2] << 16;
2688                 rsskey |= rsshash[(j * 4) + 3] << 24;
2689                 array_wr32(E1000_RSSRK(0), j, rsskey);
2690         }
2691
2692         num_rx_queues = adapter->rss_queues;
2693
2694         if (adapter->vfs_allocated_count) {
2695                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2696                 switch (hw->mac.type) {
2697                 case e1000_i350:
2698                 case e1000_82580:
2699                         num_rx_queues = 1;
2700                         shift = 0;
2701                         break;
2702                 case e1000_82576:
2703                         shift = 3;
2704                         num_rx_queues = 2;
2705                         break;
2706                 case e1000_82575:
2707                         shift = 2;
2708                         shift2 = 6;
2709                 default:
2710                         break;
2711                 }
2712         } else {
2713                 if (hw->mac.type == e1000_82575)
2714                         shift = 6;
2715         }
2716
2717         for (j = 0; j < (32 * 4); j++) {
2718                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2719                 if (shift2)
2720                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2721                 if ((j & 3) == 3)
2722                         wr32(E1000_RETA(j >> 2), reta.dword);
2723         }
2724
2725         /*
2726          * Disable raw packet checksumming so that RSS hash is placed in
2727          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2728          * offloads as they are enabled by default
2729          */
2730         rxcsum = rd32(E1000_RXCSUM);
2731         rxcsum |= E1000_RXCSUM_PCSD;
2732
2733         if (adapter->hw.mac.type >= e1000_82576)
2734                 /* Enable Receive Checksum Offload for SCTP */
2735                 rxcsum |= E1000_RXCSUM_CRCOFL;
2736
2737         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2738         wr32(E1000_RXCSUM, rxcsum);
2739
2740         /* If VMDq is enabled then we set the appropriate mode for that, else
2741          * we default to RSS so that an RSS hash is calculated per packet even
2742          * if we are only using one queue */
2743         if (adapter->vfs_allocated_count) {
2744                 if (hw->mac.type > e1000_82575) {
2745                         /* Set the default pool for the PF's first queue */
2746                         u32 vtctl = rd32(E1000_VT_CTL);
2747                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2748                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2749                         vtctl |= adapter->vfs_allocated_count <<
2750                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2751                         wr32(E1000_VT_CTL, vtctl);
2752                 }
2753                 if (adapter->rss_queues > 1)
2754                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2755                 else
2756                         mrqc = E1000_MRQC_ENABLE_VMDQ;
2757         } else {
2758                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2759         }
2760         igb_vmm_control(adapter);
2761
2762         /*
2763          * Generate RSS hash based on TCP port numbers and/or
2764          * IPv4/v6 src and dst addresses since UDP cannot be
2765          * hashed reliably due to IP fragmentation
2766          */
2767         mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2768                 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2769                 E1000_MRQC_RSS_FIELD_IPV6 |
2770                 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2771                 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2772
2773         wr32(E1000_MRQC, mrqc);
2774 }
2775
2776 /**
2777  * igb_setup_rctl - configure the receive control registers
2778  * @adapter: Board private structure
2779  **/
2780 void igb_setup_rctl(struct igb_adapter *adapter)
2781 {
2782         struct e1000_hw *hw = &adapter->hw;
2783         u32 rctl;
2784
2785         rctl = rd32(E1000_RCTL);
2786
2787         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2788         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2789
2790         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2791                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2792
2793         /*
2794          * enable stripping of CRC. It's unlikely this will break BMC
2795          * redirection as it did with e1000. Newer features require
2796          * that the HW strips the CRC.
2797          */
2798         rctl |= E1000_RCTL_SECRC;
2799
2800         /* disable store bad packets and clear size bits. */
2801         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2802
2803         /* enable LPE to prevent packets larger than max_frame_size */
2804         rctl |= E1000_RCTL_LPE;
2805
2806         /* disable queue 0 to prevent tail write w/o re-config */
2807         wr32(E1000_RXDCTL(0), 0);
2808
2809         /* Attention!!!  For SR-IOV PF driver operations you must enable
2810          * queue drop for all VF and PF queues to prevent head of line blocking
2811          * if an un-trusted VF does not provide descriptors to hardware.
2812          */
2813         if (adapter->vfs_allocated_count) {
2814                 /* set all queue drop enable bits */
2815                 wr32(E1000_QDE, ALL_QUEUES);
2816         }
2817
2818         wr32(E1000_RCTL, rctl);
2819 }
2820
2821 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2822                                    int vfn)
2823 {
2824         struct e1000_hw *hw = &adapter->hw;
2825         u32 vmolr;
2826
2827         /* if it isn't the PF check to see if VFs are enabled and
2828          * increase the size to support vlan tags */
2829         if (vfn < adapter->vfs_allocated_count &&
2830             adapter->vf_data[vfn].vlans_enabled)
2831                 size += VLAN_TAG_SIZE;
2832
2833         vmolr = rd32(E1000_VMOLR(vfn));
2834         vmolr &= ~E1000_VMOLR_RLPML_MASK;
2835         vmolr |= size | E1000_VMOLR_LPE;
2836         wr32(E1000_VMOLR(vfn), vmolr);
2837
2838         return 0;
2839 }
2840
2841 /**
2842  * igb_rlpml_set - set maximum receive packet size
2843  * @adapter: board private structure
2844  *
2845  * Configure maximum receivable packet size.
2846  **/
2847 static void igb_rlpml_set(struct igb_adapter *adapter)
2848 {
2849         u32 max_frame_size = adapter->max_frame_size;
2850         struct e1000_hw *hw = &adapter->hw;
2851         u16 pf_id = adapter->vfs_allocated_count;
2852
2853         if (adapter->vlgrp)
2854                 max_frame_size += VLAN_TAG_SIZE;
2855
2856         /* if vfs are enabled we set RLPML to the largest possible request
2857          * size and set the VMOLR RLPML to the size we need */
2858         if (pf_id) {
2859                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2860                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2861         }
2862
2863         wr32(E1000_RLPML, max_frame_size);
2864 }
2865
2866 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2867                                  int vfn, bool aupe)
2868 {
2869         struct e1000_hw *hw = &adapter->hw;
2870         u32 vmolr;
2871
2872         /*
2873          * This register exists only on 82576 and newer so if we are older then
2874          * we should exit and do nothing
2875          */
2876         if (hw->mac.type < e1000_82576)
2877                 return;
2878
2879         vmolr = rd32(E1000_VMOLR(vfn));
2880         vmolr |= E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
2881         if (aupe)
2882                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
2883         else
2884                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2885
2886         /* clear all bits that might not be set */
2887         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2888
2889         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2890                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2891         /*
2892          * for VMDq only allow the VFs and pool 0 to accept broadcast and
2893          * multicast packets
2894          */
2895         if (vfn <= adapter->vfs_allocated_count)
2896                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
2897
2898         wr32(E1000_VMOLR(vfn), vmolr);
2899 }
2900
2901 /**
2902  * igb_configure_rx_ring - Configure a receive ring after Reset
2903  * @adapter: board private structure
2904  * @ring: receive ring to be configured
2905  *
2906  * Configure the Rx unit of the MAC after a reset.
2907  **/
2908 void igb_configure_rx_ring(struct igb_adapter *adapter,
2909                            struct igb_ring *ring)
2910 {
2911         struct e1000_hw *hw = &adapter->hw;
2912         u64 rdba = ring->dma;
2913         int reg_idx = ring->reg_idx;
2914         u32 srrctl, rxdctl;
2915
2916         /* disable the queue */
2917         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2918         wr32(E1000_RXDCTL(reg_idx),
2919                         rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2920
2921         /* Set DMA base address registers */
2922         wr32(E1000_RDBAL(reg_idx),
2923              rdba & 0x00000000ffffffffULL);
2924         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2925         wr32(E1000_RDLEN(reg_idx),
2926                        ring->count * sizeof(union e1000_adv_rx_desc));
2927
2928         /* initialize head and tail */
2929         ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2930         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2931         writel(0, ring->head);
2932         writel(0, ring->tail);
2933
2934         /* set descriptor configuration */
2935         if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2936                 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2937                          E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2938 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2939                 srrctl |= IGB_RXBUFFER_16384 >>
2940                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2941 #else
2942                 srrctl |= (PAGE_SIZE / 2) >>
2943                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2944 #endif
2945                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2946         } else {
2947                 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2948                          E1000_SRRCTL_BSIZEPKT_SHIFT;
2949                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2950         }
2951         if (hw->mac.type == e1000_82580)
2952                 srrctl |= E1000_SRRCTL_TIMESTAMP;
2953         /* Only set Drop Enable if we are supporting multiple queues */
2954         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2955                 srrctl |= E1000_SRRCTL_DROP_EN;
2956
2957         wr32(E1000_SRRCTL(reg_idx), srrctl);
2958
2959         /* set filtering for VMDQ pools */
2960         igb_set_vmolr(adapter, reg_idx & 0x7, true);
2961
2962         /* enable receive descriptor fetching */
2963         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2964         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2965         rxdctl &= 0xFFF00000;
2966         rxdctl |= IGB_RX_PTHRESH;
2967         rxdctl |= IGB_RX_HTHRESH << 8;
2968         rxdctl |= IGB_RX_WTHRESH << 16;
2969         wr32(E1000_RXDCTL(reg_idx), rxdctl);
2970 }
2971
2972 /**
2973  * igb_configure_rx - Configure receive Unit after Reset
2974  * @adapter: board private structure
2975  *
2976  * Configure the Rx unit of the MAC after a reset.
2977  **/
2978 static void igb_configure_rx(struct igb_adapter *adapter)
2979 {
2980         int i;
2981
2982         /* set UTA to appropriate mode */
2983         igb_set_uta(adapter);
2984
2985         /* set the correct pool for the PF default MAC address in entry 0 */
2986         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2987                          adapter->vfs_allocated_count);
2988
2989         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2990          * the Base and Length of the Rx Descriptor Ring */
2991         for (i = 0; i < adapter->num_rx_queues; i++)
2992                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2993 }
2994
2995 /**
2996  * igb_free_tx_resources - Free Tx Resources per Queue
2997  * @tx_ring: Tx descriptor ring for a specific queue
2998  *
2999  * Free all transmit software resources
3000  **/
3001 void igb_free_tx_resources(struct igb_ring *tx_ring)
3002 {
3003         igb_clean_tx_ring(tx_ring);
3004
3005         vfree(tx_ring->buffer_info);
3006         tx_ring->buffer_info = NULL;
3007
3008         /* if not set, then don't free */
3009         if (!tx_ring->desc)
3010                 return;
3011
3012         dma_free_coherent(tx_ring->dev, tx_ring->size,
3013                           tx_ring->desc, tx_ring->dma);
3014
3015         tx_ring->desc = NULL;
3016 }
3017
3018 /**
3019  * igb_free_all_tx_resources - Free Tx Resources for All Queues
3020  * @adapter: board private structure
3021  *
3022  * Free all transmit software resources
3023  **/
3024 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3025 {
3026         int i;
3027
3028         for (i = 0; i < adapter->num_tx_queues; i++)
3029                 igb_free_tx_resources(adapter->tx_ring[i]);
3030 }
3031
3032 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3033                                     struct igb_buffer *buffer_info)
3034 {
3035         if (buffer_info->dma) {
3036                 if (buffer_info->mapped_as_page)
3037                         dma_unmap_page(tx_ring->dev,
3038                                         buffer_info->dma,
3039                                         buffer_info->length,
3040                                         DMA_TO_DEVICE);
3041                 else
3042                         dma_unmap_single(tx_ring->dev,
3043                                         buffer_info->dma,
3044                                         buffer_info->length,
3045                                         DMA_TO_DEVICE);
3046                 buffer_info->dma = 0;
3047         }
3048         if (buffer_info->skb) {
3049                 dev_kfree_skb_any(buffer_info->skb);
3050                 buffer_info->skb = NULL;
3051         }
3052         buffer_info->time_stamp = 0;
3053         buffer_info->length = 0;
3054         buffer_info->next_to_watch = 0;
3055         buffer_info->mapped_as_page = false;
3056 }
3057
3058 /**
3059  * igb_clean_tx_ring - Free Tx Buffers
3060  * @tx_ring: ring to be cleaned
3061  **/
3062 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3063 {
3064         struct igb_buffer *buffer_info;
3065         unsigned long size;
3066         unsigned int i;
3067
3068         if (!tx_ring->buffer_info)
3069                 return;
3070         /* Free all the Tx ring sk_buffs */
3071
3072         for (i = 0; i < tx_ring->count; i++) {
3073                 buffer_info = &tx_ring->buffer_info[i];
3074                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3075         }
3076
3077         size = sizeof(struct igb_buffer) * tx_ring->count;
3078         memset(tx_ring->buffer_info, 0, size);
3079
3080         /* Zero out the descriptor ring */
3081         memset(tx_ring->desc, 0, tx_ring->size);
3082
3083         tx_ring->next_to_use = 0;
3084         tx_ring->next_to_clean = 0;
3085 }
3086
3087 /**
3088  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3089  * @adapter: board private structure
3090  **/
3091 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3092 {
3093         int i;
3094
3095         for (i = 0; i < adapter->num_tx_queues; i++)
3096                 igb_clean_tx_ring(adapter->tx_ring[i]);
3097 }
3098
3099 /**
3100  * igb_free_rx_resources - Free Rx Resources
3101  * @rx_ring: ring to clean the resources from
3102  *
3103  * Free all receive software resources
3104  **/
3105 void igb_free_rx_resources(struct igb_ring *rx_ring)
3106 {
3107         igb_clean_rx_ring(rx_ring);
3108
3109         vfree(rx_ring->buffer_info);
3110         rx_ring->buffer_info = NULL;
3111
3112         /* if not set, then don't free */
3113         if (!rx_ring->desc)
3114                 return;
3115
3116         dma_free_coherent(rx_ring->dev, rx_ring->size,
3117                           rx_ring->desc, rx_ring->dma);
3118
3119         rx_ring->desc = NULL;
3120 }
3121
3122 /**
3123  * igb_free_all_rx_resources - Free Rx Resources for All Queues
3124  * @adapter: board private structure
3125  *
3126  * Free all receive software resources
3127  **/
3128 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3129 {
3130         int i;
3131
3132         for (i = 0; i < adapter->num_rx_queues; i++)
3133                 igb_free_rx_resources(adapter->rx_ring[i]);
3134 }
3135
3136 /**
3137  * igb_clean_rx_ring - Free Rx Buffers per Queue
3138  * @rx_ring: ring to free buffers from
3139  **/
3140 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3141 {
3142         struct igb_buffer *buffer_info;
3143         unsigned long size;
3144         unsigned int i;
3145
3146         if (!rx_ring->buffer_info)
3147                 return;
3148
3149         /* Free all the Rx ring sk_buffs */
3150         for (i = 0; i < rx_ring->count; i++) {
3151                 buffer_info = &rx_ring->buffer_info[i];
3152                 if (buffer_info->dma) {
3153                         dma_unmap_single(rx_ring->dev,
3154                                          buffer_info->dma,
3155                                          rx_ring->rx_buffer_len,
3156                                          DMA_FROM_DEVICE);
3157                         buffer_info->dma = 0;
3158                 }
3159
3160                 if (buffer_info->skb) {
3161                         dev_kfree_skb(buffer_info->skb);
3162                         buffer_info->skb = NULL;
3163                 }
3164                 if (buffer_info->page_dma) {
3165                         dma_unmap_page(rx_ring->dev,
3166                                        buffer_info->page_dma,
3167                                        PAGE_SIZE / 2,
3168                                        DMA_FROM_DEVICE);
3169                         buffer_info->page_dma = 0;
3170                 }
3171                 if (buffer_info->page) {
3172                         put_page(buffer_info->page);
3173                         buffer_info->page = NULL;
3174                         buffer_info->page_offset = 0;
3175                 }
3176         }
3177
3178         size = sizeof(struct igb_buffer) * rx_ring->count;
3179         memset(rx_ring->buffer_info, 0, size);
3180
3181         /* Zero out the descriptor ring */
3182         memset(rx_ring->desc, 0, rx_ring->size);
3183
3184         rx_ring->next_to_clean = 0;
3185         rx_ring->next_to_use = 0;
3186 }
3187
3188 /**
3189  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3190  * @adapter: board private structure
3191  **/
3192 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3193 {
3194         int i;
3195
3196         for (i = 0; i < adapter->num_rx_queues; i++)
3197                 igb_clean_rx_ring(adapter->rx_ring[i]);
3198 }
3199
3200 /**
3201  * igb_set_mac - Change the Ethernet Address of the NIC
3202  * @netdev: network interface device structure
3203  * @p: pointer to an address structure
3204  *
3205  * Returns 0 on success, negative on failure
3206  **/
3207 static int igb_set_mac(struct net_device *netdev, void *p)
3208 {
3209         struct igb_adapter *adapter = netdev_priv(netdev);
3210         struct e1000_hw *hw = &adapter->hw;
3211         struct sockaddr *addr = p;
3212
3213         if (!is_valid_ether_addr(addr->sa_data))
3214                 return -EADDRNOTAVAIL;
3215
3216         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3217         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3218
3219         /* set the correct pool for the new PF MAC address in entry 0 */
3220         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3221                          adapter->vfs_allocated_count);
3222
3223         return 0;
3224 }
3225
3226 /**
3227  * igb_write_mc_addr_list - write multicast addresses to MTA
3228  * @netdev: network interface device structure
3229  *
3230  * Writes multicast address list to the MTA hash table.
3231  * Returns: -ENOMEM on failure
3232  *                0 on no addresses written
3233  *                X on writing X addresses to MTA
3234  **/
3235 static int igb_write_mc_addr_list(struct net_device *netdev)
3236 {
3237         struct igb_adapter *adapter = netdev_priv(netdev);
3238         struct e1000_hw *hw = &adapter->hw;
3239         struct netdev_hw_addr *ha;
3240         u8  *mta_list;
3241         int i;
3242
3243         if (netdev_mc_empty(netdev)) {
3244                 /* nothing to program, so clear mc list */
3245                 igb_update_mc_addr_list(hw, NULL, 0);
3246                 igb_restore_vf_multicasts(adapter);
3247                 return 0;
3248         }
3249
3250         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3251         if (!mta_list)
3252                 return -ENOMEM;
3253
3254         /* The shared function expects a packed array of only addresses. */
3255         i = 0;
3256         netdev_for_each_mc_addr(ha, netdev)
3257                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3258
3259         igb_update_mc_addr_list(hw, mta_list, i);
3260         kfree(mta_list);
3261
3262         return netdev_mc_count(netdev);
3263 }
3264
3265 /**
3266  * igb_write_uc_addr_list - write unicast addresses to RAR table
3267  * @netdev: network interface device structure
3268  *
3269  * Writes unicast address list to the RAR table.
3270  * Returns: -ENOMEM on failure/insufficient address space
3271  *                0 on no addresses written
3272  *                X on writing X addresses to the RAR table
3273  **/
3274 static int igb_write_uc_addr_list(struct net_device *netdev)
3275 {
3276         struct igb_adapter *adapter = netdev_priv(netdev);
3277         struct e1000_hw *hw = &adapter->hw;
3278         unsigned int vfn = adapter->vfs_allocated_count;
3279         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3280         int count = 0;
3281
3282         /* return ENOMEM indicating insufficient memory for addresses */
3283         if (netdev_uc_count(netdev) > rar_entries)
3284                 return -ENOMEM;
3285
3286         if (!netdev_uc_empty(netdev) && rar_entries) {
3287                 struct netdev_hw_addr *ha;
3288
3289                 netdev_for_each_uc_addr(ha, netdev) {
3290                         if (!rar_entries)
3291                                 break;
3292                         igb_rar_set_qsel(adapter, ha->addr,
3293                                          rar_entries--,
3294                                          vfn);
3295                         count++;
3296                 }
3297         }
3298         /* write the addresses in reverse order to avoid write combining */
3299         for (; rar_entries > 0 ; rar_entries--) {
3300                 wr32(E1000_RAH(rar_entries), 0);
3301                 wr32(E1000_RAL(rar_entries), 0);
3302         }
3303         wrfl();
3304
3305         return count;
3306 }
3307
3308 /**
3309  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3310  * @netdev: network interface device structure
3311  *
3312  * The set_rx_mode entry point is called whenever the unicast or multicast
3313  * address lists or the network interface flags are updated.  This routine is
3314  * responsible for configuring the hardware for proper unicast, multicast,
3315  * promiscuous mode, and all-multi behavior.
3316  **/
3317 static void igb_set_rx_mode(struct net_device *netdev)
3318 {
3319         struct igb_adapter *adapter = netdev_priv(netdev);
3320         struct e1000_hw *hw = &adapter->hw;
3321         unsigned int vfn = adapter->vfs_allocated_count;
3322         u32 rctl, vmolr = 0;
3323         int count;
3324
3325         /* Check for Promiscuous and All Multicast modes */
3326         rctl = rd32(E1000_RCTL);
3327
3328         /* clear the effected bits */
3329         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3330
3331         if (netdev->flags & IFF_PROMISC) {
3332                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3333                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3334         } else {
3335                 if (netdev->flags & IFF_ALLMULTI) {
3336                         rctl |= E1000_RCTL_MPE;
3337                         vmolr |= E1000_VMOLR_MPME;
3338                 } else {
3339                         /*
3340                          * Write addresses to the MTA, if the attempt fails
3341                          * then we should just turn on promiscous mode so
3342                          * that we can at least receive multicast traffic
3343                          */
3344                         count = igb_write_mc_addr_list(netdev);
3345                         if (count < 0) {
3346                                 rctl |= E1000_RCTL_MPE;
3347                                 vmolr |= E1000_VMOLR_MPME;
3348                         } else if (count) {
3349                                 vmolr |= E1000_VMOLR_ROMPE;
3350                         }
3351                 }
3352                 /*
3353                  * Write addresses to available RAR registers, if there is not
3354                  * sufficient space to store all the addresses then enable
3355                  * unicast promiscous mode
3356                  */
3357                 count = igb_write_uc_addr_list(netdev);
3358                 if (count < 0) {
3359                         rctl |= E1000_RCTL_UPE;
3360                         vmolr |= E1000_VMOLR_ROPE;
3361                 }
3362                 rctl |= E1000_RCTL_VFE;
3363         }
3364         wr32(E1000_RCTL, rctl);
3365
3366         /*
3367          * In order to support SR-IOV and eventually VMDq it is necessary to set
3368          * the VMOLR to enable the appropriate modes.  Without this workaround
3369          * we will have issues with VLAN tag stripping not being done for frames
3370          * that are only arriving because we are the default pool
3371          */
3372         if (hw->mac.type < e1000_82576)
3373                 return;
3374
3375         vmolr |= rd32(E1000_VMOLR(vfn)) &
3376                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3377         wr32(E1000_VMOLR(vfn), vmolr);
3378         igb_restore_vf_multicasts(adapter);
3379 }
3380
3381 static void igb_check_wvbr(struct igb_adapter *adapter)
3382 {
3383         struct e1000_hw *hw = &adapter->hw;
3384         u32 wvbr = 0;
3385
3386         switch (hw->mac.type) {
3387         case e1000_82576:
3388         case e1000_i350:
3389                 if (!(wvbr = rd32(E1000_WVBR)))
3390                         return;
3391                 break;
3392         default:
3393                 break;
3394         }
3395
3396         adapter->wvbr |= wvbr;
3397 }
3398
3399 #define IGB_STAGGERED_QUEUE_OFFSET 8
3400
3401 static void igb_spoof_check(struct igb_adapter *adapter)
3402 {
3403         int j;
3404
3405         if (!adapter->wvbr)
3406                 return;
3407
3408         for(j = 0; j < adapter->vfs_allocated_count; j++) {
3409                 if (adapter->wvbr & (1 << j) ||
3410                     adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3411                         dev_warn(&adapter->pdev->dev,
3412                                 "Spoof event(s) detected on VF %d\n", j);
3413                         adapter->wvbr &=
3414                                 ~((1 << j) |
3415                                   (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3416                 }
3417         }
3418 }
3419
3420 /* Need to wait a few seconds after link up to get diagnostic information from
3421  * the phy */
3422 static void igb_update_phy_info(unsigned long data)
3423 {
3424         struct igb_adapter *adapter = (struct igb_adapter *) data;
3425         igb_get_phy_info(&adapter->hw);
3426 }
3427
3428 /**
3429  * igb_has_link - check shared code for link and determine up/down
3430  * @adapter: pointer to driver private info
3431  **/
3432 bool igb_has_link(struct igb_adapter *adapter)
3433 {
3434         struct e1000_hw *hw = &adapter->hw;
3435         bool link_active = false;
3436         s32 ret_val = 0;
3437
3438         /* get_link_status is set on LSC (link status) interrupt or
3439          * rx sequence error interrupt.  get_link_status will stay
3440          * false until the e1000_check_for_link establishes link
3441          * for copper adapters ONLY
3442          */
3443         switch (hw->phy.media_type) {
3444         case e1000_media_type_copper:
3445                 if (hw->mac.get_link_status) {
3446                         ret_val = hw->mac.ops.check_for_link(hw);
3447                         link_active = !hw->mac.get_link_status;
3448                 } else {
3449                         link_active = true;
3450                 }
3451                 break;
3452         case e1000_media_type_internal_serdes:
3453                 ret_val = hw->mac.ops.check_for_link(hw);
3454                 link_active = hw->mac.serdes_has_link;
3455                 break;
3456         default:
3457         case e1000_media_type_unknown:
3458                 break;
3459         }
3460
3461         return link_active;
3462 }
3463
3464 /**
3465  * igb_watchdog - Timer Call-back
3466  * @data: pointer to adapter cast into an unsigned long
3467  **/
3468 static void igb_watchdog(unsigned long data)
3469 {
3470         struct igb_adapter *adapter = (struct igb_adapter *)data;
3471         /* Do the rest outside of interrupt context */
3472         schedule_work(&adapter->watchdog_task);
3473 }
3474
3475 static void igb_watchdog_task(struct work_struct *work)
3476 {
3477         struct igb_adapter *adapter = container_of(work,
3478                                                    struct igb_adapter,
3479                                                    watchdog_task);
3480         struct e1000_hw *hw = &adapter->hw;
3481         struct net_device *netdev = adapter->netdev;
3482         u32 link;
3483         int i;
3484
3485         link = igb_has_link(adapter);
3486         if (link) {
3487                 if (!netif_carrier_ok(netdev)) {
3488                         u32 ctrl;
3489                         hw->mac.ops.get_speed_and_duplex(hw,
3490                                                          &adapter->link_speed,
3491                                                          &adapter->link_duplex);
3492
3493                         ctrl = rd32(E1000_CTRL);
3494                         /* Links status message must follow this format */
3495                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3496                                  "Flow Control: %s\n",
3497                                netdev->name,
3498                                adapter->link_speed,
3499                                adapter->link_duplex == FULL_DUPLEX ?
3500                                  "Full Duplex" : "Half Duplex",
3501                                ((ctrl & E1000_CTRL_TFCE) &&
3502                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3503                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
3504                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
3505
3506                         /* adjust timeout factor according to speed/duplex */
3507                         adapter->tx_timeout_factor = 1;
3508                         switch (adapter->link_speed) {
3509                         case SPEED_10:
3510                                 adapter->tx_timeout_factor = 14;
3511                                 break;
3512                         case SPEED_100:
3513                                 /* maybe add some timeout factor ? */
3514                                 break;
3515                         }
3516
3517                         netif_carrier_on(netdev);
3518
3519                         igb_ping_all_vfs(adapter);
3520                         igb_check_vf_rate_limit(adapter);
3521
3522                         /* link state has changed, schedule phy info update */
3523                         if (!test_bit(__IGB_DOWN, &adapter->state))
3524                                 mod_timer(&adapter->phy_info_timer,
3525                                           round_jiffies(jiffies + 2 * HZ));
3526                 }
3527         } else {
3528                 if (netif_carrier_ok(netdev)) {
3529                         adapter->link_speed = 0;
3530                         adapter->link_duplex = 0;
3531                         /* Links status message must follow this format */
3532                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3533                                netdev->name);
3534                         netif_carrier_off(netdev);
3535
3536                         igb_ping_all_vfs(adapter);
3537
3538                         /* link state has changed, schedule phy info update */
3539                         if (!test_bit(__IGB_DOWN, &adapter->state))
3540                                 mod_timer(&adapter->phy_info_timer,
3541                                           round_jiffies(jiffies + 2 * HZ));
3542                 }
3543         }
3544
3545         spin_lock(&adapter->stats64_lock);
3546         igb_update_stats(adapter, &adapter->stats64);
3547         spin_unlock(&adapter->stats64_lock);
3548
3549         for (i = 0; i < adapter->num_tx_queues; i++) {
3550                 struct igb_ring *tx_ring = adapter->tx_ring[i];
3551                 if (!netif_carrier_ok(netdev)) {
3552                         /* We've lost link, so the controller stops DMA,
3553                          * but we've got queued Tx work that's never going
3554                          * to get done, so reset controller to flush Tx.
3555                          * (Do the reset outside of interrupt context). */
3556                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3557                                 adapter->tx_timeout_count++;
3558                                 schedule_work(&adapter->reset_task);
3559                                 /* return immediately since reset is imminent */
3560                                 return;
3561                         }
3562                 }
3563
3564                 /* Force detection of hung controller every watchdog period */
3565                 tx_ring->detect_tx_hung = true;
3566         }
3567
3568         /* Cause software interrupt to ensure rx ring is cleaned */
3569         if (adapter->msix_entries) {
3570                 u32 eics = 0;
3571                 for (i = 0; i < adapter->num_q_vectors; i++) {
3572                         struct igb_q_vector *q_vector = adapter->q_vector[i];
3573                         eics |= q_vector->eims_value;
3574                 }
3575                 wr32(E1000_EICS, eics);
3576         } else {
3577                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3578         }
3579
3580         igb_spoof_check(adapter);
3581
3582         /* Reset the timer */
3583         if (!test_bit(__IGB_DOWN, &adapter->state))
3584                 mod_timer(&adapter->watchdog_timer,
3585                           round_jiffies(jiffies + 2 * HZ));
3586 }
3587
3588 enum latency_range {
3589         lowest_latency = 0,
3590         low_latency = 1,
3591         bulk_latency = 2,
3592         latency_invalid = 255
3593 };
3594
3595 /**
3596  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3597  *
3598  *      Stores a new ITR value based on strictly on packet size.  This
3599  *      algorithm is less sophisticated than that used in igb_update_itr,
3600  *      due to the difficulty of synchronizing statistics across multiple
3601  *      receive rings.  The divisors and thresholds used by this function
3602  *      were determined based on theoretical maximum wire speed and testing
3603  *      data, in order to minimize response time while increasing bulk
3604  *      throughput.
3605  *      This functionality is controlled by the InterruptThrottleRate module
3606  *      parameter (see igb_param.c)
3607  *      NOTE:  This function is called only when operating in a multiqueue
3608  *             receive environment.
3609  * @q_vector: pointer to q_vector
3610  **/
3611 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3612 {
3613         int new_val = q_vector->itr_val;
3614         int avg_wire_size = 0;
3615         struct igb_adapter *adapter = q_vector->adapter;
3616         struct igb_ring *ring;
3617         unsigned int packets;
3618
3619         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3620          * ints/sec - ITR timer value of 120 ticks.
3621          */
3622         if (adapter->link_speed != SPEED_1000) {
3623                 new_val = 976;
3624                 goto set_itr_val;
3625         }
3626
3627         ring = q_vector->rx_ring;
3628         if (ring) {
3629                 packets = ACCESS_ONCE(ring->total_packets);
3630
3631                 if (packets)
3632                         avg_wire_size = ring->total_bytes / packets;
3633         }
3634
3635         ring = q_vector->tx_ring;
3636         if (ring) {
3637                 packets = ACCESS_ONCE(ring->total_packets);
3638
3639                 if (packets)
3640                         avg_wire_size = max_t(u32, avg_wire_size,
3641                                               ring->total_bytes / packets);
3642         }
3643
3644         /* if avg_wire_size isn't set no work was done */
3645         if (!avg_wire_size)
3646                 goto clear_counts;
3647
3648         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3649         avg_wire_size += 24;
3650
3651         /* Don't starve jumbo frames */
3652         avg_wire_size = min(avg_wire_size, 3000);
3653
3654         /* Give a little boost to mid-size frames */
3655         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3656                 new_val = avg_wire_size / 3;
3657         else
3658                 new_val = avg_wire_size / 2;
3659
3660         /* when in itr mode 3 do not exceed 20K ints/sec */
3661         if (adapter->rx_itr_setting == 3 && new_val < 196)
3662                 new_val = 196;
3663
3664 set_itr_val:
3665         if (new_val != q_vector->itr_val) {
3666                 q_vector->itr_val = new_val;
3667                 q_vector->set_itr = 1;
3668         }
3669 clear_counts:
3670         if (q_vector->rx_ring) {
3671                 q_vector->rx_ring->total_bytes = 0;
3672                 q_vector->rx_ring->total_packets = 0;
3673         }
3674         if (q_vector->tx_ring) {
3675                 q_vector->tx_ring->total_bytes = 0;
3676                 q_vector->tx_ring->total_packets = 0;
3677         }
3678 }
3679
3680 /**
3681  * igb_update_itr - update the dynamic ITR value based on statistics
3682  *      Stores a new ITR value based on packets and byte
3683  *      counts during the last interrupt.  The advantage of per interrupt
3684  *      computation is faster updates and more accurate ITR for the current
3685  *      traffic pattern.  Constants in this function were computed
3686  *      based on theoretical maximum wire speed and thresholds were set based
3687  *      on testing data as well as attempting to minimize response time
3688  *      while increasing bulk throughput.
3689  *      this functionality is controlled by the InterruptThrottleRate module
3690  *      parameter (see igb_param.c)
3691  *      NOTE:  These calculations are only valid when operating in a single-
3692  *             queue environment.
3693  * @adapter: pointer to adapter
3694  * @itr_setting: current q_vector->itr_val
3695  * @packets: the number of packets during this measurement interval
3696  * @bytes: the number of bytes during this measurement interval
3697  **/
3698 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3699                                    int packets, int bytes)
3700 {
3701         unsigned int retval = itr_setting;
3702
3703         if (packets == 0)
3704                 goto update_itr_done;
3705
3706         switch (itr_setting) {
3707         case lowest_latency:
3708                 /* handle TSO and jumbo frames */
3709                 if (bytes/packets > 8000)
3710                         retval = bulk_latency;
3711                 else if ((packets < 5) && (bytes > 512))
3712                         retval = low_latency;
3713                 break;
3714         case low_latency:  /* 50 usec aka 20000 ints/s */
3715                 if (bytes > 10000) {
3716                         /* this if handles the TSO accounting */
3717                         if (bytes/packets > 8000) {
3718                                 retval = bulk_latency;
3719                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3720                                 retval = bulk_latency;
3721                         } else if ((packets > 35)) {
3722                                 retval = lowest_latency;
3723                         }
3724                 } else if (bytes/packets > 2000) {
3725                         retval = bulk_latency;
3726                 } else if (packets <= 2 && bytes < 512) {
3727                         retval = lowest_latency;
3728                 }
3729                 break;
3730         case bulk_latency: /* 250 usec aka 4000 ints/s */
3731                 if (bytes > 25000) {
3732                         if (packets > 35)
3733                                 retval = low_latency;
3734                 } else if (bytes < 1500) {
3735                         retval = low_latency;
3736                 }
3737                 break;
3738         }
3739
3740 update_itr_done:
3741         return retval;
3742 }
3743
3744 static void igb_set_itr(struct igb_adapter *adapter)
3745 {
3746         struct igb_q_vector *q_vector = adapter->q_vector[0];
3747         u16 current_itr;
3748         u32 new_itr = q_vector->itr_val;
3749
3750         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3751         if (adapter->link_speed != SPEED_1000) {
3752                 current_itr = 0;
3753                 new_itr = 4000;
3754                 goto set_itr_now;
3755         }
3756
3757         adapter->rx_itr = igb_update_itr(adapter,
3758                                     adapter->rx_itr,
3759                                     q_vector->rx_ring->total_packets,
3760                                     q_vector->rx_ring->total_bytes);
3761
3762         adapter->tx_itr = igb_update_itr(adapter,
3763                                     adapter->tx_itr,
3764                                     q_vector->tx_ring->total_packets,
3765                                     q_vector->tx_ring->total_bytes);
3766         current_itr = max(adapter->rx_itr, adapter->tx_itr);
3767
3768         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3769         if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3770                 current_itr = low_latency;
3771
3772         switch (current_itr) {
3773         /* counts and packets in update_itr are dependent on these numbers */
3774         case lowest_latency:
3775                 new_itr = 56;  /* aka 70,000 ints/sec */
3776                 break;
3777         case low_latency:
3778                 new_itr = 196; /* aka 20,000 ints/sec */
3779                 break;
3780         case bulk_latency:
3781                 new_itr = 980; /* aka 4,000 ints/sec */
3782                 break;
3783         default:
3784                 break;
3785         }
3786
3787 set_itr_now:
3788         q_vector->rx_ring->total_bytes = 0;
3789         q_vector->rx_ring->total_packets = 0;
3790         q_vector->tx_ring->total_bytes = 0;
3791         q_vector->tx_ring->total_packets = 0;
3792
3793         if (new_itr != q_vector->itr_val) {
3794                 /* this attempts to bias the interrupt rate towards Bulk
3795                  * by adding intermediate steps when interrupt rate is
3796                  * increasing */
3797                 new_itr = new_itr > q_vector->itr_val ?
3798                              max((new_itr * q_vector->itr_val) /
3799                                  (new_itr + (q_vector->itr_val >> 2)),
3800                                  new_itr) :
3801                              new_itr;
3802                 /* Don't write the value here; it resets the adapter's
3803                  * internal timer, and causes us to delay far longer than
3804                  * we should between interrupts.  Instead, we write the ITR
3805                  * value at the beginning of the next interrupt so the timing
3806                  * ends up being correct.
3807                  */
3808                 q_vector->itr_val = new_itr;
3809                 q_vector->set_itr = 1;
3810         }
3811 }
3812
3813 #define IGB_TX_FLAGS_CSUM               0x00000001
3814 #define IGB_TX_FLAGS_VLAN               0x00000002
3815 #define IGB_TX_FLAGS_TSO                0x00000004
3816 #define IGB_TX_FLAGS_IPV4               0x00000008
3817 #define IGB_TX_FLAGS_TSTAMP             0x00000010
3818 #define IGB_TX_FLAGS_VLAN_MASK          0xffff0000
3819 #define IGB_TX_FLAGS_VLAN_SHIFT                 16
3820
3821 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3822                               struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3823 {
3824         struct e1000_adv_tx_context_desc *context_desc;
3825         unsigned int i;
3826         int err;
3827         struct igb_buffer *buffer_info;
3828         u32 info = 0, tu_cmd = 0;
3829         u32 mss_l4len_idx;
3830         u8 l4len;
3831
3832         if (skb_header_cloned(skb)) {
3833                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3834                 if (err)
3835                         return err;
3836         }
3837
3838         l4len = tcp_hdrlen(skb);
3839         *hdr_len += l4len;
3840
3841         if (skb->protocol == htons(ETH_P_IP)) {
3842                 struct iphdr *iph = ip_hdr(skb);
3843                 iph->tot_len = 0;
3844                 iph->check = 0;
3845                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3846                                                          iph->daddr, 0,
3847                                                          IPPROTO_TCP,
3848                                                          0);
3849         } else if (skb_is_gso_v6(skb)) {
3850                 ipv6_hdr(skb)->payload_len = 0;
3851                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3852                                                        &ipv6_hdr(skb)->daddr,
3853                                                        0, IPPROTO_TCP, 0);
3854         }
3855
3856         i = tx_ring->next_to_use;
3857
3858         buffer_info = &tx_ring->buffer_info[i];
3859         context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3860         /* VLAN MACLEN IPLEN */
3861         if (tx_flags & IGB_TX_FLAGS_VLAN)
3862                 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3863         info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3864         *hdr_len += skb_network_offset(skb);
3865         info |= skb_network_header_len(skb);
3866         *hdr_len += skb_network_header_len(skb);
3867         context_desc->vlan_macip_lens = cpu_to_le32(info);
3868
3869         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3870         tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3871
3872         if (skb->protocol == htons(ETH_P_IP))
3873                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3874         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3875
3876         context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3877
3878         /* MSS L4LEN IDX */
3879         mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3880         mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3881
3882         /* For 82575, context index must be unique per ring. */
3883         if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3884                 mss_l4len_idx |= tx_ring->reg_idx << 4;
3885
3886         context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3887         context_desc->seqnum_seed = 0;
3888
3889         buffer_info->time_stamp = jiffies;
3890         buffer_info->next_to_watch = i;
3891         buffer_info->dma = 0;
3892         i++;
3893         if (i == tx_ring->count)
3894                 i = 0;
3895
3896         tx_ring->next_to_use = i;
3897
3898         return true;
3899 }
3900
3901 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3902                                    struct sk_buff *skb, u32 tx_flags)
3903 {
3904         struct e1000_adv_tx_context_desc *context_desc;
3905         struct device *dev = tx_ring->dev;
3906         struct igb_buffer *buffer_info;
3907         u32 info = 0, tu_cmd = 0;
3908         unsigned int i;
3909
3910         if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3911             (tx_flags & IGB_TX_FLAGS_VLAN)) {
3912                 i = tx_ring->next_to_use;
3913                 buffer_info = &tx_ring->buffer_info[i];
3914                 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3915
3916                 if (tx_flags & IGB_TX_FLAGS_VLAN)
3917                         info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3918
3919                 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3920                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3921                         info |= skb_network_header_len(skb);
3922
3923                 context_desc->vlan_macip_lens = cpu_to_le32(info);
3924
3925                 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3926
3927                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3928                         __be16 protocol;
3929
3930                         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3931                                 const struct vlan_ethhdr *vhdr =
3932                                           (const struct vlan_ethhdr*)skb->data;
3933
3934                                 protocol = vhdr->h_vlan_encapsulated_proto;
3935                         } else {
3936                                 protocol = skb->protocol;
3937                         }
3938
3939                         switch (protocol) {
3940                         case cpu_to_be16(ETH_P_IP):
3941                                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3942                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3943                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3944                                 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3945                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3946                                 break;
3947                         case cpu_to_be16(ETH_P_IPV6):
3948                                 /* XXX what about other V6 headers?? */
3949                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3950                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3951                                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3952                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3953                                 break;
3954                         default:
3955                                 if (unlikely(net_ratelimit()))
3956                                         dev_warn(dev,
3957                                             "partial checksum but proto=%x!\n",
3958                                             skb->protocol);
3959                                 break;
3960                         }
3961                 }
3962
3963                 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3964                 context_desc->seqnum_seed = 0;
3965                 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3966                         context_desc->mss_l4len_idx =
3967                                 cpu_to_le32(tx_ring->reg_idx << 4);
3968
3969                 buffer_info->time_stamp = jiffies;
3970                 buffer_info->next_to_watch = i;
3971                 buffer_info->dma = 0;
3972
3973                 i++;
3974                 if (i == tx_ring->count)
3975                         i = 0;
3976                 tx_ring->next_to_use = i;
3977
3978                 return true;
3979         }
3980         return false;
3981 }
3982
3983 #define IGB_MAX_TXD_PWR 16
3984 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
3985
3986 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3987                                  unsigned int first)
3988 {
3989         struct igb_buffer *buffer_info;
3990         struct device *dev = tx_ring->dev;
3991         unsigned int hlen = skb_headlen(skb);
3992         unsigned int count = 0, i;
3993         unsigned int f;
3994         u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3995
3996         i = tx_ring->next_to_use;
3997
3998         buffer_info = &tx_ring->buffer_info[i];
3999         BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
4000         buffer_info->length = hlen;
4001         /* set time_stamp *before* dma to help avoid a possible race */
4002         buffer_info->time_stamp = jiffies;
4003         buffer_info->next_to_watch = i;
4004         buffer_info->dma = dma_map_single(dev, skb->data, hlen,
4005                                           DMA_TO_DEVICE);
4006         if (dma_mapping_error(dev, buffer_info->dma))
4007                 goto dma_error;
4008
4009         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
4010                 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
4011                 unsigned int len = frag->size;
4012
4013                 count++;
4014                 i++;
4015                 if (i == tx_ring->count)
4016                         i = 0;
4017
4018                 buffer_info = &tx_ring->buffer_info[i];
4019                 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
4020                 buffer_info->length = len;
4021                 buffer_info->time_stamp = jiffies;
4022                 buffer_info->next_to_watch = i;
4023                 buffer_info->mapped_as_page = true;
4024                 buffer_info->dma = dma_map_page(dev,
4025                                                 frag->page,
4026                                                 frag->page_offset,
4027                                                 len,
4028                                                 DMA_TO_DEVICE);
4029                 if (dma_mapping_error(dev, buffer_info->dma))
4030                         goto dma_error;
4031
4032         }
4033
4034         tx_ring->buffer_info[i].skb = skb;
4035         tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
4036         /* multiply data chunks by size of headers */
4037         tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
4038         tx_ring->buffer_info[i].gso_segs = gso_segs;
4039         tx_ring->buffer_info[first].next_to_watch = i;
4040
4041         return ++count;
4042
4043 dma_error:
4044         dev_err(dev, "TX DMA map failed\n");
4045
4046         /* clear timestamp and dma mappings for failed buffer_info mapping */
4047         buffer_info->dma = 0;
4048         buffer_info->time_stamp = 0;
4049         buffer_info->length = 0;
4050         buffer_info->next_to_watch = 0;
4051         buffer_info->mapped_as_page = false;
4052
4053         /* clear timestamp and dma mappings for remaining portion of packet */
4054         while (count--) {
4055                 if (i == 0)
4056                         i = tx_ring->count;
4057                 i--;
4058                 buffer_info = &tx_ring->buffer_info[i];
4059                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4060         }
4061
4062         return 0;
4063 }
4064
4065 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
4066                                     u32 tx_flags, int count, u32 paylen,
4067                                     u8 hdr_len)
4068 {
4069         union e1000_adv_tx_desc *tx_desc;
4070         struct igb_buffer *buffer_info;
4071         u32 olinfo_status = 0, cmd_type_len;
4072         unsigned int i = tx_ring->next_to_use;
4073
4074         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4075                         E1000_ADVTXD_DCMD_DEXT);
4076
4077         if (tx_flags & IGB_TX_FLAGS_VLAN)
4078                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4079
4080         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4081                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4082
4083         if (tx_flags & IGB_TX_FLAGS_TSO) {
4084                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4085
4086                 /* insert tcp checksum */
4087                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4088
4089                 /* insert ip checksum */
4090                 if (tx_flags & IGB_TX_FLAGS_IPV4)
4091                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4092
4093         } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4094                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4095         }
4096
4097         if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4098             (tx_flags & (IGB_TX_FLAGS_CSUM |
4099                          IGB_TX_FLAGS_TSO |
4100                          IGB_TX_FLAGS_VLAN)))
4101                 olinfo_status |= tx_ring->reg_idx << 4;
4102
4103         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4104
4105         do {
4106                 buffer_info = &tx_ring->buffer_info[i];
4107                 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4108                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4109                 tx_desc->read.cmd_type_len =
4110                         cpu_to_le32(cmd_type_len | buffer_info->length);
4111                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4112                 count--;
4113                 i++;
4114                 if (i == tx_ring->count)
4115                         i = 0;
4116         } while (count > 0);
4117
4118         tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4119         /* Force memory writes to complete before letting h/w
4120          * know there are new descriptors to fetch.  (Only
4121          * applicable for weak-ordered memory model archs,
4122          * such as IA-64). */
4123         wmb();
4124
4125         tx_ring->next_to_use = i;
4126         writel(i, tx_ring->tail);
4127         /* we need this if more than one processor can write to our tail
4128          * at a time, it syncronizes IO on IA64/Altix systems */
4129         mmiowb();
4130 }
4131
4132 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4133 {
4134         struct net_device *netdev = tx_ring->netdev;
4135
4136         netif_stop_subqueue(netdev, tx_ring->queue_index);
4137
4138         /* Herbert's original patch had:
4139          *  smp_mb__after_netif_stop_queue();
4140          * but since that doesn't exist yet, just open code it. */
4141         smp_mb();
4142
4143         /* We need to check again in a case another CPU has just
4144          * made room available. */
4145         if (igb_desc_unused(tx_ring) < size)
4146                 return -EBUSY;
4147
4148         /* A reprieve! */
4149         netif_wake_subqueue(netdev, tx_ring->queue_index);
4150
4151         u64_stats_update_begin(&tx_ring->tx_syncp2);
4152         tx_ring->tx_stats.restart_queue2++;
4153         u64_stats_update_end(&tx_ring->tx_syncp2);
4154
4155         return 0;
4156 }
4157
4158 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4159 {
4160         if (igb_desc_unused(tx_ring) >= size)
4161                 return 0;
4162         return __igb_maybe_stop_tx(tx_ring, size);
4163 }
4164
4165 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4166                                     struct igb_ring *tx_ring)
4167 {
4168         int tso = 0, count;
4169         u32 tx_flags = 0;
4170         u16 first;
4171         u8 hdr_len = 0;
4172
4173         /* need: 1 descriptor per page,
4174          *       + 2 desc gap to keep tail from touching head,
4175          *       + 1 desc for skb->data,
4176          *       + 1 desc for context descriptor,
4177          * otherwise try next time */
4178         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4179                 /* this is a hard error */
4180                 return NETDEV_TX_BUSY;
4181         }
4182
4183         if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4184                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4185                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4186         }
4187
4188         if (vlan_tx_tag_present(skb)) {
4189                 tx_flags |= IGB_TX_FLAGS_VLAN;
4190                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4191         }
4192
4193         if (skb->protocol == htons(ETH_P_IP))
4194                 tx_flags |= IGB_TX_FLAGS_IPV4;
4195
4196         first = tx_ring->next_to_use;
4197         if (skb_is_gso(skb)) {
4198                 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4199
4200                 if (tso < 0) {
4201                         dev_kfree_skb_any(skb);
4202                         return NETDEV_TX_OK;
4203                 }
4204         }
4205
4206         if (tso)
4207                 tx_flags |= IGB_TX_FLAGS_TSO;
4208         else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4209                  (skb->ip_summed == CHECKSUM_PARTIAL))
4210                 tx_flags |= IGB_TX_FLAGS_CSUM;
4211
4212         /*
4213          * count reflects descriptors mapped, if 0 or less then mapping error
4214          * has occured and we need to rewind the descriptor queue
4215          */
4216         count = igb_tx_map_adv(tx_ring, skb, first);
4217         if (!count) {
4218                 dev_kfree_skb_any(skb);
4219                 tx_ring->buffer_info[first].time_stamp = 0;
4220                 tx_ring->next_to_use = first;
4221                 return NETDEV_TX_OK;
4222         }
4223
4224         igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4225
4226         /* Make sure there is space in the ring for the next send. */
4227         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4228
4229         return NETDEV_TX_OK;
4230 }
4231
4232 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4233                                       struct net_device *netdev)
4234 {
4235         struct igb_adapter *adapter = netdev_priv(netdev);
4236         struct igb_ring *tx_ring;
4237         int r_idx = 0;
4238
4239         if (test_bit(__IGB_DOWN, &adapter->state)) {
4240                 dev_kfree_skb_any(skb);
4241                 return NETDEV_TX_OK;
4242         }
4243
4244         if (skb->len <= 0) {
4245                 dev_kfree_skb_any(skb);
4246                 return NETDEV_TX_OK;
4247         }
4248
4249         r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4250         tx_ring = adapter->multi_tx_table[r_idx];
4251
4252         /* This goes back to the question of how to logically map a tx queue
4253          * to a flow.  Right now, performance is impacted slightly negatively
4254          * if using multiple tx queues.  If the stack breaks away from a
4255          * single qdisc implementation, we can look at this again. */
4256         return igb_xmit_frame_ring_adv(skb, tx_ring);
4257 }
4258
4259 /**
4260  * igb_tx_timeout - Respond to a Tx Hang
4261  * @netdev: network interface device structure
4262  **/
4263 static void igb_tx_timeout(struct net_device *netdev)
4264 {
4265         struct igb_adapter *adapter = netdev_priv(netdev);
4266         struct e1000_hw *hw = &adapter->hw;
4267
4268         /* Do the reset outside of interrupt context */
4269         adapter->tx_timeout_count++;
4270
4271         if (hw->mac.type == e1000_82580)
4272                 hw->dev_spec._82575.global_device_reset = true;
4273
4274         schedule_work(&adapter->reset_task);
4275         wr32(E1000_EICS,
4276              (adapter->eims_enable_mask & ~adapter->eims_other));
4277 }
4278
4279 static void igb_reset_task(struct work_struct *work)
4280 {
4281         struct igb_adapter *adapter;
4282         adapter = container_of(work, struct igb_adapter, reset_task);
4283
4284         igb_dump(adapter);
4285         netdev_err(adapter->netdev, "Reset adapter\n");
4286         igb_reinit_locked(adapter);
4287 }
4288
4289 /**
4290  * igb_get_stats64 - Get System Network Statistics
4291  * @netdev: network interface device structure
4292  * @stats: rtnl_link_stats64 pointer
4293  *
4294  **/
4295 static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4296                                                  struct rtnl_link_stats64 *stats)
4297 {
4298         struct igb_adapter *adapter = netdev_priv(netdev);
4299
4300         spin_lock(&adapter->stats64_lock);
4301         igb_update_stats(adapter, &adapter->stats64);
4302         memcpy(stats, &adapter->stats64, sizeof(*stats));
4303         spin_unlock(&adapter->stats64_lock);
4304
4305         return stats;
4306 }
4307
4308 /**
4309  * igb_change_mtu - Change the Maximum Transfer Unit
4310  * @netdev: network interface device structure
4311  * @new_mtu: new value for maximum frame size
4312  *
4313  * Returns 0 on success, negative on failure
4314  **/
4315 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4316 {
4317         struct igb_adapter *adapter = netdev_priv(netdev);
4318         struct pci_dev *pdev = adapter->pdev;
4319         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4320         u32 rx_buffer_len, i;
4321
4322         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4323                 dev_err(&pdev->dev, "Invalid MTU setting\n");
4324                 return -EINVAL;
4325         }
4326
4327         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4328                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4329                 return -EINVAL;
4330         }
4331
4332         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4333                 msleep(1);
4334
4335         /* igb_down has a dependency on max_frame_size */
4336         adapter->max_frame_size = max_frame;
4337
4338         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4339          * means we reserve 2 more, this pushes us to allocate from the next
4340          * larger slab size.
4341          * i.e. RXBUFFER_2048 --> size-4096 slab
4342          */
4343
4344         if (adapter->hw.mac.type == e1000_82580)
4345                 max_frame += IGB_TS_HDR_LEN;
4346
4347         if (max_frame <= IGB_RXBUFFER_1024)
4348                 rx_buffer_len = IGB_RXBUFFER_1024;
4349         else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4350                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4351         else
4352                 rx_buffer_len = IGB_RXBUFFER_128;
4353
4354         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4355              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4356                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4357
4358         if ((adapter->hw.mac.type == e1000_82580) &&
4359             (rx_buffer_len == IGB_RXBUFFER_128))
4360                 rx_buffer_len += IGB_RXBUFFER_64;
4361
4362         if (netif_running(netdev))
4363                 igb_down(adapter);
4364
4365         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4366                  netdev->mtu, new_mtu);
4367         netdev->mtu = new_mtu;
4368
4369         for (i = 0; i < adapter->num_rx_queues; i++)
4370                 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4371
4372         if (netif_running(netdev))
4373                 igb_up(adapter);
4374         else
4375                 igb_reset(adapter);
4376
4377         clear_bit(__IGB_RESETTING, &adapter->state);
4378
4379         return 0;
4380 }
4381
4382 /**
4383  * igb_update_stats - Update the board statistics counters
4384  * @adapter: board private structure
4385  **/
4386
4387 void igb_update_stats(struct igb_adapter *adapter,
4388                       struct rtnl_link_stats64 *net_stats)
4389 {
4390         struct e1000_hw *hw = &adapter->hw;
4391         struct pci_dev *pdev = adapter->pdev;
4392         u32 reg, mpc;
4393         u16 phy_tmp;
4394         int i;
4395         u64 bytes, packets;
4396         unsigned int start;
4397         u64 _bytes, _packets;
4398
4399 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4400
4401         /*
4402          * Prevent stats update while adapter is being reset, or if the pci
4403          * connection is down.
4404          */
4405         if (adapter->link_speed == 0)
4406                 return;
4407         if (pci_channel_offline(pdev))
4408                 return;
4409
4410         bytes = 0;
4411         packets = 0;
4412         for (i = 0; i < adapter->num_rx_queues; i++) {
4413                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4414                 struct igb_ring *ring = adapter->rx_ring[i];
4415
4416                 ring->rx_stats.drops += rqdpc_tmp;
4417                 net_stats->rx_fifo_errors += rqdpc_tmp;
4418
4419                 do {
4420                         start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4421                         _bytes = ring->rx_stats.bytes;
4422                         _packets = ring->rx_stats.packets;
4423                 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4424                 bytes += _bytes;
4425                 packets += _packets;
4426         }
4427
4428         net_stats->rx_bytes = bytes;
4429         net_stats->rx_packets = packets;
4430
4431         bytes = 0;
4432         packets = 0;
4433         for (i = 0; i < adapter->num_tx_queues; i++) {
4434                 struct igb_ring *ring = adapter->tx_ring[i];
4435                 do {
4436                         start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4437                         _bytes = ring->tx_stats.bytes;
4438                         _packets = ring->tx_stats.packets;
4439                 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4440                 bytes += _bytes;
4441                 packets += _packets;
4442         }
4443         net_stats->tx_bytes = bytes;
4444         net_stats->tx_packets = packets;
4445
4446         /* read stats registers */
4447         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4448         adapter->stats.gprc += rd32(E1000_GPRC);
4449         adapter->stats.gorc += rd32(E1000_GORCL);
4450         rd32(E1000_GORCH); /* clear GORCL */
4451         adapter->stats.bprc += rd32(E1000_BPRC);
4452         adapter->stats.mprc += rd32(E1000_MPRC);
4453         adapter->stats.roc += rd32(E1000_ROC);
4454
4455         adapter->stats.prc64 += rd32(E1000_PRC64);
4456         adapter->stats.prc127 += rd32(E1000_PRC127);
4457         adapter->stats.prc255 += rd32(E1000_PRC255);
4458         adapter->stats.prc511 += rd32(E1000_PRC511);
4459         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4460         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4461         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4462         adapter->stats.sec += rd32(E1000_SEC);
4463
4464         mpc = rd32(E1000_MPC);
4465         adapter->stats.mpc += mpc;
4466         net_stats->rx_fifo_errors += mpc;
4467         adapter->stats.scc += rd32(E1000_SCC);
4468         adapter->stats.ecol += rd32(E1000_ECOL);
4469         adapter->stats.mcc += rd32(E1000_MCC);
4470         adapter->stats.latecol += rd32(E1000_LATECOL);
4471         adapter->stats.dc += rd32(E1000_DC);
4472         adapter->stats.rlec += rd32(E1000_RLEC);
4473         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4474         adapter->stats.xontxc += rd32(E1000_XONTXC);
4475         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4476         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4477         adapter->stats.fcruc += rd32(E1000_FCRUC);
4478         adapter->stats.gptc += rd32(E1000_GPTC);
4479         adapter->stats.gotc += rd32(E1000_GOTCL);
4480         rd32(E1000_GOTCH); /* clear GOTCL */
4481         adapter->stats.rnbc += rd32(E1000_RNBC);
4482         adapter->stats.ruc += rd32(E1000_RUC);
4483         adapter->stats.rfc += rd32(E1000_RFC);
4484         adapter->stats.rjc += rd32(E1000_RJC);
4485         adapter->stats.tor += rd32(E1000_TORH);
4486         adapter->stats.tot += rd32(E1000_TOTH);
4487         adapter->stats.tpr += rd32(E1000_TPR);
4488
4489         adapter->stats.ptc64 += rd32(E1000_PTC64);
4490         adapter->stats.ptc127 += rd32(E1000_PTC127);
4491         adapter->stats.ptc255 += rd32(E1000_PTC255);
4492         adapter->stats.ptc511 += rd32(E1000_PTC511);
4493         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4494         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4495
4496         adapter->stats.mptc += rd32(E1000_MPTC);
4497         adapter->stats.bptc += rd32(E1000_BPTC);
4498
4499         adapter->stats.tpt += rd32(E1000_TPT);
4500         adapter->stats.colc += rd32(E1000_COLC);
4501
4502         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4503         /* read internal phy specific stats */
4504         reg = rd32(E1000_CTRL_EXT);
4505         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4506                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4507                 adapter->stats.tncrs += rd32(E1000_TNCRS);
4508         }
4509
4510         adapter->stats.tsctc += rd32(E1000_TSCTC);
4511         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4512
4513         adapter->stats.iac += rd32(E1000_IAC);
4514         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4515         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4516         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4517         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4518         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4519         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4520         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4521         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4522
4523         /* Fill out the OS statistics structure */
4524         net_stats->multicast = adapter->stats.mprc;
4525         net_stats->collisions = adapter->stats.colc;
4526
4527         /* Rx Errors */
4528
4529         /* RLEC on some newer hardware can be incorrect so build
4530          * our own version based on RUC and ROC */
4531         net_stats->rx_errors = adapter->stats.rxerrc +
4532                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4533                 adapter->stats.ruc + adapter->stats.roc +
4534                 adapter->stats.cexterr;
4535         net_stats->rx_length_errors = adapter->stats.ruc +
4536                                       adapter->stats.roc;
4537         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4538         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4539         net_stats->rx_missed_errors = adapter->stats.mpc;
4540
4541         /* Tx Errors */
4542         net_stats->tx_errors = adapter->stats.ecol +
4543                                adapter->stats.latecol;
4544         net_stats->tx_aborted_errors = adapter->stats.ecol;
4545         net_stats->tx_window_errors = adapter->stats.latecol;
4546         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4547
4548         /* Tx Dropped needs to be maintained elsewhere */
4549
4550         /* Phy Stats */
4551         if (hw->phy.media_type == e1000_media_type_copper) {
4552                 if ((adapter->link_speed == SPEED_1000) &&
4553                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4554                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4555                         adapter->phy_stats.idle_errors += phy_tmp;
4556                 }
4557         }
4558
4559         /* Management Stats */
4560         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4561         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4562         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4563 }
4564
4565 static irqreturn_t igb_msix_other(int irq, void *data)
4566 {
4567         struct igb_adapter *adapter = data;
4568         struct e1000_hw *hw = &adapter->hw;
4569         u32 icr = rd32(E1000_ICR);
4570         /* reading ICR causes bit 31 of EICR to be cleared */
4571
4572         if (icr & E1000_ICR_DRSTA)
4573                 schedule_work(&adapter->reset_task);
4574
4575         if (icr & E1000_ICR_DOUTSYNC) {
4576                 /* HW is reporting DMA is out of sync */
4577                 adapter->stats.doosync++;
4578                 /* The DMA Out of Sync is also indication of a spoof event
4579                  * in IOV mode. Check the Wrong VM Behavior register to
4580                  * see if it is really a spoof event. */
4581                 igb_check_wvbr(adapter);
4582         }
4583
4584         /* Check for a mailbox event */
4585         if (icr & E1000_ICR_VMMB)
4586                 igb_msg_task(adapter);
4587
4588         if (icr & E1000_ICR_LSC) {
4589                 hw->mac.get_link_status = 1;
4590                 /* guard against interrupt when we're going down */
4591                 if (!test_bit(__IGB_DOWN, &adapter->state))
4592                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4593         }
4594
4595         if (adapter->vfs_allocated_count)
4596                 wr32(E1000_IMS, E1000_IMS_LSC |
4597                                 E1000_IMS_VMMB |
4598                                 E1000_IMS_DOUTSYNC);
4599         else
4600                 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4601         wr32(E1000_EIMS, adapter->eims_other);
4602
4603         return IRQ_HANDLED;
4604 }
4605
4606 static void igb_write_itr(struct igb_q_vector *q_vector)
4607 {
4608         struct igb_adapter *adapter = q_vector->adapter;
4609         u32 itr_val = q_vector->itr_val & 0x7FFC;
4610
4611         if (!q_vector->set_itr)
4612                 return;
4613
4614         if (!itr_val)
4615                 itr_val = 0x4;
4616
4617         if (adapter->hw.mac.type == e1000_82575)
4618                 itr_val |= itr_val << 16;
4619         else
4620                 itr_val |= 0x8000000;
4621
4622         writel(itr_val, q_vector->itr_register);
4623         q_vector->set_itr = 0;
4624 }
4625
4626 static irqreturn_t igb_msix_ring(int irq, void *data)
4627 {
4628         struct igb_q_vector *q_vector = data;
4629
4630         /* Write the ITR value calculated from the previous interrupt. */
4631         igb_write_itr(q_vector);
4632
4633         napi_schedule(&q_vector->napi);
4634
4635         return IRQ_HANDLED;
4636 }
4637
4638 #ifdef CONFIG_IGB_DCA
4639 static void igb_update_dca(struct igb_q_vector *q_vector)
4640 {
4641         struct igb_adapter *adapter = q_vector->adapter;
4642         struct e1000_hw *hw = &adapter->hw;
4643         int cpu = get_cpu();
4644
4645         if (q_vector->cpu == cpu)
4646                 goto out_no_update;
4647
4648         if (q_vector->tx_ring) {
4649                 int q = q_vector->tx_ring->reg_idx;
4650                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4651                 if (hw->mac.type == e1000_82575) {
4652                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4653                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4654                 } else {
4655                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4656                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4657                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4658                 }
4659                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4660                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4661         }
4662         if (q_vector->rx_ring) {
4663                 int q = q_vector->rx_ring->reg_idx;
4664                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4665                 if (hw->mac.type == e1000_82575) {
4666                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4667                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4668                 } else {
4669                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4670                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4671                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4672                 }
4673                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4674                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4675                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4676                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4677         }
4678         q_vector->cpu = cpu;
4679 out_no_update:
4680         put_cpu();
4681 }
4682
4683 static void igb_setup_dca(struct igb_adapter *adapter)
4684 {
4685         struct e1000_hw *hw = &adapter->hw;
4686         int i;
4687
4688         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4689                 return;
4690
4691         /* Always use CB2 mode, difference is masked in the CB driver. */
4692         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4693
4694         for (i = 0; i < adapter->num_q_vectors; i++) {
4695                 adapter->q_vector[i]->cpu = -1;
4696                 igb_update_dca(adapter->q_vector[i]);
4697         }
4698 }
4699
4700 static int __igb_notify_dca(struct device *dev, void *data)
4701 {
4702         struct net_device *netdev = dev_get_drvdata(dev);
4703         struct igb_adapter *adapter = netdev_priv(netdev);
4704         struct pci_dev *pdev = adapter->pdev;
4705         struct e1000_hw *hw = &adapter->hw;
4706         unsigned long event = *(unsigned long *)data;
4707
4708         switch (event) {
4709         case DCA_PROVIDER_ADD:
4710                 /* if already enabled, don't do it again */
4711                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4712                         break;
4713                 if (dca_add_requester(dev) == 0) {
4714                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4715                         dev_info(&pdev->dev, "DCA enabled\n");
4716                         igb_setup_dca(adapter);
4717                         break;
4718                 }
4719                 /* Fall Through since DCA is disabled. */
4720         case DCA_PROVIDER_REMOVE:
4721                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4722                         /* without this a class_device is left
4723                          * hanging around in the sysfs model */
4724                         dca_remove_requester(dev);
4725                         dev_info(&pdev->dev, "DCA disabled\n");
4726                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4727                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4728                 }
4729                 break;
4730         }
4731
4732         return 0;
4733 }
4734
4735 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4736                           void *p)
4737 {
4738         int ret_val;
4739
4740         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4741                                          __igb_notify_dca);
4742
4743         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4744 }
4745 #endif /* CONFIG_IGB_DCA */
4746
4747 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4748 {
4749         struct e1000_hw *hw = &adapter->hw;
4750         u32 ping;
4751         int i;
4752
4753         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4754                 ping = E1000_PF_CONTROL_MSG;
4755                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4756                         ping |= E1000_VT_MSGTYPE_CTS;
4757                 igb_write_mbx(hw, &ping, 1, i);
4758         }
4759 }
4760
4761 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4762 {
4763         struct e1000_hw *hw = &adapter->hw;
4764         u32 vmolr = rd32(E1000_VMOLR(vf));
4765         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4766
4767         vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
4768                             IGB_VF_FLAG_MULTI_PROMISC);
4769         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4770
4771         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4772                 vmolr |= E1000_VMOLR_MPME;
4773                 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
4774                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4775         } else {
4776                 /*
4777                  * if we have hashes and we are clearing a multicast promisc
4778                  * flag we need to write the hashes to the MTA as this step
4779                  * was previously skipped
4780                  */
4781                 if (vf_data->num_vf_mc_hashes > 30) {
4782                         vmolr |= E1000_VMOLR_MPME;
4783                 } else if (vf_data->num_vf_mc_hashes) {
4784                         int j;
4785                         vmolr |= E1000_VMOLR_ROMPE;
4786                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4787                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4788                 }
4789         }
4790
4791         wr32(E1000_VMOLR(vf), vmolr);
4792
4793         /* there are flags left unprocessed, likely not supported */
4794         if (*msgbuf & E1000_VT_MSGINFO_MASK)
4795                 return -EINVAL;
4796
4797         return 0;
4798
4799 }
4800
4801 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4802                                   u32 *msgbuf, u32 vf)
4803 {
4804         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4805         u16 *hash_list = (u16 *)&msgbuf[1];
4806         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4807         int i;
4808
4809         /* salt away the number of multicast addresses assigned
4810          * to this VF for later use to restore when the PF multi cast
4811          * list changes
4812          */
4813         vf_data->num_vf_mc_hashes = n;
4814
4815         /* only up to 30 hash values supported */
4816         if (n > 30)
4817                 n = 30;
4818
4819         /* store the hashes for later use */
4820         for (i = 0; i < n; i++)
4821                 vf_data->vf_mc_hashes[i] = hash_list[i];
4822
4823         /* Flush and reset the mta with the new values */
4824         igb_set_rx_mode(adapter->netdev);
4825
4826         return 0;
4827 }
4828
4829 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4830 {
4831         struct e1000_hw *hw = &adapter->hw;
4832         struct vf_data_storage *vf_data;
4833         int i, j;
4834
4835         for (i = 0; i < adapter->vfs_allocated_count; i++) {
4836                 u32 vmolr = rd32(E1000_VMOLR(i));
4837                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4838
4839                 vf_data = &adapter->vf_data[i];
4840
4841                 if ((vf_data->num_vf_mc_hashes > 30) ||
4842                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4843                         vmolr |= E1000_VMOLR_MPME;
4844                 } else if (vf_data->num_vf_mc_hashes) {
4845                         vmolr |= E1000_VMOLR_ROMPE;
4846                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4847                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4848                 }
4849                 wr32(E1000_VMOLR(i), vmolr);
4850         }
4851 }
4852
4853 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4854 {
4855         struct e1000_hw *hw = &adapter->hw;
4856         u32 pool_mask, reg, vid;
4857         int i;
4858
4859         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4860
4861         /* Find the vlan filter for this id */
4862         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4863                 reg = rd32(E1000_VLVF(i));
4864
4865                 /* remove the vf from the pool */
4866                 reg &= ~pool_mask;
4867
4868                 /* if pool is empty then remove entry from vfta */
4869                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4870                     (reg & E1000_VLVF_VLANID_ENABLE)) {
4871                         reg = 0;
4872                         vid = reg & E1000_VLVF_VLANID_MASK;
4873                         igb_vfta_set(hw, vid, false);
4874                 }
4875
4876                 wr32(E1000_VLVF(i), reg);
4877         }
4878
4879         adapter->vf_data[vf].vlans_enabled = 0;
4880 }
4881
4882 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4883 {
4884         struct e1000_hw *hw = &adapter->hw;
4885         u32 reg, i;
4886
4887         /* The vlvf table only exists on 82576 hardware and newer */
4888         if (hw->mac.type < e1000_82576)
4889                 return -1;
4890
4891         /* we only need to do this if VMDq is enabled */
4892         if (!adapter->vfs_allocated_count)
4893                 return -1;
4894
4895         /* Find the vlan filter for this id */
4896         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4897                 reg = rd32(E1000_VLVF(i));
4898                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4899                     vid == (reg & E1000_VLVF_VLANID_MASK))
4900                         break;
4901         }
4902
4903         if (add) {
4904                 if (i == E1000_VLVF_ARRAY_SIZE) {
4905                         /* Did not find a matching VLAN ID entry that was
4906                          * enabled.  Search for a free filter entry, i.e.
4907                          * one without the enable bit set
4908                          */
4909                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4910                                 reg = rd32(E1000_VLVF(i));
4911                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4912                                         break;
4913                         }
4914                 }
4915                 if (i < E1000_VLVF_ARRAY_SIZE) {
4916                         /* Found an enabled/available entry */
4917                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4918
4919                         /* if !enabled we need to set this up in vfta */
4920                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4921                                 /* add VID to filter table */
4922                                 igb_vfta_set(hw, vid, true);
4923                                 reg |= E1000_VLVF_VLANID_ENABLE;
4924                         }
4925                         reg &= ~E1000_VLVF_VLANID_MASK;
4926                         reg |= vid;
4927                         wr32(E1000_VLVF(i), reg);
4928
4929                         /* do not modify RLPML for PF devices */
4930                         if (vf >= adapter->vfs_allocated_count)
4931                                 return 0;
4932
4933                         if (!adapter->vf_data[vf].vlans_enabled) {
4934                                 u32 size;
4935                                 reg = rd32(E1000_VMOLR(vf));
4936                                 size = reg & E1000_VMOLR_RLPML_MASK;
4937                                 size += 4;
4938                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4939                                 reg |= size;
4940                                 wr32(E1000_VMOLR(vf), reg);
4941                         }
4942
4943                         adapter->vf_data[vf].vlans_enabled++;
4944                         return 0;
4945                 }
4946         } else {
4947                 if (i < E1000_VLVF_ARRAY_SIZE) {
4948                         /* remove vf from the pool */
4949                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4950                         /* if pool is empty then remove entry from vfta */
4951                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4952                                 reg = 0;
4953                                 igb_vfta_set(hw, vid, false);
4954                         }
4955                         wr32(E1000_VLVF(i), reg);
4956
4957                         /* do not modify RLPML for PF devices */
4958                         if (vf >= adapter->vfs_allocated_count)
4959                                 return 0;
4960
4961                         adapter->vf_data[vf].vlans_enabled--;
4962                         if (!adapter->vf_data[vf].vlans_enabled) {
4963                                 u32 size;
4964                                 reg = rd32(E1000_VMOLR(vf));
4965                                 size = reg & E1000_VMOLR_RLPML_MASK;
4966                                 size -= 4;
4967                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4968                                 reg |= size;
4969                                 wr32(E1000_VMOLR(vf), reg);
4970                         }
4971                 }
4972         }
4973         return 0;
4974 }
4975
4976 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4977 {
4978         struct e1000_hw *hw = &adapter->hw;
4979
4980         if (vid)
4981                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4982         else
4983                 wr32(E1000_VMVIR(vf), 0);
4984 }
4985
4986 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4987                                int vf, u16 vlan, u8 qos)
4988 {
4989         int err = 0;
4990         struct igb_adapter *adapter = netdev_priv(netdev);
4991
4992         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4993                 return -EINVAL;
4994         if (vlan || qos) {
4995                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4996                 if (err)
4997                         goto out;
4998                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4999                 igb_set_vmolr(adapter, vf, !vlan);
5000                 adapter->vf_data[vf].pf_vlan = vlan;
5001                 adapter->vf_data[vf].pf_qos = qos;
5002                 dev_info(&adapter->pdev->dev,
5003                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5004                 if (test_bit(__IGB_DOWN, &adapter->state)) {
5005                         dev_warn(&adapter->pdev->dev,
5006                                  "The VF VLAN has been set,"
5007                                  " but the PF device is not up.\n");
5008                         dev_warn(&adapter->pdev->dev,
5009                                  "Bring the PF device up before"
5010                                  " attempting to use the VF device.\n");
5011                 }
5012         } else {
5013                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5014                                    false, vf);
5015                 igb_set_vmvir(adapter, vlan, vf);
5016                 igb_set_vmolr(adapter, vf, true);
5017                 adapter->vf_data[vf].pf_vlan = 0;
5018                 adapter->vf_data[vf].pf_qos = 0;
5019        }
5020 out:
5021        return err;
5022 }
5023
5024 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5025 {
5026         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5027         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5028
5029         return igb_vlvf_set(adapter, vid, add, vf);
5030 }
5031
5032 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5033 {
5034         /* clear flags - except flag that indicates PF has set the MAC */
5035         adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5036         adapter->vf_data[vf].last_nack = jiffies;
5037
5038         /* reset offloads to defaults */
5039         igb_set_vmolr(adapter, vf, true);
5040
5041         /* reset vlans for device */
5042         igb_clear_vf_vfta(adapter, vf);
5043         if (adapter->vf_data[vf].pf_vlan)
5044                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5045                                     adapter->vf_data[vf].pf_vlan,
5046                                     adapter->vf_data[vf].pf_qos);
5047         else
5048                 igb_clear_vf_vfta(adapter, vf);
5049
5050         /* reset multicast table array for vf */
5051         adapter->vf_data[vf].num_vf_mc_hashes = 0;
5052
5053         /* Flush and reset the mta with the new values */
5054         igb_set_rx_mode(adapter->netdev);
5055 }
5056
5057 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5058 {
5059         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5060
5061         /* generate a new mac address as we were hotplug removed/added */
5062         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5063                 random_ether_addr(vf_mac);
5064
5065         /* process remaining reset events */
5066         igb_vf_reset(adapter, vf);
5067 }
5068
5069 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5070 {
5071         struct e1000_hw *hw = &adapter->hw;
5072         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5073         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5074         u32 reg, msgbuf[3];
5075         u8 *addr = (u8 *)(&msgbuf[1]);
5076
5077         /* process all the same items cleared in a function level reset */
5078         igb_vf_reset(adapter, vf);
5079
5080         /* set vf mac address */
5081         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5082
5083         /* enable transmit and receive for vf */
5084         reg = rd32(E1000_VFTE);
5085         wr32(E1000_VFTE, reg | (1 << vf));
5086         reg = rd32(E1000_VFRE);
5087         wr32(E1000_VFRE, reg | (1 << vf));
5088
5089         adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5090
5091         /* reply to reset with ack and vf mac address */
5092         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5093         memcpy(addr, vf_mac, 6);
5094         igb_write_mbx(hw, msgbuf, 3, vf);
5095 }
5096
5097 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5098 {
5099         /*
5100          * The VF MAC Address is stored in a packed array of bytes
5101          * starting at the second 32 bit word of the msg array
5102          */
5103         unsigned char *addr = (char *)&msg[1];
5104         int err = -1;
5105
5106         if (is_valid_ether_addr(addr))
5107                 err = igb_set_vf_mac(adapter, vf, addr);
5108
5109         return err;
5110 }
5111
5112 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5113 {
5114         struct e1000_hw *hw = &adapter->hw;
5115         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5116         u32 msg = E1000_VT_MSGTYPE_NACK;
5117
5118         /* if device isn't clear to send it shouldn't be reading either */
5119         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5120             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5121                 igb_write_mbx(hw, &msg, 1, vf);
5122                 vf_data->last_nack = jiffies;
5123         }
5124 }
5125
5126 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5127 {
5128         struct pci_dev *pdev = adapter->pdev;
5129         u32 msgbuf[E1000_VFMAILBOX_SIZE];
5130         struct e1000_hw *hw = &adapter->hw;
5131         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5132         s32 retval;
5133
5134         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5135
5136         if (retval) {
5137                 /* if receive failed revoke VF CTS stats and restart init */
5138                 dev_err(&pdev->dev, "Error receiving message from VF\n");
5139                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5140                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5141                         return;
5142                 goto out;
5143         }
5144
5145         /* this is a message we already processed, do nothing */
5146         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5147                 return;
5148
5149         /*
5150          * until the vf completes a reset it should not be
5151          * allowed to start any configuration.
5152          */
5153
5154         if (msgbuf[0] == E1000_VF_RESET) {
5155                 igb_vf_reset_msg(adapter, vf);
5156                 return;
5157         }
5158
5159         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5160                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5161                         return;
5162                 retval = -1;
5163                 goto out;
5164         }
5165
5166         switch ((msgbuf[0] & 0xFFFF)) {
5167         case E1000_VF_SET_MAC_ADDR:
5168                 retval = -EINVAL;
5169                 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5170                         retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5171                 else
5172                         dev_warn(&pdev->dev,
5173                                  "VF %d attempted to override administratively "
5174                                  "set MAC address\nReload the VF driver to "
5175                                  "resume operations\n", vf);
5176                 break;
5177         case E1000_VF_SET_PROMISC:
5178                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5179                 break;
5180         case E1000_VF_SET_MULTICAST:
5181                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5182                 break;
5183         case E1000_VF_SET_LPE:
5184                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5185                 break;
5186         case E1000_VF_SET_VLAN:
5187                 retval = -1;
5188                 if (vf_data->pf_vlan)
5189                         dev_warn(&pdev->dev,
5190                                  "VF %d attempted to override administratively "
5191                                  "set VLAN tag\nReload the VF driver to "
5192                                  "resume operations\n", vf);
5193                 else
5194                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5195                 break;
5196         default:
5197                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5198                 retval = -1;
5199                 break;
5200         }
5201
5202         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5203 out:
5204         /* notify the VF of the results of what it sent us */
5205         if (retval)
5206                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5207         else
5208                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5209
5210         igb_write_mbx(hw, msgbuf, 1, vf);
5211 }
5212
5213 static void igb_msg_task(struct igb_adapter *adapter)
5214 {
5215         struct e1000_hw *hw = &adapter->hw;
5216         u32 vf;
5217
5218         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5219                 /* process any reset requests */
5220                 if (!igb_check_for_rst(hw, vf))
5221                         igb_vf_reset_event(adapter, vf);
5222
5223                 /* process any messages pending */
5224                 if (!igb_check_for_msg(hw, vf))
5225                         igb_rcv_msg_from_vf(adapter, vf);
5226
5227                 /* process any acks */
5228                 if (!igb_check_for_ack(hw, vf))
5229                         igb_rcv_ack_from_vf(adapter, vf);
5230         }
5231 }
5232
5233 /**
5234  *  igb_set_uta - Set unicast filter table address
5235  *  @adapter: board private structure
5236  *
5237  *  The unicast table address is a register array of 32-bit registers.
5238  *  The table is meant to be used in a way similar to how the MTA is used
5239  *  however due to certain limitations in the hardware it is necessary to
5240  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5241  *  enable bit to allow vlan tag stripping when promiscous mode is enabled
5242  **/
5243 static void igb_set_uta(struct igb_adapter *adapter)
5244 {
5245         struct e1000_hw *hw = &adapter->hw;
5246         int i;
5247
5248         /* The UTA table only exists on 82576 hardware and newer */
5249         if (hw->mac.type < e1000_82576)
5250                 return;
5251
5252         /* we only need to do this if VMDq is enabled */
5253         if (!adapter->vfs_allocated_count)
5254                 return;
5255
5256         for (i = 0; i < hw->mac.uta_reg_count; i++)
5257                 array_wr32(E1000_UTA, i, ~0);
5258 }
5259
5260 /**
5261  * igb_intr_msi - Interrupt Handler
5262  * @irq: interrupt number
5263  * @data: pointer to a network interface device structure
5264  **/
5265 static irqreturn_t igb_intr_msi(int irq, void *data)
5266 {
5267         struct igb_adapter *adapter = data;
5268         struct igb_q_vector *q_vector = adapter->q_vector[0];
5269         struct e1000_hw *hw = &adapter->hw;
5270         /* read ICR disables interrupts using IAM */
5271         u32 icr = rd32(E1000_ICR);
5272
5273         igb_write_itr(q_vector);
5274
5275         if (icr & E1000_ICR_DRSTA)
5276                 schedule_work(&adapter->reset_task);
5277
5278         if (icr & E1000_ICR_DOUTSYNC) {
5279                 /* HW is reporting DMA is out of sync */
5280                 adapter->stats.doosync++;
5281         }
5282
5283         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5284                 hw->mac.get_link_status = 1;
5285                 if (!test_bit(__IGB_DOWN, &adapter->state))
5286                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5287         }
5288
5289         napi_schedule(&q_vector->napi);
5290
5291         return IRQ_HANDLED;
5292 }
5293
5294 /**
5295  * igb_intr - Legacy Interrupt Handler
5296  * @irq: interrupt number
5297  * @data: pointer to a network interface device structure
5298  **/
5299 static irqreturn_t igb_intr(int irq, void *data)
5300 {
5301         struct igb_adapter *adapter = data;
5302         struct igb_q_vector *q_vector = adapter->q_vector[0];
5303         struct e1000_hw *hw = &adapter->hw;
5304         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5305          * need for the IMC write */
5306         u32 icr = rd32(E1000_ICR);
5307         if (!icr)
5308                 return IRQ_NONE;  /* Not our interrupt */
5309
5310         igb_write_itr(q_vector);
5311
5312         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5313          * not set, then the adapter didn't send an interrupt */
5314         if (!(icr & E1000_ICR_INT_ASSERTED))
5315                 return IRQ_NONE;
5316
5317         if (icr & E1000_ICR_DRSTA)
5318                 schedule_work(&adapter->reset_task);
5319
5320         if (icr & E1000_ICR_DOUTSYNC) {
5321                 /* HW is reporting DMA is out of sync */
5322                 adapter->stats.doosync++;
5323         }
5324
5325         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5326                 hw->mac.get_link_status = 1;
5327                 /* guard against interrupt when we're going down */
5328                 if (!test_bit(__IGB_DOWN, &adapter->state))
5329                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
5330         }
5331
5332         napi_schedule(&q_vector->napi);
5333
5334         return IRQ_HANDLED;
5335 }
5336
5337 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5338 {
5339         struct igb_adapter *adapter = q_vector->adapter;
5340         struct e1000_hw *hw = &adapter->hw;
5341
5342         if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5343             (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5344                 if (!adapter->msix_entries)
5345                         igb_set_itr(adapter);
5346                 else
5347                         igb_update_ring_itr(q_vector);
5348         }
5349
5350         if (!test_bit(__IGB_DOWN, &adapter->state)) {
5351                 if (adapter->msix_entries)
5352                         wr32(E1000_EIMS, q_vector->eims_value);
5353                 else
5354                         igb_irq_enable(adapter);
5355         }
5356 }
5357
5358 /**
5359  * igb_poll - NAPI Rx polling callback
5360  * @napi: napi polling structure
5361  * @budget: count of how many packets we should handle
5362  **/
5363 static int igb_poll(struct napi_struct *napi, int budget)
5364 {
5365         struct igb_q_vector *q_vector = container_of(napi,
5366                                                      struct igb_q_vector,
5367                                                      napi);
5368         int tx_clean_complete = 1, work_done = 0;
5369
5370 #ifdef CONFIG_IGB_DCA
5371         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5372                 igb_update_dca(q_vector);
5373 #endif
5374         if (q_vector->tx_ring)
5375                 tx_clean_complete = igb_clean_tx_irq(q_vector);
5376
5377         if (q_vector->rx_ring)
5378                 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5379
5380         if (!tx_clean_complete)
5381                 work_done = budget;
5382
5383         /* If not enough Rx work done, exit the polling mode */
5384         if (work_done < budget) {
5385                 napi_complete(napi);
5386                 igb_ring_irq_enable(q_vector);
5387         }
5388
5389         return work_done;
5390 }
5391
5392 /**
5393  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5394  * @adapter: board private structure
5395  * @shhwtstamps: timestamp structure to update
5396  * @regval: unsigned 64bit system time value.
5397  *
5398  * We need to convert the system time value stored in the RX/TXSTMP registers
5399  * into a hwtstamp which can be used by the upper level timestamping functions
5400  */
5401 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5402                                    struct skb_shared_hwtstamps *shhwtstamps,
5403                                    u64 regval)
5404 {
5405         u64 ns;
5406
5407         /*
5408          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5409          * 24 to match clock shift we setup earlier.
5410          */
5411         if (adapter->hw.mac.type == e1000_82580)
5412                 regval <<= IGB_82580_TSYNC_SHIFT;
5413
5414         ns = timecounter_cyc2time(&adapter->clock, regval);
5415         timecompare_update(&adapter->compare, ns);
5416         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5417         shhwtstamps->hwtstamp = ns_to_ktime(ns);
5418         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5419 }
5420
5421 /**
5422  * igb_tx_hwtstamp - utility function which checks for TX time stamp
5423  * @q_vector: pointer to q_vector containing needed info
5424  * @buffer: pointer to igb_buffer structure
5425  *
5426  * If we were asked to do hardware stamping and such a time stamp is
5427  * available, then it must have been for this skb here because we only
5428  * allow only one such packet into the queue.
5429  */
5430 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5431 {
5432         struct igb_adapter *adapter = q_vector->adapter;
5433         struct e1000_hw *hw = &adapter->hw;
5434         struct skb_shared_hwtstamps shhwtstamps;
5435         u64 regval;
5436
5437         /* if skb does not support hw timestamp or TX stamp not valid exit */
5438         if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
5439             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5440                 return;
5441
5442         regval = rd32(E1000_TXSTMPL);
5443         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5444
5445         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5446         skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5447 }
5448
5449 /**
5450  * igb_clean_tx_irq - Reclaim resources after transmit completes
5451  * @q_vector: pointer to q_vector containing needed info
5452  * returns true if ring is completely cleaned
5453  **/
5454 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5455 {
5456         struct igb_adapter *adapter = q_vector->adapter;
5457         struct igb_ring *tx_ring = q_vector->tx_ring;
5458         struct net_device *netdev = tx_ring->netdev;
5459         struct e1000_hw *hw = &adapter->hw;
5460         struct igb_buffer *buffer_info;
5461         union e1000_adv_tx_desc *tx_desc, *eop_desc;
5462         unsigned int total_bytes = 0, total_packets = 0;
5463         unsigned int i, eop, count = 0;
5464         bool cleaned = false;
5465
5466         i = tx_ring->next_to_clean;
5467         eop = tx_ring->buffer_info[i].next_to_watch;
5468         eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5469
5470         while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5471                (count < tx_ring->count)) {
5472                 rmb();  /* read buffer_info after eop_desc status */
5473                 for (cleaned = false; !cleaned; count++) {
5474                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5475                         buffer_info = &tx_ring->buffer_info[i];
5476                         cleaned = (i == eop);
5477
5478                         if (buffer_info->skb) {
5479                                 total_bytes += buffer_info->bytecount;
5480                                 /* gso_segs is currently only valid for tcp */
5481                                 total_packets += buffer_info->gso_segs;
5482                                 igb_tx_hwtstamp(q_vector, buffer_info);
5483                         }
5484
5485                         igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5486                         tx_desc->wb.status = 0;
5487
5488                         i++;
5489                         if (i == tx_ring->count)
5490                                 i = 0;
5491                 }
5492                 eop = tx_ring->buffer_info[i].next_to_watch;
5493                 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5494         }
5495
5496         tx_ring->next_to_clean = i;
5497
5498         if (unlikely(count &&
5499                      netif_carrier_ok(netdev) &&
5500                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5501                 /* Make sure that anybody stopping the queue after this
5502                  * sees the new next_to_clean.
5503                  */
5504                 smp_mb();
5505                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5506                     !(test_bit(__IGB_DOWN, &adapter->state))) {
5507                         netif_wake_subqueue(netdev, tx_ring->queue_index);
5508
5509                         u64_stats_update_begin(&tx_ring->tx_syncp);
5510                         tx_ring->tx_stats.restart_queue++;
5511                         u64_stats_update_end(&tx_ring->tx_syncp);
5512                 }
5513         }
5514
5515         if (tx_ring->detect_tx_hung) {
5516                 /* Detect a transmit hang in hardware, this serializes the
5517                  * check with the clearing of time_stamp and movement of i */
5518                 tx_ring->detect_tx_hung = false;
5519                 if (tx_ring->buffer_info[i].time_stamp &&
5520                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5521                                (adapter->tx_timeout_factor * HZ)) &&
5522                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5523
5524                         /* detected Tx unit hang */
5525                         dev_err(tx_ring->dev,
5526                                 "Detected Tx Unit Hang\n"
5527                                 "  Tx Queue             <%d>\n"
5528                                 "  TDH                  <%x>\n"
5529                                 "  TDT                  <%x>\n"
5530                                 "  next_to_use          <%x>\n"
5531                                 "  next_to_clean        <%x>\n"
5532                                 "buffer_info[next_to_clean]\n"
5533                                 "  time_stamp           <%lx>\n"
5534                                 "  next_to_watch        <%x>\n"
5535                                 "  jiffies              <%lx>\n"
5536                                 "  desc.status          <%x>\n",
5537                                 tx_ring->queue_index,
5538                                 readl(tx_ring->head),
5539                                 readl(tx_ring->tail),
5540                                 tx_ring->next_to_use,
5541                                 tx_ring->next_to_clean,
5542                                 tx_ring->buffer_info[eop].time_stamp,
5543                                 eop,
5544                                 jiffies,
5545                                 eop_desc->wb.status);
5546                         netif_stop_subqueue(netdev, tx_ring->queue_index);
5547                 }
5548         }
5549         tx_ring->total_bytes += total_bytes;
5550         tx_ring->total_packets += total_packets;
5551         u64_stats_update_begin(&tx_ring->tx_syncp);
5552         tx_ring->tx_stats.bytes += total_bytes;
5553         tx_ring->tx_stats.packets += total_packets;
5554         u64_stats_update_end(&tx_ring->tx_syncp);
5555         return count < tx_ring->count;
5556 }
5557
5558 /**
5559  * igb_receive_skb - helper function to handle rx indications
5560  * @q_vector: structure containing interrupt and ring information
5561  * @skb: packet to send up
5562  * @vlan_tag: vlan tag for packet
5563  **/
5564 static void igb_receive_skb(struct igb_q_vector *q_vector,
5565                             struct sk_buff *skb,
5566                             u16 vlan_tag)
5567 {
5568         struct igb_adapter *adapter = q_vector->adapter;
5569
5570         if (vlan_tag && adapter->vlgrp)
5571                 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5572                                  vlan_tag, skb);
5573         else
5574                 napi_gro_receive(&q_vector->napi, skb);
5575 }
5576
5577 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5578                                        u32 status_err, struct sk_buff *skb)
5579 {
5580         skb_checksum_none_assert(skb);
5581
5582         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5583         if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5584              (status_err & E1000_RXD_STAT_IXSM))
5585                 return;
5586
5587         /* TCP/UDP checksum error bit is set */
5588         if (status_err &
5589             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5590                 /*
5591                  * work around errata with sctp packets where the TCPE aka
5592                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5593                  * packets, (aka let the stack check the crc32c)
5594                  */
5595                 if ((skb->len == 60) &&
5596                     (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5597                         u64_stats_update_begin(&ring->rx_syncp);
5598                         ring->rx_stats.csum_err++;
5599                         u64_stats_update_end(&ring->rx_syncp);
5600                 }
5601                 /* let the stack verify checksum errors */
5602                 return;
5603         }
5604         /* It must be a TCP or UDP packet with a valid checksum */
5605         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5606                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5607
5608         dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5609 }
5610
5611 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5612                                    struct sk_buff *skb)
5613 {
5614         struct igb_adapter *adapter = q_vector->adapter;
5615         struct e1000_hw *hw = &adapter->hw;
5616         u64 regval;
5617
5618         /*
5619          * If this bit is set, then the RX registers contain the time stamp. No
5620          * other packet will be time stamped until we read these registers, so
5621          * read the registers to make them available again. Because only one
5622          * packet can be time stamped at a time, we know that the register
5623          * values must belong to this one here and therefore we don't need to
5624          * compare any of the additional attributes stored for it.
5625          *
5626          * If nothing went wrong, then it should have a shared tx_flags that we
5627          * can turn into a skb_shared_hwtstamps.
5628          */
5629         if (staterr & E1000_RXDADV_STAT_TSIP) {
5630                 u32 *stamp = (u32 *)skb->data;
5631                 regval = le32_to_cpu(*(stamp + 2));
5632                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5633                 skb_pull(skb, IGB_TS_HDR_LEN);
5634         } else {
5635                 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5636                         return;
5637
5638                 regval = rd32(E1000_RXSTMPL);
5639                 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5640         }
5641
5642         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5643 }
5644 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5645                                union e1000_adv_rx_desc *rx_desc)
5646 {
5647         /* HW will not DMA in data larger than the given buffer, even if it
5648          * parses the (NFS, of course) header to be larger.  In that case, it
5649          * fills the header buffer and spills the rest into the page.
5650          */
5651         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5652                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5653         if (hlen > rx_ring->rx_buffer_len)
5654                 hlen = rx_ring->rx_buffer_len;
5655         return hlen;
5656 }
5657
5658 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5659                                  int *work_done, int budget)
5660 {
5661         struct igb_ring *rx_ring = q_vector->rx_ring;
5662         struct net_device *netdev = rx_ring->netdev;
5663         struct device *dev = rx_ring->dev;
5664         union e1000_adv_rx_desc *rx_desc , *next_rxd;
5665         struct igb_buffer *buffer_info , *next_buffer;
5666         struct sk_buff *skb;
5667         bool cleaned = false;
5668         int cleaned_count = 0;
5669         int current_node = numa_node_id();
5670         unsigned int total_bytes = 0, total_packets = 0;
5671         unsigned int i;
5672         u32 staterr;
5673         u16 length;
5674         u16 vlan_tag;
5675
5676         i = rx_ring->next_to_clean;
5677         buffer_info = &rx_ring->buffer_info[i];
5678         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5679         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5680
5681         while (staterr & E1000_RXD_STAT_DD) {
5682                 if (*work_done >= budget)
5683                         break;
5684                 (*work_done)++;
5685                 rmb(); /* read descriptor and rx_buffer_info after status DD */
5686
5687                 skb = buffer_info->skb;
5688                 prefetch(skb->data - NET_IP_ALIGN);
5689                 buffer_info->skb = NULL;
5690
5691                 i++;
5692                 if (i == rx_ring->count)
5693                         i = 0;
5694
5695                 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5696                 prefetch(next_rxd);
5697                 next_buffer = &rx_ring->buffer_info[i];
5698
5699                 length = le16_to_cpu(rx_desc->wb.upper.length);
5700                 cleaned = true;
5701                 cleaned_count++;
5702
5703                 if (buffer_info->dma) {
5704                         dma_unmap_single(dev, buffer_info->dma,
5705                                          rx_ring->rx_buffer_len,
5706                                          DMA_FROM_DEVICE);
5707                         buffer_info->dma = 0;
5708                         if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5709                                 skb_put(skb, length);
5710                                 goto send_up;
5711                         }
5712                         skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5713                 }
5714
5715                 if (length) {
5716                         dma_unmap_page(dev, buffer_info->page_dma,
5717                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
5718                         buffer_info->page_dma = 0;
5719
5720                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5721                                                 buffer_info->page,
5722                                                 buffer_info->page_offset,
5723                                                 length);
5724
5725                         if ((page_count(buffer_info->page) != 1) ||
5726                             (page_to_nid(buffer_info->page) != current_node))
5727                                 buffer_info->page = NULL;
5728                         else
5729                                 get_page(buffer_info->page);
5730
5731                         skb->len += length;
5732                         skb->data_len += length;
5733                         skb->truesize += length;
5734                 }
5735
5736                 if (!(staterr & E1000_RXD_STAT_EOP)) {
5737                         buffer_info->skb = next_buffer->skb;
5738                         buffer_info->dma = next_buffer->dma;
5739                         next_buffer->skb = skb;
5740                         next_buffer->dma = 0;
5741                         goto next_desc;
5742                 }
5743 send_up:
5744                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5745                         dev_kfree_skb_irq(skb);
5746                         goto next_desc;
5747                 }
5748
5749                 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5750                         igb_rx_hwtstamp(q_vector, staterr, skb);
5751                 total_bytes += skb->len;
5752                 total_packets++;
5753
5754                 igb_rx_checksum_adv(rx_ring, staterr, skb);
5755
5756                 skb->protocol = eth_type_trans(skb, netdev);
5757                 skb_record_rx_queue(skb, rx_ring->queue_index);
5758
5759                 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5760                             le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5761
5762                 igb_receive_skb(q_vector, skb, vlan_tag);
5763
5764 next_desc:
5765                 rx_desc->wb.upper.status_error = 0;
5766
5767                 /* return some buffers to hardware, one at a time is too slow */
5768                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5769                         igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5770                         cleaned_count = 0;
5771                 }
5772
5773                 /* use prefetched values */
5774                 rx_desc = next_rxd;
5775                 buffer_info = next_buffer;
5776                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5777         }
5778
5779         rx_ring->next_to_clean = i;
5780         cleaned_count = igb_desc_unused(rx_ring);
5781
5782         if (cleaned_count)
5783                 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5784
5785         rx_ring->total_packets += total_packets;
5786         rx_ring->total_bytes += total_bytes;
5787         u64_stats_update_begin(&rx_ring->rx_syncp);
5788         rx_ring->rx_stats.packets += total_packets;
5789         rx_ring->rx_stats.bytes += total_bytes;
5790         u64_stats_update_end(&rx_ring->rx_syncp);
5791         return cleaned;
5792 }
5793
5794 /**
5795  * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5796  * @adapter: address of board private structure
5797  **/
5798 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5799 {
5800         struct net_device *netdev = rx_ring->netdev;
5801         union e1000_adv_rx_desc *rx_desc;
5802         struct igb_buffer *buffer_info;
5803         struct sk_buff *skb;
5804         unsigned int i;
5805         int bufsz;
5806
5807         i = rx_ring->next_to_use;
5808         buffer_info = &rx_ring->buffer_info[i];
5809
5810         bufsz = rx_ring->rx_buffer_len;
5811
5812         while (cleaned_count--) {
5813                 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5814
5815                 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5816                         if (!buffer_info->page) {
5817                                 buffer_info->page = netdev_alloc_page(netdev);
5818                                 if (unlikely(!buffer_info->page)) {
5819                                         u64_stats_update_begin(&rx_ring->rx_syncp);
5820                                         rx_ring->rx_stats.alloc_failed++;
5821                                         u64_stats_update_end(&rx_ring->rx_syncp);
5822                                         goto no_buffers;
5823                                 }
5824                                 buffer_info->page_offset = 0;
5825                         } else {
5826                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
5827                         }
5828                         buffer_info->page_dma =
5829                                 dma_map_page(rx_ring->dev, buffer_info->page,
5830                                              buffer_info->page_offset,
5831                                              PAGE_SIZE / 2,
5832                                              DMA_FROM_DEVICE);
5833                         if (dma_mapping_error(rx_ring->dev,
5834                                               buffer_info->page_dma)) {
5835                                 buffer_info->page_dma = 0;
5836                                 u64_stats_update_begin(&rx_ring->rx_syncp);
5837                                 rx_ring->rx_stats.alloc_failed++;
5838                                 u64_stats_update_end(&rx_ring->rx_syncp);
5839                                 goto no_buffers;
5840                         }
5841                 }
5842
5843                 skb = buffer_info->skb;
5844                 if (!skb) {
5845                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5846                         if (unlikely(!skb)) {
5847                                 u64_stats_update_begin(&rx_ring->rx_syncp);
5848                                 rx_ring->rx_stats.alloc_failed++;
5849                                 u64_stats_update_end(&rx_ring->rx_syncp);
5850                                 goto no_buffers;
5851                         }
5852
5853                         buffer_info->skb = skb;
5854                 }
5855                 if (!buffer_info->dma) {
5856                         buffer_info->dma = dma_map_single(rx_ring->dev,
5857                                                           skb->data,
5858                                                           bufsz,
5859                                                           DMA_FROM_DEVICE);
5860                         if (dma_mapping_error(rx_ring->dev,
5861                                               buffer_info->dma)) {
5862                                 buffer_info->dma = 0;
5863                                 u64_stats_update_begin(&rx_ring->rx_syncp);
5864                                 rx_ring->rx_stats.alloc_failed++;
5865                                 u64_stats_update_end(&rx_ring->rx_syncp);
5866                                 goto no_buffers;
5867                         }
5868                 }
5869                 /* Refresh the desc even if buffer_addrs didn't change because
5870                  * each write-back erases this info. */
5871                 if (bufsz < IGB_RXBUFFER_1024) {
5872                         rx_desc->read.pkt_addr =
5873                              cpu_to_le64(buffer_info->page_dma);
5874                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5875                 } else {
5876                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5877                         rx_desc->read.hdr_addr = 0;
5878                 }
5879
5880                 i++;
5881                 if (i == rx_ring->count)
5882                         i = 0;
5883                 buffer_info = &rx_ring->buffer_info[i];
5884         }
5885
5886 no_buffers:
5887         if (rx_ring->next_to_use != i) {
5888                 rx_ring->next_to_use = i;
5889                 if (i == 0)
5890                         i = (rx_ring->count - 1);
5891                 else
5892                         i--;
5893
5894                 /* Force memory writes to complete before letting h/w
5895                  * know there are new descriptors to fetch.  (Only
5896                  * applicable for weak-ordered memory model archs,
5897                  * such as IA-64). */
5898                 wmb();
5899                 writel(i, rx_ring->tail);
5900         }
5901 }
5902
5903 /**
5904  * igb_mii_ioctl -
5905  * @netdev:
5906  * @ifreq:
5907  * @cmd:
5908  **/
5909 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5910 {
5911         struct igb_adapter *adapter = netdev_priv(netdev);
5912         struct mii_ioctl_data *data = if_mii(ifr);
5913
5914         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5915                 return -EOPNOTSUPP;
5916
5917         switch (cmd) {
5918         case SIOCGMIIPHY:
5919                 data->phy_id = adapter->hw.phy.addr;
5920                 break;
5921         case SIOCGMIIREG:
5922                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5923                                      &data->val_out))
5924                         return -EIO;
5925                 break;
5926         case SIOCSMIIREG:
5927         default:
5928                 return -EOPNOTSUPP;
5929         }
5930         return 0;
5931 }
5932
5933 /**
5934  * igb_hwtstamp_ioctl - control hardware time stamping
5935  * @netdev:
5936  * @ifreq:
5937  * @cmd:
5938  *
5939  * Outgoing time stamping can be enabled and disabled. Play nice and
5940  * disable it when requested, although it shouldn't case any overhead
5941  * when no packet needs it. At most one packet in the queue may be
5942  * marked for time stamping, otherwise it would be impossible to tell
5943  * for sure to which packet the hardware time stamp belongs.
5944  *
5945  * Incoming time stamping has to be configured via the hardware
5946  * filters. Not all combinations are supported, in particular event
5947  * type has to be specified. Matching the kind of event packet is
5948  * not supported, with the exception of "all V2 events regardless of
5949  * level 2 or 4".
5950  *
5951  **/
5952 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5953                               struct ifreq *ifr, int cmd)
5954 {
5955         struct igb_adapter *adapter = netdev_priv(netdev);
5956         struct e1000_hw *hw = &adapter->hw;
5957         struct hwtstamp_config config;
5958         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5959         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5960         u32 tsync_rx_cfg = 0;
5961         bool is_l4 = false;
5962         bool is_l2 = false;
5963         u32 regval;
5964
5965         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5966                 return -EFAULT;
5967
5968         /* reserved for future extensions */
5969         if (config.flags)
5970                 return -EINVAL;
5971
5972         switch (config.tx_type) {
5973         case HWTSTAMP_TX_OFF:
5974                 tsync_tx_ctl = 0;
5975         case HWTSTAMP_TX_ON:
5976                 break;
5977         default:
5978                 return -ERANGE;
5979         }
5980
5981         switch (config.rx_filter) {
5982         case HWTSTAMP_FILTER_NONE:
5983                 tsync_rx_ctl = 0;
5984                 break;
5985         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5986         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5987         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5988         case HWTSTAMP_FILTER_ALL:
5989                 /*
5990                  * register TSYNCRXCFG must be set, therefore it is not
5991                  * possible to time stamp both Sync and Delay_Req messages
5992                  * => fall back to time stamping all packets
5993                  */
5994                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5995                 config.rx_filter = HWTSTAMP_FILTER_ALL;
5996                 break;
5997         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5998                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5999                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
6000                 is_l4 = true;
6001                 break;
6002         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
6003                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6004                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
6005                 is_l4 = true;
6006                 break;
6007         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6008         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
6009                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6010                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
6011                 is_l2 = true;
6012                 is_l4 = true;
6013                 config.rx_filter = HWTSTAMP_FILTER_SOME;
6014                 break;
6015         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6016         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
6017                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6018                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
6019                 is_l2 = true;
6020                 is_l4 = true;
6021                 config.rx_filter = HWTSTAMP_FILTER_SOME;
6022                 break;
6023         case HWTSTAMP_FILTER_PTP_V2_EVENT:
6024         case HWTSTAMP_FILTER_PTP_V2_SYNC:
6025         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
6026                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
6027                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
6028                 is_l2 = true;
6029                 break;
6030         default:
6031                 return -ERANGE;
6032         }
6033
6034         if (hw->mac.type == e1000_82575) {
6035                 if (tsync_rx_ctl | tsync_tx_ctl)
6036                         return -EINVAL;
6037                 return 0;
6038         }
6039
6040         /*
6041          * Per-packet timestamping only works if all packets are
6042          * timestamped, so enable timestamping in all packets as
6043          * long as one rx filter was configured.
6044          */
6045         if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6046                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6047                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6048         }
6049
6050         /* enable/disable TX */
6051         regval = rd32(E1000_TSYNCTXCTL);
6052         regval &= ~E1000_TSYNCTXCTL_ENABLED;
6053         regval |= tsync_tx_ctl;
6054         wr32(E1000_TSYNCTXCTL, regval);
6055
6056         /* enable/disable RX */
6057         regval = rd32(E1000_TSYNCRXCTL);
6058         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6059         regval |= tsync_rx_ctl;
6060         wr32(E1000_TSYNCRXCTL, regval);
6061
6062         /* define which PTP packets are time stamped */
6063         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6064
6065         /* define ethertype filter for timestamped packets */
6066         if (is_l2)
6067                 wr32(E1000_ETQF(3),
6068                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6069                                  E1000_ETQF_1588 | /* enable timestamping */
6070                                  ETH_P_1588));     /* 1588 eth protocol type */
6071         else
6072                 wr32(E1000_ETQF(3), 0);
6073
6074 #define PTP_PORT 319
6075         /* L4 Queue Filter[3]: filter by destination port and protocol */
6076         if (is_l4) {
6077                 u32 ftqf = (IPPROTO_UDP /* UDP */
6078                         | E1000_FTQF_VF_BP /* VF not compared */
6079                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6080                         | E1000_FTQF_MASK); /* mask all inputs */
6081                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6082
6083                 wr32(E1000_IMIR(3), htons(PTP_PORT));
6084                 wr32(E1000_IMIREXT(3),
6085                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6086                 if (hw->mac.type == e1000_82576) {
6087                         /* enable source port check */
6088                         wr32(E1000_SPQF(3), htons(PTP_PORT));
6089                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6090                 }
6091                 wr32(E1000_FTQF(3), ftqf);
6092         } else {
6093                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6094         }
6095         wrfl();
6096
6097         adapter->hwtstamp_config = config;
6098
6099         /* clear TX/RX time stamp registers, just to be sure */
6100         regval = rd32(E1000_TXSTMPH);
6101         regval = rd32(E1000_RXSTMPH);
6102
6103         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6104                 -EFAULT : 0;
6105 }
6106
6107 /**
6108  * igb_ioctl -
6109  * @netdev:
6110  * @ifreq:
6111  * @cmd:
6112  **/
6113 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6114 {
6115         switch (cmd) {
6116         case SIOCGMIIPHY:
6117         case SIOCGMIIREG:
6118         case SIOCSMIIREG:
6119                 return igb_mii_ioctl(netdev, ifr, cmd);
6120         case SIOCSHWTSTAMP:
6121                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6122         default:
6123                 return -EOPNOTSUPP;
6124         }
6125 }
6126
6127 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6128 {
6129         struct igb_adapter *adapter = hw->back;
6130         u16 cap_offset;
6131
6132         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6133         if (!cap_offset)
6134                 return -E1000_ERR_CONFIG;
6135
6136         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6137
6138         return 0;
6139 }
6140
6141 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6142 {
6143         struct igb_adapter *adapter = hw->back;
6144         u16 cap_offset;
6145
6146         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6147         if (!cap_offset)
6148                 return -E1000_ERR_CONFIG;
6149
6150         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6151
6152         return 0;
6153 }
6154
6155 static void igb_vlan_rx_register(struct net_device *netdev,
6156                                  struct vlan_group *grp)
6157 {
6158         struct igb_adapter *adapter = netdev_priv(netdev);
6159         struct e1000_hw *hw = &adapter->hw;
6160         u32 ctrl, rctl;
6161
6162         igb_irq_disable(adapter);
6163         adapter->vlgrp = grp;
6164
6165         if (grp) {
6166                 /* enable VLAN tag insert/strip */
6167                 ctrl = rd32(E1000_CTRL);
6168                 ctrl |= E1000_CTRL_VME;
6169                 wr32(E1000_CTRL, ctrl);
6170
6171                 /* Disable CFI check */
6172                 rctl = rd32(E1000_RCTL);
6173                 rctl &= ~E1000_RCTL_CFIEN;
6174                 wr32(E1000_RCTL, rctl);
6175         } else {
6176                 /* disable VLAN tag insert/strip */
6177                 ctrl = rd32(E1000_CTRL);
6178                 ctrl &= ~E1000_CTRL_VME;
6179                 wr32(E1000_CTRL, ctrl);
6180         }
6181
6182         igb_rlpml_set(adapter);
6183
6184         if (!test_bit(__IGB_DOWN, &adapter->state))
6185                 igb_irq_enable(adapter);
6186 }
6187
6188 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6189 {
6190         struct igb_adapter *adapter = netdev_priv(netdev);
6191         struct e1000_hw *hw = &adapter->hw;
6192         int pf_id = adapter->vfs_allocated_count;
6193
6194         /* attempt to add filter to vlvf array */
6195         igb_vlvf_set(adapter, vid, true, pf_id);
6196
6197         /* add the filter since PF can receive vlans w/o entry in vlvf */
6198         igb_vfta_set(hw, vid, true);
6199 }
6200
6201 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6202 {
6203         struct igb_adapter *adapter = netdev_priv(netdev);
6204         struct e1000_hw *hw = &adapter->hw;
6205         int pf_id = adapter->vfs_allocated_count;
6206         s32 err;
6207
6208         igb_irq_disable(adapter);
6209         vlan_group_set_device(adapter->vlgrp, vid, NULL);
6210
6211         if (!test_bit(__IGB_DOWN, &adapter->state))
6212                 igb_irq_enable(adapter);
6213
6214         /* remove vlan from VLVF table array */
6215         err = igb_vlvf_set(adapter, vid, false, pf_id);
6216
6217         /* if vid was not present in VLVF just remove it from table */
6218         if (err)
6219                 igb_vfta_set(hw, vid, false);
6220 }
6221
6222 static void igb_restore_vlan(struct igb_adapter *adapter)
6223 {
6224         igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6225
6226         if (adapter->vlgrp) {
6227                 u16 vid;
6228                 for (vid = 0; vid < VLAN_N_VID; vid++) {
6229                         if (!vlan_group_get_device(adapter->vlgrp, vid))
6230                                 continue;
6231                         igb_vlan_rx_add_vid(adapter->netdev, vid);
6232                 }
6233         }
6234 }
6235
6236 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6237 {
6238         struct pci_dev *pdev = adapter->pdev;
6239         struct e1000_mac_info *mac = &adapter->hw.mac;
6240
6241         mac->autoneg = 0;
6242
6243         /* Fiber NIC's only allow 1000 Gbps Full duplex */
6244         if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6245                 spddplx != (SPEED_1000 + DUPLEX_FULL)) {
6246                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6247                 return -EINVAL;
6248         }
6249
6250         switch (spddplx) {
6251         case SPEED_10 + DUPLEX_HALF:
6252                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6253                 break;
6254         case SPEED_10 + DUPLEX_FULL:
6255                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6256                 break;
6257         case SPEED_100 + DUPLEX_HALF:
6258                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6259                 break;
6260         case SPEED_100 + DUPLEX_FULL:
6261                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6262                 break;
6263         case SPEED_1000 + DUPLEX_FULL:
6264                 mac->autoneg = 1;
6265                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6266                 break;
6267         case SPEED_1000 + DUPLEX_HALF: /* not supported */
6268         default:
6269                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6270                 return -EINVAL;
6271         }
6272         return 0;
6273 }
6274
6275 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6276 {
6277         struct net_device *netdev = pci_get_drvdata(pdev);
6278         struct igb_adapter *adapter = netdev_priv(netdev);
6279         struct e1000_hw *hw = &adapter->hw;
6280         u32 ctrl, rctl, status;
6281         u32 wufc = adapter->wol;
6282 #ifdef CONFIG_PM
6283         int retval = 0;
6284 #endif
6285
6286         netif_device_detach(netdev);
6287
6288         if (netif_running(netdev))
6289                 igb_close(netdev);
6290
6291         igb_clear_interrupt_scheme(adapter);
6292
6293 #ifdef CONFIG_PM
6294         retval = pci_save_state(pdev);
6295         if (retval)
6296                 return retval;
6297 #endif
6298
6299         status = rd32(E1000_STATUS);
6300         if (status & E1000_STATUS_LU)
6301                 wufc &= ~E1000_WUFC_LNKC;
6302
6303         if (wufc) {
6304                 igb_setup_rctl(adapter);
6305                 igb_set_rx_mode(netdev);
6306
6307                 /* turn on all-multi mode if wake on multicast is enabled */
6308                 if (wufc & E1000_WUFC_MC) {
6309                         rctl = rd32(E1000_RCTL);
6310                         rctl |= E1000_RCTL_MPE;
6311                         wr32(E1000_RCTL, rctl);
6312                 }
6313
6314                 ctrl = rd32(E1000_CTRL);
6315                 /* advertise wake from D3Cold */
6316                 #define E1000_CTRL_ADVD3WUC 0x00100000
6317                 /* phy power management enable */
6318                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6319                 ctrl |= E1000_CTRL_ADVD3WUC;
6320                 wr32(E1000_CTRL, ctrl);
6321
6322                 /* Allow time for pending master requests to run */
6323                 igb_disable_pcie_master(hw);
6324
6325                 wr32(E1000_WUC, E1000_WUC_PME_EN);
6326                 wr32(E1000_WUFC, wufc);
6327         } else {
6328                 wr32(E1000_WUC, 0);
6329                 wr32(E1000_WUFC, 0);
6330         }
6331
6332         *enable_wake = wufc || adapter->en_mng_pt;
6333         if (!*enable_wake)
6334                 igb_power_down_link(adapter);
6335         else
6336                 igb_power_up_link(adapter);
6337
6338         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6339          * would have already happened in close and is redundant. */
6340         igb_release_hw_control(adapter);
6341
6342         pci_disable_device(pdev);
6343
6344         return 0;
6345 }
6346
6347 #ifdef CONFIG_PM
6348 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6349 {
6350         int retval;
6351         bool wake;
6352
6353         retval = __igb_shutdown(pdev, &wake);
6354         if (retval)
6355                 return retval;
6356
6357         if (wake) {
6358                 pci_prepare_to_sleep(pdev);
6359         } else {
6360                 pci_wake_from_d3(pdev, false);
6361                 pci_set_power_state(pdev, PCI_D3hot);
6362         }
6363
6364         return 0;
6365 }
6366
6367 static int igb_resume(struct pci_dev *pdev)
6368 {
6369         struct net_device *netdev = pci_get_drvdata(pdev);
6370         struct igb_adapter *adapter = netdev_priv(netdev);
6371         struct e1000_hw *hw = &adapter->hw;
6372         u32 err;
6373
6374         pci_set_power_state(pdev, PCI_D0);
6375         pci_restore_state(pdev);
6376         pci_save_state(pdev);
6377
6378         err = pci_enable_device_mem(pdev);
6379         if (err) {
6380                 dev_err(&pdev->dev,
6381                         "igb: Cannot enable PCI device from suspend\n");
6382                 return err;
6383         }
6384         pci_set_master(pdev);
6385
6386         pci_enable_wake(pdev, PCI_D3hot, 0);
6387         pci_enable_wake(pdev, PCI_D3cold, 0);
6388
6389         if (igb_init_interrupt_scheme(adapter)) {
6390                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6391                 return -ENOMEM;
6392         }
6393
6394         igb_reset(adapter);
6395
6396         /* let the f/w know that the h/w is now under the control of the
6397          * driver. */
6398         igb_get_hw_control(adapter);
6399
6400         wr32(E1000_WUS, ~0);
6401
6402         if (netif_running(netdev)) {
6403                 err = igb_open(netdev);
6404                 if (err)
6405                         return err;
6406         }
6407
6408         netif_device_attach(netdev);
6409
6410         return 0;
6411 }
6412 #endif
6413
6414 static void igb_shutdown(struct pci_dev *pdev)
6415 {
6416         bool wake;
6417
6418         __igb_shutdown(pdev, &wake);
6419
6420         if (system_state == SYSTEM_POWER_OFF) {
6421                 pci_wake_from_d3(pdev, wake);
6422                 pci_set_power_state(pdev, PCI_D3hot);
6423         }
6424 }
6425
6426 #ifdef CONFIG_NET_POLL_CONTROLLER
6427 /*
6428  * Polling 'interrupt' - used by things like netconsole to send skbs
6429  * without having to re-enable interrupts. It's not called while
6430  * the interrupt routine is executing.
6431  */
6432 static void igb_netpoll(struct net_device *netdev)
6433 {
6434         struct igb_adapter *adapter = netdev_priv(netdev);
6435         struct e1000_hw *hw = &adapter->hw;
6436         int i;
6437
6438         if (!adapter->msix_entries) {
6439                 struct igb_q_vector *q_vector = adapter->q_vector[0];
6440                 igb_irq_disable(adapter);
6441                 napi_schedule(&q_vector->napi);
6442                 return;
6443         }
6444
6445         for (i = 0; i < adapter->num_q_vectors; i++) {
6446                 struct igb_q_vector *q_vector = adapter->q_vector[i];
6447                 wr32(E1000_EIMC, q_vector->eims_value);
6448                 napi_schedule(&q_vector->napi);
6449         }
6450 }
6451 #endif /* CONFIG_NET_POLL_CONTROLLER */
6452
6453 /**
6454  * igb_io_error_detected - called when PCI error is detected
6455  * @pdev: Pointer to PCI device
6456  * @state: The current pci connection state
6457  *
6458  * This function is called after a PCI bus error affecting
6459  * this device has been detected.
6460  */
6461 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6462                                               pci_channel_state_t state)
6463 {
6464         struct net_device *netdev = pci_get_drvdata(pdev);
6465         struct igb_adapter *adapter = netdev_priv(netdev);
6466
6467         netif_device_detach(netdev);
6468
6469         if (state == pci_channel_io_perm_failure)
6470                 return PCI_ERS_RESULT_DISCONNECT;
6471
6472         if (netif_running(netdev))
6473                 igb_down(adapter);
6474         pci_disable_device(pdev);
6475
6476         /* Request a slot slot reset. */
6477         return PCI_ERS_RESULT_NEED_RESET;
6478 }
6479
6480 /**
6481  * igb_io_slot_reset - called after the pci bus has been reset.
6482  * @pdev: Pointer to PCI device
6483  *
6484  * Restart the card from scratch, as if from a cold-boot. Implementation
6485  * resembles the first-half of the igb_resume routine.
6486  */
6487 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6488 {
6489         struct net_device *netdev = pci_get_drvdata(pdev);
6490         struct igb_adapter *adapter = netdev_priv(netdev);
6491         struct e1000_hw *hw = &adapter->hw;
6492         pci_ers_result_t result;
6493         int err;
6494
6495         if (pci_enable_device_mem(pdev)) {
6496                 dev_err(&pdev->dev,
6497                         "Cannot re-enable PCI device after reset.\n");
6498                 result = PCI_ERS_RESULT_DISCONNECT;
6499         } else {
6500                 pci_set_master(pdev);
6501                 pci_restore_state(pdev);
6502                 pci_save_state(pdev);
6503
6504                 pci_enable_wake(pdev, PCI_D3hot, 0);
6505                 pci_enable_wake(pdev, PCI_D3cold, 0);
6506
6507                 igb_reset(adapter);
6508                 wr32(E1000_WUS, ~0);
6509                 result = PCI_ERS_RESULT_RECOVERED;
6510         }
6511
6512         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6513         if (err) {
6514                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6515                         "failed 0x%0x\n", err);
6516                 /* non-fatal, continue */
6517         }
6518
6519         return result;
6520 }
6521
6522 /**
6523  * igb_io_resume - called when traffic can start flowing again.
6524  * @pdev: Pointer to PCI device
6525  *
6526  * This callback is called when the error recovery driver tells us that
6527  * its OK to resume normal operation. Implementation resembles the
6528  * second-half of the igb_resume routine.
6529  */
6530 static void igb_io_resume(struct pci_dev *pdev)
6531 {
6532         struct net_device *netdev = pci_get_drvdata(pdev);
6533         struct igb_adapter *adapter = netdev_priv(netdev);
6534
6535         if (netif_running(netdev)) {
6536                 if (igb_up(adapter)) {
6537                         dev_err(&pdev->dev, "igb_up failed after reset\n");
6538                         return;
6539                 }
6540         }
6541
6542         netif_device_attach(netdev);
6543
6544         /* let the f/w know that the h/w is now under the control of the
6545          * driver. */
6546         igb_get_hw_control(adapter);
6547 }
6548
6549 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6550                              u8 qsel)
6551 {
6552         u32 rar_low, rar_high;
6553         struct e1000_hw *hw = &adapter->hw;
6554
6555         /* HW expects these in little endian so we reverse the byte order
6556          * from network order (big endian) to little endian
6557          */
6558         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6559                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6560         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6561
6562         /* Indicate to hardware the Address is Valid. */
6563         rar_high |= E1000_RAH_AV;
6564
6565         if (hw->mac.type == e1000_82575)
6566                 rar_high |= E1000_RAH_POOL_1 * qsel;
6567         else
6568                 rar_high |= E1000_RAH_POOL_1 << qsel;
6569
6570         wr32(E1000_RAL(index), rar_low);
6571         wrfl();
6572         wr32(E1000_RAH(index), rar_high);
6573         wrfl();
6574 }
6575
6576 static int igb_set_vf_mac(struct igb_adapter *adapter,
6577                           int vf, unsigned char *mac_addr)
6578 {
6579         struct e1000_hw *hw = &adapter->hw;
6580         /* VF MAC addresses start at end of receive addresses and moves
6581          * torwards the first, as a result a collision should not be possible */
6582         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6583
6584         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6585
6586         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6587
6588         return 0;
6589 }
6590
6591 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6592 {
6593         struct igb_adapter *adapter = netdev_priv(netdev);
6594         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6595                 return -EINVAL;
6596         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6597         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6598         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6599                                       " change effective.");
6600         if (test_bit(__IGB_DOWN, &adapter->state)) {
6601                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6602                          " but the PF device is not up.\n");
6603                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6604                          " attempting to use the VF device.\n");
6605         }
6606         return igb_set_vf_mac(adapter, vf, mac);
6607 }
6608
6609 static int igb_link_mbps(int internal_link_speed)
6610 {
6611         switch (internal_link_speed) {
6612         case SPEED_100:
6613                 return 100;
6614         case SPEED_1000:
6615                 return 1000;
6616         default:
6617                 return 0;
6618         }
6619 }
6620
6621 static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6622                                   int link_speed)
6623 {
6624         int rf_dec, rf_int;
6625         u32 bcnrc_val;
6626
6627         if (tx_rate != 0) {
6628                 /* Calculate the rate factor values to set */
6629                 rf_int = link_speed / tx_rate;
6630                 rf_dec = (link_speed - (rf_int * tx_rate));
6631                 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6632
6633                 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6634                 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6635                                E1000_RTTBCNRC_RF_INT_MASK);
6636                 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6637         } else {
6638                 bcnrc_val = 0;
6639         }
6640
6641         wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6642         wr32(E1000_RTTBCNRC, bcnrc_val);
6643 }
6644
6645 static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6646 {
6647         int actual_link_speed, i;
6648         bool reset_rate = false;
6649
6650         /* VF TX rate limit was not set or not supported */
6651         if ((adapter->vf_rate_link_speed == 0) ||
6652             (adapter->hw.mac.type != e1000_82576))
6653                 return;
6654
6655         actual_link_speed = igb_link_mbps(adapter->link_speed);
6656         if (actual_link_speed != adapter->vf_rate_link_speed) {
6657                 reset_rate = true;
6658                 adapter->vf_rate_link_speed = 0;
6659                 dev_info(&adapter->pdev->dev,
6660                          "Link speed has been changed. VF Transmit "
6661                          "rate is disabled\n");
6662         }
6663
6664         for (i = 0; i < adapter->vfs_allocated_count; i++) {
6665                 if (reset_rate)
6666                         adapter->vf_data[i].tx_rate = 0;
6667
6668                 igb_set_vf_rate_limit(&adapter->hw, i,
6669                                       adapter->vf_data[i].tx_rate,
6670                                       actual_link_speed);
6671         }
6672 }
6673
6674 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6675 {
6676         struct igb_adapter *adapter = netdev_priv(netdev);
6677         struct e1000_hw *hw = &adapter->hw;
6678         int actual_link_speed;
6679
6680         if (hw->mac.type != e1000_82576)
6681                 return -EOPNOTSUPP;
6682
6683         actual_link_speed = igb_link_mbps(adapter->link_speed);
6684         if ((vf >= adapter->vfs_allocated_count) ||
6685             (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6686             (tx_rate < 0) || (tx_rate > actual_link_speed))
6687                 return -EINVAL;
6688
6689         adapter->vf_rate_link_speed = actual_link_speed;
6690         adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6691         igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6692
6693         return 0;
6694 }
6695
6696 static int igb_ndo_get_vf_config(struct net_device *netdev,
6697                                  int vf, struct ifla_vf_info *ivi)
6698 {
6699         struct igb_adapter *adapter = netdev_priv(netdev);
6700         if (vf >= adapter->vfs_allocated_count)
6701                 return -EINVAL;
6702         ivi->vf = vf;
6703         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6704         ivi->tx_rate = adapter->vf_data[vf].tx_rate;
6705         ivi->vlan = adapter->vf_data[vf].pf_vlan;
6706         ivi->qos = adapter->vf_data[vf].pf_qos;
6707         return 0;
6708 }
6709
6710 static void igb_vmm_control(struct igb_adapter *adapter)
6711 {
6712         struct e1000_hw *hw = &adapter->hw;
6713         u32 reg;
6714
6715         switch (hw->mac.type) {
6716         case e1000_82575:
6717         default:
6718                 /* replication is not supported for 82575 */
6719                 return;
6720         case e1000_82576:
6721                 /* notify HW that the MAC is adding vlan tags */
6722                 reg = rd32(E1000_DTXCTL);
6723                 reg |= E1000_DTXCTL_VLAN_ADDED;
6724                 wr32(E1000_DTXCTL, reg);
6725         case e1000_82580:
6726                 /* enable replication vlan tag stripping */
6727                 reg = rd32(E1000_RPLOLR);
6728                 reg |= E1000_RPLOLR_STRVLAN;
6729                 wr32(E1000_RPLOLR, reg);
6730         case e1000_i350:
6731                 /* none of the above registers are supported by i350 */
6732                 break;
6733         }
6734
6735         if (adapter->vfs_allocated_count) {
6736                 igb_vmdq_set_loopback_pf(hw, true);
6737                 igb_vmdq_set_replication_pf(hw, true);
6738                 igb_vmdq_set_anti_spoofing_pf(hw, true,
6739                                                 adapter->vfs_allocated_count);
6740         } else {
6741                 igb_vmdq_set_loopback_pf(hw, false);
6742                 igb_vmdq_set_replication_pf(hw, false);
6743         }
6744 }
6745
6746 /* igb_main.c */