drivers/net/igb/igb_main.c

   1 /*******************************************************************************
   2
   3   Intel(R) Gigabit Ethernet Linux driver
   4   Copyright(c) 2007-2009 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28 #include <linux/module.h>
  29 #include <linux/types.h>
  30 #include <linux/init.h>
  31 #include <linux/vmalloc.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/ipv6.h>
  35 #include <linux/slab.h>
  36 #include <net/checksum.h>
  37 #include <net/ip6_checksum.h>
  38 #include <linux/net_tstamp.h>
  39 #include <linux/mii.h>
  40 #include <linux/ethtool.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/pci.h>
  43 #include <linux/pci-aspm.h>
  44 #include <linux/delay.h>
  45 #include <linux/interrupt.h>
  46 #include <linux/if_ether.h>
  47 #include <linux/aer.h>
  48 #ifdef CONFIG_IGB_DCA
  49 #include <linux/dca.h>
  50 #endif
  51 #include "igb.h"
  52
  53 #define DRV_VERSION "2.1.0-k2"
  54 char igb_driver_name[] = "igb";
  55 char igb_driver_version[] = DRV_VERSION;
  56 static const char igb_driver_string[] =
  57                                 "Intel(R) Gigabit Ethernet Network Driver";
  58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
  59
  60 static const struct e1000_info *igb_info_tbl[] = {
  61         [board_82575] = &e1000_82575_info,
  62 };
  63
  64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
  65         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
  66         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
  67         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
  68         { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
  69         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
  70         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
  71         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
  72         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
  73         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
  74         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
  75         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
  76         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
  77         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
  78         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
  79         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
  80         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
  81         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
  82         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
  83         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
  84         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
  85         /* required last entry */
  86         {0, }
  87 };
  88
  89 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
  90
  91 void igb_reset(struct igb_adapter *);
  92 static int igb_setup_all_tx_resources(struct igb_adapter *);
  93 static int igb_setup_all_rx_resources(struct igb_adapter *);
  94 static void igb_free_all_tx_resources(struct igb_adapter *);
  95 static void igb_free_all_rx_resources(struct igb_adapter *);
  96 static void igb_setup_mrqc(struct igb_adapter *);
  97 void igb_update_stats(struct igb_adapter *);
  98 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
  99 static void __devexit igb_remove(struct pci_dev *pdev);
 100 static int igb_sw_init(struct igb_adapter *);
 101 static int igb_open(struct net_device *);
 102 static int igb_close(struct net_device *);
 103 static void igb_configure_tx(struct igb_adapter *);
 104 static void igb_configure_rx(struct igb_adapter *);
 105 static void igb_clean_all_tx_rings(struct igb_adapter *);
 106 static void igb_clean_all_rx_rings(struct igb_adapter *);
 107 static void igb_clean_tx_ring(struct igb_ring *);
 108 static void igb_clean_rx_ring(struct igb_ring *);
 109 static void igb_set_rx_mode(struct net_device *);
 110 static void igb_update_phy_info(unsigned long);
 111 static void igb_watchdog(unsigned long);
 112 static void igb_watchdog_task(struct work_struct *);
 113 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
 114 static struct net_device_stats *igb_get_stats(struct net_device *);
 115 static int igb_change_mtu(struct net_device *, int);
 116 static int igb_set_mac(struct net_device *, void *);
 117 static void igb_set_uta(struct igb_adapter *adapter);
 118 static irqreturn_t igb_intr(int irq, void *);
 119 static irqreturn_t igb_intr_msi(int irq, void *);
 120 static irqreturn_t igb_msix_other(int irq, void *);
 121 static irqreturn_t igb_msix_ring(int irq, void *);
 122 #ifdef CONFIG_IGB_DCA
 123 static void igb_update_dca(struct igb_q_vector *);
 124 static void igb_setup_dca(struct igb_adapter *);
 125 #endif /* CONFIG_IGB_DCA */
 126 static bool igb_clean_tx_irq(struct igb_q_vector *);
 127 static int igb_poll(struct napi_struct *, int);
 128 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
 129 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
 130 static void igb_tx_timeout(struct net_device *);
 131 static void igb_reset_task(struct work_struct *);
 132 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
 133 static void igb_vlan_rx_add_vid(struct net_device *, u16);
 134 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
 135 static void igb_restore_vlan(struct igb_adapter *);
 136 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
 137 static void igb_ping_all_vfs(struct igb_adapter *);
 138 static void igb_msg_task(struct igb_adapter *);
 139 static void igb_vmm_control(struct igb_adapter *);
 140 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
 141 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
 142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
 143 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
 144                                int vf, u16 vlan, u8 qos);
 145 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
 146 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
 147                                  struct ifla_vf_info *ivi);
 148
 149 #ifdef CONFIG_PM
 150 static int igb_suspend(struct pci_dev *, pm_message_t);
 151 static int igb_resume(struct pci_dev *);
 152 #endif
 153 static void igb_shutdown(struct pci_dev *);
 154 #ifdef CONFIG_IGB_DCA
 155 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
 156 static struct notifier_block dca_notifier = {
 157         .notifier_call  = igb_notify_dca,
 158         .next           = NULL,
 159         .priority       = 0
 160 };
 161 #endif
 162 #ifdef CONFIG_NET_POLL_CONTROLLER
 163 /* for netdump / net console */
 164 static void igb_netpoll(struct net_device *);
 165 #endif
 166 #ifdef CONFIG_PCI_IOV
 167 static unsigned int max_vfs = 0;
 168 module_param(max_vfs, uint, 0);
 169 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
 170                  "per physical function");
 171 #endif /* CONFIG_PCI_IOV */
 172
 173 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
 174                      pci_channel_state_t);
 175 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
 176 static void igb_io_resume(struct pci_dev *);
 177
 178 static struct pci_error_handlers igb_err_handler = {
 179         .error_detected = igb_io_error_detected,
 180         .slot_reset = igb_io_slot_reset,
 181         .resume = igb_io_resume,
 182 };
 183
 184
 185 static struct pci_driver igb_driver = {
 186         .name     = igb_driver_name,
 187         .id_table = igb_pci_tbl,
 188         .probe    = igb_probe,
 189         .remove   = __devexit_p(igb_remove),
 190 #ifdef CONFIG_PM
 191         /* Power Managment Hooks */
 192         .suspend  = igb_suspend,
 193         .resume   = igb_resume,
 194 #endif
 195         .shutdown = igb_shutdown,
 196         .err_handler = &igb_err_handler
 197 };
 198
 199 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 200 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
 201 MODULE_LICENSE("GPL");
 202 MODULE_VERSION(DRV_VERSION);
 203
 204 /**
 205  * igb_read_clock - read raw cycle counter (to be used by time counter)
 206  */
 207 static cycle_t igb_read_clock(const struct cyclecounter *tc)
 208 {
 209         struct igb_adapter *adapter =
 210                 container_of(tc, struct igb_adapter, cycles);
 211         struct e1000_hw *hw = &adapter->hw;
 212         u64 stamp = 0;
 213         int shift = 0;
 214
 215         /*
 216          * The timestamp latches on lowest register read. For the 82580
 217          * the lowest register is SYSTIMR instead of SYSTIML.  However we never
 218          * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
 219          */
 220         if (hw->mac.type == e1000_82580) {
 221                 stamp = rd32(E1000_SYSTIMR) >> 8;
 222                 shift = IGB_82580_TSYNC_SHIFT;
 223         }
 224
 225         stamp |= (u64)rd32(E1000_SYSTIML) << shift;
 226         stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
 227         return stamp;
 228 }
 229
 230 /**
 231  * igb_get_hw_dev - return device
 232  * used by hardware layer to print debugging information
 233  **/
 234 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
 235 {
 236         struct igb_adapter *adapter = hw->back;
 237         return adapter->netdev;
 238 }
 239
 240 /**
 241  * igb_init_module - Driver Registration Routine
 242  *
 243  * igb_init_module is the first routine called when the driver is
 244  * loaded. All it does is register with the PCI subsystem.
 245  **/
 246 static int __init igb_init_module(void)
 247 {
 248         int ret;
 249         printk(KERN_INFO "%s - version %s\n",
 250                igb_driver_string, igb_driver_version);
 251
 252         printk(KERN_INFO "%s\n", igb_copyright);
 253
 254 #ifdef CONFIG_IGB_DCA
 255         dca_register_notify(&dca_notifier);
 256 #endif
 257         ret = pci_register_driver(&igb_driver);
 258         return ret;
 259 }
 260
 261 module_init(igb_init_module);
 262
 263 /**
 264  * igb_exit_module - Driver Exit Cleanup Routine
 265  *
 266  * igb_exit_module is called just before the driver is removed
 267  * from memory.
 268  **/
 269 static void __exit igb_exit_module(void)
 270 {
 271 #ifdef CONFIG_IGB_DCA
 272         dca_unregister_notify(&dca_notifier);
 273 #endif
 274         pci_unregister_driver(&igb_driver);
 275 }
 276
 277 module_exit(igb_exit_module);
 278
 279 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
 280 /**
 281  * igb_cache_ring_register - Descriptor ring to register mapping
 282  * @adapter: board private structure to initialize
 283  *
 284  * Once we know the feature-set enabled for the device, we'll cache
 285  * the register offset the descriptor ring is assigned to.
 286  **/
 287 static void igb_cache_ring_register(struct igb_adapter *adapter)
 288 {
 289         int i = 0, j = 0;
 290         u32 rbase_offset = adapter->vfs_allocated_count;
 291
 292         switch (adapter->hw.mac.type) {
 293         case e1000_82576:
 294                 /* The queues are allocated for virtualization such that VF 0
 295                  * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
 296                  * In order to avoid collision we start at the first free queue
 297                  * and continue consuming queues in the same sequence
 298                  */
 299                 if (adapter->vfs_allocated_count) {
 300                         for (; i < adapter->rss_queues; i++)
 301                                 adapter->rx_ring[i]->reg_idx = rbase_offset +
 302                                                                Q_IDX_82576(i);
 303                         for (; j < adapter->rss_queues; j++)
 304                                 adapter->tx_ring[j]->reg_idx = rbase_offset +
 305                                                                Q_IDX_82576(j);
 306                 }
 307         case e1000_82575:
 308         case e1000_82580:
 309         case e1000_i350:
 310         default:
 311                 for (; i < adapter->num_rx_queues; i++)
 312                         adapter->rx_ring[i]->reg_idx = rbase_offset + i;
 313                 for (; j < adapter->num_tx_queues; j++)
 314                         adapter->tx_ring[j]->reg_idx = rbase_offset + j;
 315                 break;
 316         }
 317 }
 318
 319 static void igb_free_queues(struct igb_adapter *adapter)
 320 {
 321         int i;
 322
 323         for (i = 0; i < adapter->num_tx_queues; i++) {
 324                 kfree(adapter->tx_ring[i]);
 325                 adapter->tx_ring[i] = NULL;
 326         }
 327         for (i = 0; i < adapter->num_rx_queues; i++) {
 328                 kfree(adapter->rx_ring[i]);
 329                 adapter->rx_ring[i] = NULL;
 330         }
 331         adapter->num_rx_queues = 0;
 332         adapter->num_tx_queues = 0;
 333 }
 334
 335 /**
 336  * igb_alloc_queues - Allocate memory for all rings
 337  * @adapter: board private structure to initialize
 338  *
 339  * We allocate one ring per queue at run-time since we don't know the
 340  * number of queues at compile-time.
 341  **/
 342 static int igb_alloc_queues(struct igb_adapter *adapter)
 343 {
 344         struct igb_ring *ring;
 345         int i;
 346
 347         for (i = 0; i < adapter->num_tx_queues; i++) {
 348                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 349                 if (!ring)
 350                         goto err;
 351                 ring->count = adapter->tx_ring_count;
 352                 ring->queue_index = i;
 353                 ring->dev = &adapter->pdev->dev;
 354                 ring->netdev = adapter->netdev;
 355                 /* For 82575, context index must be unique per ring. */
 356                 if (adapter->hw.mac.type == e1000_82575)
 357                         ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
 358                 adapter->tx_ring[i] = ring;
 359         }
 360
 361         for (i = 0; i < adapter->num_rx_queues; i++) {
 362                 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
 363                 if (!ring)
 364                         goto err;
 365                 ring->count = adapter->rx_ring_count;
 366                 ring->queue_index = i;
 367                 ring->dev = &adapter->pdev->dev;
 368                 ring->netdev = adapter->netdev;
 369                 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
 370                 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
 371                 /* set flag indicating ring supports SCTP checksum offload */
 372                 if (adapter->hw.mac.type >= e1000_82576)
 373                         ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
 374                 adapter->rx_ring[i] = ring;
 375         }
 376
 377         igb_cache_ring_register(adapter);
 378
 379         return 0;
 380
 381 err:
 382         igb_free_queues(adapter);
 383
 384         return -ENOMEM;
 385 }
 386
 387 #define IGB_N0_QUEUE -1
 388 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
 389 {
 390         u32 msixbm = 0;
 391         struct igb_adapter *adapter = q_vector->adapter;
 392         struct e1000_hw *hw = &adapter->hw;
 393         u32 ivar, index;
 394         int rx_queue = IGB_N0_QUEUE;
 395         int tx_queue = IGB_N0_QUEUE;
 396
 397         if (q_vector->rx_ring)
 398                 rx_queue = q_vector->rx_ring->reg_idx;
 399         if (q_vector->tx_ring)
 400                 tx_queue = q_vector->tx_ring->reg_idx;
 401
 402         switch (hw->mac.type) {
 403         case e1000_82575:
 404                 /* The 82575 assigns vectors using a bitmask, which matches the
 405                    bitmask for the EICR/EIMS/EIMC registers.  To assign one
 406                    or more queues to a vector, we write the appropriate bits
 407                    into the MSIXBM register for that vector. */
 408                 if (rx_queue > IGB_N0_QUEUE)
 409                         msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
 410                 if (tx_queue > IGB_N0_QUEUE)
 411                         msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
 412                 if (!adapter->msix_entries && msix_vector == 0)
 413                         msixbm |= E1000_EIMS_OTHER;
 414                 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
 415                 q_vector->eims_value = msixbm;
 416                 break;
 417         case e1000_82576:
 418                 /* 82576 uses a table-based method for assigning vectors.
 419                    Each queue has a single entry in the table to which we write
 420                    a vector number along with a "valid" bit.  Sadly, the layout
 421                    of the table is somewhat counterintuitive. */
 422                 if (rx_queue > IGB_N0_QUEUE) {
 423                         index = (rx_queue & 0x7);
 424                         ivar = array_rd32(E1000_IVAR0, index);
 425                         if (rx_queue < 8) {
 426                                 /* vector goes into low byte of register */
 427                                 ivar = ivar & 0xFFFFFF00;
 428                                 ivar |= msix_vector | E1000_IVAR_VALID;
 429                         } else {
 430                                 /* vector goes into third byte of register */
 431                                 ivar = ivar & 0xFF00FFFF;
 432                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 433                         }
 434                         array_wr32(E1000_IVAR0, index, ivar);
 435                 }
 436                 if (tx_queue > IGB_N0_QUEUE) {
 437                         index = (tx_queue & 0x7);
 438                         ivar = array_rd32(E1000_IVAR0, index);
 439                         if (tx_queue < 8) {
 440                                 /* vector goes into second byte of register */
 441                                 ivar = ivar & 0xFFFF00FF;
 442                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 443                         } else {
 444                                 /* vector goes into high byte of register */
 445                                 ivar = ivar & 0x00FFFFFF;
 446                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 447                         }
 448                         array_wr32(E1000_IVAR0, index, ivar);
 449                 }
 450                 q_vector->eims_value = 1 << msix_vector;
 451                 break;
 452         case e1000_82580:
 453         case e1000_i350:
 454                 /* 82580 uses the same table-based approach as 82576 but has fewer
 455                    entries as a result we carry over for queues greater than 4. */
 456                 if (rx_queue > IGB_N0_QUEUE) {
 457                         index = (rx_queue >> 1);
 458                         ivar = array_rd32(E1000_IVAR0, index);
 459                         if (rx_queue & 0x1) {
 460                                 /* vector goes into third byte of register */
 461                                 ivar = ivar & 0xFF00FFFF;
 462                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 463                         } else {
 464                                 /* vector goes into low byte of register */
 465                                 ivar = ivar & 0xFFFFFF00;
 466                                 ivar |= msix_vector | E1000_IVAR_VALID;
 467                         }
 468                         array_wr32(E1000_IVAR0, index, ivar);
 469                 }
 470                 if (tx_queue > IGB_N0_QUEUE) {
 471                         index = (tx_queue >> 1);
 472                         ivar = array_rd32(E1000_IVAR0, index);
 473                         if (tx_queue & 0x1) {
 474                                 /* vector goes into high byte of register */
 475                                 ivar = ivar & 0x00FFFFFF;
 476                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 477                         } else {
 478                                 /* vector goes into second byte of register */
 479                                 ivar = ivar & 0xFFFF00FF;
 480                                 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 481                         }
 482                         array_wr32(E1000_IVAR0, index, ivar);
 483                 }
 484                 q_vector->eims_value = 1 << msix_vector;
 485                 break;
 486         default:
 487                 BUG();
 488                 break;
 489         }
 490
 491         /* add q_vector eims value to global eims_enable_mask */
 492         adapter->eims_enable_mask |= q_vector->eims_value;
 493
 494         /* configure q_vector to set itr on first interrupt */
 495         q_vector->set_itr = 1;
 496 }
 497
 498 /**
 499  * igb_configure_msix - Configure MSI-X hardware
 500  *
 501  * igb_configure_msix sets up the hardware to properly
 502  * generate MSI-X interrupts.
 503  **/
 504 static void igb_configure_msix(struct igb_adapter *adapter)
 505 {
 506         u32 tmp;
 507         int i, vector = 0;
 508         struct e1000_hw *hw = &adapter->hw;
 509
 510         adapter->eims_enable_mask = 0;
 511
 512         /* set vector for other causes, i.e. link changes */
 513         switch (hw->mac.type) {
 514         case e1000_82575:
 515                 tmp = rd32(E1000_CTRL_EXT);
 516                 /* enable MSI-X PBA support*/
 517                 tmp |= E1000_CTRL_EXT_PBA_CLR;
 518
 519                 /* Auto-Mask interrupts upon ICR read. */
 520                 tmp |= E1000_CTRL_EXT_EIAME;
 521                 tmp |= E1000_CTRL_EXT_IRCA;
 522
 523                 wr32(E1000_CTRL_EXT, tmp);
 524
 525                 /* enable msix_other interrupt */
 526                 array_wr32(E1000_MSIXBM(0), vector++,
 527                                       E1000_EIMS_OTHER);
 528                 adapter->eims_other = E1000_EIMS_OTHER;
 529
 530                 break;
 531
 532         case e1000_82576:
 533         case e1000_82580:
 534         case e1000_i350:
 535                 /* Turn on MSI-X capability first, or our settings
 536                  * won't stick.  And it will take days to debug. */
 537                 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
 538                                 E1000_GPIE_PBA | E1000_GPIE_EIAME |
 539                                 E1000_GPIE_NSICR);
 540
 541                 /* enable msix_other interrupt */
 542                 adapter->eims_other = 1 << vector;
 543                 tmp = (vector++ | E1000_IVAR_VALID) << 8;
 544
 545                 wr32(E1000_IVAR_MISC, tmp);
 546                 break;
 547         default:
 548                 /* do nothing, since nothing else supports MSI-X */
 549                 break;
 550         } /* switch (hw->mac.type) */
 551
 552         adapter->eims_enable_mask |= adapter->eims_other;
 553
 554         for (i = 0; i < adapter->num_q_vectors; i++)
 555                 igb_assign_vector(adapter->q_vector[i], vector++);
 556
 557         wrfl();
 558 }
 559
 560 /**
 561  * igb_request_msix - Initialize MSI-X interrupts
 562  *
 563  * igb_request_msix allocates MSI-X vectors and requests interrupts from the
 564  * kernel.
 565  **/
 566 static int igb_request_msix(struct igb_adapter *adapter)
 567 {
 568         struct net_device *netdev = adapter->netdev;
 569         struct e1000_hw *hw = &adapter->hw;
 570         int i, err = 0, vector = 0;
 571
 572         err = request_irq(adapter->msix_entries[vector].vector,
 573                           igb_msix_other, 0, netdev->name, adapter);
 574         if (err)
 575                 goto out;
 576         vector++;
 577
 578         for (i = 0; i < adapter->num_q_vectors; i++) {
 579                 struct igb_q_vector *q_vector = adapter->q_vector[i];
 580
 581                 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
 582
 583                 if (q_vector->rx_ring && q_vector->tx_ring)
 584                         sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
 585                                 q_vector->rx_ring->queue_index);
 586                 else if (q_vector->tx_ring)
 587                         sprintf(q_vector->name, "%s-tx-%u", netdev->name,
 588                                 q_vector->tx_ring->queue_index);
 589                 else if (q_vector->rx_ring)
 590                         sprintf(q_vector->name, "%s-rx-%u", netdev->name,
 591                                 q_vector->rx_ring->queue_index);
 592                 else
 593                         sprintf(q_vector->name, "%s-unused", netdev->name);
 594
 595                 err = request_irq(adapter->msix_entries[vector].vector,
 596                                   igb_msix_ring, 0, q_vector->name,
 597                                   q_vector);
 598                 if (err)
 599                         goto out;
 600                 vector++;
 601         }
 602
 603         igb_configure_msix(adapter);
 604         return 0;
 605 out:
 606         return err;
 607 }
 608
 609 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
 610 {
 611         if (adapter->msix_entries) {
 612                 pci_disable_msix(adapter->pdev);
 613                 kfree(adapter->msix_entries);
 614                 adapter->msix_entries = NULL;
 615         } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
 616                 pci_disable_msi(adapter->pdev);
 617         }
 618 }
 619
 620 /**
 621  * igb_free_q_vectors - Free memory allocated for interrupt vectors
 622  * @adapter: board private structure to initialize
 623  *
 624  * This function frees the memory allocated to the q_vectors.  In addition if
 625  * NAPI is enabled it will delete any references to the NAPI struct prior
 626  * to freeing the q_vector.
 627  **/
 628 static void igb_free_q_vectors(struct igb_adapter *adapter)
 629 {
 630         int v_idx;
 631
 632         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 633                 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 634                 adapter->q_vector[v_idx] = NULL;
 635                 if (!q_vector)
 636                         continue;
 637                 netif_napi_del(&q_vector->napi);
 638                 kfree(q_vector);
 639         }
 640         adapter->num_q_vectors = 0;
 641 }
 642
 643 /**
 644  * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 645  *
 646  * This function resets the device so that it has 0 rx queues, tx queues, and
 647  * MSI-X interrupts allocated.
 648  */
 649 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
 650 {
 651         igb_free_queues(adapter);
 652         igb_free_q_vectors(adapter);
 653         igb_reset_interrupt_capability(adapter);
 654 }
 655
 656 /**
 657  * igb_set_interrupt_capability - set MSI or MSI-X if supported
 658  *
 659  * Attempt to configure interrupts using the best available
 660  * capabilities of the hardware and kernel.
 661  **/
 662 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
 663 {
 664         int err;
 665         int numvecs, i;
 666
 667         /* Number of supported queues. */
 668         adapter->num_rx_queues = adapter->rss_queues;
 669         adapter->num_tx_queues = adapter->rss_queues;
 670
 671         /* start with one vector for every rx queue */
 672         numvecs = adapter->num_rx_queues;
 673
 674         /* if tx handler is separate add 1 for every tx queue */
 675         if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
 676                 numvecs += adapter->num_tx_queues;
 677
 678         /* store the number of vectors reserved for queues */
 679         adapter->num_q_vectors = numvecs;
 680
 681         /* add 1 vector for link status interrupts */
 682         numvecs++;
 683         adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
 684                                         GFP_KERNEL);
 685         if (!adapter->msix_entries)
 686                 goto msi_only;
 687
 688         for (i = 0; i < numvecs; i++)
 689                 adapter->msix_entries[i].entry = i;
 690
 691         err = pci_enable_msix(adapter->pdev,
 692                               adapter->msix_entries,
 693                               numvecs);
 694         if (err == 0)
 695                 goto out;
 696
 697         igb_reset_interrupt_capability(adapter);
 698
 699         /* If we can't do MSI-X, try MSI */
 700 msi_only:
 701 #ifdef CONFIG_PCI_IOV
 702         /* disable SR-IOV for non MSI-X configurations */
 703         if (adapter->vf_data) {
 704                 struct e1000_hw *hw = &adapter->hw;
 705                 /* disable iov and allow time for transactions to clear */
 706                 pci_disable_sriov(adapter->pdev);
 707                 msleep(500);
 708
 709                 kfree(adapter->vf_data);
 710                 adapter->vf_data = NULL;
 711                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
 712                 msleep(100);
 713                 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
 714         }
 715 #endif
 716         adapter->vfs_allocated_count = 0;
 717         adapter->rss_queues = 1;
 718         adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
 719         adapter->num_rx_queues = 1;
 720         adapter->num_tx_queues = 1;
 721         adapter->num_q_vectors = 1;
 722         if (!pci_enable_msi(adapter->pdev))
 723                 adapter->flags |= IGB_FLAG_HAS_MSI;
 724 out:
 725         /* Notify the stack of the (possibly) reduced Tx Queue count. */
 726         adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
 727         return;
 728 }
 729
 730 /**
 731  * igb_alloc_q_vectors - Allocate memory for interrupt vectors
 732  * @adapter: board private structure to initialize
 733  *
 734  * We allocate one q_vector per queue interrupt.  If allocation fails we
 735  * return -ENOMEM.
 736  **/
 737 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
 738 {
 739         struct igb_q_vector *q_vector;
 740         struct e1000_hw *hw = &adapter->hw;
 741         int v_idx;
 742
 743         for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
 744                 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
 745                 if (!q_vector)
 746                         goto err_out;
 747                 q_vector->adapter = adapter;
 748                 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
 749                 q_vector->itr_val = IGB_START_ITR;
 750                 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
 751                 adapter->q_vector[v_idx] = q_vector;
 752         }
 753         return 0;
 754
 755 err_out:
 756         igb_free_q_vectors(adapter);
 757         return -ENOMEM;
 758 }
 759
 760 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
 761                                       int ring_idx, int v_idx)
 762 {
 763         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 764
 765         q_vector->rx_ring = adapter->rx_ring[ring_idx];
 766         q_vector->rx_ring->q_vector = q_vector;
 767         q_vector->itr_val = adapter->rx_itr_setting;
 768         if (q_vector->itr_val && q_vector->itr_val <= 3)
 769                 q_vector->itr_val = IGB_START_ITR;
 770 }
 771
 772 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
 773                                       int ring_idx, int v_idx)
 774 {
 775         struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
 776
 777         q_vector->tx_ring = adapter->tx_ring[ring_idx];
 778         q_vector->tx_ring->q_vector = q_vector;
 779         q_vector->itr_val = adapter->tx_itr_setting;
 780         if (q_vector->itr_val && q_vector->itr_val <= 3)
 781                 q_vector->itr_val = IGB_START_ITR;
 782 }
 783
 784 /**
 785  * igb_map_ring_to_vector - maps allocated queues to vectors
 786  *
 787  * This function maps the recently allocated queues to vectors.
 788  **/
 789 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
 790 {
 791         int i;
 792         int v_idx = 0;
 793
 794         if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
 795             (adapter->num_q_vectors < adapter->num_tx_queues))
 796                 return -ENOMEM;
 797
 798         if (adapter->num_q_vectors >=
 799             (adapter->num_rx_queues + adapter->num_tx_queues)) {
 800                 for (i = 0; i < adapter->num_rx_queues; i++)
 801                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 802                 for (i = 0; i < adapter->num_tx_queues; i++)
 803                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 804         } else {
 805                 for (i = 0; i < adapter->num_rx_queues; i++) {
 806                         if (i < adapter->num_tx_queues)
 807                                 igb_map_tx_ring_to_vector(adapter, i, v_idx);
 808                         igb_map_rx_ring_to_vector(adapter, i, v_idx++);
 809                 }
 810                 for (; i < adapter->num_tx_queues; i++)
 811                         igb_map_tx_ring_to_vector(adapter, i, v_idx++);
 812         }
 813         return 0;
 814 }
 815
 816 /**
 817  * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
 818  *
 819  * This function initializes the interrupts and allocates all of the queues.
 820  **/
 821 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
 822 {
 823         struct pci_dev *pdev = adapter->pdev;
 824         int err;
 825
 826         igb_set_interrupt_capability(adapter);
 827
 828         err = igb_alloc_q_vectors(adapter);
 829         if (err) {
 830                 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
 831                 goto err_alloc_q_vectors;
 832         }
 833
 834         err = igb_alloc_queues(adapter);
 835         if (err) {
 836                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
 837                 goto err_alloc_queues;
 838         }
 839
 840         err = igb_map_ring_to_vector(adapter);
 841         if (err) {
 842                 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
 843                 goto err_map_queues;
 844         }
 845
 846
 847         return 0;
 848 err_map_queues:
 849         igb_free_queues(adapter);
 850 err_alloc_queues:
 851         igb_free_q_vectors(adapter);
 852 err_alloc_q_vectors:
 853         igb_reset_interrupt_capability(adapter);
 854         return err;
 855 }
 856
 857 /**
 858  * igb_request_irq - initialize interrupts
 859  *
 860  * Attempts to configure interrupts using the best available
 861  * capabilities of the hardware and kernel.
 862  **/
 863 static int igb_request_irq(struct igb_adapter *adapter)
 864 {
 865         struct net_device *netdev = adapter->netdev;
 866         struct pci_dev *pdev = adapter->pdev;
 867         int err = 0;
 868
 869         if (adapter->msix_entries) {
 870                 err = igb_request_msix(adapter);
 871                 if (!err)
 872                         goto request_done;
 873                 /* fall back to MSI */
 874                 igb_clear_interrupt_scheme(adapter);
 875                 if (!pci_enable_msi(adapter->pdev))
 876                         adapter->flags |= IGB_FLAG_HAS_MSI;
 877                 igb_free_all_tx_resources(adapter);
 878                 igb_free_all_rx_resources(adapter);
 879                 adapter->num_tx_queues = 1;
 880                 adapter->num_rx_queues = 1;
 881                 adapter->num_q_vectors = 1;
 882                 err = igb_alloc_q_vectors(adapter);
 883                 if (err) {
 884                         dev_err(&pdev->dev,
 885                                 "Unable to allocate memory for vectors\n");
 886                         goto request_done;
 887                 }
 888                 err = igb_alloc_queues(adapter);
 889                 if (err) {
 890                         dev_err(&pdev->dev,
 891                                 "Unable to allocate memory for queues\n");
 892                         igb_free_q_vectors(adapter);
 893                         goto request_done;
 894                 }
 895                 igb_setup_all_tx_resources(adapter);
 896                 igb_setup_all_rx_resources(adapter);
 897         } else {
 898                 igb_assign_vector(adapter->q_vector[0], 0);
 899         }
 900
 901         if (adapter->flags & IGB_FLAG_HAS_MSI) {
 902                 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
 903                                   netdev->name, adapter);
 904                 if (!err)
 905                         goto request_done;
 906
 907                 /* fall back to legacy interrupts */
 908                 igb_reset_interrupt_capability(adapter);
 909                 adapter->flags &= ~IGB_FLAG_HAS_MSI;
 910         }
 911
 912         err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
 913                           netdev->name, adapter);
 914
 915         if (err)
 916                 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
 917                         err);
 918
 919 request_done:
 920         return err;
 921 }
 922
 923 static void igb_free_irq(struct igb_adapter *adapter)
 924 {
 925         if (adapter->msix_entries) {
 926                 int vector = 0, i;
 927
 928                 free_irq(adapter->msix_entries[vector++].vector, adapter);
 929
 930                 for (i = 0; i < adapter->num_q_vectors; i++) {
 931                         struct igb_q_vector *q_vector = adapter->q_vector[i];
 932                         free_irq(adapter->msix_entries[vector++].vector,
 933                                  q_vector);
 934                 }
 935         } else {
 936                 free_irq(adapter->pdev->irq, adapter);
 937         }
 938 }
 939
 940 /**
 941  * igb_irq_disable - Mask off interrupt generation on the NIC
 942  * @adapter: board private structure
 943  **/
 944 static void igb_irq_disable(struct igb_adapter *adapter)
 945 {
 946         struct e1000_hw *hw = &adapter->hw;
 947
 948         /*
 949          * we need to be careful when disabling interrupts.  The VFs are also
 950          * mapped into these registers and so clearing the bits can cause
 951          * issues on the VF drivers so we only need to clear what we set
 952          */
 953         if (adapter->msix_entries) {
 954                 u32 regval = rd32(E1000_EIAM);
 955                 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
 956                 wr32(E1000_EIMC, adapter->eims_enable_mask);
 957                 regval = rd32(E1000_EIAC);
 958                 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
 959         }
 960
 961         wr32(E1000_IAM, 0);
 962         wr32(E1000_IMC, ~0);
 963         wrfl();
 964         synchronize_irq(adapter->pdev->irq);
 965 }
 966
 967 /**
 968  * igb_irq_enable - Enable default interrupt generation settings
 969  * @adapter: board private structure
 970  **/
 971 static void igb_irq_enable(struct igb_adapter *adapter)
 972 {
 973         struct e1000_hw *hw = &adapter->hw;
 974
 975         if (adapter->msix_entries) {
 976                 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
 977                 u32 regval = rd32(E1000_EIAC);
 978                 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
 979                 regval = rd32(E1000_EIAM);
 980                 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
 981                 wr32(E1000_EIMS, adapter->eims_enable_mask);
 982                 if (adapter->vfs_allocated_count) {
 983                         wr32(E1000_MBVFIMR, 0xFF);
 984                         ims |= E1000_IMS_VMMB;
 985                 }
 986                 if (adapter->hw.mac.type == e1000_82580)
 987                         ims |= E1000_IMS_DRSTA;
 988
 989                 wr32(E1000_IMS, ims);
 990         } else {
 991                 wr32(E1000_IMS, IMS_ENABLE_MASK |
 992                                 E1000_IMS_DRSTA);
 993                 wr32(E1000_IAM, IMS_ENABLE_MASK |
 994                                 E1000_IMS_DRSTA);
 995         }
 996 }
 997
 998 static void igb_update_mng_vlan(struct igb_adapter *adapter)
 999 {
1000         struct e1000_hw *hw = &adapter->hw;
1001         u16 vid = adapter->hw.mng_cookie.vlan_id;
1002         u16 old_vid = adapter->mng_vlan_id;
1003
1004         if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1005                 /* add VID to filter table */
1006                 igb_vfta_set(hw, vid, true);
1007                 adapter->mng_vlan_id = vid;
1008         } else {
1009                 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1010         }
1011
1012         if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1013             (vid != old_vid) &&
1014             !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1015                 /* remove VID from filter table */
1016                 igb_vfta_set(hw, old_vid, false);
1017         }
1018 }
1019
1020 /**
1021  * igb_release_hw_control - release control of the h/w to f/w
1022  * @adapter: address of board private structure
1023  *
1024  * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1025  * For ASF and Pass Through versions of f/w this means that the
1026  * driver is no longer loaded.
1027  *
1028  **/
1029 static void igb_release_hw_control(struct igb_adapter *adapter)
1030 {
1031         struct e1000_hw *hw = &adapter->hw;
1032         u32 ctrl_ext;
1033
1034         /* Let firmware take over control of h/w */
1035         ctrl_ext = rd32(E1000_CTRL_EXT);
1036         wr32(E1000_CTRL_EXT,
1037                         ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1038 }
1039
1040 /**
1041  * igb_get_hw_control - get control of the h/w from f/w
1042  * @adapter: address of board private structure
1043  *
1044  * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1045  * For ASF and Pass Through versions of f/w this means that
1046  * the driver is loaded.
1047  *
1048  **/
1049 static void igb_get_hw_control(struct igb_adapter *adapter)
1050 {
1051         struct e1000_hw *hw = &adapter->hw;
1052         u32 ctrl_ext;
1053
1054         /* Let firmware know the driver has taken over */
1055         ctrl_ext = rd32(E1000_CTRL_EXT);
1056         wr32(E1000_CTRL_EXT,
1057                         ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1058 }
1059
1060 /**
1061  * igb_configure - configure the hardware for RX and TX
1062  * @adapter: private board structure
1063  **/
1064 static void igb_configure(struct igb_adapter *adapter)
1065 {
1066         struct net_device *netdev = adapter->netdev;
1067         int i;
1068
1069         igb_get_hw_control(adapter);
1070         igb_set_rx_mode(netdev);
1071
1072         igb_restore_vlan(adapter);
1073
1074         igb_setup_tctl(adapter);
1075         igb_setup_mrqc(adapter);
1076         igb_setup_rctl(adapter);
1077
1078         igb_configure_tx(adapter);
1079         igb_configure_rx(adapter);
1080
1081         igb_rx_fifo_flush_82575(&adapter->hw);
1082
1083         /* call igb_desc_unused which always leaves
1084          * at least 1 descriptor unused to make sure
1085          * next_to_use != next_to_clean */
1086         for (i = 0; i < adapter->num_rx_queues; i++) {
1087                 struct igb_ring *ring = adapter->rx_ring[i];
1088                 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1089         }
1090 }
1091
1092 /**
1093  * igb_power_up_link - Power up the phy/serdes link
1094  * @adapter: address of board private structure
1095  **/
1096 void igb_power_up_link(struct igb_adapter *adapter)
1097 {
1098         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1099                 igb_power_up_phy_copper(&adapter->hw);
1100         else
1101                 igb_power_up_serdes_link_82575(&adapter->hw);
1102 }
1103
1104 /**
1105  * igb_power_down_link - Power down the phy/serdes link
1106  * @adapter: address of board private structure
1107  */
1108 static void igb_power_down_link(struct igb_adapter *adapter)
1109 {
1110         if (adapter->hw.phy.media_type == e1000_media_type_copper)
1111                 igb_power_down_phy_copper_82575(&adapter->hw);
1112         else
1113                 igb_shutdown_serdes_link_82575(&adapter->hw);
1114 }
1115
1116 /**
1117  * igb_up - Open the interface and prepare it to handle traffic
1118  * @adapter: board private structure
1119  **/
1120 int igb_up(struct igb_adapter *adapter)
1121 {
1122         struct e1000_hw *hw = &adapter->hw;
1123         int i;
1124
1125         /* hardware has been reset, we need to reload some things */
1126         igb_configure(adapter);
1127
1128         clear_bit(__IGB_DOWN, &adapter->state);
1129
1130         for (i = 0; i < adapter->num_q_vectors; i++) {
1131                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1132                 napi_enable(&q_vector->napi);
1133         }
1134         if (adapter->msix_entries)
1135                 igb_configure_msix(adapter);
1136         else
1137                 igb_assign_vector(adapter->q_vector[0], 0);
1138
1139         /* Clear any pending interrupts. */
1140         rd32(E1000_ICR);
1141         igb_irq_enable(adapter);
1142
1143         /* notify VFs that reset has been completed */
1144         if (adapter->vfs_allocated_count) {
1145                 u32 reg_data = rd32(E1000_CTRL_EXT);
1146                 reg_data |= E1000_CTRL_EXT_PFRSTD;
1147                 wr32(E1000_CTRL_EXT, reg_data);
1148         }
1149
1150         netif_tx_start_all_queues(adapter->netdev);
1151
1152         /* start the watchdog. */
1153         hw->mac.get_link_status = 1;
1154         schedule_work(&adapter->watchdog_task);
1155
1156         return 0;
1157 }
1158
1159 void igb_down(struct igb_adapter *adapter)
1160 {
1161         struct net_device *netdev = adapter->netdev;
1162         struct e1000_hw *hw = &adapter->hw;
1163         u32 tctl, rctl;
1164         int i;
1165
1166         /* signal that we're down so the interrupt handler does not
1167          * reschedule our watchdog timer */
1168         set_bit(__IGB_DOWN, &adapter->state);
1169
1170         /* disable receives in the hardware */
1171         rctl = rd32(E1000_RCTL);
1172         wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1173         /* flush and sleep below */
1174
1175         netif_tx_stop_all_queues(netdev);
1176
1177         /* disable transmits in the hardware */
1178         tctl = rd32(E1000_TCTL);
1179         tctl &= ~E1000_TCTL_EN;
1180         wr32(E1000_TCTL, tctl);
1181         /* flush both disables and wait for them to finish */
1182         wrfl();
1183         msleep(10);
1184
1185         for (i = 0; i < adapter->num_q_vectors; i++) {
1186                 struct igb_q_vector *q_vector = adapter->q_vector[i];
1187                 napi_disable(&q_vector->napi);
1188         }
1189
1190         igb_irq_disable(adapter);
1191
1192         del_timer_sync(&adapter->watchdog_timer);
1193         del_timer_sync(&adapter->phy_info_timer);
1194
1195         netif_carrier_off(netdev);
1196
1197         /* record the stats before reset*/
1198         igb_update_stats(adapter);
1199
1200         adapter->link_speed = 0;
1201         adapter->link_duplex = 0;
1202
1203         if (!pci_channel_offline(adapter->pdev))
1204                 igb_reset(adapter);
1205         igb_clean_all_tx_rings(adapter);
1206         igb_clean_all_rx_rings(adapter);
1207 #ifdef CONFIG_IGB_DCA
1208
1209         /* since we reset the hardware DCA settings were cleared */
1210         igb_setup_dca(adapter);
1211 #endif
1212 }
1213
1214 void igb_reinit_locked(struct igb_adapter *adapter)
1215 {
1216         WARN_ON(in_interrupt());
1217         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1218                 msleep(1);
1219         igb_down(adapter);
1220         igb_up(adapter);
1221         clear_bit(__IGB_RESETTING, &adapter->state);
1222 }
1223
1224 void igb_reset(struct igb_adapter *adapter)
1225 {
1226         struct pci_dev *pdev = adapter->pdev;
1227         struct e1000_hw *hw = &adapter->hw;
1228         struct e1000_mac_info *mac = &hw->mac;
1229         struct e1000_fc_info *fc = &hw->fc;
1230         u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1231         u16 hwm;
1232
1233         /* Repartition Pba for greater than 9k mtu
1234          * To take effect CTRL.RST is required.
1235          */
1236         switch (mac->type) {
1237         case e1000_i350:
1238         case e1000_82580:
1239                 pba = rd32(E1000_RXPBS);
1240                 pba = igb_rxpbs_adjust_82580(pba);
1241                 break;
1242         case e1000_82576:
1243                 pba = rd32(E1000_RXPBS);
1244                 pba &= E1000_RXPBS_SIZE_MASK_82576;
1245                 break;
1246         case e1000_82575:
1247         default:
1248                 pba = E1000_PBA_34K;
1249                 break;
1250         }
1251
1252         if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1253             (mac->type < e1000_82576)) {
1254                 /* adjust PBA for jumbo frames */
1255                 wr32(E1000_PBA, pba);
1256
1257                 /* To maintain wire speed transmits, the Tx FIFO should be
1258                  * large enough to accommodate two full transmit packets,
1259                  * rounded up to the next 1KB and expressed in KB.  Likewise,
1260                  * the Rx FIFO should be large enough to accommodate at least
1261                  * one full receive packet and is similarly rounded up and
1262                  * expressed in KB. */
1263                 pba = rd32(E1000_PBA);
1264                 /* upper 16 bits has Tx packet buffer allocation size in KB */
1265                 tx_space = pba >> 16;
1266                 /* lower 16 bits has Rx packet buffer allocation size in KB */
1267                 pba &= 0xffff;
1268                 /* the tx fifo also stores 16 bytes of information about the tx
1269                  * but don't include ethernet FCS because hardware appends it */
1270                 min_tx_space = (adapter->max_frame_size +
1271                                 sizeof(union e1000_adv_tx_desc) -
1272                                 ETH_FCS_LEN) * 2;
1273                 min_tx_space = ALIGN(min_tx_space, 1024);
1274                 min_tx_space >>= 10;
1275                 /* software strips receive CRC, so leave room for it */
1276                 min_rx_space = adapter->max_frame_size;
1277                 min_rx_space = ALIGN(min_rx_space, 1024);
1278                 min_rx_space >>= 10;
1279
1280                 /* If current Tx allocation is less than the min Tx FIFO size,
1281                  * and the min Tx FIFO size is less than the current Rx FIFO
1282                  * allocation, take space away from current Rx allocation */
1283                 if (tx_space < min_tx_space &&
1284                     ((min_tx_space - tx_space) < pba)) {
1285                         pba = pba - (min_tx_space - tx_space);
1286
1287                         /* if short on rx space, rx wins and must trump tx
1288                          * adjustment */
1289                         if (pba < min_rx_space)
1290                                 pba = min_rx_space;
1291                 }
1292                 wr32(E1000_PBA, pba);
1293         }
1294
1295         /* flow control settings */
1296         /* The high water mark must be low enough to fit one full frame
1297          * (or the size used for early receive) above it in the Rx FIFO.
1298          * Set it to the lower of:
1299          * - 90% of the Rx FIFO size, or
1300          * - the full Rx FIFO size minus one full frame */
1301         hwm = min(((pba << 10) * 9 / 10),
1302                         ((pba << 10) - 2 * adapter->max_frame_size));
1303
1304         fc->high_water = hwm & 0xFFF0;  /* 16-byte granularity */
1305         fc->low_water = fc->high_water - 16;
1306         fc->pause_time = 0xFFFF;
1307         fc->send_xon = 1;
1308         fc->current_mode = fc->requested_mode;
1309
1310         /* disable receive for all VFs and wait one second */
1311         if (adapter->vfs_allocated_count) {
1312                 int i;
1313                 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1314                         adapter->vf_data[i].flags = 0;
1315
1316                 /* ping all the active vfs to let them know we are going down */
1317                 igb_ping_all_vfs(adapter);
1318
1319                 /* disable transmits and receives */
1320                 wr32(E1000_VFRE, 0);
1321                 wr32(E1000_VFTE, 0);
1322         }
1323
1324         /* Allow time for pending master requests to run */
1325         hw->mac.ops.reset_hw(hw);
1326         wr32(E1000_WUC, 0);
1327
1328         if (hw->mac.ops.init_hw(hw))
1329                 dev_err(&pdev->dev, "Hardware Error\n");
1330
1331         if (hw->mac.type == e1000_82580) {
1332                 u32 reg = rd32(E1000_PCIEMISC);
1333                 wr32(E1000_PCIEMISC,
1334                                 reg & ~E1000_PCIEMISC_LX_DECISION);
1335         }
1336         if (!netif_running(adapter->netdev))
1337                 igb_power_down_link(adapter);
1338
1339         igb_update_mng_vlan(adapter);
1340
1341         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1342         wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1343
1344         igb_get_phy_info(hw);
1345 }
1346
1347 static const struct net_device_ops igb_netdev_ops = {
1348         .ndo_open               = igb_open,
1349         .ndo_stop               = igb_close,
1350         .ndo_start_xmit         = igb_xmit_frame_adv,
1351         .ndo_get_stats          = igb_get_stats,
1352         .ndo_set_rx_mode        = igb_set_rx_mode,
1353         .ndo_set_multicast_list = igb_set_rx_mode,
1354         .ndo_set_mac_address    = igb_set_mac,
1355         .ndo_change_mtu         = igb_change_mtu,
1356         .ndo_do_ioctl           = igb_ioctl,
1357         .ndo_tx_timeout         = igb_tx_timeout,
1358         .ndo_validate_addr      = eth_validate_addr,
1359         .ndo_vlan_rx_register   = igb_vlan_rx_register,
1360         .ndo_vlan_rx_add_vid    = igb_vlan_rx_add_vid,
1361         .ndo_vlan_rx_kill_vid   = igb_vlan_rx_kill_vid,
1362         .ndo_set_vf_mac         = igb_ndo_set_vf_mac,
1363         .ndo_set_vf_vlan        = igb_ndo_set_vf_vlan,
1364         .ndo_set_vf_tx_rate     = igb_ndo_set_vf_bw,
1365         .ndo_get_vf_config      = igb_ndo_get_vf_config,
1366 #ifdef CONFIG_NET_POLL_CONTROLLER
1367         .ndo_poll_controller    = igb_netpoll,
1368 #endif
1369 };
1370
1371 /**
1372  * igb_probe - Device Initialization Routine
1373  * @pdev: PCI device information struct
1374  * @ent: entry in igb_pci_tbl
1375  *
1376  * Returns 0 on success, negative on failure
1377  *
1378  * igb_probe initializes an adapter identified by a pci_dev structure.
1379  * The OS initialization, configuring of the adapter private structure,
1380  * and a hardware reset occur.
1381  **/
1382 static int __devinit igb_probe(struct pci_dev *pdev,
1383                                const struct pci_device_id *ent)
1384 {
1385         struct net_device *netdev;
1386         struct igb_adapter *adapter;
1387         struct e1000_hw *hw;
1388         u16 eeprom_data = 0;
1389         static int global_quad_port_a; /* global quad port a indication */
1390         const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1391         unsigned long mmio_start, mmio_len;
1392         int err, pci_using_dac;
1393         u16 eeprom_apme_mask = IGB_EEPROM_APME;
1394         u32 part_num;
1395
1396         err = pci_enable_device_mem(pdev);
1397         if (err)
1398                 return err;
1399
1400         pci_using_dac = 0;
1401         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1402         if (!err) {
1403                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1404                 if (!err)
1405                         pci_using_dac = 1;
1406         } else {
1407                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1408                 if (err) {
1409                         err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1410                         if (err) {
1411                                 dev_err(&pdev->dev, "No usable DMA "
1412                                         "configuration, aborting\n");
1413                                 goto err_dma;
1414                         }
1415                 }
1416         }
1417
1418         err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1419                                            IORESOURCE_MEM),
1420                                            igb_driver_name);
1421         if (err)
1422                 goto err_pci_reg;
1423
1424         pci_enable_pcie_error_reporting(pdev);
1425
1426         pci_set_master(pdev);
1427         pci_save_state(pdev);
1428
1429         err = -ENOMEM;
1430         netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1431                                    IGB_ABS_MAX_TX_QUEUES);
1432         if (!netdev)
1433                 goto err_alloc_etherdev;
1434
1435         SET_NETDEV_DEV(netdev, &pdev->dev);
1436
1437         pci_set_drvdata(pdev, netdev);
1438         adapter = netdev_priv(netdev);
1439         adapter->netdev = netdev;
1440         adapter->pdev = pdev;
1441         hw = &adapter->hw;
1442         hw->back = adapter;
1443         adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1444
1445         mmio_start = pci_resource_start(pdev, 0);
1446         mmio_len = pci_resource_len(pdev, 0);
1447
1448         err = -EIO;
1449         hw->hw_addr = ioremap(mmio_start, mmio_len);
1450         if (!hw->hw_addr)
1451                 goto err_ioremap;
1452
1453         netdev->netdev_ops = &igb_netdev_ops;
1454         igb_set_ethtool_ops(netdev);
1455         netdev->watchdog_timeo = 5 * HZ;
1456
1457         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1458
1459         netdev->mem_start = mmio_start;
1460         netdev->mem_end = mmio_start + mmio_len;
1461
1462         /* PCI config space info */
1463         hw->vendor_id = pdev->vendor;
1464         hw->device_id = pdev->device;
1465         hw->revision_id = pdev->revision;
1466         hw->subsystem_vendor_id = pdev->subsystem_vendor;
1467         hw->subsystem_device_id = pdev->subsystem_device;
1468
1469         /* Copy the default MAC, PHY and NVM function pointers */
1470         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1471         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1472         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1473         /* Initialize skew-specific constants */
1474         err = ei->get_invariants(hw);
1475         if (err)
1476                 goto err_sw_init;
1477
1478         /* setup the private structure */
1479         err = igb_sw_init(adapter);
1480         if (err)
1481                 goto err_sw_init;
1482
1483         igb_get_bus_info_pcie(hw);
1484
1485         hw->phy.autoneg_wait_to_complete = false;
1486
1487         /* Copper options */
1488         if (hw->phy.media_type == e1000_media_type_copper) {
1489                 hw->phy.mdix = AUTO_ALL_MODES;
1490                 hw->phy.disable_polarity_correction = false;
1491                 hw->phy.ms_type = e1000_ms_hw_default;
1492         }
1493
1494         if (igb_check_reset_block(hw))
1495                 dev_info(&pdev->dev,
1496                         "PHY reset is blocked due to SOL/IDER session.\n");
1497
1498         netdev->features = NETIF_F_SG |
1499                            NETIF_F_IP_CSUM |
1500                            NETIF_F_HW_VLAN_TX |
1501                            NETIF_F_HW_VLAN_RX |
1502                            NETIF_F_HW_VLAN_FILTER;
1503
1504         netdev->features |= NETIF_F_IPV6_CSUM;
1505         netdev->features |= NETIF_F_TSO;
1506         netdev->features |= NETIF_F_TSO6;
1507         netdev->features |= NETIF_F_GRO;
1508
1509         netdev->vlan_features |= NETIF_F_TSO;
1510         netdev->vlan_features |= NETIF_F_TSO6;
1511         netdev->vlan_features |= NETIF_F_IP_CSUM;
1512         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1513         netdev->vlan_features |= NETIF_F_SG;
1514
1515         if (pci_using_dac)
1516                 netdev->features |= NETIF_F_HIGHDMA;
1517
1518         if (hw->mac.type >= e1000_82576)
1519                 netdev->features |= NETIF_F_SCTP_CSUM;
1520
1521         adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1522
1523         /* before reading the NVM, reset the controller to put the device in a
1524          * known good starting state */
1525         hw->mac.ops.reset_hw(hw);
1526
1527         /* make sure the NVM is good */
1528         if (igb_validate_nvm_checksum(hw) < 0) {
1529                 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1530                 err = -EIO;
1531                 goto err_eeprom;
1532         }
1533
1534         /* copy the MAC address out of the NVM */
1535         if (hw->mac.ops.read_mac_addr(hw))
1536                 dev_err(&pdev->dev, "NVM Read Error\n");
1537
1538         memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1539         memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1540
1541         if (!is_valid_ether_addr(netdev->perm_addr)) {
1542                 dev_err(&pdev->dev, "Invalid MAC Address\n");
1543                 err = -EIO;
1544                 goto err_eeprom;
1545         }
1546
1547         setup_timer(&adapter->watchdog_timer, &igb_watchdog,
1548                     (unsigned long) adapter);
1549         setup_timer(&adapter->phy_info_timer, &igb_update_phy_info,
1550                     (unsigned long) adapter);
1551
1552         INIT_WORK(&adapter->reset_task, igb_reset_task);
1553         INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1554
1555         /* Initialize link properties that are user-changeable */
1556         adapter->fc_autoneg = true;
1557         hw->mac.autoneg = true;
1558         hw->phy.autoneg_advertised = 0x2f;
1559
1560         hw->fc.requested_mode = e1000_fc_default;
1561         hw->fc.current_mode = e1000_fc_default;
1562
1563         igb_validate_mdi_setting(hw);
1564
1565         /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1566          * enable the ACPI Magic Packet filter
1567          */
1568
1569         if (hw->bus.func == 0)
1570                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1571         else if (hw->mac.type == e1000_82580)
1572                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1573                                  NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1574                                  &eeprom_data);
1575         else if (hw->bus.func == 1)
1576                 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1577
1578         if (eeprom_data & eeprom_apme_mask)
1579                 adapter->eeprom_wol |= E1000_WUFC_MAG;
1580
1581         /* now that we have the eeprom settings, apply the special cases where
1582          * the eeprom may be wrong or the board simply won't support wake on
1583          * lan on a particular port */
1584         switch (pdev->device) {
1585         case E1000_DEV_ID_82575GB_QUAD_COPPER:
1586                 adapter->eeprom_wol = 0;
1587                 break;
1588         case E1000_DEV_ID_82575EB_FIBER_SERDES:
1589         case E1000_DEV_ID_82576_FIBER:
1590         case E1000_DEV_ID_82576_SERDES:
1591                 /* Wake events only supported on port A for dual fiber
1592                  * regardless of eeprom setting */
1593                 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1594                         adapter->eeprom_wol = 0;
1595                 break;
1596         case E1000_DEV_ID_82576_QUAD_COPPER:
1597         case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1598                 /* if quad port adapter, disable WoL on all but port A */
1599                 if (global_quad_port_a != 0)
1600                         adapter->eeprom_wol = 0;
1601                 else
1602                         adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1603                 /* Reset for multiple quad port adapters */
1604                 if (++global_quad_port_a == 4)
1605                         global_quad_port_a = 0;
1606                 break;
1607         }
1608
1609         /* initialize the wol settings based on the eeprom settings */
1610         adapter->wol = adapter->eeprom_wol;
1611         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1612
1613         /* reset the hardware with the new settings */
1614         igb_reset(adapter);
1615
1616         /* let the f/w know that the h/w is now under the control of the
1617          * driver. */
1618         igb_get_hw_control(adapter);
1619
1620         strcpy(netdev->name, "eth%d");
1621         err = register_netdev(netdev);
1622         if (err)
1623                 goto err_register;
1624
1625         /* carrier off reporting is important to ethtool even BEFORE open */
1626         netif_carrier_off(netdev);
1627
1628 #ifdef CONFIG_IGB_DCA
1629         if (dca_add_requester(&pdev->dev) == 0) {
1630                 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1631                 dev_info(&pdev->dev, "DCA enabled\n");
1632                 igb_setup_dca(adapter);
1633         }
1634
1635 #endif
1636         dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1637         /* print bus type/speed/width info */
1638         dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1639                  netdev->name,
1640                  ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1641                   (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1642                                                             "unknown"),
1643                  ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1644                   (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1645                   (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1646                    "unknown"),
1647                  netdev->dev_addr);
1648
1649         igb_read_part_num(hw, &part_num);
1650         dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1651                 (part_num >> 8), (part_num & 0xff));
1652
1653         dev_info(&pdev->dev,
1654                 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1655                 adapter->msix_entries ? "MSI-X" :
1656                 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
1657                 adapter->num_rx_queues, adapter->num_tx_queues);
1658
1659         return 0;
1660
1661 err_register:
1662         igb_release_hw_control(adapter);
1663 err_eeprom:
1664         if (!igb_check_reset_block(hw))
1665                 igb_reset_phy(hw);
1666
1667         if (hw->flash_address)
1668                 iounmap(hw->flash_address);
1669 err_sw_init:
1670         igb_clear_interrupt_scheme(adapter);
1671         iounmap(hw->hw_addr);
1672 err_ioremap:
1673         free_netdev(netdev);
1674 err_alloc_etherdev:
1675         pci_release_selected_regions(pdev,
1676                                      pci_select_bars(pdev, IORESOURCE_MEM));
1677 err_pci_reg:
1678 err_dma:
1679         pci_disable_device(pdev);
1680         return err;
1681 }
1682
1683 /**
1684  * igb_remove - Device Removal Routine
1685  * @pdev: PCI device information struct
1686  *
1687  * igb_remove is called by the PCI subsystem to alert the driver
1688  * that it should release a PCI device.  The could be caused by a
1689  * Hot-Plug event, or because the driver is going to be removed from
1690  * memory.
1691  **/
1692 static void __devexit igb_remove(struct pci_dev *pdev)
1693 {
1694         struct net_device *netdev = pci_get_drvdata(pdev);
1695         struct igb_adapter *adapter = netdev_priv(netdev);
1696         struct e1000_hw *hw = &adapter->hw;
1697
1698         /* flush_scheduled work may reschedule our watchdog task, so
1699          * explicitly disable watchdog tasks from being rescheduled  */
1700         set_bit(__IGB_DOWN, &adapter->state);
1701         del_timer_sync(&adapter->watchdog_timer);
1702         del_timer_sync(&adapter->phy_info_timer);
1703
1704         flush_scheduled_work();
1705
1706 #ifdef CONFIG_IGB_DCA
1707         if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
1708                 dev_info(&pdev->dev, "DCA disabled\n");
1709                 dca_remove_requester(&pdev->dev);
1710                 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
1711                 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
1712         }
1713 #endif
1714
1715         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
1716          * would have already happened in close and is redundant. */
1717         igb_release_hw_control(adapter);
1718
1719         unregister_netdev(netdev);
1720
1721         igb_clear_interrupt_scheme(adapter);
1722
1723 #ifdef CONFIG_PCI_IOV
1724         /* reclaim resources allocated to VFs */
1725         if (adapter->vf_data) {
1726                 /* disable iov and allow time for transactions to clear */
1727                 pci_disable_sriov(pdev);
1728                 msleep(500);
1729
1730                 kfree(adapter->vf_data);
1731                 adapter->vf_data = NULL;
1732                 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1733                 msleep(100);
1734                 dev_info(&pdev->dev, "IOV Disabled\n");
1735         }
1736 #endif
1737
1738         iounmap(hw->hw_addr);
1739         if (hw->flash_address)
1740                 iounmap(hw->flash_address);
1741         pci_release_selected_regions(pdev,
1742                                      pci_select_bars(pdev, IORESOURCE_MEM));
1743
1744         free_netdev(netdev);
1745
1746         pci_disable_pcie_error_reporting(pdev);
1747
1748         pci_disable_device(pdev);
1749 }
1750
1751 /**
1752  * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
1753  * @adapter: board private structure to initialize
1754  *
1755  * This function initializes the vf specific data storage and then attempts to
1756  * allocate the VFs.  The reason for ordering it this way is because it is much
1757  * mor expensive time wise to disable SR-IOV than it is to allocate and free
1758  * the memory for the VFs.
1759  **/
1760 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
1761 {
1762 #ifdef CONFIG_PCI_IOV
1763         struct pci_dev *pdev = adapter->pdev;
1764
1765         if (adapter->vfs_allocated_count > 7)
1766                 adapter->vfs_allocated_count = 7;
1767
1768         if (adapter->vfs_allocated_count) {
1769                 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
1770                                            sizeof(struct vf_data_storage),
1771                                            GFP_KERNEL);
1772                 /* if allocation failed then we do not support SR-IOV */
1773                 if (!adapter->vf_data) {
1774                         adapter->vfs_allocated_count = 0;
1775                         dev_err(&pdev->dev, "Unable to allocate memory for VF "
1776                                 "Data Storage\n");
1777                 }
1778         }
1779
1780         if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
1781                 kfree(adapter->vf_data);
1782                 adapter->vf_data = NULL;
1783 #endif /* CONFIG_PCI_IOV */
1784                 adapter->vfs_allocated_count = 0;
1785 #ifdef CONFIG_PCI_IOV
1786         } else {
1787                 unsigned char mac_addr[ETH_ALEN];
1788                 int i;
1789                 dev_info(&pdev->dev, "%d vfs allocated\n",
1790                          adapter->vfs_allocated_count);
1791                 for (i = 0; i < adapter->vfs_allocated_count; i++) {
1792                         random_ether_addr(mac_addr);
1793                         igb_set_vf_mac(adapter, i, mac_addr);
1794                 }
1795         }
1796 #endif /* CONFIG_PCI_IOV */
1797 }
1798
1799
1800 /**
1801  * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
1802  * @adapter: board private structure to initialize
1803  *
1804  * igb_init_hw_timer initializes the function pointer and values for the hw
1805  * timer found in hardware.
1806  **/
1807 static void igb_init_hw_timer(struct igb_adapter *adapter)
1808 {
1809         struct e1000_hw *hw = &adapter->hw;
1810
1811         switch (hw->mac.type) {
1812         case e1000_i350:
1813         case e1000_82580:
1814                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1815                 adapter->cycles.read = igb_read_clock;
1816                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1817                 adapter->cycles.mult = 1;
1818                 /*
1819                  * The 82580 timesync updates the system timer every 8ns by 8ns
1820                  * and the value cannot be shifted.  Instead we need to shift
1821                  * the registers to generate a 64bit timer value.  As a result
1822                  * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
1823                  * 24 in order to generate a larger value for synchronization.
1824                  */
1825                 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
1826                 /* disable system timer temporarily by setting bit 31 */
1827                 wr32(E1000_TSAUXC, 0x80000000);
1828                 wrfl();
1829
1830                 /* Set registers so that rollover occurs soon to test this. */
1831                 wr32(E1000_SYSTIMR, 0x00000000);
1832                 wr32(E1000_SYSTIML, 0x80000000);
1833                 wr32(E1000_SYSTIMH, 0x000000FF);
1834                 wrfl();
1835
1836                 /* enable system timer by clearing bit 31 */
1837                 wr32(E1000_TSAUXC, 0x0);
1838                 wrfl();
1839
1840                 timecounter_init(&adapter->clock,
1841                                  &adapter->cycles,
1842                                  ktime_to_ns(ktime_get_real()));
1843                 /*
1844                  * Synchronize our NIC clock against system wall clock. NIC
1845                  * time stamp reading requires ~3us per sample, each sample
1846                  * was pretty stable even under load => only require 10
1847                  * samples for each offset comparison.
1848                  */
1849                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1850                 adapter->compare.source = &adapter->clock;
1851                 adapter->compare.target = ktime_get_real;
1852                 adapter->compare.num_samples = 10;
1853                 timecompare_update(&adapter->compare, 0);
1854                 break;
1855         case e1000_82576:
1856                 /*
1857                  * Initialize hardware timer: we keep it running just in case
1858                  * that some program needs it later on.
1859                  */
1860                 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
1861                 adapter->cycles.read = igb_read_clock;
1862                 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
1863                 adapter->cycles.mult = 1;
1864                 /**
1865                  * Scale the NIC clock cycle by a large factor so that
1866                  * relatively small clock corrections can be added or
1867                  * substracted at each clock tick. The drawbacks of a large
1868                  * factor are a) that the clock register overflows more quickly
1869                  * (not such a big deal) and b) that the increment per tick has
1870                  * to fit into 24 bits.  As a result we need to use a shift of
1871                  * 19 so we can fit a value of 16 into the TIMINCA register.
1872                  */
1873                 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
1874                 wr32(E1000_TIMINCA,
1875                                 (1 << E1000_TIMINCA_16NS_SHIFT) |
1876                                 (16 << IGB_82576_TSYNC_SHIFT));
1877
1878                 /* Set registers so that rollover occurs soon to test this. */
1879                 wr32(E1000_SYSTIML, 0x00000000);
1880                 wr32(E1000_SYSTIMH, 0xFF800000);
1881                 wrfl();
1882
1883                 timecounter_init(&adapter->clock,
1884                                  &adapter->cycles,
1885                                  ktime_to_ns(ktime_get_real()));
1886                 /*
1887                  * Synchronize our NIC clock against system wall clock. NIC
1888                  * time stamp reading requires ~3us per sample, each sample
1889                  * was pretty stable even under load => only require 10
1890                  * samples for each offset comparison.
1891                  */
1892                 memset(&adapter->compare, 0, sizeof(adapter->compare));
1893                 adapter->compare.source = &adapter->clock;
1894                 adapter->compare.target = ktime_get_real;
1895                 adapter->compare.num_samples = 10;
1896                 timecompare_update(&adapter->compare, 0);
1897                 break;
1898         case e1000_82575:
1899                 /* 82575 does not support timesync */
1900         default:
1901                 break;
1902         }
1903
1904 }
1905
1906 /**
1907  * igb_sw_init - Initialize general software structures (struct igb_adapter)
1908  * @adapter: board private structure to initialize
1909  *
1910  * igb_sw_init initializes the Adapter private data structure.
1911  * Fields are initialized based on PCI device information and
1912  * OS network device settings (MTU size).
1913  **/
1914 static int __devinit igb_sw_init(struct igb_adapter *adapter)
1915 {
1916         struct e1000_hw *hw = &adapter->hw;
1917         struct net_device *netdev = adapter->netdev;
1918         struct pci_dev *pdev = adapter->pdev;
1919
1920         pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
1921
1922         adapter->tx_ring_count = IGB_DEFAULT_TXD;
1923         adapter->rx_ring_count = IGB_DEFAULT_RXD;
1924         adapter->rx_itr_setting = IGB_DEFAULT_ITR;
1925         adapter->tx_itr_setting = IGB_DEFAULT_ITR;
1926
1927         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1928         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1929
1930 #ifdef CONFIG_PCI_IOV
1931         if (hw->mac.type == e1000_82576)
1932                 adapter->vfs_allocated_count = max_vfs;
1933
1934 #endif /* CONFIG_PCI_IOV */
1935         adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
1936
1937         /*
1938          * if rss_queues > 4 or vfs are going to be allocated with rss_queues
1939          * then we should combine the queues into a queue pair in order to
1940          * conserve interrupts due to limited supply
1941          */
1942         if ((adapter->rss_queues > 4) ||
1943             ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
1944                 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1945
1946         /* This call may decrease the number of queues */
1947         if (igb_init_interrupt_scheme(adapter)) {
1948                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1949                 return -ENOMEM;
1950         }
1951
1952         igb_init_hw_timer(adapter);
1953         igb_probe_vfs(adapter);
1954
1955         /* Explicitly disable IRQ since the NIC can be in any state. */
1956         igb_irq_disable(adapter);
1957
1958         set_bit(__IGB_DOWN, &adapter->state);
1959         return 0;
1960 }
1961
1962 /**
1963  * igb_open - Called when a network interface is made active
1964  * @netdev: network interface device structure
1965  *
1966  * Returns 0 on success, negative value on failure
1967  *
1968  * The open entry point is called when a network interface is made
1969  * active by the system (IFF_UP).  At this point all resources needed
1970  * for transmit and receive operations are allocated, the interrupt
1971  * handler is registered with the OS, the watchdog timer is started,
1972  * and the stack is notified that the interface is ready.
1973  **/
1974 static int igb_open(struct net_device *netdev)
1975 {
1976         struct igb_adapter *adapter = netdev_priv(netdev);
1977         struct e1000_hw *hw = &adapter->hw;
1978         int err;
1979         int i;
1980
1981         /* disallow open during test */
1982         if (test_bit(__IGB_TESTING, &adapter->state))
1983                 return -EBUSY;
1984
1985         netif_carrier_off(netdev);
1986
1987         /* allocate transmit descriptors */
1988         err = igb_setup_all_tx_resources(adapter);
1989         if (err)
1990                 goto err_setup_tx;
1991
1992         /* allocate receive descriptors */
1993         err = igb_setup_all_rx_resources(adapter);
1994         if (err)
1995                 goto err_setup_rx;
1996
1997         igb_power_up_link(adapter);
1998
1999         /* before we allocate an interrupt, we must be ready to handle it.
2000          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2001          * as soon as we call pci_request_irq, so we have to setup our
2002          * clean_rx handler before we do so.  */
2003         igb_configure(adapter);
2004
2005         err = igb_request_irq(adapter);
2006         if (err)
2007                 goto err_req_irq;
2008
2009         /* From here on the code is the same as igb_up() */
2010         clear_bit(__IGB_DOWN, &adapter->state);
2011
2012         for (i = 0; i < adapter->num_q_vectors; i++) {
2013                 struct igb_q_vector *q_vector = adapter->q_vector[i];
2014                 napi_enable(&q_vector->napi);
2015         }
2016
2017         /* Clear any pending interrupts. */
2018         rd32(E1000_ICR);
2019
2020         igb_irq_enable(adapter);
2021
2022         /* notify VFs that reset has been completed */
2023         if (adapter->vfs_allocated_count) {
2024                 u32 reg_data = rd32(E1000_CTRL_EXT);
2025                 reg_data |= E1000_CTRL_EXT_PFRSTD;
2026                 wr32(E1000_CTRL_EXT, reg_data);
2027         }
2028
2029         netif_tx_start_all_queues(netdev);
2030
2031         /* start the watchdog. */
2032         hw->mac.get_link_status = 1;
2033         schedule_work(&adapter->watchdog_task);
2034
2035         return 0;
2036
2037 err_req_irq:
2038         igb_release_hw_control(adapter);
2039         igb_power_down_link(adapter);
2040         igb_free_all_rx_resources(adapter);
2041 err_setup_rx:
2042         igb_free_all_tx_resources(adapter);
2043 err_setup_tx:
2044         igb_reset(adapter);
2045
2046         return err;
2047 }
2048
2049 /**
2050  * igb_close - Disables a network interface
2051  * @netdev: network interface device structure
2052  *
2053  * Returns 0, this is not allowed to fail
2054  *
2055  * The close entry point is called when an interface is de-activated
2056  * by the OS.  The hardware is still under the driver's control, but
2057  * needs to be disabled.  A global MAC reset is issued to stop the
2058  * hardware, and all transmit and receive resources are freed.
2059  **/
2060 static int igb_close(struct net_device *netdev)
2061 {
2062         struct igb_adapter *adapter = netdev_priv(netdev);
2063
2064         WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2065         igb_down(adapter);
2066
2067         igb_free_irq(adapter);
2068
2069         igb_free_all_tx_resources(adapter);
2070         igb_free_all_rx_resources(adapter);
2071
2072         return 0;
2073 }
2074
2075 /**
2076  * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2077  * @tx_ring: tx descriptor ring (for a specific queue) to setup
2078  *
2079  * Return 0 on success, negative on failure
2080  **/
2081 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2082 {
2083         struct device *dev = tx_ring->dev;
2084         int size;
2085
2086         size = sizeof(struct igb_buffer) * tx_ring->count;
2087         tx_ring->buffer_info = vmalloc(size);
2088         if (!tx_ring->buffer_info)
2089                 goto err;
2090         memset(tx_ring->buffer_info, 0, size);
2091
2092         /* round up to nearest 4K */
2093         tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2094         tx_ring->size = ALIGN(tx_ring->size, 4096);
2095
2096         tx_ring->desc = dma_alloc_coherent(dev,
2097                                            tx_ring->size,
2098                                            &tx_ring->dma,
2099                                            GFP_KERNEL);
2100
2101         if (!tx_ring->desc)
2102                 goto err;
2103
2104         tx_ring->next_to_use = 0;
2105         tx_ring->next_to_clean = 0;
2106         return 0;
2107
2108 err:
2109         vfree(tx_ring->buffer_info);
2110         dev_err(dev,
2111                 "Unable to allocate memory for the transmit descriptor ring\n");
2112         return -ENOMEM;
2113 }
2114
2115 /**
2116  * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2117  *                                (Descriptors) for all queues
2118  * @adapter: board private structure
2119  *
2120  * Return 0 on success, negative on failure
2121  **/
2122 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2123 {
2124         struct pci_dev *pdev = adapter->pdev;
2125         int i, err = 0;
2126
2127         for (i = 0; i < adapter->num_tx_queues; i++) {
2128                 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2129                 if (err) {
2130                         dev_err(&pdev->dev,
2131                                 "Allocation for Tx Queue %u failed\n", i);
2132                         for (i--; i >= 0; i--)
2133                                 igb_free_tx_resources(adapter->tx_ring[i]);
2134                         break;
2135                 }
2136         }
2137
2138         for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2139                 int r_idx = i % adapter->num_tx_queues;
2140                 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2141         }
2142         return err;
2143 }
2144
2145 /**
2146  * igb_setup_tctl - configure the transmit control registers
2147  * @adapter: Board private structure
2148  **/
2149 void igb_setup_tctl(struct igb_adapter *adapter)
2150 {
2151         struct e1000_hw *hw = &adapter->hw;
2152         u32 tctl;
2153
2154         /* disable queue 0 which is enabled by default on 82575 and 82576 */
2155         wr32(E1000_TXDCTL(0), 0);
2156
2157         /* Program the Transmit Control Register */
2158         tctl = rd32(E1000_TCTL);
2159         tctl &= ~E1000_TCTL_CT;
2160         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2161                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2162
2163         igb_config_collision_dist(hw);
2164
2165         /* Enable transmits */
2166         tctl |= E1000_TCTL_EN;
2167
2168         wr32(E1000_TCTL, tctl);
2169 }
2170
2171 /**
2172  * igb_configure_tx_ring - Configure transmit ring after Reset
2173  * @adapter: board private structure
2174  * @ring: tx ring to configure
2175  *
2176  * Configure a transmit ring after a reset.
2177  **/
2178 void igb_configure_tx_ring(struct igb_adapter *adapter,
2179                            struct igb_ring *ring)
2180 {
2181         struct e1000_hw *hw = &adapter->hw;
2182         u32 txdctl;
2183         u64 tdba = ring->dma;
2184         int reg_idx = ring->reg_idx;
2185
2186         /* disable the queue */
2187         txdctl = rd32(E1000_TXDCTL(reg_idx));
2188         wr32(E1000_TXDCTL(reg_idx),
2189                         txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2190         wrfl();
2191         mdelay(10);
2192
2193         wr32(E1000_TDLEN(reg_idx),
2194                         ring->count * sizeof(union e1000_adv_tx_desc));
2195         wr32(E1000_TDBAL(reg_idx),
2196                         tdba & 0x00000000ffffffffULL);
2197         wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2198
2199         ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2200         ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2201         writel(0, ring->head);
2202         writel(0, ring->tail);
2203
2204         txdctl |= IGB_TX_PTHRESH;
2205         txdctl |= IGB_TX_HTHRESH << 8;
2206         txdctl |= IGB_TX_WTHRESH << 16;
2207
2208         txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2209         wr32(E1000_TXDCTL(reg_idx), txdctl);
2210 }
2211
2212 /**
2213  * igb_configure_tx - Configure transmit Unit after Reset
2214  * @adapter: board private structure
2215  *
2216  * Configure the Tx unit of the MAC after a reset.
2217  **/
2218 static void igb_configure_tx(struct igb_adapter *adapter)
2219 {
2220         int i;
2221
2222         for (i = 0; i < adapter->num_tx_queues; i++)
2223                 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2224 }
2225
2226 /**
2227  * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2228  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2229  *
2230  * Returns 0 on success, negative on failure
2231  **/
2232 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2233 {
2234         struct device *dev = rx_ring->dev;
2235         int size, desc_len;
2236
2237         size = sizeof(struct igb_buffer) * rx_ring->count;
2238         rx_ring->buffer_info = vmalloc(size);
2239         if (!rx_ring->buffer_info)
2240                 goto err;
2241         memset(rx_ring->buffer_info, 0, size);
2242
2243         desc_len = sizeof(union e1000_adv_rx_desc);
2244
2245         /* Round up to nearest 4K */
2246         rx_ring->size = rx_ring->count * desc_len;
2247         rx_ring->size = ALIGN(rx_ring->size, 4096);
2248
2249         rx_ring->desc = dma_alloc_coherent(dev,
2250                                            rx_ring->size,
2251                                            &rx_ring->dma,
2252                                            GFP_KERNEL);
2253
2254         if (!rx_ring->desc)
2255                 goto err;
2256
2257         rx_ring->next_to_clean = 0;
2258         rx_ring->next_to_use = 0;
2259
2260         return 0;
2261
2262 err:
2263         vfree(rx_ring->buffer_info);
2264         rx_ring->buffer_info = NULL;
2265         dev_err(dev, "Unable to allocate memory for the receive descriptor"
2266                 " ring\n");
2267         return -ENOMEM;
2268 }
2269
2270 /**
2271  * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2272  *                                (Descriptors) for all queues
2273  * @adapter: board private structure
2274  *
2275  * Return 0 on success, negative on failure
2276  **/
2277 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2278 {
2279         struct pci_dev *pdev = adapter->pdev;
2280         int i, err = 0;
2281
2282         for (i = 0; i < adapter->num_rx_queues; i++) {
2283                 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2284                 if (err) {
2285                         dev_err(&pdev->dev,
2286                                 "Allocation for Rx Queue %u failed\n", i);
2287                         for (i--; i >= 0; i--)
2288                                 igb_free_rx_resources(adapter->rx_ring[i]);
2289                         break;
2290                 }
2291         }
2292
2293         return err;
2294 }
2295
2296 /**
2297  * igb_setup_mrqc - configure the multiple receive queue control registers
2298  * @adapter: Board private structure
2299  **/
2300 static void igb_setup_mrqc(struct igb_adapter *adapter)
2301 {
2302         struct e1000_hw *hw = &adapter->hw;
2303         u32 mrqc, rxcsum;
2304         u32 j, num_rx_queues, shift = 0, shift2 = 0;
2305         union e1000_reta {
2306                 u32 dword;
2307                 u8  bytes[4];
2308         } reta;
2309         static const u8 rsshash[40] = {
2310                 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2311                 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2312                 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2313                 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2314
2315         /* Fill out hash function seeds */
2316         for (j = 0; j < 10; j++) {
2317                 u32 rsskey = rsshash[(j * 4)];
2318                 rsskey |= rsshash[(j * 4) + 1] << 8;
2319                 rsskey |= rsshash[(j * 4) + 2] << 16;
2320                 rsskey |= rsshash[(j * 4) + 3] << 24;
2321                 array_wr32(E1000_RSSRK(0), j, rsskey);
2322         }
2323
2324         num_rx_queues = adapter->rss_queues;
2325
2326         if (adapter->vfs_allocated_count) {
2327                 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2328                 switch (hw->mac.type) {
2329                 case e1000_i350:
2330                 case e1000_82580:
2331                         num_rx_queues = 1;
2332                         shift = 0;
2333                         break;
2334                 case e1000_82576:
2335                         shift = 3;
2336                         num_rx_queues = 2;
2337                         break;
2338                 case e1000_82575:
2339                         shift = 2;
2340                         shift2 = 6;
2341                 default:
2342                         break;
2343                 }
2344         } else {
2345                 if (hw->mac.type == e1000_82575)
2346                         shift = 6;
2347         }
2348
2349         for (j = 0; j < (32 * 4); j++) {
2350                 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2351                 if (shift2)
2352                         reta.bytes[j & 3] |= num_rx_queues << shift2;
2353                 if ((j & 3) == 3)
2354                         wr32(E1000_RETA(j >> 2), reta.dword);
2355         }
2356
2357         /*
2358          * Disable raw packet checksumming so that RSS hash is placed in
2359          * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
2360          * offloads as they are enabled by default
2361          */
2362         rxcsum = rd32(E1000_RXCSUM);
2363         rxcsum |= E1000_RXCSUM_PCSD;
2364
2365         if (adapter->hw.mac.type >= e1000_82576)
2366                 /* Enable Receive Checksum Offload for SCTP */
2367                 rxcsum |= E1000_RXCSUM_CRCOFL;
2368
2369         /* Don't need to set TUOFL or IPOFL, they default to 1 */
2370         wr32(E1000_RXCSUM, rxcsum);
2371
2372         /* If VMDq is enabled then we set the appropriate mode for that, else
2373          * we default to RSS so that an RSS hash is calculated per packet even
2374          * if we are only using one queue */
2375         if (adapter->vfs_allocated_count) {
2376                 if (hw->mac.type > e1000_82575) {
2377                         /* Set the default pool for the PF's first queue */
2378                         u32 vtctl = rd32(E1000_VT_CTL);
2379                         vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2380                                    E1000_VT_CTL_DISABLE_DEF_POOL);
2381                         vtctl |= adapter->vfs_allocated_count <<
2382                                 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2383                         wr32(E1000_VT_CTL, vtctl);
2384                 }
2385                 if (adapter->rss_queues > 1)
2386                         mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2387                 else
2388                         mrqc = E1000_MRQC_ENABLE_VMDQ;
2389         } else {
2390                 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2391         }
2392         igb_vmm_control(adapter);
2393
2394         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 |
2395                  E1000_MRQC_RSS_FIELD_IPV4_TCP);
2396         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 |
2397                  E1000_MRQC_RSS_FIELD_IPV6_TCP);
2398         mrqc |= (E1000_MRQC_RSS_FIELD_IPV4_UDP |
2399                  E1000_MRQC_RSS_FIELD_IPV6_UDP);
2400         mrqc |= (E1000_MRQC_RSS_FIELD_IPV6_UDP_EX |
2401                  E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
2402
2403         wr32(E1000_MRQC, mrqc);
2404 }
2405
2406 /**
2407  * igb_setup_rctl - configure the receive control registers
2408  * @adapter: Board private structure
2409  **/
2410 void igb_setup_rctl(struct igb_adapter *adapter)
2411 {
2412         struct e1000_hw *hw = &adapter->hw;
2413         u32 rctl;
2414
2415         rctl = rd32(E1000_RCTL);
2416
2417         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2418         rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2419
2420         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2421                 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2422
2423         /*
2424          * enable stripping of CRC. It's unlikely this will break BMC
2425          * redirection as it did with e1000. Newer features require
2426          * that the HW strips the CRC.
2427          */
2428         rctl |= E1000_RCTL_SECRC;
2429
2430         /* disable store bad packets and clear size bits. */
2431         rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2432
2433         /* enable LPE to prevent packets larger than max_frame_size */
2434         rctl |= E1000_RCTL_LPE;
2435
2436         /* disable queue 0 to prevent tail write w/o re-config */
2437         wr32(E1000_RXDCTL(0), 0);
2438
2439         /* Attention!!!  For SR-IOV PF driver operations you must enable
2440          * queue drop for all VF and PF queues to prevent head of line blocking
2441          * if an un-trusted VF does not provide descriptors to hardware.
2442          */
2443         if (adapter->vfs_allocated_count) {
2444                 /* set all queue drop enable bits */
2445                 wr32(E1000_QDE, ALL_QUEUES);
2446         }
2447
2448         wr32(E1000_RCTL, rctl);
2449 }
2450
2451 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2452                                    int vfn)
2453 {
2454         struct e1000_hw *hw = &adapter->hw;
2455         u32 vmolr;
2456
2457         /* if it isn't the PF check to see if VFs are enabled and
2458          * increase the size to support vlan tags */
2459         if (vfn < adapter->vfs_allocated_count &&
2460             adapter->vf_data[vfn].vlans_enabled)
2461                 size += VLAN_TAG_SIZE;
2462
2463         vmolr = rd32(E1000_VMOLR(vfn));
2464         vmolr &= ~E1000_VMOLR_RLPML_MASK;
2465         vmolr |= size | E1000_VMOLR_LPE;
2466         wr32(E1000_VMOLR(vfn), vmolr);
2467
2468         return 0;
2469 }
2470
2471 /**
2472  * igb_rlpml_set - set maximum receive packet size
2473  * @adapter: board private structure
2474  *
2475  * Configure maximum receivable packet size.
2476  **/
2477 static void igb_rlpml_set(struct igb_adapter *adapter)
2478 {
2479         u32 max_frame_size = adapter->max_frame_size;
2480         struct e1000_hw *hw = &adapter->hw;
2481         u16 pf_id = adapter->vfs_allocated_count;
2482
2483         if (adapter->vlgrp)
2484                 max_frame_size += VLAN_TAG_SIZE;
2485
2486         /* if vfs are enabled we set RLPML to the largest possible request
2487          * size and set the VMOLR RLPML to the size we need */
2488         if (pf_id) {
2489                 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2490                 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2491         }
2492
2493         wr32(E1000_RLPML, max_frame_size);
2494 }
2495
2496 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2497                                  int vfn, bool aupe)
2498 {
2499         struct e1000_hw *hw = &adapter->hw;
2500         u32 vmolr;
2501
2502         /*
2503          * This register exists only on 82576 and newer so if we are older then
2504          * we should exit and do nothing
2505          */
2506         if (hw->mac.type < e1000_82576)
2507                 return;
2508
2509         vmolr = rd32(E1000_VMOLR(vfn));
2510         vmolr |= E1000_VMOLR_STRVLAN;      /* Strip vlan tags */
2511         if (aupe)
2512                 vmolr |= E1000_VMOLR_AUPE;        /* Accept untagged packets */
2513         else
2514                 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2515
2516         /* clear all bits that might not be set */
2517         vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2518
2519         if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2520                 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2521         /*
2522          * for VMDq only allow the VFs and pool 0 to accept broadcast and
2523          * multicast packets
2524          */
2525         if (vfn <= adapter->vfs_allocated_count)
2526                 vmolr |= E1000_VMOLR_BAM;          /* Accept broadcast */
2527
2528         wr32(E1000_VMOLR(vfn), vmolr);
2529 }
2530
2531 /**
2532  * igb_configure_rx_ring - Configure a receive ring after Reset
2533  * @adapter: board private structure
2534  * @ring: receive ring to be configured
2535  *
2536  * Configure the Rx unit of the MAC after a reset.
2537  **/
2538 void igb_configure_rx_ring(struct igb_adapter *adapter,
2539                            struct igb_ring *ring)
2540 {
2541         struct e1000_hw *hw = &adapter->hw;
2542         u64 rdba = ring->dma;
2543         int reg_idx = ring->reg_idx;
2544         u32 srrctl, rxdctl;
2545
2546         /* disable the queue */
2547         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2548         wr32(E1000_RXDCTL(reg_idx),
2549                         rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2550
2551         /* Set DMA base address registers */
2552         wr32(E1000_RDBAL(reg_idx),
2553              rdba & 0x00000000ffffffffULL);
2554         wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2555         wr32(E1000_RDLEN(reg_idx),
2556                        ring->count * sizeof(union e1000_adv_rx_desc));
2557
2558         /* initialize head and tail */
2559         ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2560         ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2561         writel(0, ring->head);
2562         writel(0, ring->tail);
2563
2564         /* set descriptor configuration */
2565         if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2566                 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2567                          E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2568 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2569                 srrctl |= IGB_RXBUFFER_16384 >>
2570                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2571 #else
2572                 srrctl |= (PAGE_SIZE / 2) >>
2573                           E1000_SRRCTL_BSIZEPKT_SHIFT;
2574 #endif
2575                 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2576         } else {
2577                 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2578                          E1000_SRRCTL_BSIZEPKT_SHIFT;
2579                 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2580         }
2581         if (hw->mac.type == e1000_82580)
2582                 srrctl |= E1000_SRRCTL_TIMESTAMP;
2583         /* Only set Drop Enable if we are supporting multiple queues */
2584         if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2585                 srrctl |= E1000_SRRCTL_DROP_EN;
2586
2587         wr32(E1000_SRRCTL(reg_idx), srrctl);
2588
2589         /* set filtering for VMDQ pools */
2590         igb_set_vmolr(adapter, reg_idx & 0x7, true);
2591
2592         /* enable receive descriptor fetching */
2593         rxdctl = rd32(E1000_RXDCTL(reg_idx));
2594         rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2595         rxdctl &= 0xFFF00000;
2596         rxdctl |= IGB_RX_PTHRESH;
2597         rxdctl |= IGB_RX_HTHRESH << 8;
2598         rxdctl |= IGB_RX_WTHRESH << 16;
2599         wr32(E1000_RXDCTL(reg_idx), rxdctl);
2600 }
2601
2602 /**
2603  * igb_configure_rx - Configure receive Unit after Reset
2604  * @adapter: board private structure
2605  *
2606  * Configure the Rx unit of the MAC after a reset.
2607  **/
2608 static void igb_configure_rx(struct igb_adapter *adapter)
2609 {
2610         int i;
2611
2612         /* set UTA to appropriate mode */
2613         igb_set_uta(adapter);
2614
2615         /* set the correct pool for the PF default MAC address in entry 0 */
2616         igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2617                          adapter->vfs_allocated_count);
2618
2619         /* Setup the HW Rx Head and Tail Descriptor Pointers and
2620          * the Base and Length of the Rx Descriptor Ring */
2621         for (i = 0; i < adapter->num_rx_queues; i++)
2622                 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2623 }
2624
2625 /**
2626  * igb_free_tx_resources - Free Tx Resources per Queue
2627  * @tx_ring: Tx descriptor ring for a specific queue
2628  *
2629  * Free all transmit software resources
2630  **/
2631 void igb_free_tx_resources(struct igb_ring *tx_ring)
2632 {
2633         igb_clean_tx_ring(tx_ring);
2634
2635         vfree(tx_ring->buffer_info);
2636         tx_ring->buffer_info = NULL;
2637
2638         /* if not set, then don't free */
2639         if (!tx_ring->desc)
2640                 return;
2641
2642         dma_free_coherent(tx_ring->dev, tx_ring->size,
2643                           tx_ring->desc, tx_ring->dma);
2644
2645         tx_ring->desc = NULL;
2646 }
2647
2648 /**
2649  * igb_free_all_tx_resources - Free Tx Resources for All Queues
2650  * @adapter: board private structure
2651  *
2652  * Free all transmit software resources
2653  **/
2654 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
2655 {
2656         int i;
2657
2658         for (i = 0; i < adapter->num_tx_queues; i++)
2659                 igb_free_tx_resources(adapter->tx_ring[i]);
2660 }
2661
2662 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
2663                                     struct igb_buffer *buffer_info)
2664 {
2665         if (buffer_info->dma) {
2666                 if (buffer_info->mapped_as_page)
2667                         dma_unmap_page(tx_ring->dev,
2668                                         buffer_info->dma,
2669                                         buffer_info->length,
2670                                         DMA_TO_DEVICE);
2671                 else
2672                         dma_unmap_single(tx_ring->dev,
2673                                         buffer_info->dma,
2674                                         buffer_info->length,
2675                                         DMA_TO_DEVICE);
2676                 buffer_info->dma = 0;
2677         }
2678         if (buffer_info->skb) {
2679                 dev_kfree_skb_any(buffer_info->skb);
2680                 buffer_info->skb = NULL;
2681         }
2682         buffer_info->time_stamp = 0;
2683         buffer_info->length = 0;
2684         buffer_info->next_to_watch = 0;
2685         buffer_info->mapped_as_page = false;
2686 }
2687
2688 /**
2689  * igb_clean_tx_ring - Free Tx Buffers
2690  * @tx_ring: ring to be cleaned
2691  **/
2692 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
2693 {
2694         struct igb_buffer *buffer_info;
2695         unsigned long size;
2696         unsigned int i;
2697
2698         if (!tx_ring->buffer_info)
2699                 return;
2700         /* Free all the Tx ring sk_buffs */
2701
2702         for (i = 0; i < tx_ring->count; i++) {
2703                 buffer_info = &tx_ring->buffer_info[i];
2704                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
2705         }
2706
2707         size = sizeof(struct igb_buffer) * tx_ring->count;
2708         memset(tx_ring->buffer_info, 0, size);
2709
2710         /* Zero out the descriptor ring */
2711         memset(tx_ring->desc, 0, tx_ring->size);
2712
2713         tx_ring->next_to_use = 0;
2714         tx_ring->next_to_clean = 0;
2715 }
2716
2717 /**
2718  * igb_clean_all_tx_rings - Free Tx Buffers for all queues
2719  * @adapter: board private structure
2720  **/
2721 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
2722 {
2723         int i;
2724
2725         for (i = 0; i < adapter->num_tx_queues; i++)
2726                 igb_clean_tx_ring(adapter->tx_ring[i]);
2727 }
2728
2729 /**
2730  * igb_free_rx_resources - Free Rx Resources
2731  * @rx_ring: ring to clean the resources from
2732  *
2733  * Free all receive software resources
2734  **/
2735 void igb_free_rx_resources(struct igb_ring *rx_ring)
2736 {
2737         igb_clean_rx_ring(rx_ring);
2738
2739         vfree(rx_ring->buffer_info);
2740         rx_ring->buffer_info = NULL;
2741
2742         /* if not set, then don't free */
2743         if (!rx_ring->desc)
2744                 return;
2745
2746         dma_free_coherent(rx_ring->dev, rx_ring->size,
2747                           rx_ring->desc, rx_ring->dma);
2748
2749         rx_ring->desc = NULL;
2750 }
2751
2752 /**
2753  * igb_free_all_rx_resources - Free Rx Resources for All Queues
2754  * @adapter: board private structure
2755  *
2756  * Free all receive software resources
2757  **/
2758 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
2759 {
2760         int i;
2761
2762         for (i = 0; i < adapter->num_rx_queues; i++)
2763                 igb_free_rx_resources(adapter->rx_ring[i]);
2764 }
2765
2766 /**
2767  * igb_clean_rx_ring - Free Rx Buffers per Queue
2768  * @rx_ring: ring to free buffers from
2769  **/
2770 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
2771 {
2772         struct igb_buffer *buffer_info;
2773         unsigned long size;
2774         unsigned int i;
2775
2776         if (!rx_ring->buffer_info)
2777                 return;
2778
2779         /* Free all the Rx ring sk_buffs */
2780         for (i = 0; i < rx_ring->count; i++) {
2781                 buffer_info = &rx_ring->buffer_info[i];
2782                 if (buffer_info->dma) {
2783                         dma_unmap_single(rx_ring->dev,
2784                                          buffer_info->dma,
2785                                          rx_ring->rx_buffer_len,
2786                                          DMA_FROM_DEVICE);
2787                         buffer_info->dma = 0;
2788                 }
2789
2790                 if (buffer_info->skb) {
2791                         dev_kfree_skb(buffer_info->skb);
2792                         buffer_info->skb = NULL;
2793                 }
2794                 if (buffer_info->page_dma) {
2795                         dma_unmap_page(rx_ring->dev,
2796                                        buffer_info->page_dma,
2797                                        PAGE_SIZE / 2,
2798                                        DMA_FROM_DEVICE);
2799                         buffer_info->page_dma = 0;
2800                 }
2801                 if (buffer_info->page) {
2802                         put_page(buffer_info->page);
2803                         buffer_info->page = NULL;
2804                         buffer_info->page_offset = 0;
2805                 }
2806         }
2807
2808         size = sizeof(struct igb_buffer) * rx_ring->count;
2809         memset(rx_ring->buffer_info, 0, size);
2810
2811         /* Zero out the descriptor ring */
2812         memset(rx_ring->desc, 0, rx_ring->size);
2813
2814         rx_ring->next_to_clean = 0;
2815         rx_ring->next_to_use = 0;
2816 }
2817
2818 /**
2819  * igb_clean_all_rx_rings - Free Rx Buffers for all queues
2820  * @adapter: board private structure
2821  **/
2822 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
2823 {
2824         int i;
2825
2826         for (i = 0; i < adapter->num_rx_queues; i++)
2827                 igb_clean_rx_ring(adapter->rx_ring[i]);
2828 }
2829
2830 /**
2831  * igb_set_mac - Change the Ethernet Address of the NIC
2832  * @netdev: network interface device structure
2833  * @p: pointer to an address structure
2834  *
2835  * Returns 0 on success, negative on failure
2836  **/
2837 static int igb_set_mac(struct net_device *netdev, void *p)
2838 {
2839         struct igb_adapter *adapter = netdev_priv(netdev);
2840         struct e1000_hw *hw = &adapter->hw;
2841         struct sockaddr *addr = p;
2842
2843         if (!is_valid_ether_addr(addr->sa_data))
2844                 return -EADDRNOTAVAIL;
2845
2846         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2847         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
2848
2849         /* set the correct pool for the new PF MAC address in entry 0 */
2850         igb_rar_set_qsel(adapter, hw->mac.addr, 0,
2851                          adapter->vfs_allocated_count);
2852
2853         return 0;
2854 }
2855
2856 /**
2857  * igb_write_mc_addr_list - write multicast addresses to MTA
2858  * @netdev: network interface device structure
2859  *
2860  * Writes multicast address list to the MTA hash table.
2861  * Returns: -ENOMEM on failure
2862  *                0 on no addresses written
2863  *                X on writing X addresses to MTA
2864  **/
2865 static int igb_write_mc_addr_list(struct net_device *netdev)
2866 {
2867         struct igb_adapter *adapter = netdev_priv(netdev);
2868         struct e1000_hw *hw = &adapter->hw;
2869         struct netdev_hw_addr *ha;
2870         u8  *mta_list;
2871         int i;
2872
2873         if (netdev_mc_empty(netdev)) {
2874                 /* nothing to program, so clear mc list */
2875                 igb_update_mc_addr_list(hw, NULL, 0);
2876                 igb_restore_vf_multicasts(adapter);
2877                 return 0;
2878         }
2879
2880         mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
2881         if (!mta_list)
2882                 return -ENOMEM;
2883
2884         /* The shared function expects a packed array of only addresses. */
2885         i = 0;
2886         netdev_for_each_mc_addr(ha, netdev)
2887                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
2888
2889         igb_update_mc_addr_list(hw, mta_list, i);
2890         kfree(mta_list);
2891
2892         return netdev_mc_count(netdev);
2893 }
2894
2895 /**
2896  * igb_write_uc_addr_list - write unicast addresses to RAR table
2897  * @netdev: network interface device structure
2898  *
2899  * Writes unicast address list to the RAR table.
2900  * Returns: -ENOMEM on failure/insufficient address space
2901  *                0 on no addresses written
2902  *                X on writing X addresses to the RAR table
2903  **/
2904 static int igb_write_uc_addr_list(struct net_device *netdev)
2905 {
2906         struct igb_adapter *adapter = netdev_priv(netdev);
2907         struct e1000_hw *hw = &adapter->hw;
2908         unsigned int vfn = adapter->vfs_allocated_count;
2909         unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
2910         int count = 0;
2911
2912         /* return ENOMEM indicating insufficient memory for addresses */
2913         if (netdev_uc_count(netdev) > rar_entries)
2914                 return -ENOMEM;
2915
2916         if (!netdev_uc_empty(netdev) && rar_entries) {
2917                 struct netdev_hw_addr *ha;
2918
2919                 netdev_for_each_uc_addr(ha, netdev) {
2920                         if (!rar_entries)
2921                                 break;
2922                         igb_rar_set_qsel(adapter, ha->addr,
2923                                          rar_entries--,
2924                                          vfn);
2925                         count++;
2926                 }
2927         }
2928         /* write the addresses in reverse order to avoid write combining */
2929         for (; rar_entries > 0 ; rar_entries--) {
2930                 wr32(E1000_RAH(rar_entries), 0);
2931                 wr32(E1000_RAL(rar_entries), 0);
2932         }
2933         wrfl();
2934
2935         return count;
2936 }
2937
2938 /**
2939  * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
2940  * @netdev: network interface device structure
2941  *
2942  * The set_rx_mode entry point is called whenever the unicast or multicast
2943  * address lists or the network interface flags are updated.  This routine is
2944  * responsible for configuring the hardware for proper unicast, multicast,
2945  * promiscuous mode, and all-multi behavior.
2946  **/
2947 static void igb_set_rx_mode(struct net_device *netdev)
2948 {
2949         struct igb_adapter *adapter = netdev_priv(netdev);
2950         struct e1000_hw *hw = &adapter->hw;
2951         unsigned int vfn = adapter->vfs_allocated_count;
2952         u32 rctl, vmolr = 0;
2953         int count;
2954
2955         /* Check for Promiscuous and All Multicast modes */
2956         rctl = rd32(E1000_RCTL);
2957
2958         /* clear the effected bits */
2959         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
2960
2961         if (netdev->flags & IFF_PROMISC) {
2962                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2963                 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
2964         } else {
2965                 if (netdev->flags & IFF_ALLMULTI) {
2966                         rctl |= E1000_RCTL_MPE;
2967                         vmolr |= E1000_VMOLR_MPME;
2968                 } else {
2969                         /*
2970                          * Write addresses to the MTA, if the attempt fails
2971                          * then we should just turn on promiscous mode so
2972                          * that we can at least receive multicast traffic
2973                          */
2974                         count = igb_write_mc_addr_list(netdev);
2975                         if (count < 0) {
2976                                 rctl |= E1000_RCTL_MPE;
2977                                 vmolr |= E1000_VMOLR_MPME;
2978                         } else if (count) {
2979                                 vmolr |= E1000_VMOLR_ROMPE;
2980                         }
2981                 }
2982                 /*
2983                  * Write addresses to available RAR registers, if there is not
2984                  * sufficient space to store all the addresses then enable
2985                  * unicast promiscous mode
2986                  */
2987                 count = igb_write_uc_addr_list(netdev);
2988                 if (count < 0) {
2989                         rctl |= E1000_RCTL_UPE;
2990                         vmolr |= E1000_VMOLR_ROPE;
2991                 }
2992                 rctl |= E1000_RCTL_VFE;
2993         }
2994         wr32(E1000_RCTL, rctl);
2995
2996         /*
2997          * In order to support SR-IOV and eventually VMDq it is necessary to set
2998          * the VMOLR to enable the appropriate modes.  Without this workaround
2999          * we will have issues with VLAN tag stripping not being done for frames
3000          * that are only arriving because we are the default pool
3001          */
3002         if (hw->mac.type < e1000_82576)
3003                 return;
3004
3005         vmolr |= rd32(E1000_VMOLR(vfn)) &
3006                  ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3007         wr32(E1000_VMOLR(vfn), vmolr);
3008         igb_restore_vf_multicasts(adapter);
3009 }
3010
3011 /* Need to wait a few seconds after link up to get diagnostic information from
3012  * the phy */
3013 static void igb_update_phy_info(unsigned long data)
3014 {
3015         struct igb_adapter *adapter = (struct igb_adapter *) data;
3016         igb_get_phy_info(&adapter->hw);
3017 }
3018
3019 /**
3020  * igb_has_link - check shared code for link and determine up/down
3021  * @adapter: pointer to driver private info
3022  **/
3023 bool igb_has_link(struct igb_adapter *adapter)
3024 {
3025         struct e1000_hw *hw = &adapter->hw;
3026         bool link_active = false;
3027         s32 ret_val = 0;
3028
3029         /* get_link_status is set on LSC (link status) interrupt or
3030          * rx sequence error interrupt.  get_link_status will stay
3031          * false until the e1000_check_for_link establishes link
3032          * for copper adapters ONLY
3033          */
3034         switch (hw->phy.media_type) {
3035         case e1000_media_type_copper:
3036                 if (hw->mac.get_link_status) {
3037                         ret_val = hw->mac.ops.check_for_link(hw);
3038                         link_active = !hw->mac.get_link_status;
3039                 } else {
3040                         link_active = true;
3041                 }
3042                 break;
3043         case e1000_media_type_internal_serdes:
3044                 ret_val = hw->mac.ops.check_for_link(hw);
3045                 link_active = hw->mac.serdes_has_link;
3046                 break;
3047         default:
3048         case e1000_media_type_unknown:
3049                 break;
3050         }
3051
3052         return link_active;
3053 }
3054
3055 /**
3056  * igb_watchdog - Timer Call-back
3057  * @data: pointer to adapter cast into an unsigned long
3058  **/
3059 static void igb_watchdog(unsigned long data)
3060 {
3061         struct igb_adapter *adapter = (struct igb_adapter *)data;
3062         /* Do the rest outside of interrupt context */
3063         schedule_work(&adapter->watchdog_task);
3064 }
3065
3066 static void igb_watchdog_task(struct work_struct *work)
3067 {
3068         struct igb_adapter *adapter = container_of(work,
3069                                                    struct igb_adapter,
3070                                                    watchdog_task);
3071         struct e1000_hw *hw = &adapter->hw;
3072         struct net_device *netdev = adapter->netdev;
3073         u32 link;
3074         int i;
3075
3076         link = igb_has_link(adapter);
3077         if (link) {
3078                 if (!netif_carrier_ok(netdev)) {
3079                         u32 ctrl;
3080                         hw->mac.ops.get_speed_and_duplex(hw,
3081                                                          &adapter->link_speed,
3082                                                          &adapter->link_duplex);
3083
3084                         ctrl = rd32(E1000_CTRL);
3085                         /* Links status message must follow this format */
3086                         printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3087                                  "Flow Control: %s\n",
3088                                netdev->name,
3089                                adapter->link_speed,
3090                                adapter->link_duplex == FULL_DUPLEX ?
3091                                  "Full Duplex" : "Half Duplex",
3092                                ((ctrl & E1000_CTRL_TFCE) &&
3093                                 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3094                                ((ctrl & E1000_CTRL_RFCE) ?  "RX" :
3095                                ((ctrl & E1000_CTRL_TFCE) ?  "TX" : "None")));
3096
3097                         /* adjust timeout factor according to speed/duplex */
3098                         adapter->tx_timeout_factor = 1;
3099                         switch (adapter->link_speed) {
3100                         case SPEED_10:
3101                                 adapter->tx_timeout_factor = 14;
3102                                 break;
3103                         case SPEED_100:
3104                                 /* maybe add some timeout factor ? */
3105                                 break;
3106                         }
3107
3108                         netif_carrier_on(netdev);
3109
3110                         igb_ping_all_vfs(adapter);
3111
3112                         /* link state has changed, schedule phy info update */
3113                         if (!test_bit(__IGB_DOWN, &adapter->state))
3114                                 mod_timer(&adapter->phy_info_timer,
3115                                           round_jiffies(jiffies + 2 * HZ));
3116                 }
3117         } else {
3118                 if (netif_carrier_ok(netdev)) {
3119                         adapter->link_speed = 0;
3120                         adapter->link_duplex = 0;
3121                         /* Links status message must follow this format */
3122                         printk(KERN_INFO "igb: %s NIC Link is Down\n",
3123                                netdev->name);
3124                         netif_carrier_off(netdev);
3125
3126                         igb_ping_all_vfs(adapter);
3127
3128                         /* link state has changed, schedule phy info update */
3129                         if (!test_bit(__IGB_DOWN, &adapter->state))
3130                                 mod_timer(&adapter->phy_info_timer,
3131                                           round_jiffies(jiffies + 2 * HZ));
3132                 }
3133         }
3134
3135         igb_update_stats(adapter);
3136
3137         for (i = 0; i < adapter->num_tx_queues; i++) {
3138                 struct igb_ring *tx_ring = adapter->tx_ring[i];
3139                 if (!netif_carrier_ok(netdev)) {
3140                         /* We've lost link, so the controller stops DMA,
3141                          * but we've got queued Tx work that's never going
3142                          * to get done, so reset controller to flush Tx.
3143                          * (Do the reset outside of interrupt context). */
3144                         if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3145                                 adapter->tx_timeout_count++;
3146                                 schedule_work(&adapter->reset_task);
3147                                 /* return immediately since reset is imminent */
3148                                 return;
3149                         }
3150                 }
3151
3152                 /* Force detection of hung controller every watchdog period */
3153                 tx_ring->detect_tx_hung = true;
3154         }
3155
3156         /* Cause software interrupt to ensure rx ring is cleaned */
3157         if (adapter->msix_entries) {
3158                 u32 eics = 0;
3159                 for (i = 0; i < adapter->num_q_vectors; i++) {
3160                         struct igb_q_vector *q_vector = adapter->q_vector[i];
3161                         eics |= q_vector->eims_value;
3162                 }
3163                 wr32(E1000_EICS, eics);
3164         } else {
3165                 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3166         }
3167
3168         /* Reset the timer */
3169         if (!test_bit(__IGB_DOWN, &adapter->state))
3170                 mod_timer(&adapter->watchdog_timer,
3171                           round_jiffies(jiffies + 2 * HZ));
3172 }
3173
3174 enum latency_range {
3175         lowest_latency = 0,
3176         low_latency = 1,
3177         bulk_latency = 2,
3178         latency_invalid = 255
3179 };
3180
3181 /**
3182  * igb_update_ring_itr - update the dynamic ITR value based on packet size
3183  *
3184  *      Stores a new ITR value based on strictly on packet size.  This
3185  *      algorithm is less sophisticated than that used in igb_update_itr,
3186  *      due to the difficulty of synchronizing statistics across multiple
3187  *      receive rings.  The divisors and thresholds used by this fuction
3188  *      were determined based on theoretical maximum wire speed and testing
3189  *      data, in order to minimize response time while increasing bulk
3190  *      throughput.
3191  *      This functionality is controlled by the InterruptThrottleRate module
3192  *      parameter (see igb_param.c)
3193  *      NOTE:  This function is called only when operating in a multiqueue
3194  *             receive environment.
3195  * @q_vector: pointer to q_vector
3196  **/
3197 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3198 {
3199         int new_val = q_vector->itr_val;
3200         int avg_wire_size = 0;
3201         struct igb_adapter *adapter = q_vector->adapter;
3202
3203         /* For non-gigabit speeds, just fix the interrupt rate at 4000
3204          * ints/sec - ITR timer value of 120 ticks.
3205          */
3206         if (adapter->link_speed != SPEED_1000) {
3207                 new_val = 976;
3208                 goto set_itr_val;
3209         }
3210
3211         if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3212                 struct igb_ring *ring = q_vector->rx_ring;
3213                 avg_wire_size = ring->total_bytes / ring->total_packets;
3214         }
3215
3216         if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3217                 struct igb_ring *ring = q_vector->tx_ring;
3218                 avg_wire_size = max_t(u32, avg_wire_size,
3219                                       (ring->total_bytes /
3220                                        ring->total_packets));
3221         }
3222
3223         /* if avg_wire_size isn't set no work was done */
3224         if (!avg_wire_size)
3225                 goto clear_counts;
3226
3227         /* Add 24 bytes to size to account for CRC, preamble, and gap */
3228         avg_wire_size += 24;
3229
3230         /* Don't starve jumbo frames */
3231         avg_wire_size = min(avg_wire_size, 3000);
3232
3233         /* Give a little boost to mid-size frames */
3234         if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3235                 new_val = avg_wire_size / 3;
3236         else
3237                 new_val = avg_wire_size / 2;
3238
3239         /* when in itr mode 3 do not exceed 20K ints/sec */
3240         if (adapter->rx_itr_setting == 3 && new_val < 196)
3241                 new_val = 196;
3242
3243 set_itr_val:
3244         if (new_val != q_vector->itr_val) {
3245                 q_vector->itr_val = new_val;
3246                 q_vector->set_itr = 1;
3247         }
3248 clear_counts:
3249         if (q_vector->rx_ring) {
3250                 q_vector->rx_ring->total_bytes = 0;
3251                 q_vector->rx_ring->total_packets = 0;
3252         }
3253         if (q_vector->tx_ring) {
3254                 q_vector->tx_ring->total_bytes = 0;
3255                 q_vector->tx_ring->total_packets = 0;
3256         }
3257 }
3258
3259 /**
3260  * igb_update_itr - update the dynamic ITR value based on statistics
3261  *      Stores a new ITR value based on packets and byte
3262  *      counts during the last interrupt.  The advantage of per interrupt
3263  *      computation is faster updates and more accurate ITR for the current
3264  *      traffic pattern.  Constants in this function were computed
3265  *      based on theoretical maximum wire speed and thresholds were set based
3266  *      on testing data as well as attempting to minimize response time
3267  *      while increasing bulk throughput.
3268  *      this functionality is controlled by the InterruptThrottleRate module
3269  *      parameter (see igb_param.c)
3270  *      NOTE:  These calculations are only valid when operating in a single-
3271  *             queue environment.
3272  * @adapter: pointer to adapter
3273  * @itr_setting: current q_vector->itr_val
3274  * @packets: the number of packets during this measurement interval
3275  * @bytes: the number of bytes during this measurement interval
3276  **/
3277 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3278                                    int packets, int bytes)
3279 {
3280         unsigned int retval = itr_setting;
3281
3282         if (packets == 0)
3283                 goto update_itr_done;
3284
3285         switch (itr_setting) {
3286         case lowest_latency:
3287                 /* handle TSO and jumbo frames */
3288                 if (bytes/packets > 8000)
3289                         retval = bulk_latency;
3290                 else if ((packets < 5) && (bytes > 512))
3291                         retval = low_latency;
3292                 break;
3293         case low_latency:  /* 50 usec aka 20000 ints/s */
3294                 if (bytes > 10000) {
3295                         /* this if handles the TSO accounting */
3296                         if (bytes/packets > 8000) {
3297                                 retval = bulk_latency;
3298                         } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3299                                 retval = bulk_latency;
3300                         } else if ((packets > 35)) {
3301                                 retval = lowest_latency;
3302                         }
3303                 } else if (bytes/packets > 2000) {
3304                         retval = bulk_latency;
3305                 } else if (packets <= 2 && bytes < 512) {
3306                         retval = lowest_latency;
3307                 }
3308                 break;
3309         case bulk_latency: /* 250 usec aka 4000 ints/s */
3310                 if (bytes > 25000) {
3311                         if (packets > 35)
3312                                 retval = low_latency;
3313                 } else if (bytes < 1500) {
3314                         retval = low_latency;
3315                 }
3316                 break;
3317         }
3318
3319 update_itr_done:
3320         return retval;
3321 }
3322
3323 static void igb_set_itr(struct igb_adapter *adapter)
3324 {
3325         struct igb_q_vector *q_vector = adapter->q_vector[0];
3326         u16 current_itr;
3327         u32 new_itr = q_vector->itr_val;
3328
3329         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3330         if (adapter->link_speed != SPEED_1000) {
3331                 current_itr = 0;
3332                 new_itr = 4000;
3333                 goto set_itr_now;
3334         }
3335
3336         adapter->rx_itr = igb_update_itr(adapter,
3337                                     adapter->rx_itr,
3338                                     q_vector->rx_ring->total_packets,
3339                                     q_vector->rx_ring->total_bytes);
3340
3341         adapter->tx_itr = igb_update_itr(adapter,
3342                                     adapter->tx_itr,
3343                                     q_vector->tx_ring->total_packets,
3344                                     q_vector->tx_ring->total_bytes);
3345         current_itr = max(adapter->rx_itr, adapter->tx_itr);
3346
3347         /* conservative mode (itr 3) eliminates the lowest_latency setting */
3348         if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3349                 current_itr = low_latency;
3350
3351         switch (current_itr) {
3352         /* counts and packets in update_itr are dependent on these numbers */
3353         case lowest_latency:
3354                 new_itr = 56;  /* aka 70,000 ints/sec */
3355                 break;
3356         case low_latency:
3357                 new_itr = 196; /* aka 20,000 ints/sec */
3358                 break;
3359         case bulk_latency:
3360                 new_itr = 980; /* aka 4,000 ints/sec */
3361                 break;
3362         default:
3363                 break;
3364         }
3365
3366 set_itr_now:
3367         q_vector->rx_ring->total_bytes = 0;
3368         q_vector->rx_ring->total_packets = 0;
3369         q_vector->tx_ring->total_bytes = 0;
3370         q_vector->tx_ring->total_packets = 0;
3371
3372         if (new_itr != q_vector->itr_val) {
3373                 /* this attempts to bias the interrupt rate towards Bulk
3374                  * by adding intermediate steps when interrupt rate is
3375                  * increasing */
3376                 new_itr = new_itr > q_vector->itr_val ?
3377                              max((new_itr * q_vector->itr_val) /
3378                                  (new_itr + (q_vector->itr_val >> 2)),
3379                                  new_itr) :
3380                              new_itr;
3381                 /* Don't write the value here; it resets the adapter's
3382                  * internal timer, and causes us to delay far longer than
3383                  * we should between interrupts.  Instead, we write the ITR
3384                  * value at the beginning of the next interrupt so the timing
3385                  * ends up being correct.
3386                  */
3387                 q_vector->itr_val = new_itr;
3388                 q_vector->set_itr = 1;
3389         }
3390
3391         return;
3392 }
3393
3394 #define IGB_TX_FLAGS_CSUM               0x00000001
3395 #define IGB_TX_FLAGS_VLAN               0x00000002
3396 #define IGB_TX_FLAGS_TSO                0x00000004
3397 #define IGB_TX_FLAGS_IPV4               0x00000008
3398 #define IGB_TX_FLAGS_TSTAMP             0x00000010
3399 #define IGB_TX_FLAGS_VLAN_MASK          0xffff0000
3400 #define IGB_TX_FLAGS_VLAN_SHIFT                 16
3401
3402 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3403                               struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3404 {
3405         struct e1000_adv_tx_context_desc *context_desc;
3406         unsigned int i;
3407         int err;
3408         struct igb_buffer *buffer_info;
3409         u32 info = 0, tu_cmd = 0;
3410         u32 mss_l4len_idx;
3411         u8 l4len;
3412
3413         if (skb_header_cloned(skb)) {
3414                 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3415                 if (err)
3416                         return err;
3417         }
3418
3419         l4len = tcp_hdrlen(skb);
3420         *hdr_len += l4len;
3421
3422         if (skb->protocol == htons(ETH_P_IP)) {
3423                 struct iphdr *iph = ip_hdr(skb);
3424                 iph->tot_len = 0;
3425                 iph->check = 0;
3426                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3427                                                          iph->daddr, 0,
3428                                                          IPPROTO_TCP,
3429                                                          0);
3430         } else if (skb_is_gso_v6(skb)) {
3431                 ipv6_hdr(skb)->payload_len = 0;
3432                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3433                                                        &ipv6_hdr(skb)->daddr,
3434                                                        0, IPPROTO_TCP, 0);
3435         }
3436
3437         i = tx_ring->next_to_use;
3438
3439         buffer_info = &tx_ring->buffer_info[i];
3440         context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3441         /* VLAN MACLEN IPLEN */
3442         if (tx_flags & IGB_TX_FLAGS_VLAN)
3443                 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3444         info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3445         *hdr_len += skb_network_offset(skb);
3446         info |= skb_network_header_len(skb);
3447         *hdr_len += skb_network_header_len(skb);
3448         context_desc->vlan_macip_lens = cpu_to_le32(info);
3449
3450         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3451         tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3452
3453         if (skb->protocol == htons(ETH_P_IP))
3454                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3455         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3456
3457         context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3458
3459         /* MSS L4LEN IDX */
3460         mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3461         mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3462
3463         /* For 82575, context index must be unique per ring. */
3464         if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3465                 mss_l4len_idx |= tx_ring->reg_idx << 4;
3466
3467         context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3468         context_desc->seqnum_seed = 0;
3469
3470         buffer_info->time_stamp = jiffies;
3471         buffer_info->next_to_watch = i;
3472         buffer_info->dma = 0;
3473         i++;
3474         if (i == tx_ring->count)
3475                 i = 0;
3476
3477         tx_ring->next_to_use = i;
3478
3479         return true;
3480 }
3481
3482 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3483                                    struct sk_buff *skb, u32 tx_flags)
3484 {
3485         struct e1000_adv_tx_context_desc *context_desc;
3486         struct device *dev = tx_ring->dev;
3487         struct igb_buffer *buffer_info;
3488         u32 info = 0, tu_cmd = 0;
3489         unsigned int i;
3490
3491         if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3492             (tx_flags & IGB_TX_FLAGS_VLAN)) {
3493                 i = tx_ring->next_to_use;
3494                 buffer_info = &tx_ring->buffer_info[i];
3495                 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3496
3497                 if (tx_flags & IGB_TX_FLAGS_VLAN)
3498                         info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3499
3500                 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3501                 if (skb->ip_summed == CHECKSUM_PARTIAL)
3502                         info |= skb_network_header_len(skb);
3503
3504                 context_desc->vlan_macip_lens = cpu_to_le32(info);
3505
3506                 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3507
3508                 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3509                         __be16 protocol;
3510
3511                         if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3512                                 const struct vlan_ethhdr *vhdr =
3513                                           (const struct vlan_ethhdr*)skb->data;
3514
3515                                 protocol = vhdr->h_vlan_encapsulated_proto;
3516                         } else {
3517                                 protocol = skb->protocol;
3518                         }
3519
3520                         switch (protocol) {
3521                         case cpu_to_be16(ETH_P_IP):
3522                                 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3523                                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3524                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3525                                 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3526                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3527                                 break;
3528                         case cpu_to_be16(ETH_P_IPV6):
3529                                 /* XXX what about other V6 headers?? */
3530                                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3531                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3532                                 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3533                                         tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3534                                 break;
3535                         default:
3536                                 if (unlikely(net_ratelimit()))
3537                                         dev_warn(dev,
3538                                             "partial checksum but proto=%x!\n",
3539                                             skb->protocol);
3540                                 break;
3541                         }
3542                 }
3543
3544                 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3545                 context_desc->seqnum_seed = 0;
3546                 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3547                         context_desc->mss_l4len_idx =
3548                                 cpu_to_le32(tx_ring->reg_idx << 4);
3549
3550                 buffer_info->time_stamp = jiffies;
3551                 buffer_info->next_to_watch = i;
3552                 buffer_info->dma = 0;
3553
3554                 i++;
3555                 if (i == tx_ring->count)
3556                         i = 0;
3557                 tx_ring->next_to_use = i;
3558
3559                 return true;
3560         }
3561         return false;
3562 }
3563
3564 #define IGB_MAX_TXD_PWR 16
3565 #define IGB_MAX_DATA_PER_TXD    (1<<IGB_MAX_TXD_PWR)
3566
3567 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3568                                  unsigned int first)
3569 {
3570         struct igb_buffer *buffer_info;
3571         struct device *dev = tx_ring->dev;
3572         unsigned int len = skb_headlen(skb);
3573         unsigned int count = 0, i;
3574         unsigned int f;
3575
3576         i = tx_ring->next_to_use;
3577
3578         buffer_info = &tx_ring->buffer_info[i];
3579         BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3580         buffer_info->length = len;
3581         /* set time_stamp *before* dma to help avoid a possible race */
3582         buffer_info->time_stamp = jiffies;
3583         buffer_info->next_to_watch = i;
3584         buffer_info->dma = dma_map_single(dev, skb->data, len,
3585                                           DMA_TO_DEVICE);
3586         if (dma_mapping_error(dev, buffer_info->dma))
3587                 goto dma_error;
3588
3589         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3590                 struct skb_frag_struct *frag;
3591
3592                 count++;
3593                 i++;
3594                 if (i == tx_ring->count)
3595                         i = 0;
3596
3597                 frag = &skb_shinfo(skb)->frags[f];
3598                 len = frag->size;
3599
3600                 buffer_info = &tx_ring->buffer_info[i];
3601                 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3602                 buffer_info->length = len;
3603                 buffer_info->time_stamp = jiffies;
3604                 buffer_info->next_to_watch = i;
3605                 buffer_info->mapped_as_page = true;
3606                 buffer_info->dma = dma_map_page(dev,
3607                                                 frag->page,
3608                                                 frag->page_offset,
3609                                                 len,
3610                                                 DMA_TO_DEVICE);
3611                 if (dma_mapping_error(dev, buffer_info->dma))
3612                         goto dma_error;
3613
3614         }
3615
3616         tx_ring->buffer_info[i].skb = skb;
3617         tx_ring->buffer_info[i].gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3618         tx_ring->buffer_info[first].next_to_watch = i;
3619
3620         return ++count;
3621
3622 dma_error:
3623         dev_err(dev, "TX DMA map failed\n");
3624
3625         /* clear timestamp and dma mappings for failed buffer_info mapping */
3626         buffer_info->dma = 0;
3627         buffer_info->time_stamp = 0;
3628         buffer_info->length = 0;
3629         buffer_info->next_to_watch = 0;
3630         buffer_info->mapped_as_page = false;
3631
3632         /* clear timestamp and dma mappings for remaining portion of packet */
3633         while (count--) {
3634                 if (i == 0)
3635                         i = tx_ring->count;
3636                 i--;
3637                 buffer_info = &tx_ring->buffer_info[i];
3638                 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3639         }
3640
3641         return 0;
3642 }
3643
3644 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3645                                     u32 tx_flags, int count, u32 paylen,
3646                                     u8 hdr_len)
3647 {
3648         union e1000_adv_tx_desc *tx_desc;
3649         struct igb_buffer *buffer_info;
3650         u32 olinfo_status = 0, cmd_type_len;
3651         unsigned int i = tx_ring->next_to_use;
3652
3653         cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3654                         E1000_ADVTXD_DCMD_DEXT);
3655
3656         if (tx_flags & IGB_TX_FLAGS_VLAN)
3657                 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
3658
3659         if (tx_flags & IGB_TX_FLAGS_TSTAMP)
3660                 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
3661
3662         if (tx_flags & IGB_TX_FLAGS_TSO) {
3663                 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
3664
3665                 /* insert tcp checksum */
3666                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3667
3668                 /* insert ip checksum */
3669                 if (tx_flags & IGB_TX_FLAGS_IPV4)
3670                         olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
3671
3672         } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
3673                 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
3674         }
3675
3676         if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
3677             (tx_flags & (IGB_TX_FLAGS_CSUM |
3678                          IGB_TX_FLAGS_TSO |
3679                          IGB_TX_FLAGS_VLAN)))
3680                 olinfo_status |= tx_ring->reg_idx << 4;
3681
3682         olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
3683
3684         do {
3685                 buffer_info = &tx_ring->buffer_info[i];
3686                 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
3687                 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
3688                 tx_desc->read.cmd_type_len =
3689                         cpu_to_le32(cmd_type_len | buffer_info->length);
3690                 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3691                 count--;
3692                 i++;
3693                 if (i == tx_ring->count)
3694                         i = 0;
3695         } while (count > 0);
3696
3697         tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
3698         /* Force memory writes to complete before letting h/w
3699          * know there are new descriptors to fetch.  (Only
3700          * applicable for weak-ordered memory model archs,
3701          * such as IA-64). */
3702         wmb();
3703
3704         tx_ring->next_to_use = i;
3705         writel(i, tx_ring->tail);
3706         /* we need this if more than one processor can write to our tail
3707          * at a time, it syncronizes IO on IA64/Altix systems */
3708         mmiowb();
3709 }
3710
3711 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3712 {
3713         struct net_device *netdev = tx_ring->netdev;
3714
3715         netif_stop_subqueue(netdev, tx_ring->queue_index);
3716
3717         /* Herbert's original patch had:
3718          *  smp_mb__after_netif_stop_queue();
3719          * but since that doesn't exist yet, just open code it. */
3720         smp_mb();
3721
3722         /* We need to check again in a case another CPU has just
3723          * made room available. */
3724         if (igb_desc_unused(tx_ring) < size)
3725                 return -EBUSY;
3726
3727         /* A reprieve! */
3728         netif_wake_subqueue(netdev, tx_ring->queue_index);
3729         tx_ring->tx_stats.restart_queue++;
3730         return 0;
3731 }
3732
3733 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
3734 {
3735         if (igb_desc_unused(tx_ring) >= size)
3736                 return 0;
3737         return __igb_maybe_stop_tx(tx_ring, size);
3738 }
3739
3740 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
3741                                     struct igb_ring *tx_ring)
3742 {
3743         struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
3744         int tso = 0, count;
3745         u32 tx_flags = 0;
3746         u16 first;
3747         u8 hdr_len = 0;
3748         union skb_shared_tx *shtx = skb_tx(skb);
3749
3750         /* need: 1 descriptor per page,
3751          *       + 2 desc gap to keep tail from touching head,
3752          *       + 1 desc for skb->data,
3753          *       + 1 desc for context descriptor,
3754          * otherwise try next time */
3755         if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
3756                 /* this is a hard error */
3757                 return NETDEV_TX_BUSY;
3758         }
3759
3760         if (unlikely(shtx->hardware)) {
3761                 shtx->in_progress = 1;
3762                 tx_flags |= IGB_TX_FLAGS_TSTAMP;
3763         }
3764
3765         if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
3766                 tx_flags |= IGB_TX_FLAGS_VLAN;
3767                 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
3768         }
3769
3770         if (skb->protocol == htons(ETH_P_IP))
3771                 tx_flags |= IGB_TX_FLAGS_IPV4;
3772
3773         first = tx_ring->next_to_use;
3774         if (skb_is_gso(skb)) {
3775                 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
3776
3777                 if (tso < 0) {
3778                         dev_kfree_skb_any(skb);
3779                         return NETDEV_TX_OK;
3780                 }
3781         }
3782
3783         if (tso)
3784                 tx_flags |= IGB_TX_FLAGS_TSO;
3785         else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
3786                  (skb->ip_summed == CHECKSUM_PARTIAL))
3787                 tx_flags |= IGB_TX_FLAGS_CSUM;
3788
3789         /*
3790          * count reflects descriptors mapped, if 0 or less then mapping error
3791          * has occured and we need to rewind the descriptor queue
3792          */
3793         count = igb_tx_map_adv(tx_ring, skb, first);
3794         if (!count) {
3795                 dev_kfree_skb_any(skb);
3796                 tx_ring->buffer_info[first].time_stamp = 0;
3797                 tx_ring->next_to_use = first;
3798                 return NETDEV_TX_OK;
3799         }
3800
3801         igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
3802
3803         /* Make sure there is space in the ring for the next send. */
3804         igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
3805
3806         return NETDEV_TX_OK;
3807 }
3808
3809 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
3810                                       struct net_device *netdev)
3811 {
3812         struct igb_adapter *adapter = netdev_priv(netdev);
3813         struct igb_ring *tx_ring;
3814         int r_idx = 0;
3815
3816         if (test_bit(__IGB_DOWN, &adapter->state)) {
3817                 dev_kfree_skb_any(skb);
3818                 return NETDEV_TX_OK;
3819         }
3820
3821         if (skb->len <= 0) {
3822                 dev_kfree_skb_any(skb);
3823                 return NETDEV_TX_OK;
3824         }
3825
3826         r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
3827         tx_ring = adapter->multi_tx_table[r_idx];
3828
3829         /* This goes back to the question of how to logically map a tx queue
3830          * to a flow.  Right now, performance is impacted slightly negatively
3831          * if using multiple tx queues.  If the stack breaks away from a
3832          * single qdisc implementation, we can look at this again. */
3833         return igb_xmit_frame_ring_adv(skb, tx_ring);
3834 }
3835
3836 /**
3837  * igb_tx_timeout - Respond to a Tx Hang
3838  * @netdev: network interface device structure
3839  **/
3840 static void igb_tx_timeout(struct net_device *netdev)
3841 {
3842         struct igb_adapter *adapter = netdev_priv(netdev);
3843         struct e1000_hw *hw = &adapter->hw;
3844
3845         /* Do the reset outside of interrupt context */
3846         adapter->tx_timeout_count++;
3847
3848         if (hw->mac.type == e1000_82580)
3849                 hw->dev_spec._82575.global_device_reset = true;
3850
3851         schedule_work(&adapter->reset_task);
3852         wr32(E1000_EICS,
3853              (adapter->eims_enable_mask & ~adapter->eims_other));
3854 }
3855
3856 static void igb_reset_task(struct work_struct *work)
3857 {
3858         struct igb_adapter *adapter;
3859         adapter = container_of(work, struct igb_adapter, reset_task);
3860
3861         igb_reinit_locked(adapter);
3862 }
3863
3864 /**
3865  * igb_get_stats - Get System Network Statistics
3866  * @netdev: network interface device structure
3867  *
3868  * Returns the address of the device statistics structure.
3869  * The statistics are actually updated from the timer callback.
3870  **/
3871 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
3872 {
3873         /* only return the current stats */
3874         return &netdev->stats;
3875 }
3876
3877 /**
3878  * igb_change_mtu - Change the Maximum Transfer Unit
3879  * @netdev: network interface device structure
3880  * @new_mtu: new value for maximum frame size
3881  *
3882  * Returns 0 on success, negative on failure
3883  **/
3884 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
3885 {
3886         struct igb_adapter *adapter = netdev_priv(netdev);
3887         struct pci_dev *pdev = adapter->pdev;
3888         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3889         u32 rx_buffer_len, i;
3890
3891         if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
3892                 dev_err(&pdev->dev, "Invalid MTU setting\n");
3893                 return -EINVAL;
3894         }
3895
3896         if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
3897                 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
3898                 return -EINVAL;
3899         }
3900
3901         while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
3902                 msleep(1);
3903
3904         /* igb_down has a dependency on max_frame_size */
3905         adapter->max_frame_size = max_frame;
3906
3907         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
3908          * means we reserve 2 more, this pushes us to allocate from the next
3909          * larger slab size.
3910          * i.e. RXBUFFER_2048 --> size-4096 slab
3911          */
3912
3913         if (adapter->hw.mac.type == e1000_82580)
3914                 max_frame += IGB_TS_HDR_LEN;
3915
3916         if (max_frame <= IGB_RXBUFFER_1024)
3917                 rx_buffer_len = IGB_RXBUFFER_1024;
3918         else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
3919                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
3920         else
3921                 rx_buffer_len = IGB_RXBUFFER_128;
3922
3923         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
3924              (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
3925                 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
3926
3927         if ((adapter->hw.mac.type == e1000_82580) &&
3928             (rx_buffer_len == IGB_RXBUFFER_128))
3929                 rx_buffer_len += IGB_RXBUFFER_64;
3930
3931         if (netif_running(netdev))
3932                 igb_down(adapter);
3933
3934         dev_info(&pdev->dev, "changing MTU from %d to %d\n",
3935                  netdev->mtu, new_mtu);
3936         netdev->mtu = new_mtu;
3937
3938         for (i = 0; i < adapter->num_rx_queues; i++)
3939                 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
3940
3941         if (netif_running(netdev))
3942                 igb_up(adapter);
3943         else
3944                 igb_reset(adapter);
3945
3946         clear_bit(__IGB_RESETTING, &adapter->state);
3947
3948         return 0;
3949 }
3950
3951 /**
3952  * igb_update_stats - Update the board statistics counters
3953  * @adapter: board private structure
3954  **/
3955
3956 void igb_update_stats(struct igb_adapter *adapter)
3957 {
3958         struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
3959         struct e1000_hw *hw = &adapter->hw;
3960         struct pci_dev *pdev = adapter->pdev;
3961         u32 reg, mpc;
3962         u16 phy_tmp;
3963         int i;
3964         u64 bytes, packets;
3965
3966 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
3967
3968         /*
3969          * Prevent stats update while adapter is being reset, or if the pci
3970          * connection is down.
3971          */
3972         if (adapter->link_speed == 0)
3973                 return;
3974         if (pci_channel_offline(pdev))
3975                 return;
3976
3977         bytes = 0;
3978         packets = 0;
3979         for (i = 0; i < adapter->num_rx_queues; i++) {
3980                 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
3981                 struct igb_ring *ring = adapter->rx_ring[i];
3982                 ring->rx_stats.drops += rqdpc_tmp;
3983                 net_stats->rx_fifo_errors += rqdpc_tmp;
3984                 bytes += ring->rx_stats.bytes;
3985                 packets += ring->rx_stats.packets;
3986         }
3987
3988         net_stats->rx_bytes = bytes;
3989         net_stats->rx_packets = packets;
3990
3991         bytes = 0;
3992         packets = 0;
3993         for (i = 0; i < adapter->num_tx_queues; i++) {
3994                 struct igb_ring *ring = adapter->tx_ring[i];
3995                 bytes += ring->tx_stats.bytes;
3996                 packets += ring->tx_stats.packets;
3997         }
3998         net_stats->tx_bytes = bytes;
3999         net_stats->tx_packets = packets;
4000
4001         /* read stats registers */
4002         adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4003         adapter->stats.gprc += rd32(E1000_GPRC);
4004         adapter->stats.gorc += rd32(E1000_GORCL);
4005         rd32(E1000_GORCH); /* clear GORCL */
4006         adapter->stats.bprc += rd32(E1000_BPRC);
4007         adapter->stats.mprc += rd32(E1000_MPRC);
4008         adapter->stats.roc += rd32(E1000_ROC);
4009
4010         adapter->stats.prc64 += rd32(E1000_PRC64);
4011         adapter->stats.prc127 += rd32(E1000_PRC127);
4012         adapter->stats.prc255 += rd32(E1000_PRC255);
4013         adapter->stats.prc511 += rd32(E1000_PRC511);
4014         adapter->stats.prc1023 += rd32(E1000_PRC1023);
4015         adapter->stats.prc1522 += rd32(E1000_PRC1522);
4016         adapter->stats.symerrs += rd32(E1000_SYMERRS);
4017         adapter->stats.sec += rd32(E1000_SEC);
4018
4019         mpc = rd32(E1000_MPC);
4020         adapter->stats.mpc += mpc;
4021         net_stats->rx_fifo_errors += mpc;
4022         adapter->stats.scc += rd32(E1000_SCC);
4023         adapter->stats.ecol += rd32(E1000_ECOL);
4024         adapter->stats.mcc += rd32(E1000_MCC);
4025         adapter->stats.latecol += rd32(E1000_LATECOL);
4026         adapter->stats.dc += rd32(E1000_DC);
4027         adapter->stats.rlec += rd32(E1000_RLEC);
4028         adapter->stats.xonrxc += rd32(E1000_XONRXC);
4029         adapter->stats.xontxc += rd32(E1000_XONTXC);
4030         adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4031         adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4032         adapter->stats.fcruc += rd32(E1000_FCRUC);
4033         adapter->stats.gptc += rd32(E1000_GPTC);
4034         adapter->stats.gotc += rd32(E1000_GOTCL);
4035         rd32(E1000_GOTCH); /* clear GOTCL */
4036         adapter->stats.rnbc += rd32(E1000_RNBC);
4037         adapter->stats.ruc += rd32(E1000_RUC);
4038         adapter->stats.rfc += rd32(E1000_RFC);
4039         adapter->stats.rjc += rd32(E1000_RJC);
4040         adapter->stats.tor += rd32(E1000_TORH);
4041         adapter->stats.tot += rd32(E1000_TOTH);
4042         adapter->stats.tpr += rd32(E1000_TPR);
4043
4044         adapter->stats.ptc64 += rd32(E1000_PTC64);
4045         adapter->stats.ptc127 += rd32(E1000_PTC127);
4046         adapter->stats.ptc255 += rd32(E1000_PTC255);
4047         adapter->stats.ptc511 += rd32(E1000_PTC511);
4048         adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4049         adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4050
4051         adapter->stats.mptc += rd32(E1000_MPTC);
4052         adapter->stats.bptc += rd32(E1000_BPTC);
4053
4054         adapter->stats.tpt += rd32(E1000_TPT);
4055         adapter->stats.colc += rd32(E1000_COLC);
4056
4057         adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4058         /* read internal phy specific stats */
4059         reg = rd32(E1000_CTRL_EXT);
4060         if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4061                 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4062                 adapter->stats.tncrs += rd32(E1000_TNCRS);
4063         }
4064
4065         adapter->stats.tsctc += rd32(E1000_TSCTC);
4066         adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4067
4068         adapter->stats.iac += rd32(E1000_IAC);
4069         adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4070         adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4071         adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4072         adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4073         adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4074         adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4075         adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4076         adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4077
4078         /* Fill out the OS statistics structure */
4079         net_stats->multicast = adapter->stats.mprc;
4080         net_stats->collisions = adapter->stats.colc;
4081
4082         /* Rx Errors */
4083
4084         /* RLEC on some newer hardware can be incorrect so build
4085          * our own version based on RUC and ROC */
4086         net_stats->rx_errors = adapter->stats.rxerrc +
4087                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4088                 adapter->stats.ruc + adapter->stats.roc +
4089                 adapter->stats.cexterr;
4090         net_stats->rx_length_errors = adapter->stats.ruc +
4091                                       adapter->stats.roc;
4092         net_stats->rx_crc_errors = adapter->stats.crcerrs;
4093         net_stats->rx_frame_errors = adapter->stats.algnerrc;
4094         net_stats->rx_missed_errors = adapter->stats.mpc;
4095
4096         /* Tx Errors */
4097         net_stats->tx_errors = adapter->stats.ecol +
4098                                adapter->stats.latecol;
4099         net_stats->tx_aborted_errors = adapter->stats.ecol;
4100         net_stats->tx_window_errors = adapter->stats.latecol;
4101         net_stats->tx_carrier_errors = adapter->stats.tncrs;
4102
4103         /* Tx Dropped needs to be maintained elsewhere */
4104
4105         /* Phy Stats */
4106         if (hw->phy.media_type == e1000_media_type_copper) {
4107                 if ((adapter->link_speed == SPEED_1000) &&
4108                    (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4109                         phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4110                         adapter->phy_stats.idle_errors += phy_tmp;
4111                 }
4112         }
4113
4114         /* Management Stats */
4115         adapter->stats.mgptc += rd32(E1000_MGTPTC);
4116         adapter->stats.mgprc += rd32(E1000_MGTPRC);
4117         adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4118 }
4119
4120 static irqreturn_t igb_msix_other(int irq, void *data)
4121 {
4122         struct igb_adapter *adapter = data;
4123         struct e1000_hw *hw = &adapter->hw;
4124         u32 icr = rd32(E1000_ICR);
4125         /* reading ICR causes bit 31 of EICR to be cleared */
4126
4127         if (icr & E1000_ICR_DRSTA)
4128                 schedule_work(&adapter->reset_task);
4129
4130         if (icr & E1000_ICR_DOUTSYNC) {
4131                 /* HW is reporting DMA is out of sync */
4132                 adapter->stats.doosync++;
4133         }
4134
4135         /* Check for a mailbox event */
4136         if (icr & E1000_ICR_VMMB)
4137                 igb_msg_task(adapter);
4138
4139         if (icr & E1000_ICR_LSC) {
4140                 hw->mac.get_link_status = 1;
4141                 /* guard against interrupt when we're going down */
4142                 if (!test_bit(__IGB_DOWN, &adapter->state))
4143                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4144         }
4145
4146         if (adapter->vfs_allocated_count)
4147                 wr32(E1000_IMS, E1000_IMS_LSC |
4148                                 E1000_IMS_VMMB |
4149                                 E1000_IMS_DOUTSYNC);
4150         else
4151                 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4152         wr32(E1000_EIMS, adapter->eims_other);
4153
4154         return IRQ_HANDLED;
4155 }
4156
4157 static void igb_write_itr(struct igb_q_vector *q_vector)
4158 {
4159         struct igb_adapter *adapter = q_vector->adapter;
4160         u32 itr_val = q_vector->itr_val & 0x7FFC;
4161
4162         if (!q_vector->set_itr)
4163                 return;
4164
4165         if (!itr_val)
4166                 itr_val = 0x4;
4167
4168         if (adapter->hw.mac.type == e1000_82575)
4169                 itr_val |= itr_val << 16;
4170         else
4171                 itr_val |= 0x8000000;
4172
4173         writel(itr_val, q_vector->itr_register);
4174         q_vector->set_itr = 0;
4175 }
4176
4177 static irqreturn_t igb_msix_ring(int irq, void *data)
4178 {
4179         struct igb_q_vector *q_vector = data;
4180
4181         /* Write the ITR value calculated from the previous interrupt. */
4182         igb_write_itr(q_vector);
4183
4184         napi_schedule(&q_vector->napi);
4185
4186         return IRQ_HANDLED;
4187 }
4188
4189 #ifdef CONFIG_IGB_DCA
4190 static void igb_update_dca(struct igb_q_vector *q_vector)
4191 {
4192         struct igb_adapter *adapter = q_vector->adapter;
4193         struct e1000_hw *hw = &adapter->hw;
4194         int cpu = get_cpu();
4195
4196         if (q_vector->cpu == cpu)
4197                 goto out_no_update;
4198
4199         if (q_vector->tx_ring) {
4200                 int q = q_vector->tx_ring->reg_idx;
4201                 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4202                 if (hw->mac.type == e1000_82575) {
4203                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4204                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4205                 } else {
4206                         dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4207                         dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4208                                       E1000_DCA_TXCTRL_CPUID_SHIFT;
4209                 }
4210                 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4211                 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4212         }
4213         if (q_vector->rx_ring) {
4214                 int q = q_vector->rx_ring->reg_idx;
4215                 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4216                 if (hw->mac.type == e1000_82575) {
4217                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4218                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4219                 } else {
4220                         dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4221                         dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4222                                       E1000_DCA_RXCTRL_CPUID_SHIFT;
4223                 }
4224                 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4225                 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4226                 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4227                 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4228         }
4229         q_vector->cpu = cpu;
4230 out_no_update:
4231         put_cpu();
4232 }
4233
4234 static void igb_setup_dca(struct igb_adapter *adapter)
4235 {
4236         struct e1000_hw *hw = &adapter->hw;
4237         int i;
4238
4239         if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4240                 return;
4241
4242         /* Always use CB2 mode, difference is masked in the CB driver. */
4243         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4244
4245         for (i = 0; i < adapter->num_q_vectors; i++) {
4246                 adapter->q_vector[i]->cpu = -1;
4247                 igb_update_dca(adapter->q_vector[i]);
4248         }
4249 }
4250
4251 static int __igb_notify_dca(struct device *dev, void *data)
4252 {
4253         struct net_device *netdev = dev_get_drvdata(dev);
4254         struct igb_adapter *adapter = netdev_priv(netdev);
4255         struct pci_dev *pdev = adapter->pdev;
4256         struct e1000_hw *hw = &adapter->hw;
4257         unsigned long event = *(unsigned long *)data;
4258
4259         switch (event) {
4260         case DCA_PROVIDER_ADD:
4261                 /* if already enabled, don't do it again */
4262                 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4263                         break;
4264                 if (dca_add_requester(dev) == 0) {
4265                         adapter->flags |= IGB_FLAG_DCA_ENABLED;
4266                         dev_info(&pdev->dev, "DCA enabled\n");
4267                         igb_setup_dca(adapter);
4268                         break;
4269                 }
4270                 /* Fall Through since DCA is disabled. */
4271         case DCA_PROVIDER_REMOVE:
4272                 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4273                         /* without this a class_device is left
4274                          * hanging around in the sysfs model */
4275                         dca_remove_requester(dev);
4276                         dev_info(&pdev->dev, "DCA disabled\n");
4277                         adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4278                         wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4279                 }
4280                 break;
4281         }
4282
4283         return 0;
4284 }
4285
4286 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4287                           void *p)
4288 {
4289         int ret_val;
4290
4291         ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4292                                          __igb_notify_dca);
4293
4294         return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4295 }
4296 #endif /* CONFIG_IGB_DCA */
4297
4298 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4299 {
4300         struct e1000_hw *hw = &adapter->hw;
4301         u32 ping;
4302         int i;
4303
4304         for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4305                 ping = E1000_PF_CONTROL_MSG;
4306                 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4307                         ping |= E1000_VT_MSGTYPE_CTS;
4308                 igb_write_mbx(hw, &ping, 1, i);
4309         }
4310 }
4311
4312 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4313 {
4314         struct e1000_hw *hw = &adapter->hw;
4315         u32 vmolr = rd32(E1000_VMOLR(vf));
4316         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4317
4318         vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4319                             IGB_VF_FLAG_MULTI_PROMISC);
4320         vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4321
4322         if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4323                 vmolr |= E1000_VMOLR_MPME;
4324                 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4325         } else {
4326                 /*
4327                  * if we have hashes and we are clearing a multicast promisc
4328                  * flag we need to write the hashes to the MTA as this step
4329                  * was previously skipped
4330                  */
4331                 if (vf_data->num_vf_mc_hashes > 30) {
4332                         vmolr |= E1000_VMOLR_MPME;
4333                 } else if (vf_data->num_vf_mc_hashes) {
4334                         int j;
4335                         vmolr |= E1000_VMOLR_ROMPE;
4336                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4337                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4338                 }
4339         }
4340
4341         wr32(E1000_VMOLR(vf), vmolr);
4342
4343         /* there are flags left unprocessed, likely not supported */
4344         if (*msgbuf & E1000_VT_MSGINFO_MASK)
4345                 return -EINVAL;
4346
4347         return 0;
4348
4349 }
4350
4351 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4352                                   u32 *msgbuf, u32 vf)
4353 {
4354         int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4355         u16 *hash_list = (u16 *)&msgbuf[1];
4356         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4357         int i;
4358
4359         /* salt away the number of multicast addresses assigned
4360          * to this VF for later use to restore when the PF multi cast
4361          * list changes
4362          */
4363         vf_data->num_vf_mc_hashes = n;
4364
4365         /* only up to 30 hash values supported */
4366         if (n > 30)
4367                 n = 30;
4368
4369         /* store the hashes for later use */
4370         for (i = 0; i < n; i++)
4371                 vf_data->vf_mc_hashes[i] = hash_list[i];
4372
4373         /* Flush and reset the mta with the new values */
4374         igb_set_rx_mode(adapter->netdev);
4375
4376         return 0;
4377 }
4378
4379 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4380 {
4381         struct e1000_hw *hw = &adapter->hw;
4382         struct vf_data_storage *vf_data;
4383         int i, j;
4384
4385         for (i = 0; i < adapter->vfs_allocated_count; i++) {
4386                 u32 vmolr = rd32(E1000_VMOLR(i));
4387                 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4388
4389                 vf_data = &adapter->vf_data[i];
4390
4391                 if ((vf_data->num_vf_mc_hashes > 30) ||
4392                     (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4393                         vmolr |= E1000_VMOLR_MPME;
4394                 } else if (vf_data->num_vf_mc_hashes) {
4395                         vmolr |= E1000_VMOLR_ROMPE;
4396                         for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4397                                 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4398                 }
4399                 wr32(E1000_VMOLR(i), vmolr);
4400         }
4401 }
4402
4403 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4404 {
4405         struct e1000_hw *hw = &adapter->hw;
4406         u32 pool_mask, reg, vid;
4407         int i;
4408
4409         pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4410
4411         /* Find the vlan filter for this id */
4412         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4413                 reg = rd32(E1000_VLVF(i));
4414
4415                 /* remove the vf from the pool */
4416                 reg &= ~pool_mask;
4417
4418                 /* if pool is empty then remove entry from vfta */
4419                 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4420                     (reg & E1000_VLVF_VLANID_ENABLE)) {
4421                         reg = 0;
4422                         vid = reg & E1000_VLVF_VLANID_MASK;
4423                         igb_vfta_set(hw, vid, false);
4424                 }
4425
4426                 wr32(E1000_VLVF(i), reg);
4427         }
4428
4429         adapter->vf_data[vf].vlans_enabled = 0;
4430 }
4431
4432 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4433 {
4434         struct e1000_hw *hw = &adapter->hw;
4435         u32 reg, i;
4436
4437         /* The vlvf table only exists on 82576 hardware and newer */
4438         if (hw->mac.type < e1000_82576)
4439                 return -1;
4440
4441         /* we only need to do this if VMDq is enabled */
4442         if (!adapter->vfs_allocated_count)
4443                 return -1;
4444
4445         /* Find the vlan filter for this id */
4446         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4447                 reg = rd32(E1000_VLVF(i));
4448                 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4449                     vid == (reg & E1000_VLVF_VLANID_MASK))
4450                         break;
4451         }
4452
4453         if (add) {
4454                 if (i == E1000_VLVF_ARRAY_SIZE) {
4455                         /* Did not find a matching VLAN ID entry that was
4456                          * enabled.  Search for a free filter entry, i.e.
4457                          * one without the enable bit set
4458                          */
4459                         for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4460                                 reg = rd32(E1000_VLVF(i));
4461                                 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4462                                         break;
4463                         }
4464                 }
4465                 if (i < E1000_VLVF_ARRAY_SIZE) {
4466                         /* Found an enabled/available entry */
4467                         reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4468
4469                         /* if !enabled we need to set this up in vfta */
4470                         if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4471                                 /* add VID to filter table */
4472                                 igb_vfta_set(hw, vid, true);
4473                                 reg |= E1000_VLVF_VLANID_ENABLE;
4474                         }
4475                         reg &= ~E1000_VLVF_VLANID_MASK;
4476                         reg |= vid;
4477                         wr32(E1000_VLVF(i), reg);
4478
4479                         /* do not modify RLPML for PF devices */
4480                         if (vf >= adapter->vfs_allocated_count)
4481                                 return 0;
4482
4483                         if (!adapter->vf_data[vf].vlans_enabled) {
4484                                 u32 size;
4485                                 reg = rd32(E1000_VMOLR(vf));
4486                                 size = reg & E1000_VMOLR_RLPML_MASK;
4487                                 size += 4;
4488                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4489                                 reg |= size;
4490                                 wr32(E1000_VMOLR(vf), reg);
4491                         }
4492
4493                         adapter->vf_data[vf].vlans_enabled++;
4494                         return 0;
4495                 }
4496         } else {
4497                 if (i < E1000_VLVF_ARRAY_SIZE) {
4498                         /* remove vf from the pool */
4499                         reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4500                         /* if pool is empty then remove entry from vfta */
4501                         if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4502                                 reg = 0;
4503                                 igb_vfta_set(hw, vid, false);
4504                         }
4505                         wr32(E1000_VLVF(i), reg);
4506
4507                         /* do not modify RLPML for PF devices */
4508                         if (vf >= adapter->vfs_allocated_count)
4509                                 return 0;
4510
4511                         adapter->vf_data[vf].vlans_enabled--;
4512                         if (!adapter->vf_data[vf].vlans_enabled) {
4513                                 u32 size;
4514                                 reg = rd32(E1000_VMOLR(vf));
4515                                 size = reg & E1000_VMOLR_RLPML_MASK;
4516                                 size -= 4;
4517                                 reg &= ~E1000_VMOLR_RLPML_MASK;
4518                                 reg |= size;
4519                                 wr32(E1000_VMOLR(vf), reg);
4520                         }
4521                 }
4522         }
4523         return 0;
4524 }
4525
4526 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4527 {
4528         struct e1000_hw *hw = &adapter->hw;
4529
4530         if (vid)
4531                 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4532         else
4533                 wr32(E1000_VMVIR(vf), 0);
4534 }
4535
4536 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4537                                int vf, u16 vlan, u8 qos)
4538 {
4539         int err = 0;
4540         struct igb_adapter *adapter = netdev_priv(netdev);
4541
4542         if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4543                 return -EINVAL;
4544         if (vlan || qos) {
4545                 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4546                 if (err)
4547                         goto out;
4548                 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4549                 igb_set_vmolr(adapter, vf, !vlan);
4550                 adapter->vf_data[vf].pf_vlan = vlan;
4551                 adapter->vf_data[vf].pf_qos = qos;
4552                 dev_info(&adapter->pdev->dev,
4553                          "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4554                 if (test_bit(__IGB_DOWN, &adapter->state)) {
4555                         dev_warn(&adapter->pdev->dev,
4556                                  "The VF VLAN has been set,"
4557                                  " but the PF device is not up.\n");
4558                         dev_warn(&adapter->pdev->dev,
4559                                  "Bring the PF device up before"
4560                                  " attempting to use the VF device.\n");
4561                 }
4562         } else {
4563                 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4564                                    false, vf);
4565                 igb_set_vmvir(adapter, vlan, vf);
4566                 igb_set_vmolr(adapter, vf, true);
4567                 adapter->vf_data[vf].pf_vlan = 0;
4568                 adapter->vf_data[vf].pf_qos = 0;
4569        }
4570 out:
4571        return err;
4572 }
4573
4574 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4575 {
4576         int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4577         int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4578
4579         return igb_vlvf_set(adapter, vid, add, vf);
4580 }
4581
4582 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4583 {
4584         /* clear flags */
4585         adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4586         adapter->vf_data[vf].last_nack = jiffies;
4587
4588         /* reset offloads to defaults */
4589         igb_set_vmolr(adapter, vf, true);
4590
4591         /* reset vlans for device */
4592         igb_clear_vf_vfta(adapter, vf);
4593         if (adapter->vf_data[vf].pf_vlan)
4594                 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4595                                     adapter->vf_data[vf].pf_vlan,
4596                                     adapter->vf_data[vf].pf_qos);
4597         else
4598                 igb_clear_vf_vfta(adapter, vf);
4599
4600         /* reset multicast table array for vf */
4601         adapter->vf_data[vf].num_vf_mc_hashes = 0;
4602
4603         /* Flush and reset the mta with the new values */
4604         igb_set_rx_mode(adapter->netdev);
4605 }
4606
4607 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4608 {
4609         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4610
4611         /* generate a new mac address as we were hotplug removed/added */
4612         if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4613                 random_ether_addr(vf_mac);
4614
4615         /* process remaining reset events */
4616         igb_vf_reset(adapter, vf);
4617 }
4618
4619 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4620 {
4621         struct e1000_hw *hw = &adapter->hw;
4622         unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4623         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4624         u32 reg, msgbuf[3];
4625         u8 *addr = (u8 *)(&msgbuf[1]);
4626
4627         /* process all the same items cleared in a function level reset */
4628         igb_vf_reset(adapter, vf);
4629
4630         /* set vf mac address */
4631         igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4632
4633         /* enable transmit and receive for vf */
4634         reg = rd32(E1000_VFTE);
4635         wr32(E1000_VFTE, reg | (1 << vf));
4636         reg = rd32(E1000_VFRE);
4637         wr32(E1000_VFRE, reg | (1 << vf));
4638
4639         adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4640
4641         /* reply to reset with ack and vf mac address */
4642         msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4643         memcpy(addr, vf_mac, 6);
4644         igb_write_mbx(hw, msgbuf, 3, vf);
4645 }
4646
4647 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4648 {
4649         unsigned char *addr = (char *)&msg[1];
4650         int err = -1;
4651
4652         if (is_valid_ether_addr(addr))
4653                 err = igb_set_vf_mac(adapter, vf, addr);
4654
4655         return err;
4656 }
4657
4658 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
4659 {
4660         struct e1000_hw *hw = &adapter->hw;
4661         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4662         u32 msg = E1000_VT_MSGTYPE_NACK;
4663
4664         /* if device isn't clear to send it shouldn't be reading either */
4665         if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
4666             time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
4667                 igb_write_mbx(hw, &msg, 1, vf);
4668                 vf_data->last_nack = jiffies;
4669         }
4670 }
4671
4672 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
4673 {
4674         struct pci_dev *pdev = adapter->pdev;
4675         u32 msgbuf[E1000_VFMAILBOX_SIZE];
4676         struct e1000_hw *hw = &adapter->hw;
4677         struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4678         s32 retval;
4679
4680         retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
4681
4682         if (retval) {
4683                 /* if receive failed revoke VF CTS stats and restart init */
4684                 dev_err(&pdev->dev, "Error receiving message from VF\n");
4685                 vf_data->flags &= ~IGB_VF_FLAG_CTS;
4686                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4687                         return;
4688                 goto out;
4689         }
4690
4691         /* this is a message we already processed, do nothing */
4692         if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
4693                 return;
4694
4695         /*
4696          * until the vf completes a reset it should not be
4697          * allowed to start any configuration.
4698          */
4699
4700         if (msgbuf[0] == E1000_VF_RESET) {
4701                 igb_vf_reset_msg(adapter, vf);
4702                 return;
4703         }
4704
4705         if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
4706                 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
4707                         return;
4708                 retval = -1;
4709                 goto out;
4710         }
4711
4712         switch ((msgbuf[0] & 0xFFFF)) {
4713         case E1000_VF_SET_MAC_ADDR:
4714                 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
4715                 break;
4716         case E1000_VF_SET_PROMISC:
4717                 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
4718                 break;
4719         case E1000_VF_SET_MULTICAST:
4720                 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
4721                 break;
4722         case E1000_VF_SET_LPE:
4723                 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
4724                 break;
4725         case E1000_VF_SET_VLAN:
4726                 if (adapter->vf_data[vf].pf_vlan)
4727                         retval = -1;
4728                 else
4729                         retval = igb_set_vf_vlan(adapter, msgbuf, vf);
4730                 break;
4731         default:
4732                 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
4733                 retval = -1;
4734                 break;
4735         }
4736
4737         msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
4738 out:
4739         /* notify the VF of the results of what it sent us */
4740         if (retval)
4741                 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
4742         else
4743                 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
4744
4745         igb_write_mbx(hw, msgbuf, 1, vf);
4746 }
4747
4748 static void igb_msg_task(struct igb_adapter *adapter)
4749 {
4750         struct e1000_hw *hw = &adapter->hw;
4751         u32 vf;
4752
4753         for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
4754                 /* process any reset requests */
4755                 if (!igb_check_for_rst(hw, vf))
4756                         igb_vf_reset_event(adapter, vf);
4757
4758                 /* process any messages pending */
4759                 if (!igb_check_for_msg(hw, vf))
4760                         igb_rcv_msg_from_vf(adapter, vf);
4761
4762                 /* process any acks */
4763                 if (!igb_check_for_ack(hw, vf))
4764                         igb_rcv_ack_from_vf(adapter, vf);
4765         }
4766 }
4767
4768 /**
4769  *  igb_set_uta - Set unicast filter table address
4770  *  @adapter: board private structure
4771  *
4772  *  The unicast table address is a register array of 32-bit registers.
4773  *  The table is meant to be used in a way similar to how the MTA is used
4774  *  however due to certain limitations in the hardware it is necessary to
4775  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
4776  *  enable bit to allow vlan tag stripping when promiscous mode is enabled
4777  **/
4778 static void igb_set_uta(struct igb_adapter *adapter)
4779 {
4780         struct e1000_hw *hw = &adapter->hw;
4781         int i;
4782
4783         /* The UTA table only exists on 82576 hardware and newer */
4784         if (hw->mac.type < e1000_82576)
4785                 return;
4786
4787         /* we only need to do this if VMDq is enabled */
4788         if (!adapter->vfs_allocated_count)
4789                 return;
4790
4791         for (i = 0; i < hw->mac.uta_reg_count; i++)
4792                 array_wr32(E1000_UTA, i, ~0);
4793 }
4794
4795 /**
4796  * igb_intr_msi - Interrupt Handler
4797  * @irq: interrupt number
4798  * @data: pointer to a network interface device structure
4799  **/
4800 static irqreturn_t igb_intr_msi(int irq, void *data)
4801 {
4802         struct igb_adapter *adapter = data;
4803         struct igb_q_vector *q_vector = adapter->q_vector[0];
4804         struct e1000_hw *hw = &adapter->hw;
4805         /* read ICR disables interrupts using IAM */
4806         u32 icr = rd32(E1000_ICR);
4807
4808         igb_write_itr(q_vector);
4809
4810         if (icr & E1000_ICR_DRSTA)
4811                 schedule_work(&adapter->reset_task);
4812
4813         if (icr & E1000_ICR_DOUTSYNC) {
4814                 /* HW is reporting DMA is out of sync */
4815                 adapter->stats.doosync++;
4816         }
4817
4818         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4819                 hw->mac.get_link_status = 1;
4820                 if (!test_bit(__IGB_DOWN, &adapter->state))
4821                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4822         }
4823
4824         napi_schedule(&q_vector->napi);
4825
4826         return IRQ_HANDLED;
4827 }
4828
4829 /**
4830  * igb_intr - Legacy Interrupt Handler
4831  * @irq: interrupt number
4832  * @data: pointer to a network interface device structure
4833  **/
4834 static irqreturn_t igb_intr(int irq, void *data)
4835 {
4836         struct igb_adapter *adapter = data;
4837         struct igb_q_vector *q_vector = adapter->q_vector[0];
4838         struct e1000_hw *hw = &adapter->hw;
4839         /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
4840          * need for the IMC write */
4841         u32 icr = rd32(E1000_ICR);
4842         if (!icr)
4843                 return IRQ_NONE;  /* Not our interrupt */
4844
4845         igb_write_itr(q_vector);
4846
4847         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
4848          * not set, then the adapter didn't send an interrupt */
4849         if (!(icr & E1000_ICR_INT_ASSERTED))
4850                 return IRQ_NONE;
4851
4852         if (icr & E1000_ICR_DRSTA)
4853                 schedule_work(&adapter->reset_task);
4854
4855         if (icr & E1000_ICR_DOUTSYNC) {
4856                 /* HW is reporting DMA is out of sync */
4857                 adapter->stats.doosync++;
4858         }
4859
4860         if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
4861                 hw->mac.get_link_status = 1;
4862                 /* guard against interrupt when we're going down */
4863                 if (!test_bit(__IGB_DOWN, &adapter->state))
4864                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
4865         }
4866
4867         napi_schedule(&q_vector->napi);
4868
4869         return IRQ_HANDLED;
4870 }
4871
4872 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
4873 {
4874         struct igb_adapter *adapter = q_vector->adapter;
4875         struct e1000_hw *hw = &adapter->hw;
4876
4877         if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
4878             (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
4879                 if (!adapter->msix_entries)
4880                         igb_set_itr(adapter);
4881                 else
4882                         igb_update_ring_itr(q_vector);
4883         }
4884
4885         if (!test_bit(__IGB_DOWN, &adapter->state)) {
4886                 if (adapter->msix_entries)
4887                         wr32(E1000_EIMS, q_vector->eims_value);
4888                 else
4889                         igb_irq_enable(adapter);
4890         }
4891 }
4892
4893 /**
4894  * igb_poll - NAPI Rx polling callback
4895  * @napi: napi polling structure
4896  * @budget: count of how many packets we should handle
4897  **/
4898 static int igb_poll(struct napi_struct *napi, int budget)
4899 {
4900         struct igb_q_vector *q_vector = container_of(napi,
4901                                                      struct igb_q_vector,
4902                                                      napi);
4903         int tx_clean_complete = 1, work_done = 0;
4904
4905 #ifdef CONFIG_IGB_DCA
4906         if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
4907                 igb_update_dca(q_vector);
4908 #endif
4909         if (q_vector->tx_ring)
4910                 tx_clean_complete = igb_clean_tx_irq(q_vector);
4911
4912         if (q_vector->rx_ring)
4913                 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
4914
4915         if (!tx_clean_complete)
4916                 work_done = budget;
4917
4918         /* If not enough Rx work done, exit the polling mode */
4919         if (work_done < budget) {
4920                 napi_complete(napi);
4921                 igb_ring_irq_enable(q_vector);
4922         }
4923
4924         return work_done;
4925 }
4926
4927 /**
4928  * igb_systim_to_hwtstamp - convert system time value to hw timestamp
4929  * @adapter: board private structure
4930  * @shhwtstamps: timestamp structure to update
4931  * @regval: unsigned 64bit system time value.
4932  *
4933  * We need to convert the system time value stored in the RX/TXSTMP registers
4934  * into a hwtstamp which can be used by the upper level timestamping functions
4935  */
4936 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
4937                                    struct skb_shared_hwtstamps *shhwtstamps,
4938                                    u64 regval)
4939 {
4940         u64 ns;
4941
4942         /*
4943          * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
4944          * 24 to match clock shift we setup earlier.
4945          */
4946         if (adapter->hw.mac.type == e1000_82580)
4947                 regval <<= IGB_82580_TSYNC_SHIFT;
4948
4949         ns = timecounter_cyc2time(&adapter->clock, regval);
4950         timecompare_update(&adapter->compare, ns);
4951         memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
4952         shhwtstamps->hwtstamp = ns_to_ktime(ns);
4953         shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
4954 }
4955
4956 /**
4957  * igb_tx_hwtstamp - utility function which checks for TX time stamp
4958  * @q_vector: pointer to q_vector containing needed info
4959  * @skb: packet that was just sent
4960  *
4961  * If we were asked to do hardware stamping and such a time stamp is
4962  * available, then it must have been for this skb here because we only
4963  * allow only one such packet into the queue.
4964  */
4965 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct sk_buff *skb)
4966 {
4967         struct igb_adapter *adapter = q_vector->adapter;
4968         union skb_shared_tx *shtx = skb_tx(skb);
4969         struct e1000_hw *hw = &adapter->hw;
4970         struct skb_shared_hwtstamps shhwtstamps;
4971         u64 regval;
4972
4973         /* if skb does not support hw timestamp or TX stamp not valid exit */
4974         if (likely(!shtx->hardware) ||
4975             !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
4976                 return;
4977
4978         regval = rd32(E1000_TXSTMPL);
4979         regval |= (u64)rd32(E1000_TXSTMPH) << 32;
4980
4981         igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
4982         skb_tstamp_tx(skb, &shhwtstamps);
4983 }
4984
4985 /**
4986  * igb_clean_tx_irq - Reclaim resources after transmit completes
4987  * @q_vector: pointer to q_vector containing needed info
4988  * returns true if ring is completely cleaned
4989  **/
4990 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
4991 {
4992         struct igb_adapter *adapter = q_vector->adapter;
4993         struct igb_ring *tx_ring = q_vector->tx_ring;
4994         struct net_device *netdev = tx_ring->netdev;
4995         struct e1000_hw *hw = &adapter->hw;
4996         struct igb_buffer *buffer_info;
4997         struct sk_buff *skb;
4998         union e1000_adv_tx_desc *tx_desc, *eop_desc;
4999         unsigned int total_bytes = 0, total_packets = 0;
5000         unsigned int i, eop, count = 0;
5001         bool cleaned = false;
5002
5003         i = tx_ring->next_to_clean;
5004         eop = tx_ring->buffer_info[i].next_to_watch;
5005         eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5006
5007         while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5008                (count < tx_ring->count)) {
5009                 for (cleaned = false; !cleaned; count++) {
5010                         tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5011                         buffer_info = &tx_ring->buffer_info[i];
5012                         cleaned = (i == eop);
5013                         skb = buffer_info->skb;
5014
5015                         if (skb) {
5016                                 unsigned int segs, bytecount;
5017                                 /* gso_segs is currently only valid for tcp */
5018                                 segs = buffer_info->gso_segs;
5019                                 /* multiply data chunks by size of headers */
5020                                 bytecount = ((segs - 1) * skb_headlen(skb)) +
5021                                             skb->len;
5022                                 total_packets += segs;
5023                                 total_bytes += bytecount;
5024
5025                                 igb_tx_hwtstamp(q_vector, skb);
5026                         }
5027
5028                         igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5029                         tx_desc->wb.status = 0;
5030
5031                         i++;
5032                         if (i == tx_ring->count)
5033                                 i = 0;
5034                 }
5035                 eop = tx_ring->buffer_info[i].next_to_watch;
5036                 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5037         }
5038
5039         tx_ring->next_to_clean = i;
5040
5041         if (unlikely(count &&
5042                      netif_carrier_ok(netdev) &&
5043                      igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5044                 /* Make sure that anybody stopping the queue after this
5045                  * sees the new next_to_clean.
5046                  */
5047                 smp_mb();
5048                 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5049                     !(test_bit(__IGB_DOWN, &adapter->state))) {
5050                         netif_wake_subqueue(netdev, tx_ring->queue_index);
5051                         tx_ring->tx_stats.restart_queue++;
5052                 }
5053         }
5054
5055         if (tx_ring->detect_tx_hung) {
5056                 /* Detect a transmit hang in hardware, this serializes the
5057                  * check with the clearing of time_stamp and movement of i */
5058                 tx_ring->detect_tx_hung = false;
5059                 if (tx_ring->buffer_info[i].time_stamp &&
5060                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5061                                (adapter->tx_timeout_factor * HZ)) &&
5062                     !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5063
5064                         /* detected Tx unit hang */
5065                         dev_err(tx_ring->dev,
5066                                 "Detected Tx Unit Hang\n"
5067                                 "  Tx Queue             <%d>\n"
5068                                 "  TDH                  <%x>\n"
5069                                 "  TDT                  <%x>\n"
5070                                 "  next_to_use          <%x>\n"
5071                                 "  next_to_clean        <%x>\n"
5072                                 "buffer_info[next_to_clean]\n"
5073                                 "  time_stamp           <%lx>\n"
5074                                 "  next_to_watch        <%x>\n"
5075                                 "  jiffies              <%lx>\n"
5076                                 "  desc.status          <%x>\n",
5077                                 tx_ring->queue_index,
5078                                 readl(tx_ring->head),
5079                                 readl(tx_ring->tail),
5080                                 tx_ring->next_to_use,
5081                                 tx_ring->next_to_clean,
5082                                 tx_ring->buffer_info[eop].time_stamp,
5083                                 eop,
5084                                 jiffies,
5085                                 eop_desc->wb.status);
5086                         netif_stop_subqueue(netdev, tx_ring->queue_index);
5087                 }
5088         }
5089         tx_ring->total_bytes += total_bytes;
5090         tx_ring->total_packets += total_packets;
5091         tx_ring->tx_stats.bytes += total_bytes;
5092         tx_ring->tx_stats.packets += total_packets;
5093         return (count < tx_ring->count);
5094 }
5095
5096 /**
5097  * igb_receive_skb - helper function to handle rx indications
5098  * @q_vector: structure containing interrupt and ring information
5099  * @skb: packet to send up
5100  * @vlan_tag: vlan tag for packet
5101  **/
5102 static void igb_receive_skb(struct igb_q_vector *q_vector,
5103                             struct sk_buff *skb,
5104                             u16 vlan_tag)
5105 {
5106         struct igb_adapter *adapter = q_vector->adapter;
5107
5108         if (vlan_tag && adapter->vlgrp)
5109                 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5110                                  vlan_tag, skb);
5111         else
5112                 napi_gro_receive(&q_vector->napi, skb);
5113 }
5114
5115 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5116                                        u32 status_err, struct sk_buff *skb)
5117 {
5118         skb->ip_summed = CHECKSUM_NONE;
5119
5120         /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5121         if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5122              (status_err & E1000_RXD_STAT_IXSM))
5123                 return;
5124
5125         /* TCP/UDP checksum error bit is set */
5126         if (status_err &
5127             (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5128                 /*
5129                  * work around errata with sctp packets where the TCPE aka
5130                  * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5131                  * packets, (aka let the stack check the crc32c)
5132                  */
5133                 if ((skb->len == 60) &&
5134                     (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5135                         ring->rx_stats.csum_err++;
5136
5137                 /* let the stack verify checksum errors */
5138                 return;
5139         }
5140         /* It must be a TCP or UDP packet with a valid checksum */
5141         if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5142                 skb->ip_summed = CHECKSUM_UNNECESSARY;
5143
5144         dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5145 }
5146
5147 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5148                                    struct sk_buff *skb)
5149 {
5150         struct igb_adapter *adapter = q_vector->adapter;
5151         struct e1000_hw *hw = &adapter->hw;
5152         u64 regval;
5153
5154         /*
5155          * If this bit is set, then the RX registers contain the time stamp. No
5156          * other packet will be time stamped until we read these registers, so
5157          * read the registers to make them available again. Because only one
5158          * packet can be time stamped at a time, we know that the register
5159          * values must belong to this one here and therefore we don't need to
5160          * compare any of the additional attributes stored for it.
5161          *
5162          * If nothing went wrong, then it should have a skb_shared_tx that we
5163          * can turn into a skb_shared_hwtstamps.
5164          */
5165         if (staterr & E1000_RXDADV_STAT_TSIP) {
5166                 u32 *stamp = (u32 *)skb->data;
5167                 regval = le32_to_cpu(*(stamp + 2));
5168                 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5169                 skb_pull(skb, IGB_TS_HDR_LEN);
5170         } else {
5171                 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5172                         return;
5173
5174                 regval = rd32(E1000_RXSTMPL);
5175                 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5176         }
5177
5178         igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5179 }
5180 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5181                                union e1000_adv_rx_desc *rx_desc)
5182 {
5183         /* HW will not DMA in data larger than the given buffer, even if it
5184          * parses the (NFS, of course) header to be larger.  In that case, it
5185          * fills the header buffer and spills the rest into the page.
5186          */
5187         u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5188                    E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5189         if (hlen > rx_ring->rx_buffer_len)
5190                 hlen = rx_ring->rx_buffer_len;
5191         return hlen;
5192 }
5193
5194 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5195                                  int *work_done, int budget)
5196 {
5197         struct igb_ring *rx_ring = q_vector->rx_ring;
5198         struct net_device *netdev = rx_ring->netdev;
5199         struct device *dev = rx_ring->dev;
5200         union e1000_adv_rx_desc *rx_desc , *next_rxd;
5201         struct igb_buffer *buffer_info , *next_buffer;
5202         struct sk_buff *skb;
5203         bool cleaned = false;
5204         int cleaned_count = 0;
5205         int current_node = numa_node_id();
5206         unsigned int total_bytes = 0, total_packets = 0;
5207         unsigned int i;
5208         u32 staterr;
5209         u16 length;
5210         u16 vlan_tag;
5211
5212         i = rx_ring->next_to_clean;
5213         buffer_info = &rx_ring->buffer_info[i];
5214         rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5215         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5216
5217         while (staterr & E1000_RXD_STAT_DD) {
5218                 if (*work_done >= budget)
5219                         break;
5220                 (*work_done)++;
5221
5222                 skb = buffer_info->skb;
5223                 prefetch(skb->data - NET_IP_ALIGN);
5224                 buffer_info->skb = NULL;
5225
5226                 i++;
5227                 if (i == rx_ring->count)
5228                         i = 0;
5229
5230                 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5231                 prefetch(next_rxd);
5232                 next_buffer = &rx_ring->buffer_info[i];
5233
5234                 length = le16_to_cpu(rx_desc->wb.upper.length);
5235                 cleaned = true;
5236                 cleaned_count++;
5237
5238                 if (buffer_info->dma) {
5239                         dma_unmap_single(dev, buffer_info->dma,
5240                                          rx_ring->rx_buffer_len,
5241                                          DMA_FROM_DEVICE);
5242                         buffer_info->dma = 0;
5243                         if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5244                                 skb_put(skb, length);
5245                                 goto send_up;
5246                         }
5247                         skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5248                 }
5249
5250                 if (length) {
5251                         dma_unmap_page(dev, buffer_info->page_dma,
5252                                        PAGE_SIZE / 2, DMA_FROM_DEVICE);
5253                         buffer_info->page_dma = 0;
5254
5255                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5256                                                 buffer_info->page,
5257                                                 buffer_info->page_offset,
5258                                                 length);
5259
5260                         if ((page_count(buffer_info->page) != 1) ||
5261                             (page_to_nid(buffer_info->page) != current_node))
5262                                 buffer_info->page = NULL;
5263                         else
5264                                 get_page(buffer_info->page);
5265
5266                         skb->len += length;
5267                         skb->data_len += length;
5268                         skb->truesize += length;
5269                 }
5270
5271                 if (!(staterr & E1000_RXD_STAT_EOP)) {
5272                         buffer_info->skb = next_buffer->skb;
5273                         buffer_info->dma = next_buffer->dma;
5274                         next_buffer->skb = skb;
5275                         next_buffer->dma = 0;
5276                         goto next_desc;
5277                 }
5278 send_up:
5279                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5280                         dev_kfree_skb_irq(skb);
5281                         goto next_desc;
5282                 }
5283
5284                 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5285                         igb_rx_hwtstamp(q_vector, staterr, skb);
5286                 total_bytes += skb->len;
5287                 total_packets++;
5288
5289                 igb_rx_checksum_adv(rx_ring, staterr, skb);
5290
5291                 skb->protocol = eth_type_trans(skb, netdev);
5292                 skb_record_rx_queue(skb, rx_ring->queue_index);
5293
5294                 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5295                             le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5296
5297                 igb_receive_skb(q_vector, skb, vlan_tag);
5298
5299 next_desc:
5300                 rx_desc->wb.upper.status_error = 0;
5301
5302                 /* return some buffers to hardware, one at a time is too slow */
5303                 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5304                         igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5305                         cleaned_count = 0;
5306                 }
5307
5308                 /* use prefetched values */
5309                 rx_desc = next_rxd;
5310                 buffer_info = next_buffer;
5311                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5312         }
5313
5314         rx_ring->next_to_clean = i;
5315         cleaned_count = igb_desc_unused(rx_ring);
5316
5317         if (cleaned_count)
5318                 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5319
5320         rx_ring->total_packets += total_packets;
5321         rx_ring->total_bytes += total_bytes;
5322         rx_ring->rx_stats.packets += total_packets;
5323         rx_ring->rx_stats.bytes += total_bytes;
5324         return cleaned;
5325 }
5326
5327 /**
5328  * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5329  * @adapter: address of board private structure
5330  **/
5331 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5332 {
5333         struct net_device *netdev = rx_ring->netdev;
5334         union e1000_adv_rx_desc *rx_desc;
5335         struct igb_buffer *buffer_info;
5336         struct sk_buff *skb;
5337         unsigned int i;
5338         int bufsz;
5339
5340         i = rx_ring->next_to_use;
5341         buffer_info = &rx_ring->buffer_info[i];
5342
5343         bufsz = rx_ring->rx_buffer_len;
5344
5345         while (cleaned_count--) {
5346                 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5347
5348                 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5349                         if (!buffer_info->page) {
5350                                 buffer_info->page = netdev_alloc_page(netdev);
5351                                 if (!buffer_info->page) {
5352                                         rx_ring->rx_stats.alloc_failed++;
5353                                         goto no_buffers;
5354                                 }
5355                                 buffer_info->page_offset = 0;
5356                         } else {
5357                                 buffer_info->page_offset ^= PAGE_SIZE / 2;
5358                         }
5359                         buffer_info->page_dma =
5360                                 dma_map_page(rx_ring->dev, buffer_info->page,
5361                                              buffer_info->page_offset,
5362                                              PAGE_SIZE / 2,
5363                                              DMA_FROM_DEVICE);
5364                         if (dma_mapping_error(rx_ring->dev,
5365                                               buffer_info->page_dma)) {
5366                                 buffer_info->page_dma = 0;
5367                                 rx_ring->rx_stats.alloc_failed++;
5368                                 goto no_buffers;
5369                         }
5370                 }
5371
5372                 skb = buffer_info->skb;
5373                 if (!skb) {
5374                         skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5375                         if (!skb) {
5376                                 rx_ring->rx_stats.alloc_failed++;
5377                                 goto no_buffers;
5378                         }
5379
5380                         buffer_info->skb = skb;
5381                 }
5382                 if (!buffer_info->dma) {
5383                         buffer_info->dma = dma_map_single(rx_ring->dev,
5384                                                           skb->data,
5385                                                           bufsz,
5386                                                           DMA_FROM_DEVICE);
5387                         if (dma_mapping_error(rx_ring->dev,
5388                                               buffer_info->dma)) {
5389                                 buffer_info->dma = 0;
5390                                 rx_ring->rx_stats.alloc_failed++;
5391                                 goto no_buffers;
5392                         }
5393                 }
5394                 /* Refresh the desc even if buffer_addrs didn't change because
5395                  * each write-back erases this info. */
5396                 if (bufsz < IGB_RXBUFFER_1024) {
5397                         rx_desc->read.pkt_addr =
5398                              cpu_to_le64(buffer_info->page_dma);
5399                         rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5400                 } else {
5401                         rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5402                         rx_desc->read.hdr_addr = 0;
5403                 }
5404
5405                 i++;
5406                 if (i == rx_ring->count)
5407                         i = 0;
5408                 buffer_info = &rx_ring->buffer_info[i];
5409         }
5410
5411 no_buffers:
5412         if (rx_ring->next_to_use != i) {
5413                 rx_ring->next_to_use = i;
5414                 if (i == 0)
5415                         i = (rx_ring->count - 1);
5416                 else
5417                         i--;
5418
5419                 /* Force memory writes to complete before letting h/w
5420                  * know there are new descriptors to fetch.  (Only
5421                  * applicable for weak-ordered memory model archs,
5422                  * such as IA-64). */
5423                 wmb();
5424                 writel(i, rx_ring->tail);
5425         }
5426 }
5427
5428 /**
5429  * igb_mii_ioctl -
5430  * @netdev:
5431  * @ifreq:
5432  * @cmd:
5433  **/
5434 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5435 {
5436         struct igb_adapter *adapter = netdev_priv(netdev);
5437         struct mii_ioctl_data *data = if_mii(ifr);
5438
5439         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5440                 return -EOPNOTSUPP;
5441
5442         switch (cmd) {
5443         case SIOCGMIIPHY:
5444                 data->phy_id = adapter->hw.phy.addr;
5445                 break;
5446         case SIOCGMIIREG:
5447                 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5448                                      &data->val_out))
5449                         return -EIO;
5450                 break;
5451         case SIOCSMIIREG:
5452         default:
5453                 return -EOPNOTSUPP;
5454         }
5455         return 0;
5456 }
5457
5458 /**
5459  * igb_hwtstamp_ioctl - control hardware time stamping
5460  * @netdev:
5461  * @ifreq:
5462  * @cmd:
5463  *
5464  * Outgoing time stamping can be enabled and disabled. Play nice and
5465  * disable it when requested, although it shouldn't case any overhead
5466  * when no packet needs it. At most one packet in the queue may be
5467  * marked for time stamping, otherwise it would be impossible to tell
5468  * for sure to which packet the hardware time stamp belongs.
5469  *
5470  * Incoming time stamping has to be configured via the hardware
5471  * filters. Not all combinations are supported, in particular event
5472  * type has to be specified. Matching the kind of event packet is
5473  * not supported, with the exception of "all V2 events regardless of
5474  * level 2 or 4".
5475  *
5476  **/
5477 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5478                               struct ifreq *ifr, int cmd)
5479 {
5480         struct igb_adapter *adapter = netdev_priv(netdev);
5481         struct e1000_hw *hw = &adapter->hw;
5482         struct hwtstamp_config config;
5483         u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5484         u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5485         u32 tsync_rx_cfg = 0;
5486         bool is_l4 = false;
5487         bool is_l2 = false;
5488         u32 regval;
5489
5490         if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5491                 return -EFAULT;
5492
5493         /* reserved for future extensions */
5494         if (config.flags)
5495                 return -EINVAL;
5496
5497         switch (config.tx_type) {
5498         case HWTSTAMP_TX_OFF:
5499                 tsync_tx_ctl = 0;
5500         case HWTSTAMP_TX_ON:
5501                 break;
5502         default:
5503                 return -ERANGE;
5504         }
5505
5506         switch (config.rx_filter) {
5507         case HWTSTAMP_FILTER_NONE:
5508                 tsync_rx_ctl = 0;
5509                 break;
5510         case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5511         case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5512         case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5513         case HWTSTAMP_FILTER_ALL:
5514                 /*
5515                  * register TSYNCRXCFG must be set, therefore it is not
5516                  * possible to time stamp both Sync and Delay_Req messages
5517                  * => fall back to time stamping all packets
5518                  */
5519                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5520                 config.rx_filter = HWTSTAMP_FILTER_ALL;
5521                 break;
5522         case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5523                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5524                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5525                 is_l4 = true;
5526                 break;
5527         case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5528                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5529                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5530                 is_l4 = true;
5531                 break;
5532         case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5533         case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5534                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5535                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5536                 is_l2 = true;
5537                 is_l4 = true;
5538                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5539                 break;
5540         case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5541         case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5542                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5543                 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5544                 is_l2 = true;
5545                 is_l4 = true;
5546                 config.rx_filter = HWTSTAMP_FILTER_SOME;
5547                 break;
5548         case HWTSTAMP_FILTER_PTP_V2_EVENT:
5549         case HWTSTAMP_FILTER_PTP_V2_SYNC:
5550         case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5551                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5552                 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5553                 is_l2 = true;
5554                 break;
5555         default:
5556                 return -ERANGE;
5557         }
5558
5559         if (hw->mac.type == e1000_82575) {
5560                 if (tsync_rx_ctl | tsync_tx_ctl)
5561                         return -EINVAL;
5562                 return 0;
5563         }
5564
5565         /*
5566          * Per-packet timestamping only works if all packets are
5567          * timestamped, so enable timestamping in all packets as
5568          * long as one rx filter was configured.
5569          */
5570         if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5571                 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5572                 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5573         }
5574
5575         /* enable/disable TX */
5576         regval = rd32(E1000_TSYNCTXCTL);
5577         regval &= ~E1000_TSYNCTXCTL_ENABLED;
5578         regval |= tsync_tx_ctl;
5579         wr32(E1000_TSYNCTXCTL, regval);
5580
5581         /* enable/disable RX */
5582         regval = rd32(E1000_TSYNCRXCTL);
5583         regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5584         regval |= tsync_rx_ctl;
5585         wr32(E1000_TSYNCRXCTL, regval);
5586
5587         /* define which PTP packets are time stamped */
5588         wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5589
5590         /* define ethertype filter for timestamped packets */
5591         if (is_l2)
5592                 wr32(E1000_ETQF(3),
5593                                 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5594                                  E1000_ETQF_1588 | /* enable timestamping */
5595                                  ETH_P_1588));     /* 1588 eth protocol type */
5596         else
5597                 wr32(E1000_ETQF(3), 0);
5598
5599 #define PTP_PORT 319
5600         /* L4 Queue Filter[3]: filter by destination port and protocol */
5601         if (is_l4) {
5602                 u32 ftqf = (IPPROTO_UDP /* UDP */
5603                         | E1000_FTQF_VF_BP /* VF not compared */
5604                         | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5605                         | E1000_FTQF_MASK); /* mask all inputs */
5606                 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5607
5608                 wr32(E1000_IMIR(3), htons(PTP_PORT));
5609                 wr32(E1000_IMIREXT(3),
5610                      (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5611                 if (hw->mac.type == e1000_82576) {
5612                         /* enable source port check */
5613                         wr32(E1000_SPQF(3), htons(PTP_PORT));
5614                         ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5615                 }
5616                 wr32(E1000_FTQF(3), ftqf);
5617         } else {
5618                 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5619         }
5620         wrfl();
5621
5622         adapter->hwtstamp_config = config;
5623
5624         /* clear TX/RX time stamp registers, just to be sure */
5625         regval = rd32(E1000_TXSTMPH);
5626         regval = rd32(E1000_RXSTMPH);
5627
5628         return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5629                 -EFAULT : 0;
5630 }
5631
5632 /**
5633  * igb_ioctl -
5634  * @netdev:
5635  * @ifreq:
5636  * @cmd:
5637  **/
5638 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5639 {
5640         switch (cmd) {
5641         case SIOCGMIIPHY:
5642         case SIOCGMIIREG:
5643         case SIOCSMIIREG:
5644                 return igb_mii_ioctl(netdev, ifr, cmd);
5645         case SIOCSHWTSTAMP:
5646                 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5647         default:
5648                 return -EOPNOTSUPP;
5649         }
5650 }
5651
5652 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5653 {
5654         struct igb_adapter *adapter = hw->back;
5655         u16 cap_offset;
5656
5657         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5658         if (!cap_offset)
5659                 return -E1000_ERR_CONFIG;
5660
5661         pci_read_config_word(adapter->pdev, cap_offset + reg, value);
5662
5663         return 0;
5664 }
5665
5666 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5667 {
5668         struct igb_adapter *adapter = hw->back;
5669         u16 cap_offset;
5670
5671         cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
5672         if (!cap_offset)
5673                 return -E1000_ERR_CONFIG;
5674
5675         pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
5676
5677         return 0;
5678 }
5679
5680 static void igb_vlan_rx_register(struct net_device *netdev,
5681                                  struct vlan_group *grp)
5682 {
5683         struct igb_adapter *adapter = netdev_priv(netdev);
5684         struct e1000_hw *hw = &adapter->hw;
5685         u32 ctrl, rctl;
5686
5687         igb_irq_disable(adapter);
5688         adapter->vlgrp = grp;
5689
5690         if (grp) {
5691                 /* enable VLAN tag insert/strip */
5692                 ctrl = rd32(E1000_CTRL);
5693                 ctrl |= E1000_CTRL_VME;
5694                 wr32(E1000_CTRL, ctrl);
5695
5696                 /* Disable CFI check */
5697                 rctl = rd32(E1000_RCTL);
5698                 rctl &= ~E1000_RCTL_CFIEN;
5699                 wr32(E1000_RCTL, rctl);
5700         } else {
5701                 /* disable VLAN tag insert/strip */
5702                 ctrl = rd32(E1000_CTRL);
5703                 ctrl &= ~E1000_CTRL_VME;
5704                 wr32(E1000_CTRL, ctrl);
5705         }
5706
5707         igb_rlpml_set(adapter);
5708
5709         if (!test_bit(__IGB_DOWN, &adapter->state))
5710                 igb_irq_enable(adapter);
5711 }
5712
5713 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
5714 {
5715         struct igb_adapter *adapter = netdev_priv(netdev);
5716         struct e1000_hw *hw = &adapter->hw;
5717         int pf_id = adapter->vfs_allocated_count;
5718
5719         /* attempt to add filter to vlvf array */
5720         igb_vlvf_set(adapter, vid, true, pf_id);
5721
5722         /* add the filter since PF can receive vlans w/o entry in vlvf */
5723         igb_vfta_set(hw, vid, true);
5724 }
5725
5726 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
5727 {
5728         struct igb_adapter *adapter = netdev_priv(netdev);
5729         struct e1000_hw *hw = &adapter->hw;
5730         int pf_id = adapter->vfs_allocated_count;
5731         s32 err;
5732
5733         igb_irq_disable(adapter);
5734         vlan_group_set_device(adapter->vlgrp, vid, NULL);
5735
5736         if (!test_bit(__IGB_DOWN, &adapter->state))
5737                 igb_irq_enable(adapter);
5738
5739         /* remove vlan from VLVF table array */
5740         err = igb_vlvf_set(adapter, vid, false, pf_id);
5741
5742         /* if vid was not present in VLVF just remove it from table */
5743         if (err)
5744                 igb_vfta_set(hw, vid, false);
5745 }
5746
5747 static void igb_restore_vlan(struct igb_adapter *adapter)
5748 {
5749         igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
5750
5751         if (adapter->vlgrp) {
5752                 u16 vid;
5753                 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
5754                         if (!vlan_group_get_device(adapter->vlgrp, vid))
5755                                 continue;
5756                         igb_vlan_rx_add_vid(adapter->netdev, vid);
5757                 }
5758         }
5759 }
5760
5761 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
5762 {
5763         struct pci_dev *pdev = adapter->pdev;
5764         struct e1000_mac_info *mac = &adapter->hw.mac;
5765
5766         mac->autoneg = 0;
5767
5768         switch (spddplx) {
5769         case SPEED_10 + DUPLEX_HALF:
5770                 mac->forced_speed_duplex = ADVERTISE_10_HALF;
5771                 break;
5772         case SPEED_10 + DUPLEX_FULL:
5773                 mac->forced_speed_duplex = ADVERTISE_10_FULL;
5774                 break;
5775         case SPEED_100 + DUPLEX_HALF:
5776                 mac->forced_speed_duplex = ADVERTISE_100_HALF;
5777                 break;
5778         case SPEED_100 + DUPLEX_FULL:
5779                 mac->forced_speed_duplex = ADVERTISE_100_FULL;
5780                 break;
5781         case SPEED_1000 + DUPLEX_FULL:
5782                 mac->autoneg = 1;
5783                 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
5784                 break;
5785         case SPEED_1000 + DUPLEX_HALF: /* not supported */
5786         default:
5787                 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
5788                 return -EINVAL;
5789         }
5790         return 0;
5791 }
5792
5793 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
5794 {
5795         struct net_device *netdev = pci_get_drvdata(pdev);
5796         struct igb_adapter *adapter = netdev_priv(netdev);
5797         struct e1000_hw *hw = &adapter->hw;
5798         u32 ctrl, rctl, status;
5799         u32 wufc = adapter->wol;
5800 #ifdef CONFIG_PM
5801         int retval = 0;
5802 #endif
5803
5804         netif_device_detach(netdev);
5805
5806         if (netif_running(netdev))
5807                 igb_close(netdev);
5808
5809         igb_clear_interrupt_scheme(adapter);
5810
5811 #ifdef CONFIG_PM
5812         retval = pci_save_state(pdev);
5813         if (retval)
5814                 return retval;
5815 #endif
5816
5817         status = rd32(E1000_STATUS);
5818         if (status & E1000_STATUS_LU)
5819                 wufc &= ~E1000_WUFC_LNKC;
5820
5821         if (wufc) {
5822                 igb_setup_rctl(adapter);
5823                 igb_set_rx_mode(netdev);
5824
5825                 /* turn on all-multi mode if wake on multicast is enabled */
5826                 if (wufc & E1000_WUFC_MC) {
5827                         rctl = rd32(E1000_RCTL);
5828                         rctl |= E1000_RCTL_MPE;
5829                         wr32(E1000_RCTL, rctl);
5830                 }
5831
5832                 ctrl = rd32(E1000_CTRL);
5833                 /* advertise wake from D3Cold */
5834                 #define E1000_CTRL_ADVD3WUC 0x00100000
5835                 /* phy power management enable */
5836                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5837                 ctrl |= E1000_CTRL_ADVD3WUC;
5838                 wr32(E1000_CTRL, ctrl);
5839
5840                 /* Allow time for pending master requests to run */
5841                 igb_disable_pcie_master(hw);
5842
5843                 wr32(E1000_WUC, E1000_WUC_PME_EN);
5844                 wr32(E1000_WUFC, wufc);
5845         } else {
5846                 wr32(E1000_WUC, 0);
5847                 wr32(E1000_WUFC, 0);
5848         }
5849
5850         *enable_wake = wufc || adapter->en_mng_pt;
5851         if (!*enable_wake)
5852                 igb_power_down_link(adapter);
5853         else
5854                 igb_power_up_link(adapter);
5855
5856         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5857          * would have already happened in close and is redundant. */
5858         igb_release_hw_control(adapter);
5859
5860         pci_disable_device(pdev);
5861
5862         return 0;
5863 }
5864
5865 #ifdef CONFIG_PM
5866 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
5867 {
5868         int retval;
5869         bool wake;
5870
5871         retval = __igb_shutdown(pdev, &wake);
5872         if (retval)
5873                 return retval;
5874
5875         if (wake) {
5876                 pci_prepare_to_sleep(pdev);
5877         } else {
5878                 pci_wake_from_d3(pdev, false);
5879                 pci_set_power_state(pdev, PCI_D3hot);
5880         }
5881
5882         return 0;
5883 }
5884
5885 static int igb_resume(struct pci_dev *pdev)
5886 {
5887         struct net_device *netdev = pci_get_drvdata(pdev);
5888         struct igb_adapter *adapter = netdev_priv(netdev);
5889         struct e1000_hw *hw = &adapter->hw;
5890         u32 err;
5891
5892         pci_set_power_state(pdev, PCI_D0);
5893         pci_restore_state(pdev);
5894         pci_save_state(pdev);
5895
5896         err = pci_enable_device_mem(pdev);
5897         if (err) {
5898                 dev_err(&pdev->dev,
5899                         "igb: Cannot enable PCI device from suspend\n");
5900                 return err;
5901         }
5902         pci_set_master(pdev);
5903
5904         pci_enable_wake(pdev, PCI_D3hot, 0);
5905         pci_enable_wake(pdev, PCI_D3cold, 0);
5906
5907         if (igb_init_interrupt_scheme(adapter)) {
5908                 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
5909                 return -ENOMEM;
5910         }
5911
5912         igb_reset(adapter);
5913
5914         /* let the f/w know that the h/w is now under the control of the
5915          * driver. */
5916         igb_get_hw_control(adapter);
5917
5918         wr32(E1000_WUS, ~0);
5919
5920         if (netif_running(netdev)) {
5921                 err = igb_open(netdev);
5922                 if (err)
5923                         return err;
5924         }
5925
5926         netif_device_attach(netdev);
5927
5928         return 0;
5929 }
5930 #endif
5931
5932 static void igb_shutdown(struct pci_dev *pdev)
5933 {
5934         bool wake;
5935
5936         __igb_shutdown(pdev, &wake);
5937
5938         if (system_state == SYSTEM_POWER_OFF) {
5939                 pci_wake_from_d3(pdev, wake);
5940                 pci_set_power_state(pdev, PCI_D3hot);
5941         }
5942 }
5943
5944 #ifdef CONFIG_NET_POLL_CONTROLLER
5945 /*
5946  * Polling 'interrupt' - used by things like netconsole to send skbs
5947  * without having to re-enable interrupts. It's not called while
5948  * the interrupt routine is executing.
5949  */
5950 static void igb_netpoll(struct net_device *netdev)
5951 {
5952         struct igb_adapter *adapter = netdev_priv(netdev);
5953         struct e1000_hw *hw = &adapter->hw;
5954         int i;
5955
5956         if (!adapter->msix_entries) {
5957                 struct igb_q_vector *q_vector = adapter->q_vector[0];
5958                 igb_irq_disable(adapter);
5959                 napi_schedule(&q_vector->napi);
5960                 return;
5961         }
5962
5963         for (i = 0; i < adapter->num_q_vectors; i++) {
5964                 struct igb_q_vector *q_vector = adapter->q_vector[i];
5965                 wr32(E1000_EIMC, q_vector->eims_value);
5966                 napi_schedule(&q_vector->napi);
5967         }
5968 }
5969 #endif /* CONFIG_NET_POLL_CONTROLLER */
5970
5971 /**
5972  * igb_io_error_detected - called when PCI error is detected
5973  * @pdev: Pointer to PCI device
5974  * @state: The current pci connection state
5975  *
5976  * This function is called after a PCI bus error affecting
5977  * this device has been detected.
5978  */
5979 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
5980                                               pci_channel_state_t state)
5981 {
5982         struct net_device *netdev = pci_get_drvdata(pdev);
5983         struct igb_adapter *adapter = netdev_priv(netdev);
5984
5985         netif_device_detach(netdev);
5986
5987         if (state == pci_channel_io_perm_failure)
5988                 return PCI_ERS_RESULT_DISCONNECT;
5989
5990         if (netif_running(netdev))
5991                 igb_down(adapter);
5992         pci_disable_device(pdev);
5993
5994         /* Request a slot slot reset. */
5995         return PCI_ERS_RESULT_NEED_RESET;
5996 }
5997
5998 /**
5999  * igb_io_slot_reset - called after the pci bus has been reset.
6000  * @pdev: Pointer to PCI device
6001  *
6002  * Restart the card from scratch, as if from a cold-boot. Implementation
6003  * resembles the first-half of the igb_resume routine.
6004  */
6005 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6006 {
6007         struct net_device *netdev = pci_get_drvdata(pdev);
6008         struct igb_adapter *adapter = netdev_priv(netdev);
6009         struct e1000_hw *hw = &adapter->hw;
6010         pci_ers_result_t result;
6011         int err;
6012
6013         if (pci_enable_device_mem(pdev)) {
6014                 dev_err(&pdev->dev,
6015                         "Cannot re-enable PCI device after reset.\n");
6016                 result = PCI_ERS_RESULT_DISCONNECT;
6017         } else {
6018                 pci_set_master(pdev);
6019                 pci_restore_state(pdev);
6020                 pci_save_state(pdev);
6021
6022                 pci_enable_wake(pdev, PCI_D3hot, 0);
6023                 pci_enable_wake(pdev, PCI_D3cold, 0);
6024
6025                 igb_reset(adapter);
6026                 wr32(E1000_WUS, ~0);
6027                 result = PCI_ERS_RESULT_RECOVERED;
6028         }
6029
6030         err = pci_cleanup_aer_uncorrect_error_status(pdev);
6031         if (err) {
6032                 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6033                         "failed 0x%0x\n", err);
6034                 /* non-fatal, continue */
6035         }
6036
6037         return result;
6038 }
6039
6040 /**
6041  * igb_io_resume - called when traffic can start flowing again.
6042  * @pdev: Pointer to PCI device
6043  *
6044  * This callback is called when the error recovery driver tells us that
6045  * its OK to resume normal operation. Implementation resembles the
6046  * second-half of the igb_resume routine.
6047  */
6048 static void igb_io_resume(struct pci_dev *pdev)
6049 {
6050         struct net_device *netdev = pci_get_drvdata(pdev);
6051         struct igb_adapter *adapter = netdev_priv(netdev);
6052
6053         if (netif_running(netdev)) {
6054                 if (igb_up(adapter)) {
6055                         dev_err(&pdev->dev, "igb_up failed after reset\n");
6056                         return;
6057                 }
6058         }
6059
6060         netif_device_attach(netdev);
6061
6062         /* let the f/w know that the h/w is now under the control of the
6063          * driver. */
6064         igb_get_hw_control(adapter);
6065 }
6066
6067 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6068                              u8 qsel)
6069 {
6070         u32 rar_low, rar_high;
6071         struct e1000_hw *hw = &adapter->hw;
6072
6073         /* HW expects these in little endian so we reverse the byte order
6074          * from network order (big endian) to little endian
6075          */
6076         rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6077                   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6078         rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6079
6080         /* Indicate to hardware the Address is Valid. */
6081         rar_high |= E1000_RAH_AV;
6082
6083         if (hw->mac.type == e1000_82575)
6084                 rar_high |= E1000_RAH_POOL_1 * qsel;
6085         else
6086                 rar_high |= E1000_RAH_POOL_1 << qsel;
6087
6088         wr32(E1000_RAL(index), rar_low);
6089         wrfl();
6090         wr32(E1000_RAH(index), rar_high);
6091         wrfl();
6092 }
6093
6094 static int igb_set_vf_mac(struct igb_adapter *adapter,
6095                           int vf, unsigned char *mac_addr)
6096 {
6097         struct e1000_hw *hw = &adapter->hw;
6098         /* VF MAC addresses start at end of receive addresses and moves
6099          * torwards the first, as a result a collision should not be possible */
6100         int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6101
6102         memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6103
6104         igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6105
6106         return 0;
6107 }
6108
6109 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6110 {
6111         struct igb_adapter *adapter = netdev_priv(netdev);
6112         if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6113                 return -EINVAL;
6114         adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6115         dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6116         dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6117                                       " change effective.");
6118         if (test_bit(__IGB_DOWN, &adapter->state)) {
6119                 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6120                          " but the PF device is not up.\n");
6121                 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6122                          " attempting to use the VF device.\n");
6123         }
6124         return igb_set_vf_mac(adapter, vf, mac);
6125 }
6126
6127 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6128 {
6129         return -EOPNOTSUPP;
6130 }
6131
6132 static int igb_ndo_get_vf_config(struct net_device *netdev,
6133                                  int vf, struct ifla_vf_info *ivi)
6134 {
6135         struct igb_adapter *adapter = netdev_priv(netdev);
6136         if (vf >= adapter->vfs_allocated_count)
6137                 return -EINVAL;
6138         ivi->vf = vf;
6139         memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6140         ivi->tx_rate = 0;
6141         ivi->vlan = adapter->vf_data[vf].pf_vlan;
6142         ivi->qos = adapter->vf_data[vf].pf_qos;
6143         return 0;
6144 }
6145
6146 static void igb_vmm_control(struct igb_adapter *adapter)
6147 {
6148         struct e1000_hw *hw = &adapter->hw;
6149         u32 reg;
6150
6151         switch (hw->mac.type) {
6152         case e1000_82575:
6153         default:
6154                 /* replication is not supported for 82575 */
6155                 return;
6156         case e1000_82576:
6157                 /* notify HW that the MAC is adding vlan tags */
6158                 reg = rd32(E1000_DTXCTL);
6159                 reg |= E1000_DTXCTL_VLAN_ADDED;
6160                 wr32(E1000_DTXCTL, reg);
6161         case e1000_82580:
6162                 /* enable replication vlan tag stripping */
6163                 reg = rd32(E1000_RPLOLR);
6164                 reg |= E1000_RPLOLR_STRVLAN;
6165                 wr32(E1000_RPLOLR, reg);
6166         case e1000_i350:
6167                 /* none of the above registers are supported by i350 */
6168                 break;
6169         }
6170
6171         if (adapter->vfs_allocated_count) {
6172                 igb_vmdq_set_loopback_pf(hw, true);
6173                 igb_vmdq_set_replication_pf(hw, true);
6174         } else {
6175                 igb_vmdq_set_loopback_pf(hw, false);
6176                 igb_vmdq_set_replication_pf(hw, false);
6177         }
6178 }
6179
6180 /* igb_main.c */