drivers/net/ethernet/intel/e1000e/netdev.c

   1 /*******************************************************************************
   2
   3   Intel PRO/1000 Linux driver
   4   Copyright(c) 1999 - 2012 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   Linux NICS <linux.nics@intel.com>
  24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/module.h>
  32 #include <linux/types.h>
  33 #include <linux/init.h>
  34 #include <linux/pci.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/pagemap.h>
  37 #include <linux/delay.h>
  38 #include <linux/netdevice.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/tcp.h>
  41 #include <linux/ipv6.h>
  42 #include <linux/slab.h>
  43 #include <net/checksum.h>
  44 #include <net/ip6_checksum.h>
  45 #include <linux/mii.h>
  46 #include <linux/ethtool.h>
  47 #include <linux/if_vlan.h>
  48 #include <linux/cpu.h>
  49 #include <linux/smp.h>
  50 #include <linux/pm_qos.h>
  51 #include <linux/pm_runtime.h>
  52 #include <linux/aer.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "e1000.h"
  56
  57 #define DRV_EXTRAVERSION "-k"
  58
  59 #define DRV_VERSION "2.1.4" DRV_EXTRAVERSION
  60 char e1000e_driver_name[] = "e1000e";
  61 const char e1000e_driver_version[] = DRV_VERSION;
  62
  63 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  64 static int debug = -1;
  65 module_param(debug, int, 0);
  66 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  67
  68 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
  69
  70 static const struct e1000_info *e1000_info_tbl[] = {
  71         [board_82571]           = &e1000_82571_info,
  72         [board_82572]           = &e1000_82572_info,
  73         [board_82573]           = &e1000_82573_info,
  74         [board_82574]           = &e1000_82574_info,
  75         [board_82583]           = &e1000_82583_info,
  76         [board_80003es2lan]     = &e1000_es2_info,
  77         [board_ich8lan]         = &e1000_ich8_info,
  78         [board_ich9lan]         = &e1000_ich9_info,
  79         [board_ich10lan]        = &e1000_ich10_info,
  80         [board_pchlan]          = &e1000_pch_info,
  81         [board_pch2lan]         = &e1000_pch2_info,
  82         [board_pch_lpt]         = &e1000_pch_lpt_info,
  83 };
  84
  85 struct e1000_reg_info {
  86         u32 ofs;
  87         char *name;
  88 };
  89
  90 #define E1000_RDFH      0x02410 /* Rx Data FIFO Head - RW */
  91 #define E1000_RDFT      0x02418 /* Rx Data FIFO Tail - RW */
  92 #define E1000_RDFHS     0x02420 /* Rx Data FIFO Head Saved - RW */
  93 #define E1000_RDFTS     0x02428 /* Rx Data FIFO Tail Saved - RW */
  94 #define E1000_RDFPC     0x02430 /* Rx Data FIFO Packet Count - RW */
  95
  96 #define E1000_TDFH      0x03410 /* Tx Data FIFO Head - RW */
  97 #define E1000_TDFT      0x03418 /* Tx Data FIFO Tail - RW */
  98 #define E1000_TDFHS     0x03420 /* Tx Data FIFO Head Saved - RW */
  99 #define E1000_TDFTS     0x03428 /* Tx Data FIFO Tail Saved - RW */
 100 #define E1000_TDFPC     0x03430 /* Tx Data FIFO Packet Count - RW */
 101
 102 static const struct e1000_reg_info e1000_reg_info_tbl[] = {
 103
 104         /* General Registers */
 105         {E1000_CTRL, "CTRL"},
 106         {E1000_STATUS, "STATUS"},
 107         {E1000_CTRL_EXT, "CTRL_EXT"},
 108
 109         /* Interrupt Registers */
 110         {E1000_ICR, "ICR"},
 111
 112         /* Rx Registers */
 113         {E1000_RCTL, "RCTL"},
 114         {E1000_RDLEN(0), "RDLEN"},
 115         {E1000_RDH(0), "RDH"},
 116         {E1000_RDT(0), "RDT"},
 117         {E1000_RDTR, "RDTR"},
 118         {E1000_RXDCTL(0), "RXDCTL"},
 119         {E1000_ERT, "ERT"},
 120         {E1000_RDBAL(0), "RDBAL"},
 121         {E1000_RDBAH(0), "RDBAH"},
 122         {E1000_RDFH, "RDFH"},
 123         {E1000_RDFT, "RDFT"},
 124         {E1000_RDFHS, "RDFHS"},
 125         {E1000_RDFTS, "RDFTS"},
 126         {E1000_RDFPC, "RDFPC"},
 127
 128         /* Tx Registers */
 129         {E1000_TCTL, "TCTL"},
 130         {E1000_TDBAL(0), "TDBAL"},
 131         {E1000_TDBAH(0), "TDBAH"},
 132         {E1000_TDLEN(0), "TDLEN"},
 133         {E1000_TDH(0), "TDH"},
 134         {E1000_TDT(0), "TDT"},
 135         {E1000_TIDV, "TIDV"},
 136         {E1000_TXDCTL(0), "TXDCTL"},
 137         {E1000_TADV, "TADV"},
 138         {E1000_TARC(0), "TARC"},
 139         {E1000_TDFH, "TDFH"},
 140         {E1000_TDFT, "TDFT"},
 141         {E1000_TDFHS, "TDFHS"},
 142         {E1000_TDFTS, "TDFTS"},
 143         {E1000_TDFPC, "TDFPC"},
 144
 145         /* List Terminator */
 146         {0, NULL}
 147 };
 148
 149 /**
 150  * e1000_regdump - register printout routine
 151  * @hw: pointer to the HW structure
 152  * @reginfo: pointer to the register info table
 153  **/
 154 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
 155 {
 156         int n = 0;
 157         char rname[16];
 158         u32 regs[8];
 159
 160         switch (reginfo->ofs) {
 161         case E1000_RXDCTL(0):
 162                 for (n = 0; n < 2; n++)
 163                         regs[n] = __er32(hw, E1000_RXDCTL(n));
 164                 break;
 165         case E1000_TXDCTL(0):
 166                 for (n = 0; n < 2; n++)
 167                         regs[n] = __er32(hw, E1000_TXDCTL(n));
 168                 break;
 169         case E1000_TARC(0):
 170                 for (n = 0; n < 2; n++)
 171                         regs[n] = __er32(hw, E1000_TARC(n));
 172                 break;
 173         default:
 174                 pr_info("%-15s %08x\n",
 175                         reginfo->name, __er32(hw, reginfo->ofs));
 176                 return;
 177         }
 178
 179         snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
 180         pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
 181 }
 182
 183 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
 184                                  struct e1000_buffer *bi)
 185 {
 186         int i;
 187         struct e1000_ps_page *ps_page;
 188
 189         for (i = 0; i < adapter->rx_ps_pages; i++) {
 190                 ps_page = &bi->ps_pages[i];
 191
 192                 if (ps_page->page) {
 193                         pr_info("packet dump for ps_page %d:\n", i);
 194                         print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
 195                                        16, 1, page_address(ps_page->page),
 196                                        PAGE_SIZE, true);
 197                 }
 198         }
 199 }
 200
 201 /**
 202  * e1000e_dump - Print registers, Tx-ring and Rx-ring
 203  * @adapter: board private structure
 204  **/
 205 static void e1000e_dump(struct e1000_adapter *adapter)
 206 {
 207         struct net_device *netdev = adapter->netdev;
 208         struct e1000_hw *hw = &adapter->hw;
 209         struct e1000_reg_info *reginfo;
 210         struct e1000_ring *tx_ring = adapter->tx_ring;
 211         struct e1000_tx_desc *tx_desc;
 212         struct my_u0 {
 213                 __le64 a;
 214                 __le64 b;
 215         } *u0;
 216         struct e1000_buffer *buffer_info;
 217         struct e1000_ring *rx_ring = adapter->rx_ring;
 218         union e1000_rx_desc_packet_split *rx_desc_ps;
 219         union e1000_rx_desc_extended *rx_desc;
 220         struct my_u1 {
 221                 __le64 a;
 222                 __le64 b;
 223                 __le64 c;
 224                 __le64 d;
 225         } *u1;
 226         u32 staterr;
 227         int i = 0;
 228
 229         if (!netif_msg_hw(adapter))
 230                 return;
 231
 232         /* Print netdevice Info */
 233         if (netdev) {
 234                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 235                 pr_info("Device Name     state            trans_start      last_rx\n");
 236                 pr_info("%-15s %016lX %016lX %016lX\n",
 237                         netdev->name, netdev->state, netdev->trans_start,
 238                         netdev->last_rx);
 239         }
 240
 241         /* Print Registers */
 242         dev_info(&adapter->pdev->dev, "Register Dump\n");
 243         pr_info(" Register Name   Value\n");
 244         for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
 245              reginfo->name; reginfo++) {
 246                 e1000_regdump(hw, reginfo);
 247         }
 248
 249         /* Print Tx Ring Summary */
 250         if (!netdev || !netif_running(netdev))
 251                 return;
 252
 253         dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
 254         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
 255         buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
 256         pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
 257                 0, tx_ring->next_to_use, tx_ring->next_to_clean,
 258                 (unsigned long long)buffer_info->dma,
 259                 buffer_info->length,
 260                 buffer_info->next_to_watch,
 261                 (unsigned long long)buffer_info->time_stamp);
 262
 263         /* Print Tx Ring */
 264         if (!netif_msg_tx_done(adapter))
 265                 goto rx_ring_summary;
 266
 267         dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
 268
 269         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
 270          *
 271          * Legacy Transmit Descriptor
 272          *   +--------------------------------------------------------------+
 273          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
 274          *   +--------------------------------------------------------------+
 275          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
 276          *   +--------------------------------------------------------------+
 277          *   63       48 47        36 35    32 31     24 23    16 15        0
 278          *
 279          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
 280          *   63      48 47    40 39       32 31             16 15    8 7      0
 281          *   +----------------------------------------------------------------+
 282          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
 283          *   +----------------------------------------------------------------+
 284          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
 285          *   +----------------------------------------------------------------+
 286          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
 287          *
 288          * Extended Data Descriptor (DTYP=0x1)
 289          *   +----------------------------------------------------------------+
 290          * 0 |                     Buffer Address [63:0]                      |
 291          *   +----------------------------------------------------------------+
 292          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
 293          *   +----------------------------------------------------------------+
 294          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
 295          */
 296         pr_info("Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Legacy format\n");
 297         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Context format\n");
 298         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Data format\n");
 299         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 300                 const char *next_desc;
 301                 tx_desc = E1000_TX_DESC(*tx_ring, i);
 302                 buffer_info = &tx_ring->buffer_info[i];
 303                 u0 = (struct my_u0 *)tx_desc;
 304                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
 305                         next_desc = " NTC/U";
 306                 else if (i == tx_ring->next_to_use)
 307                         next_desc = " NTU";
 308                 else if (i == tx_ring->next_to_clean)
 309                         next_desc = " NTC";
 310                 else
 311                         next_desc = "";
 312                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p%s\n",
 313                         (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
 314                          ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
 315                         i,
 316                         (unsigned long long)le64_to_cpu(u0->a),
 317                         (unsigned long long)le64_to_cpu(u0->b),
 318                         (unsigned long long)buffer_info->dma,
 319                         buffer_info->length, buffer_info->next_to_watch,
 320                         (unsigned long long)buffer_info->time_stamp,
 321                         buffer_info->skb, next_desc);
 322
 323                 if (netif_msg_pktdata(adapter) && buffer_info->skb)
 324                         print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
 325                                        16, 1, buffer_info->skb->data,
 326                                        buffer_info->skb->len, true);
 327         }
 328
 329         /* Print Rx Ring Summary */
 330 rx_ring_summary:
 331         dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
 332         pr_info("Queue [NTU] [NTC]\n");
 333         pr_info(" %5d %5X %5X\n",
 334                 0, rx_ring->next_to_use, rx_ring->next_to_clean);
 335
 336         /* Print Rx Ring */
 337         if (!netif_msg_rx_status(adapter))
 338                 return;
 339
 340         dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
 341         switch (adapter->rx_ps_pages) {
 342         case 1:
 343         case 2:
 344         case 3:
 345                 /* [Extended] Packet Split Receive Descriptor Format
 346                  *
 347                  *    +-----------------------------------------------------+
 348                  *  0 |                Buffer Address 0 [63:0]              |
 349                  *    +-----------------------------------------------------+
 350                  *  8 |                Buffer Address 1 [63:0]              |
 351                  *    +-----------------------------------------------------+
 352                  * 16 |                Buffer Address 2 [63:0]              |
 353                  *    +-----------------------------------------------------+
 354                  * 24 |                Buffer Address 3 [63:0]              |
 355                  *    +-----------------------------------------------------+
 356                  */
 357                 pr_info("R  [desc]      [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] [bi->skb] <-- Ext Pkt Split format\n");
 358                 /* [Extended] Receive Descriptor (Write-Back) Format
 359                  *
 360                  *   63       48 47    32 31     13 12    8 7    4 3        0
 361                  *   +------------------------------------------------------+
 362                  * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
 363                  *   | Checksum | Ident  |         | Queue |      |  Type   |
 364                  *   +------------------------------------------------------+
 365                  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 366                  *   +------------------------------------------------------+
 367                  *   63       48 47    32 31            20 19               0
 368                  */
 369                 pr_info("RWB[desc]      [ck ipid mrqhsh] [vl   l0 ee  es] [ l3  l2  l1 hs] [reserved      ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
 370                 for (i = 0; i < rx_ring->count; i++) {
 371                         const char *next_desc;
 372                         buffer_info = &rx_ring->buffer_info[i];
 373                         rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
 374                         u1 = (struct my_u1 *)rx_desc_ps;
 375                         staterr =
 376                             le32_to_cpu(rx_desc_ps->wb.middle.status_error);
 377
 378                         if (i == rx_ring->next_to_use)
 379                                 next_desc = " NTU";
 380                         else if (i == rx_ring->next_to_clean)
 381                                 next_desc = " NTC";
 382                         else
 383                                 next_desc = "";
 384
 385                         if (staterr & E1000_RXD_STAT_DD) {
 386                                 /* Descriptor Done */
 387                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX ---------------- %p%s\n",
 388                                         "RWB", i,
 389                                         (unsigned long long)le64_to_cpu(u1->a),
 390                                         (unsigned long long)le64_to_cpu(u1->b),
 391                                         (unsigned long long)le64_to_cpu(u1->c),
 392                                         (unsigned long long)le64_to_cpu(u1->d),
 393                                         buffer_info->skb, next_desc);
 394                         } else {
 395                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX %016llX %p%s\n",
 396                                         "R  ", i,
 397                                         (unsigned long long)le64_to_cpu(u1->a),
 398                                         (unsigned long long)le64_to_cpu(u1->b),
 399                                         (unsigned long long)le64_to_cpu(u1->c),
 400                                         (unsigned long long)le64_to_cpu(u1->d),
 401                                         (unsigned long long)buffer_info->dma,
 402                                         buffer_info->skb, next_desc);
 403
 404                                 if (netif_msg_pktdata(adapter))
 405                                         e1000e_dump_ps_pages(adapter,
 406                                                              buffer_info);
 407                         }
 408                 }
 409                 break;
 410         default:
 411         case 0:
 412                 /* Extended Receive Descriptor (Read) Format
 413                  *
 414                  *   +-----------------------------------------------------+
 415                  * 0 |                Buffer Address [63:0]                |
 416                  *   +-----------------------------------------------------+
 417                  * 8 |                      Reserved                       |
 418                  *   +-----------------------------------------------------+
 419                  */
 420                 pr_info("R  [desc]      [buf addr 63:0 ] [reserved 63:0 ] [bi->dma       ] [bi->skb] <-- Ext (Read) format\n");
 421                 /* Extended Receive Descriptor (Write-Back) Format
 422                  *
 423                  *   63       48 47    32 31    24 23            4 3        0
 424                  *   +------------------------------------------------------+
 425                  *   |     RSS Hash      |        |               |         |
 426                  * 0 +-------------------+  Rsvd  |   Reserved    | MRQ RSS |
 427                  *   | Packet   | IP     |        |               |  Type   |
 428                  *   | Checksum | Ident  |        |               |         |
 429                  *   +------------------------------------------------------+
 430                  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 431                  *   +------------------------------------------------------+
 432                  *   63       48 47    32 31            20 19               0
 433                  */
 434                 pr_info("RWB[desc]      [cs ipid    mrq] [vt   ln xe  xs] [bi->skb] <-- Ext (Write-Back) format\n");
 435
 436                 for (i = 0; i < rx_ring->count; i++) {
 437                         const char *next_desc;
 438
 439                         buffer_info = &rx_ring->buffer_info[i];
 440                         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
 441                         u1 = (struct my_u1 *)rx_desc;
 442                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 443
 444                         if (i == rx_ring->next_to_use)
 445                                 next_desc = " NTU";
 446                         else if (i == rx_ring->next_to_clean)
 447                                 next_desc = " NTC";
 448                         else
 449                                 next_desc = "";
 450
 451                         if (staterr & E1000_RXD_STAT_DD) {
 452                                 /* Descriptor Done */
 453                                 pr_info("%s[0x%03X]     %016llX %016llX ---------------- %p%s\n",
 454                                         "RWB", i,
 455                                         (unsigned long long)le64_to_cpu(u1->a),
 456                                         (unsigned long long)le64_to_cpu(u1->b),
 457                                         buffer_info->skb, next_desc);
 458                         } else {
 459                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %p%s\n",
 460                                         "R  ", i,
 461                                         (unsigned long long)le64_to_cpu(u1->a),
 462                                         (unsigned long long)le64_to_cpu(u1->b),
 463                                         (unsigned long long)buffer_info->dma,
 464                                         buffer_info->skb, next_desc);
 465
 466                                 if (netif_msg_pktdata(adapter) &&
 467                                     buffer_info->skb)
 468                                         print_hex_dump(KERN_INFO, "",
 469                                                        DUMP_PREFIX_ADDRESS, 16,
 470                                                        1,
 471                                                        buffer_info->skb->data,
 472                                                        adapter->rx_buffer_len,
 473                                                        true);
 474                         }
 475                 }
 476         }
 477 }
 478
 479 /**
 480  * e1000_desc_unused - calculate if we have unused descriptors
 481  **/
 482 static int e1000_desc_unused(struct e1000_ring *ring)
 483 {
 484         if (ring->next_to_clean > ring->next_to_use)
 485                 return ring->next_to_clean - ring->next_to_use - 1;
 486
 487         return ring->count + ring->next_to_clean - ring->next_to_use - 1;
 488 }
 489
 490 /**
 491  * e1000_receive_skb - helper function to handle Rx indications
 492  * @adapter: board private structure
 493  * @status: descriptor status field as written by hardware
 494  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 495  * @skb: pointer to sk_buff to be indicated to stack
 496  **/
 497 static void e1000_receive_skb(struct e1000_adapter *adapter,
 498                               struct net_device *netdev, struct sk_buff *skb,
 499                               u8 status, __le16 vlan)
 500 {
 501         u16 tag = le16_to_cpu(vlan);
 502         skb->protocol = eth_type_trans(skb, netdev);
 503
 504         if (status & E1000_RXD_STAT_VP)
 505                 __vlan_hwaccel_put_tag(skb, tag);
 506
 507         napi_gro_receive(&adapter->napi, skb);
 508 }
 509
 510 /**
 511  * e1000_rx_checksum - Receive Checksum Offload
 512  * @adapter: board private structure
 513  * @status_err: receive descriptor status and error fields
 514  * @csum: receive descriptor csum field
 515  * @sk_buff: socket buffer with received data
 516  **/
 517 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
 518                               struct sk_buff *skb)
 519 {
 520         u16 status = (u16)status_err;
 521         u8 errors = (u8)(status_err >> 24);
 522
 523         skb_checksum_none_assert(skb);
 524
 525         /* Rx checksum disabled */
 526         if (!(adapter->netdev->features & NETIF_F_RXCSUM))
 527                 return;
 528
 529         /* Ignore Checksum bit is set */
 530         if (status & E1000_RXD_STAT_IXSM)
 531                 return;
 532
 533         /* TCP/UDP checksum error bit or IP checksum error bit is set */
 534         if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
 535                 /* let the stack verify checksum errors */
 536                 adapter->hw_csum_err++;
 537                 return;
 538         }
 539
 540         /* TCP/UDP Checksum has not been calculated */
 541         if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
 542                 return;
 543
 544         /* It must be a TCP or UDP packet with a valid checksum */
 545         skb->ip_summed = CHECKSUM_UNNECESSARY;
 546         adapter->hw_csum_good++;
 547 }
 548
 549 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
 550 {
 551         struct e1000_adapter *adapter = rx_ring->adapter;
 552         struct e1000_hw *hw = &adapter->hw;
 553         s32 ret_val = __ew32_prepare(hw);
 554
 555         writel(i, rx_ring->tail);
 556
 557         if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
 558                 u32 rctl = er32(RCTL);
 559                 ew32(RCTL, rctl & ~E1000_RCTL_EN);
 560                 e_err("ME firmware caused invalid RDT - resetting\n");
 561                 schedule_work(&adapter->reset_task);
 562         }
 563 }
 564
 565 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
 566 {
 567         struct e1000_adapter *adapter = tx_ring->adapter;
 568         struct e1000_hw *hw = &adapter->hw;
 569         s32 ret_val = __ew32_prepare(hw);
 570
 571         writel(i, tx_ring->tail);
 572
 573         if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
 574                 u32 tctl = er32(TCTL);
 575                 ew32(TCTL, tctl & ~E1000_TCTL_EN);
 576                 e_err("ME firmware caused invalid TDT - resetting\n");
 577                 schedule_work(&adapter->reset_task);
 578         }
 579 }
 580
 581 /**
 582  * e1000_alloc_rx_buffers - Replace used receive buffers
 583  * @rx_ring: Rx descriptor ring
 584  **/
 585 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
 586                                    int cleaned_count, gfp_t gfp)
 587 {
 588         struct e1000_adapter *adapter = rx_ring->adapter;
 589         struct net_device *netdev = adapter->netdev;
 590         struct pci_dev *pdev = adapter->pdev;
 591         union e1000_rx_desc_extended *rx_desc;
 592         struct e1000_buffer *buffer_info;
 593         struct sk_buff *skb;
 594         unsigned int i;
 595         unsigned int bufsz = adapter->rx_buffer_len;
 596
 597         i = rx_ring->next_to_use;
 598         buffer_info = &rx_ring->buffer_info[i];
 599
 600         while (cleaned_count--) {
 601                 skb = buffer_info->skb;
 602                 if (skb) {
 603                         skb_trim(skb, 0);
 604                         goto map_skb;
 605                 }
 606
 607                 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
 608                 if (!skb) {
 609                         /* Better luck next round */
 610                         adapter->alloc_rx_buff_failed++;
 611                         break;
 612                 }
 613
 614                 buffer_info->skb = skb;
 615 map_skb:
 616                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 617                                                   adapter->rx_buffer_len,
 618                                                   DMA_FROM_DEVICE);
 619                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 620                         dev_err(&pdev->dev, "Rx DMA map failed\n");
 621                         adapter->rx_dma_failed++;
 622                         break;
 623                 }
 624
 625                 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
 626                 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
 627
 628                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
 629                         /* Force memory writes to complete before letting h/w
 630                          * know there are new descriptors to fetch.  (Only
 631                          * applicable for weak-ordered memory model archs,
 632                          * such as IA-64).
 633                          */
 634                         wmb();
 635                         if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
 636                                 e1000e_update_rdt_wa(rx_ring, i);
 637                         else
 638                                 writel(i, rx_ring->tail);
 639                 }
 640                 i++;
 641                 if (i == rx_ring->count)
 642                         i = 0;
 643                 buffer_info = &rx_ring->buffer_info[i];
 644         }
 645
 646         rx_ring->next_to_use = i;
 647 }
 648
 649 /**
 650  * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
 651  * @rx_ring: Rx descriptor ring
 652  **/
 653 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
 654                                       int cleaned_count, gfp_t gfp)
 655 {
 656         struct e1000_adapter *adapter = rx_ring->adapter;
 657         struct net_device *netdev = adapter->netdev;
 658         struct pci_dev *pdev = adapter->pdev;
 659         union e1000_rx_desc_packet_split *rx_desc;
 660         struct e1000_buffer *buffer_info;
 661         struct e1000_ps_page *ps_page;
 662         struct sk_buff *skb;
 663         unsigned int i, j;
 664
 665         i = rx_ring->next_to_use;
 666         buffer_info = &rx_ring->buffer_info[i];
 667
 668         while (cleaned_count--) {
 669                 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
 670
 671                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
 672                         ps_page = &buffer_info->ps_pages[j];
 673                         if (j >= adapter->rx_ps_pages) {
 674                                 /* all unused desc entries get hw null ptr */
 675                                 rx_desc->read.buffer_addr[j + 1] =
 676                                     ~cpu_to_le64(0);
 677                                 continue;
 678                         }
 679                         if (!ps_page->page) {
 680                                 ps_page->page = alloc_page(gfp);
 681                                 if (!ps_page->page) {
 682                                         adapter->alloc_rx_buff_failed++;
 683                                         goto no_buffers;
 684                                 }
 685                                 ps_page->dma = dma_map_page(&pdev->dev,
 686                                                             ps_page->page,
 687                                                             0, PAGE_SIZE,
 688                                                             DMA_FROM_DEVICE);
 689                                 if (dma_mapping_error(&pdev->dev,
 690                                                       ps_page->dma)) {
 691                                         dev_err(&adapter->pdev->dev,
 692                                                 "Rx DMA page map failed\n");
 693                                         adapter->rx_dma_failed++;
 694                                         goto no_buffers;
 695                                 }
 696                         }
 697                         /* Refresh the desc even if buffer_addrs
 698                          * didn't change because each write-back
 699                          * erases this info.
 700                          */
 701                         rx_desc->read.buffer_addr[j + 1] =
 702                             cpu_to_le64(ps_page->dma);
 703                 }
 704
 705                 skb = __netdev_alloc_skb_ip_align(netdev,
 706                                                   adapter->rx_ps_bsize0,
 707                                                   gfp);
 708
 709                 if (!skb) {
 710                         adapter->alloc_rx_buff_failed++;
 711                         break;
 712                 }
 713
 714                 buffer_info->skb = skb;
 715                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 716                                                   adapter->rx_ps_bsize0,
 717                                                   DMA_FROM_DEVICE);
 718                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 719                         dev_err(&pdev->dev, "Rx DMA map failed\n");
 720                         adapter->rx_dma_failed++;
 721                         /* cleanup skb */
 722                         dev_kfree_skb_any(skb);
 723                         buffer_info->skb = NULL;
 724                         break;
 725                 }
 726
 727                 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
 728
 729                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
 730                         /* Force memory writes to complete before letting h/w
 731                          * know there are new descriptors to fetch.  (Only
 732                          * applicable for weak-ordered memory model archs,
 733                          * such as IA-64).
 734                          */
 735                         wmb();
 736                         if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
 737                                 e1000e_update_rdt_wa(rx_ring, i << 1);
 738                         else
 739                                 writel(i << 1, rx_ring->tail);
 740                 }
 741
 742                 i++;
 743                 if (i == rx_ring->count)
 744                         i = 0;
 745                 buffer_info = &rx_ring->buffer_info[i];
 746         }
 747
 748 no_buffers:
 749         rx_ring->next_to_use = i;
 750 }
 751
 752 /**
 753  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
 754  * @rx_ring: Rx descriptor ring
 755  * @cleaned_count: number of buffers to allocate this pass
 756  **/
 757
 758 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
 759                                          int cleaned_count, gfp_t gfp)
 760 {
 761         struct e1000_adapter *adapter = rx_ring->adapter;
 762         struct net_device *netdev = adapter->netdev;
 763         struct pci_dev *pdev = adapter->pdev;
 764         union e1000_rx_desc_extended *rx_desc;
 765         struct e1000_buffer *buffer_info;
 766         struct sk_buff *skb;
 767         unsigned int i;
 768         unsigned int bufsz = 256 - 16 /* for skb_reserve */;
 769
 770         i = rx_ring->next_to_use;
 771         buffer_info = &rx_ring->buffer_info[i];
 772
 773         while (cleaned_count--) {
 774                 skb = buffer_info->skb;
 775                 if (skb) {
 776                         skb_trim(skb, 0);
 777                         goto check_page;
 778                 }
 779
 780                 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
 781                 if (unlikely(!skb)) {
 782                         /* Better luck next round */
 783                         adapter->alloc_rx_buff_failed++;
 784                         break;
 785                 }
 786
 787                 buffer_info->skb = skb;
 788 check_page:
 789                 /* allocate a new page if necessary */
 790                 if (!buffer_info->page) {
 791                         buffer_info->page = alloc_page(gfp);
 792                         if (unlikely(!buffer_info->page)) {
 793                                 adapter->alloc_rx_buff_failed++;
 794                                 break;
 795                         }
 796                 }
 797
 798                 if (!buffer_info->dma)
 799                         buffer_info->dma = dma_map_page(&pdev->dev,
 800                                                         buffer_info->page, 0,
 801                                                         PAGE_SIZE,
 802                                                         DMA_FROM_DEVICE);
 803
 804                 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
 805                 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
 806
 807                 if (unlikely(++i == rx_ring->count))
 808                         i = 0;
 809                 buffer_info = &rx_ring->buffer_info[i];
 810         }
 811
 812         if (likely(rx_ring->next_to_use != i)) {
 813                 rx_ring->next_to_use = i;
 814                 if (unlikely(i-- == 0))
 815                         i = (rx_ring->count - 1);
 816
 817                 /* Force memory writes to complete before letting h/w
 818                  * know there are new descriptors to fetch.  (Only
 819                  * applicable for weak-ordered memory model archs,
 820                  * such as IA-64).
 821                  */
 822                 wmb();
 823                 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
 824                         e1000e_update_rdt_wa(rx_ring, i);
 825                 else
 826                         writel(i, rx_ring->tail);
 827         }
 828 }
 829
 830 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
 831                                  struct sk_buff *skb)
 832 {
 833         if (netdev->features & NETIF_F_RXHASH)
 834                 skb->rxhash = le32_to_cpu(rss);
 835 }
 836
 837 /**
 838  * e1000_clean_rx_irq - Send received data up the network stack
 839  * @rx_ring: Rx descriptor ring
 840  *
 841  * the return value indicates whether actual cleaning was done, there
 842  * is no guarantee that everything was cleaned
 843  **/
 844 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
 845                                int work_to_do)
 846 {
 847         struct e1000_adapter *adapter = rx_ring->adapter;
 848         struct net_device *netdev = adapter->netdev;
 849         struct pci_dev *pdev = adapter->pdev;
 850         struct e1000_hw *hw = &adapter->hw;
 851         union e1000_rx_desc_extended *rx_desc, *next_rxd;
 852         struct e1000_buffer *buffer_info, *next_buffer;
 853         u32 length, staterr;
 854         unsigned int i;
 855         int cleaned_count = 0;
 856         bool cleaned = false;
 857         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 858
 859         i = rx_ring->next_to_clean;
 860         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
 861         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 862         buffer_info = &rx_ring->buffer_info[i];
 863
 864         while (staterr & E1000_RXD_STAT_DD) {
 865                 struct sk_buff *skb;
 866
 867                 if (*work_done >= work_to_do)
 868                         break;
 869                 (*work_done)++;
 870                 rmb();  /* read descriptor and rx_buffer_info after status DD */
 871
 872                 skb = buffer_info->skb;
 873                 buffer_info->skb = NULL;
 874
 875                 prefetch(skb->data - NET_IP_ALIGN);
 876
 877                 i++;
 878                 if (i == rx_ring->count)
 879                         i = 0;
 880                 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
 881                 prefetch(next_rxd);
 882
 883                 next_buffer = &rx_ring->buffer_info[i];
 884
 885                 cleaned = true;
 886                 cleaned_count++;
 887                 dma_unmap_single(&pdev->dev,
 888                                  buffer_info->dma,
 889                                  adapter->rx_buffer_len,
 890                                  DMA_FROM_DEVICE);
 891                 buffer_info->dma = 0;
 892
 893                 length = le16_to_cpu(rx_desc->wb.upper.length);
 894
 895                 /* !EOP means multiple descriptors were used to store a single
 896                  * packet, if that's the case we need to toss it.  In fact, we
 897                  * need to toss every packet with the EOP bit clear and the
 898                  * next frame that _does_ have the EOP bit set, as it is by
 899                  * definition only a frame fragment
 900                  */
 901                 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
 902                         adapter->flags2 |= FLAG2_IS_DISCARDING;
 903
 904                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
 905                         /* All receives must fit into a single buffer */
 906                         e_dbg("Receive packet consumed multiple buffers\n");
 907                         /* recycle */
 908                         buffer_info->skb = skb;
 909                         if (staterr & E1000_RXD_STAT_EOP)
 910                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
 911                         goto next_desc;
 912                 }
 913
 914                 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
 915                              !(netdev->features & NETIF_F_RXALL))) {
 916                         /* recycle */
 917                         buffer_info->skb = skb;
 918                         goto next_desc;
 919                 }
 920
 921                 /* adjust length to remove Ethernet CRC */
 922                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
 923                         /* If configured to store CRC, don't subtract FCS,
 924                          * but keep the FCS bytes out of the total_rx_bytes
 925                          * counter
 926                          */
 927                         if (netdev->features & NETIF_F_RXFCS)
 928                                 total_rx_bytes -= 4;
 929                         else
 930                                 length -= 4;
 931                 }
 932
 933                 total_rx_bytes += length;
 934                 total_rx_packets++;
 935
 936                 /* code added for copybreak, this should improve
 937                  * performance for small packets with large amounts
 938                  * of reassembly being done in the stack
 939                  */
 940                 if (length < copybreak) {
 941                         struct sk_buff *new_skb =
 942                             netdev_alloc_skb_ip_align(netdev, length);
 943                         if (new_skb) {
 944                                 skb_copy_to_linear_data_offset(new_skb,
 945                                                                -NET_IP_ALIGN,
 946                                                                (skb->data -
 947                                                                 NET_IP_ALIGN),
 948                                                                (length +
 949                                                                 NET_IP_ALIGN));
 950                                 /* save the skb in buffer_info as good */
 951                                 buffer_info->skb = skb;
 952                                 skb = new_skb;
 953                         }
 954                         /* else just continue with the old one */
 955                 }
 956                 /* end copybreak code */
 957                 skb_put(skb, length);
 958
 959                 /* Receive Checksum Offload */
 960                 e1000_rx_checksum(adapter, staterr, skb);
 961
 962                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
 963
 964                 e1000_receive_skb(adapter, netdev, skb, staterr,
 965                                   rx_desc->wb.upper.vlan);
 966
 967 next_desc:
 968                 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
 969
 970                 /* return some buffers to hardware, one at a time is too slow */
 971                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
 972                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
 973                                               GFP_ATOMIC);
 974                         cleaned_count = 0;
 975                 }
 976
 977                 /* use prefetched values */
 978                 rx_desc = next_rxd;
 979                 buffer_info = next_buffer;
 980
 981                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 982         }
 983         rx_ring->next_to_clean = i;
 984
 985         cleaned_count = e1000_desc_unused(rx_ring);
 986         if (cleaned_count)
 987                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
 988
 989         adapter->total_rx_bytes += total_rx_bytes;
 990         adapter->total_rx_packets += total_rx_packets;
 991         return cleaned;
 992 }
 993
 994 static void e1000_put_txbuf(struct e1000_ring *tx_ring,
 995                             struct e1000_buffer *buffer_info)
 996 {
 997         struct e1000_adapter *adapter = tx_ring->adapter;
 998
 999         if (buffer_info->dma) {
1000                 if (buffer_info->mapped_as_page)
1001                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1002                                        buffer_info->length, DMA_TO_DEVICE);
1003                 else
1004                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1005                                          buffer_info->length, DMA_TO_DEVICE);
1006                 buffer_info->dma = 0;
1007         }
1008         if (buffer_info->skb) {
1009                 dev_kfree_skb_any(buffer_info->skb);
1010                 buffer_info->skb = NULL;
1011         }
1012         buffer_info->time_stamp = 0;
1013 }
1014
1015 static void e1000_print_hw_hang(struct work_struct *work)
1016 {
1017         struct e1000_adapter *adapter = container_of(work,
1018                                                      struct e1000_adapter,
1019                                                      print_hang_task);
1020         struct net_device *netdev = adapter->netdev;
1021         struct e1000_ring *tx_ring = adapter->tx_ring;
1022         unsigned int i = tx_ring->next_to_clean;
1023         unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1024         struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
1025         struct e1000_hw *hw = &adapter->hw;
1026         u16 phy_status, phy_1000t_status, phy_ext_status;
1027         u16 pci_status;
1028
1029         if (test_bit(__E1000_DOWN, &adapter->state))
1030                 return;
1031
1032         if (!adapter->tx_hang_recheck &&
1033             (adapter->flags2 & FLAG2_DMA_BURST)) {
1034                 /* May be block on write-back, flush and detect again
1035                  * flush pending descriptor writebacks to memory
1036                  */
1037                 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1038                 /* execute the writes immediately */
1039                 e1e_flush();
1040                 /* Due to rare timing issues, write to TIDV again to ensure
1041                  * the write is successful
1042                  */
1043                 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1044                 /* execute the writes immediately */
1045                 e1e_flush();
1046                 adapter->tx_hang_recheck = true;
1047                 return;
1048         }
1049         /* Real hang detected */
1050         adapter->tx_hang_recheck = false;
1051         netif_stop_queue(netdev);
1052
1053         e1e_rphy(hw, PHY_STATUS, &phy_status);
1054         e1e_rphy(hw, PHY_1000T_STATUS, &phy_1000t_status);
1055         e1e_rphy(hw, PHY_EXT_STATUS, &phy_ext_status);
1056
1057         pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1058
1059         /* detected Hardware unit hang */
1060         e_err("Detected Hardware Unit Hang:\n"
1061               "  TDH                  <%x>\n"
1062               "  TDT                  <%x>\n"
1063               "  next_to_use          <%x>\n"
1064               "  next_to_clean        <%x>\n"
1065               "buffer_info[next_to_clean]:\n"
1066               "  time_stamp           <%lx>\n"
1067               "  next_to_watch        <%x>\n"
1068               "  jiffies              <%lx>\n"
1069               "  next_to_watch.status <%x>\n"
1070               "MAC Status             <%x>\n"
1071               "PHY Status             <%x>\n"
1072               "PHY 1000BASE-T Status  <%x>\n"
1073               "PHY Extended Status    <%x>\n"
1074               "PCI Status             <%x>\n",
1075               readl(tx_ring->head),
1076               readl(tx_ring->tail),
1077               tx_ring->next_to_use,
1078               tx_ring->next_to_clean,
1079               tx_ring->buffer_info[eop].time_stamp,
1080               eop,
1081               jiffies,
1082               eop_desc->upper.fields.status,
1083               er32(STATUS),
1084               phy_status,
1085               phy_1000t_status,
1086               phy_ext_status,
1087               pci_status);
1088
1089         /* Suggest workaround for known h/w issue */
1090         if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1091                 e_err("Try turning off Tx pause (flow control) via ethtool\n");
1092 }
1093
1094 /**
1095  * e1000_clean_tx_irq - Reclaim resources after transmit completes
1096  * @tx_ring: Tx descriptor ring
1097  *
1098  * the return value indicates whether actual cleaning was done, there
1099  * is no guarantee that everything was cleaned
1100  **/
1101 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
1102 {
1103         struct e1000_adapter *adapter = tx_ring->adapter;
1104         struct net_device *netdev = adapter->netdev;
1105         struct e1000_hw *hw = &adapter->hw;
1106         struct e1000_tx_desc *tx_desc, *eop_desc;
1107         struct e1000_buffer *buffer_info;
1108         unsigned int i, eop;
1109         unsigned int count = 0;
1110         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
1111         unsigned int bytes_compl = 0, pkts_compl = 0;
1112
1113         i = tx_ring->next_to_clean;
1114         eop = tx_ring->buffer_info[i].next_to_watch;
1115         eop_desc = E1000_TX_DESC(*tx_ring, eop);
1116
1117         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1118                (count < tx_ring->count)) {
1119                 bool cleaned = false;
1120                 rmb(); /* read buffer_info after eop_desc */
1121                 for (; !cleaned; count++) {
1122                         tx_desc = E1000_TX_DESC(*tx_ring, i);
1123                         buffer_info = &tx_ring->buffer_info[i];
1124                         cleaned = (i == eop);
1125
1126                         if (cleaned) {
1127                                 total_tx_packets += buffer_info->segs;
1128                                 total_tx_bytes += buffer_info->bytecount;
1129                                 if (buffer_info->skb) {
1130                                         bytes_compl += buffer_info->skb->len;
1131                                         pkts_compl++;
1132                                 }
1133                         }
1134
1135                         e1000_put_txbuf(tx_ring, buffer_info);
1136                         tx_desc->upper.data = 0;
1137
1138                         i++;
1139                         if (i == tx_ring->count)
1140                                 i = 0;
1141                 }
1142
1143                 if (i == tx_ring->next_to_use)
1144                         break;
1145                 eop = tx_ring->buffer_info[i].next_to_watch;
1146                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1147         }
1148
1149         tx_ring->next_to_clean = i;
1150
1151         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1152
1153 #define TX_WAKE_THRESHOLD 32
1154         if (count && netif_carrier_ok(netdev) &&
1155             e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
1156                 /* Make sure that anybody stopping the queue after this
1157                  * sees the new next_to_clean.
1158                  */
1159                 smp_mb();
1160
1161                 if (netif_queue_stopped(netdev) &&
1162                     !(test_bit(__E1000_DOWN, &adapter->state))) {
1163                         netif_wake_queue(netdev);
1164                         ++adapter->restart_queue;
1165                 }
1166         }
1167
1168         if (adapter->detect_tx_hung) {
1169                 /* Detect a transmit hang in hardware, this serializes the
1170                  * check with the clearing of time_stamp and movement of i
1171                  */
1172                 adapter->detect_tx_hung = false;
1173                 if (tx_ring->buffer_info[i].time_stamp &&
1174                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp
1175                                + (adapter->tx_timeout_factor * HZ)) &&
1176                     !(er32(STATUS) & E1000_STATUS_TXOFF))
1177                         schedule_work(&adapter->print_hang_task);
1178                 else
1179                         adapter->tx_hang_recheck = false;
1180         }
1181         adapter->total_tx_bytes += total_tx_bytes;
1182         adapter->total_tx_packets += total_tx_packets;
1183         return count < tx_ring->count;
1184 }
1185
1186 /**
1187  * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
1188  * @rx_ring: Rx descriptor ring
1189  *
1190  * the return value indicates whether actual cleaning was done, there
1191  * is no guarantee that everything was cleaned
1192  **/
1193 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1194                                   int work_to_do)
1195 {
1196         struct e1000_adapter *adapter = rx_ring->adapter;
1197         struct e1000_hw *hw = &adapter->hw;
1198         union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1199         struct net_device *netdev = adapter->netdev;
1200         struct pci_dev *pdev = adapter->pdev;
1201         struct e1000_buffer *buffer_info, *next_buffer;
1202         struct e1000_ps_page *ps_page;
1203         struct sk_buff *skb;
1204         unsigned int i, j;
1205         u32 length, staterr;
1206         int cleaned_count = 0;
1207         bool cleaned = false;
1208         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1209
1210         i = rx_ring->next_to_clean;
1211         rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1212         staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1213         buffer_info = &rx_ring->buffer_info[i];
1214
1215         while (staterr & E1000_RXD_STAT_DD) {
1216                 if (*work_done >= work_to_do)
1217                         break;
1218                 (*work_done)++;
1219                 skb = buffer_info->skb;
1220                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1221
1222                 /* in the packet split case this is header only */
1223                 prefetch(skb->data - NET_IP_ALIGN);
1224
1225                 i++;
1226                 if (i == rx_ring->count)
1227                         i = 0;
1228                 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1229                 prefetch(next_rxd);
1230
1231                 next_buffer = &rx_ring->buffer_info[i];
1232
1233                 cleaned = true;
1234                 cleaned_count++;
1235                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1236                                  adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
1237                 buffer_info->dma = 0;
1238
1239                 /* see !EOP comment in other Rx routine */
1240                 if (!(staterr & E1000_RXD_STAT_EOP))
1241                         adapter->flags2 |= FLAG2_IS_DISCARDING;
1242
1243                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
1244                         e_dbg("Packet Split buffers didn't pick up the full packet\n");
1245                         dev_kfree_skb_irq(skb);
1246                         if (staterr & E1000_RXD_STAT_EOP)
1247                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1248                         goto next_desc;
1249                 }
1250
1251                 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1252                              !(netdev->features & NETIF_F_RXALL))) {
1253                         dev_kfree_skb_irq(skb);
1254                         goto next_desc;
1255                 }
1256
1257                 length = le16_to_cpu(rx_desc->wb.middle.length0);
1258
1259                 if (!length) {
1260                         e_dbg("Last part of the packet spanning multiple descriptors\n");
1261                         dev_kfree_skb_irq(skb);
1262                         goto next_desc;
1263                 }
1264
1265                 /* Good Receive */
1266                 skb_put(skb, length);
1267
1268                 {
1269                         /* this looks ugly, but it seems compiler issues make
1270                          * it more efficient than reusing j
1271                          */
1272                         int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
1273
1274                         /* page alloc/put takes too long and effects small
1275                          * packet throughput, so unsplit small packets and
1276                          * save the alloc/put only valid in softirq (napi)
1277                          * context to call kmap_*
1278                          */
1279                         if (l1 && (l1 <= copybreak) &&
1280                             ((length + l1) <= adapter->rx_ps_bsize0)) {
1281                                 u8 *vaddr;
1282
1283                                 ps_page = &buffer_info->ps_pages[0];
1284
1285                                 /* there is no documentation about how to call
1286                                  * kmap_atomic, so we can't hold the mapping
1287                                  * very long
1288                                  */
1289                                 dma_sync_single_for_cpu(&pdev->dev,
1290                                                         ps_page->dma,
1291                                                         PAGE_SIZE,
1292                                                         DMA_FROM_DEVICE);
1293                                 vaddr = kmap_atomic(ps_page->page);
1294                                 memcpy(skb_tail_pointer(skb), vaddr, l1);
1295                                 kunmap_atomic(vaddr);
1296                                 dma_sync_single_for_device(&pdev->dev,
1297                                                            ps_page->dma,
1298                                                            PAGE_SIZE,
1299                                                            DMA_FROM_DEVICE);
1300
1301                                 /* remove the CRC */
1302                                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1303                                         if (!(netdev->features & NETIF_F_RXFCS))
1304                                                 l1 -= 4;
1305                                 }
1306
1307                                 skb_put(skb, l1);
1308                                 goto copydone;
1309                         } /* if */
1310                 }
1311
1312                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1313                         length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1314                         if (!length)
1315                                 break;
1316
1317                         ps_page = &buffer_info->ps_pages[j];
1318                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1319                                        DMA_FROM_DEVICE);
1320                         ps_page->dma = 0;
1321                         skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1322                         ps_page->page = NULL;
1323                         skb->len += length;
1324                         skb->data_len += length;
1325                         skb->truesize += PAGE_SIZE;
1326                 }
1327
1328                 /* strip the ethernet crc, problem is we're using pages now so
1329                  * this whole operation can get a little cpu intensive
1330                  */
1331                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1332                         if (!(netdev->features & NETIF_F_RXFCS))
1333                                 pskb_trim(skb, skb->len - 4);
1334                 }
1335
1336 copydone:
1337                 total_rx_bytes += skb->len;
1338                 total_rx_packets++;
1339
1340                 e1000_rx_checksum(adapter, staterr, skb);
1341
1342                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1343
1344                 if (rx_desc->wb.upper.header_status &
1345                            cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
1346                         adapter->rx_hdr_split++;
1347
1348                 e1000_receive_skb(adapter, netdev, skb,
1349                                   staterr, rx_desc->wb.middle.vlan);
1350
1351 next_desc:
1352                 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1353                 buffer_info->skb = NULL;
1354
1355                 /* return some buffers to hardware, one at a time is too slow */
1356                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1357                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
1358                                               GFP_ATOMIC);
1359                         cleaned_count = 0;
1360                 }
1361
1362                 /* use prefetched values */
1363                 rx_desc = next_rxd;
1364                 buffer_info = next_buffer;
1365
1366                 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1367         }
1368         rx_ring->next_to_clean = i;
1369
1370         cleaned_count = e1000_desc_unused(rx_ring);
1371         if (cleaned_count)
1372                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1373
1374         adapter->total_rx_bytes += total_rx_bytes;
1375         adapter->total_rx_packets += total_rx_packets;
1376         return cleaned;
1377 }
1378
1379 /**
1380  * e1000_consume_page - helper function
1381  **/
1382 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
1383                                u16 length)
1384 {
1385         bi->page = NULL;
1386         skb->len += length;
1387         skb->data_len += length;
1388         skb->truesize += PAGE_SIZE;
1389 }
1390
1391 /**
1392  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1393  * @adapter: board private structure
1394  *
1395  * the return value indicates whether actual cleaning was done, there
1396  * is no guarantee that everything was cleaned
1397  **/
1398 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1399                                      int work_to_do)
1400 {
1401         struct e1000_adapter *adapter = rx_ring->adapter;
1402         struct net_device *netdev = adapter->netdev;
1403         struct pci_dev *pdev = adapter->pdev;
1404         union e1000_rx_desc_extended *rx_desc, *next_rxd;
1405         struct e1000_buffer *buffer_info, *next_buffer;
1406         u32 length, staterr;
1407         unsigned int i;
1408         int cleaned_count = 0;
1409         bool cleaned = false;
1410         unsigned int total_rx_bytes=0, total_rx_packets=0;
1411
1412         i = rx_ring->next_to_clean;
1413         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1414         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1415         buffer_info = &rx_ring->buffer_info[i];
1416
1417         while (staterr & E1000_RXD_STAT_DD) {
1418                 struct sk_buff *skb;
1419
1420                 if (*work_done >= work_to_do)
1421                         break;
1422                 (*work_done)++;
1423                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1424
1425                 skb = buffer_info->skb;
1426                 buffer_info->skb = NULL;
1427
1428                 ++i;
1429                 if (i == rx_ring->count)
1430                         i = 0;
1431                 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
1432                 prefetch(next_rxd);
1433
1434                 next_buffer = &rx_ring->buffer_info[i];
1435
1436                 cleaned = true;
1437                 cleaned_count++;
1438                 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1439                                DMA_FROM_DEVICE);
1440                 buffer_info->dma = 0;
1441
1442                 length = le16_to_cpu(rx_desc->wb.upper.length);
1443
1444                 /* errors is only valid for DD + EOP descriptors */
1445                 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
1446                              ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1447                               !(netdev->features & NETIF_F_RXALL)))) {
1448                         /* recycle both page and skb */
1449                         buffer_info->skb = skb;
1450                         /* an error means any chain goes out the window too */
1451                         if (rx_ring->rx_skb_top)
1452                                 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1453                         rx_ring->rx_skb_top = NULL;
1454                         goto next_desc;
1455                 }
1456
1457 #define rxtop (rx_ring->rx_skb_top)
1458                 if (!(staterr & E1000_RXD_STAT_EOP)) {
1459                         /* this descriptor is only the beginning (or middle) */
1460                         if (!rxtop) {
1461                                 /* this is the beginning of a chain */
1462                                 rxtop = skb;
1463                                 skb_fill_page_desc(rxtop, 0, buffer_info->page,
1464                                                    0, length);
1465                         } else {
1466                                 /* this is the middle of a chain */
1467                                 skb_fill_page_desc(rxtop,
1468                                     skb_shinfo(rxtop)->nr_frags,
1469                                     buffer_info->page, 0, length);
1470                                 /* re-use the skb, only consumed the page */
1471                                 buffer_info->skb = skb;
1472                         }
1473                         e1000_consume_page(buffer_info, rxtop, length);
1474                         goto next_desc;
1475                 } else {
1476                         if (rxtop) {
1477                                 /* end of the chain */
1478                                 skb_fill_page_desc(rxtop,
1479                                     skb_shinfo(rxtop)->nr_frags,
1480                                     buffer_info->page, 0, length);
1481                                 /* re-use the current skb, we only consumed the
1482                                  * page
1483                                  */
1484                                 buffer_info->skb = skb;
1485                                 skb = rxtop;
1486                                 rxtop = NULL;
1487                                 e1000_consume_page(buffer_info, skb, length);
1488                         } else {
1489                                 /* no chain, got EOP, this buf is the packet
1490                                  * copybreak to save the put_page/alloc_page
1491                                  */
1492                                 if (length <= copybreak &&
1493                                     skb_tailroom(skb) >= length) {
1494                                         u8 *vaddr;
1495                                         vaddr = kmap_atomic(buffer_info->page);
1496                                         memcpy(skb_tail_pointer(skb), vaddr,
1497                                                length);
1498                                         kunmap_atomic(vaddr);
1499                                         /* re-use the page, so don't erase
1500                                          * buffer_info->page
1501                                          */
1502                                         skb_put(skb, length);
1503                                 } else {
1504                                         skb_fill_page_desc(skb, 0,
1505                                                            buffer_info->page, 0,
1506                                                            length);
1507                                         e1000_consume_page(buffer_info, skb,
1508                                                            length);
1509                                 }
1510                         }
1511                 }
1512
1513                 /* Receive Checksum Offload */
1514                 e1000_rx_checksum(adapter, staterr, skb);
1515
1516                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1517
1518                 /* probably a little skewed due to removing CRC */
1519                 total_rx_bytes += skb->len;
1520                 total_rx_packets++;
1521
1522                 /* eth type trans needs skb->data to point to something */
1523                 if (!pskb_may_pull(skb, ETH_HLEN)) {
1524                         e_err("pskb_may_pull failed.\n");
1525                         dev_kfree_skb_irq(skb);
1526                         goto next_desc;
1527                 }
1528
1529                 e1000_receive_skb(adapter, netdev, skb, staterr,
1530                                   rx_desc->wb.upper.vlan);
1531
1532 next_desc:
1533                 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1534
1535                 /* return some buffers to hardware, one at a time is too slow */
1536                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
1537                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
1538                                               GFP_ATOMIC);
1539                         cleaned_count = 0;
1540                 }
1541
1542                 /* use prefetched values */
1543                 rx_desc = next_rxd;
1544                 buffer_info = next_buffer;
1545
1546                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1547         }
1548         rx_ring->next_to_clean = i;
1549
1550         cleaned_count = e1000_desc_unused(rx_ring);
1551         if (cleaned_count)
1552                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1553
1554         adapter->total_rx_bytes += total_rx_bytes;
1555         adapter->total_rx_packets += total_rx_packets;
1556         return cleaned;
1557 }
1558
1559 /**
1560  * e1000_clean_rx_ring - Free Rx Buffers per Queue
1561  * @rx_ring: Rx descriptor ring
1562  **/
1563 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
1564 {
1565         struct e1000_adapter *adapter = rx_ring->adapter;
1566         struct e1000_buffer *buffer_info;
1567         struct e1000_ps_page *ps_page;
1568         struct pci_dev *pdev = adapter->pdev;
1569         unsigned int i, j;
1570
1571         /* Free all the Rx ring sk_buffs */
1572         for (i = 0; i < rx_ring->count; i++) {
1573                 buffer_info = &rx_ring->buffer_info[i];
1574                 if (buffer_info->dma) {
1575                         if (adapter->clean_rx == e1000_clean_rx_irq)
1576                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1577                                                  adapter->rx_buffer_len,
1578                                                  DMA_FROM_DEVICE);
1579                         else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
1580                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
1581                                                PAGE_SIZE,
1582                                                DMA_FROM_DEVICE);
1583                         else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
1584                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1585                                                  adapter->rx_ps_bsize0,
1586                                                  DMA_FROM_DEVICE);
1587                         buffer_info->dma = 0;
1588                 }
1589
1590                 if (buffer_info->page) {
1591                         put_page(buffer_info->page);
1592                         buffer_info->page = NULL;
1593                 }
1594
1595                 if (buffer_info->skb) {
1596                         dev_kfree_skb(buffer_info->skb);
1597                         buffer_info->skb = NULL;
1598                 }
1599
1600                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1601                         ps_page = &buffer_info->ps_pages[j];
1602                         if (!ps_page->page)
1603                                 break;
1604                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1605                                        DMA_FROM_DEVICE);
1606                         ps_page->dma = 0;
1607                         put_page(ps_page->page);
1608                         ps_page->page = NULL;
1609                 }
1610         }
1611
1612         /* there also may be some cached data from a chained receive */
1613         if (rx_ring->rx_skb_top) {
1614                 dev_kfree_skb(rx_ring->rx_skb_top);
1615                 rx_ring->rx_skb_top = NULL;
1616         }
1617
1618         /* Zero out the descriptor ring */
1619         memset(rx_ring->desc, 0, rx_ring->size);
1620
1621         rx_ring->next_to_clean = 0;
1622         rx_ring->next_to_use = 0;
1623         adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1624
1625         writel(0, rx_ring->head);
1626         if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1627                 e1000e_update_rdt_wa(rx_ring, 0);
1628         else
1629                 writel(0, rx_ring->tail);
1630 }
1631
1632 static void e1000e_downshift_workaround(struct work_struct *work)
1633 {
1634         struct e1000_adapter *adapter = container_of(work,
1635                                         struct e1000_adapter, downshift_task);
1636
1637         if (test_bit(__E1000_DOWN, &adapter->state))
1638                 return;
1639
1640         e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1641 }
1642
1643 /**
1644  * e1000_intr_msi - Interrupt Handler
1645  * @irq: interrupt number
1646  * @data: pointer to a network interface device structure
1647  **/
1648 static irqreturn_t e1000_intr_msi(int irq, void *data)
1649 {
1650         struct net_device *netdev = data;
1651         struct e1000_adapter *adapter = netdev_priv(netdev);
1652         struct e1000_hw *hw = &adapter->hw;
1653         u32 icr = er32(ICR);
1654
1655         /* read ICR disables interrupts using IAM */
1656         if (icr & E1000_ICR_LSC) {
1657                 hw->mac.get_link_status = true;
1658                 /* ICH8 workaround-- Call gig speed drop workaround on cable
1659                  * disconnect (LSC) before accessing any PHY registers
1660                  */
1661                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1662                     (!(er32(STATUS) & E1000_STATUS_LU)))
1663                         schedule_work(&adapter->downshift_task);
1664
1665                 /* 80003ES2LAN workaround-- For packet buffer work-around on
1666                  * link down event; disable receives here in the ISR and reset
1667                  * adapter in watchdog
1668                  */
1669                 if (netif_carrier_ok(netdev) &&
1670                     adapter->flags & FLAG_RX_NEEDS_RESTART) {
1671                         /* disable receives */
1672                         u32 rctl = er32(RCTL);
1673                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1674                         adapter->flags |= FLAG_RX_RESTART_NOW;
1675                 }
1676                 /* guard against interrupt when we're going down */
1677                 if (!test_bit(__E1000_DOWN, &adapter->state))
1678                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1679         }
1680
1681         if (napi_schedule_prep(&adapter->napi)) {
1682                 adapter->total_tx_bytes = 0;
1683                 adapter->total_tx_packets = 0;
1684                 adapter->total_rx_bytes = 0;
1685                 adapter->total_rx_packets = 0;
1686                 __napi_schedule(&adapter->napi);
1687         }
1688
1689         return IRQ_HANDLED;
1690 }
1691
1692 /**
1693  * e1000_intr - Interrupt Handler
1694  * @irq: interrupt number
1695  * @data: pointer to a network interface device structure
1696  **/
1697 static irqreturn_t e1000_intr(int irq, void *data)
1698 {
1699         struct net_device *netdev = data;
1700         struct e1000_adapter *adapter = netdev_priv(netdev);
1701         struct e1000_hw *hw = &adapter->hw;
1702         u32 rctl, icr = er32(ICR);
1703
1704         if (!icr || test_bit(__E1000_DOWN, &adapter->state))
1705                 return IRQ_NONE;  /* Not our interrupt */
1706
1707         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
1708          * not set, then the adapter didn't send an interrupt
1709          */
1710         if (!(icr & E1000_ICR_INT_ASSERTED))
1711                 return IRQ_NONE;
1712
1713         /* Interrupt Auto-Mask...upon reading ICR,
1714          * interrupts are masked.  No need for the
1715          * IMC write
1716          */
1717
1718         if (icr & E1000_ICR_LSC) {
1719                 hw->mac.get_link_status = true;
1720                 /* ICH8 workaround-- Call gig speed drop workaround on cable
1721                  * disconnect (LSC) before accessing any PHY registers
1722                  */
1723                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1724                     (!(er32(STATUS) & E1000_STATUS_LU)))
1725                         schedule_work(&adapter->downshift_task);
1726
1727                 /* 80003ES2LAN workaround--
1728                  * For packet buffer work-around on link down event;
1729                  * disable receives here in the ISR and
1730                  * reset adapter in watchdog
1731                  */
1732                 if (netif_carrier_ok(netdev) &&
1733                     (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1734                         /* disable receives */
1735                         rctl = er32(RCTL);
1736                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1737                         adapter->flags |= FLAG_RX_RESTART_NOW;
1738                 }
1739                 /* guard against interrupt when we're going down */
1740                 if (!test_bit(__E1000_DOWN, &adapter->state))
1741                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1742         }
1743
1744         if (napi_schedule_prep(&adapter->napi)) {
1745                 adapter->total_tx_bytes = 0;
1746                 adapter->total_tx_packets = 0;
1747                 adapter->total_rx_bytes = 0;
1748                 adapter->total_rx_packets = 0;
1749                 __napi_schedule(&adapter->napi);
1750         }
1751
1752         return IRQ_HANDLED;
1753 }
1754
1755 static irqreturn_t e1000_msix_other(int irq, void *data)
1756 {
1757         struct net_device *netdev = data;
1758         struct e1000_adapter *adapter = netdev_priv(netdev);
1759         struct e1000_hw *hw = &adapter->hw;
1760         u32 icr = er32(ICR);
1761
1762         if (!(icr & E1000_ICR_INT_ASSERTED)) {
1763                 if (!test_bit(__E1000_DOWN, &adapter->state))
1764                         ew32(IMS, E1000_IMS_OTHER);
1765                 return IRQ_NONE;
1766         }
1767
1768         if (icr & adapter->eiac_mask)
1769                 ew32(ICS, (icr & adapter->eiac_mask));
1770
1771         if (icr & E1000_ICR_OTHER) {
1772                 if (!(icr & E1000_ICR_LSC))
1773                         goto no_link_interrupt;
1774                 hw->mac.get_link_status = true;
1775                 /* guard against interrupt when we're going down */
1776                 if (!test_bit(__E1000_DOWN, &adapter->state))
1777                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1778         }
1779
1780 no_link_interrupt:
1781         if (!test_bit(__E1000_DOWN, &adapter->state))
1782                 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
1783
1784         return IRQ_HANDLED;
1785 }
1786
1787
1788 static irqreturn_t e1000_intr_msix_tx(int irq, void *data)
1789 {
1790         struct net_device *netdev = data;
1791         struct e1000_adapter *adapter = netdev_priv(netdev);
1792         struct e1000_hw *hw = &adapter->hw;
1793         struct e1000_ring *tx_ring = adapter->tx_ring;
1794
1795
1796         adapter->total_tx_bytes = 0;
1797         adapter->total_tx_packets = 0;
1798
1799         if (!e1000_clean_tx_irq(tx_ring))
1800                 /* Ring was not completely cleaned, so fire another interrupt */
1801                 ew32(ICS, tx_ring->ims_val);
1802
1803         return IRQ_HANDLED;
1804 }
1805
1806 static irqreturn_t e1000_intr_msix_rx(int irq, void *data)
1807 {
1808         struct net_device *netdev = data;
1809         struct e1000_adapter *adapter = netdev_priv(netdev);
1810         struct e1000_ring *rx_ring = adapter->rx_ring;
1811
1812         /* Write the ITR value calculated at the end of the
1813          * previous interrupt.
1814          */
1815         if (rx_ring->set_itr) {
1816                 writel(1000000000 / (rx_ring->itr_val * 256),
1817                        rx_ring->itr_register);
1818                 rx_ring->set_itr = 0;
1819         }
1820
1821         if (napi_schedule_prep(&adapter->napi)) {
1822                 adapter->total_rx_bytes = 0;
1823                 adapter->total_rx_packets = 0;
1824                 __napi_schedule(&adapter->napi);
1825         }
1826         return IRQ_HANDLED;
1827 }
1828
1829 /**
1830  * e1000_configure_msix - Configure MSI-X hardware
1831  *
1832  * e1000_configure_msix sets up the hardware to properly
1833  * generate MSI-X interrupts.
1834  **/
1835 static void e1000_configure_msix(struct e1000_adapter *adapter)
1836 {
1837         struct e1000_hw *hw = &adapter->hw;
1838         struct e1000_ring *rx_ring = adapter->rx_ring;
1839         struct e1000_ring *tx_ring = adapter->tx_ring;
1840         int vector = 0;
1841         u32 ctrl_ext, ivar = 0;
1842
1843         adapter->eiac_mask = 0;
1844
1845         /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1846         if (hw->mac.type == e1000_82574) {
1847                 u32 rfctl = er32(RFCTL);
1848                 rfctl |= E1000_RFCTL_ACK_DIS;
1849                 ew32(RFCTL, rfctl);
1850         }
1851
1852 #define E1000_IVAR_INT_ALLOC_VALID      0x8
1853         /* Configure Rx vector */
1854         rx_ring->ims_val = E1000_IMS_RXQ0;
1855         adapter->eiac_mask |= rx_ring->ims_val;
1856         if (rx_ring->itr_val)
1857                 writel(1000000000 / (rx_ring->itr_val * 256),
1858                        rx_ring->itr_register);
1859         else
1860                 writel(1, rx_ring->itr_register);
1861         ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1862
1863         /* Configure Tx vector */
1864         tx_ring->ims_val = E1000_IMS_TXQ0;
1865         vector++;
1866         if (tx_ring->itr_val)
1867                 writel(1000000000 / (tx_ring->itr_val * 256),
1868                        tx_ring->itr_register);
1869         else
1870                 writel(1, tx_ring->itr_register);
1871         adapter->eiac_mask |= tx_ring->ims_val;
1872         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1873
1874         /* set vector for Other Causes, e.g. link changes */
1875         vector++;
1876         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
1877         if (rx_ring->itr_val)
1878                 writel(1000000000 / (rx_ring->itr_val * 256),
1879                        hw->hw_addr + E1000_EITR_82574(vector));
1880         else
1881                 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
1882
1883         /* Cause Tx interrupts on every write back */
1884         ivar |= (1 << 31);
1885
1886         ew32(IVAR, ivar);
1887
1888         /* enable MSI-X PBA support */
1889         ctrl_ext = er32(CTRL_EXT);
1890         ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
1891
1892         /* Auto-Mask Other interrupts upon ICR read */
1893 #define E1000_EIAC_MASK_82574   0x01F00000
1894         ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
1895         ctrl_ext |= E1000_CTRL_EXT_EIAME;
1896         ew32(CTRL_EXT, ctrl_ext);
1897         e1e_flush();
1898 }
1899
1900 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
1901 {
1902         if (adapter->msix_entries) {
1903                 pci_disable_msix(adapter->pdev);
1904                 kfree(adapter->msix_entries);
1905                 adapter->msix_entries = NULL;
1906         } else if (adapter->flags & FLAG_MSI_ENABLED) {
1907                 pci_disable_msi(adapter->pdev);
1908                 adapter->flags &= ~FLAG_MSI_ENABLED;
1909         }
1910 }
1911
1912 /**
1913  * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
1914  *
1915  * Attempt to configure interrupts using the best available
1916  * capabilities of the hardware and kernel.
1917  **/
1918 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
1919 {
1920         int err;
1921         int i;
1922
1923         switch (adapter->int_mode) {
1924         case E1000E_INT_MODE_MSIX:
1925                 if (adapter->flags & FLAG_HAS_MSIX) {
1926                         adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
1927                         adapter->msix_entries = kcalloc(adapter->num_vectors,
1928                                                       sizeof(struct msix_entry),
1929                                                       GFP_KERNEL);
1930                         if (adapter->msix_entries) {
1931                                 for (i = 0; i < adapter->num_vectors; i++)
1932                                         adapter->msix_entries[i].entry = i;
1933
1934                                 err = pci_enable_msix(adapter->pdev,
1935                                                       adapter->msix_entries,
1936                                                       adapter->num_vectors);
1937                                 if (err == 0)
1938                                         return;
1939                         }
1940                         /* MSI-X failed, so fall through and try MSI */
1941                         e_err("Failed to initialize MSI-X interrupts.  Falling back to MSI interrupts.\n");
1942                         e1000e_reset_interrupt_capability(adapter);
1943                 }
1944                 adapter->int_mode = E1000E_INT_MODE_MSI;
1945                 /* Fall through */
1946         case E1000E_INT_MODE_MSI:
1947                 if (!pci_enable_msi(adapter->pdev)) {
1948                         adapter->flags |= FLAG_MSI_ENABLED;
1949                 } else {
1950                         adapter->int_mode = E1000E_INT_MODE_LEGACY;
1951                         e_err("Failed to initialize MSI interrupts.  Falling back to legacy interrupts.\n");
1952                 }
1953                 /* Fall through */
1954         case E1000E_INT_MODE_LEGACY:
1955                 /* Don't do anything; this is the system default */
1956                 break;
1957         }
1958
1959         /* store the number of vectors being used */
1960         adapter->num_vectors = 1;
1961 }
1962
1963 /**
1964  * e1000_request_msix - Initialize MSI-X interrupts
1965  *
1966  * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
1967  * kernel.
1968  **/
1969 static int e1000_request_msix(struct e1000_adapter *adapter)
1970 {
1971         struct net_device *netdev = adapter->netdev;
1972         int err = 0, vector = 0;
1973
1974         if (strlen(netdev->name) < (IFNAMSIZ - 5))
1975                 snprintf(adapter->rx_ring->name,
1976                          sizeof(adapter->rx_ring->name) - 1,
1977                          "%s-rx-0", netdev->name);
1978         else
1979                 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1980         err = request_irq(adapter->msix_entries[vector].vector,
1981                           e1000_intr_msix_rx, 0, adapter->rx_ring->name,
1982                           netdev);
1983         if (err)
1984                 return err;
1985         adapter->rx_ring->itr_register = adapter->hw.hw_addr +
1986             E1000_EITR_82574(vector);
1987         adapter->rx_ring->itr_val = adapter->itr;
1988         vector++;
1989
1990         if (strlen(netdev->name) < (IFNAMSIZ - 5))
1991                 snprintf(adapter->tx_ring->name,
1992                          sizeof(adapter->tx_ring->name) - 1,
1993                          "%s-tx-0", netdev->name);
1994         else
1995                 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1996         err = request_irq(adapter->msix_entries[vector].vector,
1997                           e1000_intr_msix_tx, 0, adapter->tx_ring->name,
1998                           netdev);
1999         if (err)
2000                 return err;
2001         adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2002             E1000_EITR_82574(vector);
2003         adapter->tx_ring->itr_val = adapter->itr;
2004         vector++;
2005
2006         err = request_irq(adapter->msix_entries[vector].vector,
2007                           e1000_msix_other, 0, netdev->name, netdev);
2008         if (err)
2009                 return err;
2010
2011         e1000_configure_msix(adapter);
2012
2013         return 0;
2014 }
2015
2016 /**
2017  * e1000_request_irq - initialize interrupts
2018  *
2019  * Attempts to configure interrupts using the best available
2020  * capabilities of the hardware and kernel.
2021  **/
2022 static int e1000_request_irq(struct e1000_adapter *adapter)
2023 {
2024         struct net_device *netdev = adapter->netdev;
2025         int err;
2026
2027         if (adapter->msix_entries) {
2028                 err = e1000_request_msix(adapter);
2029                 if (!err)
2030                         return err;
2031                 /* fall back to MSI */
2032                 e1000e_reset_interrupt_capability(adapter);
2033                 adapter->int_mode = E1000E_INT_MODE_MSI;
2034                 e1000e_set_interrupt_capability(adapter);
2035         }
2036         if (adapter->flags & FLAG_MSI_ENABLED) {
2037                 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
2038                                   netdev->name, netdev);
2039                 if (!err)
2040                         return err;
2041
2042                 /* fall back to legacy interrupt */
2043                 e1000e_reset_interrupt_capability(adapter);
2044                 adapter->int_mode = E1000E_INT_MODE_LEGACY;
2045         }
2046
2047         err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
2048                           netdev->name, netdev);
2049         if (err)
2050                 e_err("Unable to allocate interrupt, Error: %d\n", err);
2051
2052         return err;
2053 }
2054
2055 static void e1000_free_irq(struct e1000_adapter *adapter)
2056 {
2057         struct net_device *netdev = adapter->netdev;
2058
2059         if (adapter->msix_entries) {
2060                 int vector = 0;
2061
2062                 free_irq(adapter->msix_entries[vector].vector, netdev);
2063                 vector++;
2064
2065                 free_irq(adapter->msix_entries[vector].vector, netdev);
2066                 vector++;
2067
2068                 /* Other Causes interrupt vector */
2069                 free_irq(adapter->msix_entries[vector].vector, netdev);
2070                 return;
2071         }
2072
2073         free_irq(adapter->pdev->irq, netdev);
2074 }
2075
2076 /**
2077  * e1000_irq_disable - Mask off interrupt generation on the NIC
2078  **/
2079 static void e1000_irq_disable(struct e1000_adapter *adapter)
2080 {
2081         struct e1000_hw *hw = &adapter->hw;
2082
2083         ew32(IMC, ~0);
2084         if (adapter->msix_entries)
2085                 ew32(EIAC_82574, 0);
2086         e1e_flush();
2087
2088         if (adapter->msix_entries) {
2089                 int i;
2090                 for (i = 0; i < adapter->num_vectors; i++)
2091                         synchronize_irq(adapter->msix_entries[i].vector);
2092         } else {
2093                 synchronize_irq(adapter->pdev->irq);
2094         }
2095 }
2096
2097 /**
2098  * e1000_irq_enable - Enable default interrupt generation settings
2099  **/
2100 static void e1000_irq_enable(struct e1000_adapter *adapter)
2101 {
2102         struct e1000_hw *hw = &adapter->hw;
2103
2104         if (adapter->msix_entries) {
2105                 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2106                 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
2107         } else {
2108                 ew32(IMS, IMS_ENABLE_MASK);
2109         }
2110         e1e_flush();
2111 }
2112
2113 /**
2114  * e1000e_get_hw_control - get control of the h/w from f/w
2115  * @adapter: address of board private structure
2116  *
2117  * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2118  * For ASF and Pass Through versions of f/w this means that
2119  * the driver is loaded. For AMT version (only with 82573)
2120  * of the f/w this means that the network i/f is open.
2121  **/
2122 void e1000e_get_hw_control(struct e1000_adapter *adapter)
2123 {
2124         struct e1000_hw *hw = &adapter->hw;
2125         u32 ctrl_ext;
2126         u32 swsm;
2127
2128         /* Let firmware know the driver has taken over */
2129         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2130                 swsm = er32(SWSM);
2131                 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2132         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2133                 ctrl_ext = er32(CTRL_EXT);
2134                 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2135         }
2136 }
2137
2138 /**
2139  * e1000e_release_hw_control - release control of the h/w to f/w
2140  * @adapter: address of board private structure
2141  *
2142  * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2143  * For ASF and Pass Through versions of f/w this means that the
2144  * driver is no longer loaded. For AMT version (only with 82573) i
2145  * of the f/w this means that the network i/f is closed.
2146  *
2147  **/
2148 void e1000e_release_hw_control(struct e1000_adapter *adapter)
2149 {
2150         struct e1000_hw *hw = &adapter->hw;
2151         u32 ctrl_ext;
2152         u32 swsm;
2153
2154         /* Let firmware taken over control of h/w */
2155         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2156                 swsm = er32(SWSM);
2157                 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2158         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2159                 ctrl_ext = er32(CTRL_EXT);
2160                 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2161         }
2162 }
2163
2164 /**
2165  * e1000_alloc_ring_dma - allocate memory for a ring structure
2166  **/
2167 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2168                                 struct e1000_ring *ring)
2169 {
2170         struct pci_dev *pdev = adapter->pdev;
2171
2172         ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2173                                         GFP_KERNEL);
2174         if (!ring->desc)
2175                 return -ENOMEM;
2176
2177         return 0;
2178 }
2179
2180 /**
2181  * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
2182  * @tx_ring: Tx descriptor ring
2183  *
2184  * Return 0 on success, negative on failure
2185  **/
2186 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
2187 {
2188         struct e1000_adapter *adapter = tx_ring->adapter;
2189         int err = -ENOMEM, size;
2190
2191         size = sizeof(struct e1000_buffer) * tx_ring->count;
2192         tx_ring->buffer_info = vzalloc(size);
2193         if (!tx_ring->buffer_info)
2194                 goto err;
2195
2196         /* round up to nearest 4K */
2197         tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2198         tx_ring->size = ALIGN(tx_ring->size, 4096);
2199
2200         err = e1000_alloc_ring_dma(adapter, tx_ring);
2201         if (err)
2202                 goto err;
2203
2204         tx_ring->next_to_use = 0;
2205         tx_ring->next_to_clean = 0;
2206
2207         return 0;
2208 err:
2209         vfree(tx_ring->buffer_info);
2210         e_err("Unable to allocate memory for the transmit descriptor ring\n");
2211         return err;
2212 }
2213
2214 /**
2215  * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
2216  * @rx_ring: Rx descriptor ring
2217  *
2218  * Returns 0 on success, negative on failure
2219  **/
2220 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
2221 {
2222         struct e1000_adapter *adapter = rx_ring->adapter;
2223         struct e1000_buffer *buffer_info;
2224         int i, size, desc_len, err = -ENOMEM;
2225
2226         size = sizeof(struct e1000_buffer) * rx_ring->count;
2227         rx_ring->buffer_info = vzalloc(size);
2228         if (!rx_ring->buffer_info)
2229                 goto err;
2230
2231         for (i = 0; i < rx_ring->count; i++) {
2232                 buffer_info = &rx_ring->buffer_info[i];
2233                 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2234                                                 sizeof(struct e1000_ps_page),
2235                                                 GFP_KERNEL);
2236                 if (!buffer_info->ps_pages)
2237                         goto err_pages;
2238         }
2239
2240         desc_len = sizeof(union e1000_rx_desc_packet_split);
2241
2242         /* Round up to nearest 4K */
2243         rx_ring->size = rx_ring->count * desc_len;
2244         rx_ring->size = ALIGN(rx_ring->size, 4096);
2245
2246         err = e1000_alloc_ring_dma(adapter, rx_ring);
2247         if (err)
2248                 goto err_pages;
2249
2250         rx_ring->next_to_clean = 0;
2251         rx_ring->next_to_use = 0;
2252         rx_ring->rx_skb_top = NULL;
2253
2254         return 0;
2255
2256 err_pages:
2257         for (i = 0; i < rx_ring->count; i++) {
2258                 buffer_info = &rx_ring->buffer_info[i];
2259                 kfree(buffer_info->ps_pages);
2260         }
2261 err:
2262         vfree(rx_ring->buffer_info);
2263         e_err("Unable to allocate memory for the receive descriptor ring\n");
2264         return err;
2265 }
2266
2267 /**
2268  * e1000_clean_tx_ring - Free Tx Buffers
2269  * @tx_ring: Tx descriptor ring
2270  **/
2271 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
2272 {
2273         struct e1000_adapter *adapter = tx_ring->adapter;
2274         struct e1000_buffer *buffer_info;
2275         unsigned long size;
2276         unsigned int i;
2277
2278         for (i = 0; i < tx_ring->count; i++) {
2279                 buffer_info = &tx_ring->buffer_info[i];
2280                 e1000_put_txbuf(tx_ring, buffer_info);
2281         }
2282
2283         netdev_reset_queue(adapter->netdev);
2284         size = sizeof(struct e1000_buffer) * tx_ring->count;
2285         memset(tx_ring->buffer_info, 0, size);
2286
2287         memset(tx_ring->desc, 0, tx_ring->size);
2288
2289         tx_ring->next_to_use = 0;
2290         tx_ring->next_to_clean = 0;
2291
2292         writel(0, tx_ring->head);
2293         if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2294                 e1000e_update_tdt_wa(tx_ring, 0);
2295         else
2296                 writel(0, tx_ring->tail);
2297 }
2298
2299 /**
2300  * e1000e_free_tx_resources - Free Tx Resources per Queue
2301  * @tx_ring: Tx descriptor ring
2302  *
2303  * Free all transmit software resources
2304  **/
2305 void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
2306 {
2307         struct e1000_adapter *adapter = tx_ring->adapter;
2308         struct pci_dev *pdev = adapter->pdev;
2309
2310         e1000_clean_tx_ring(tx_ring);
2311
2312         vfree(tx_ring->buffer_info);
2313         tx_ring->buffer_info = NULL;
2314
2315         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2316                           tx_ring->dma);
2317         tx_ring->desc = NULL;
2318 }
2319
2320 /**
2321  * e1000e_free_rx_resources - Free Rx Resources
2322  * @rx_ring: Rx descriptor ring
2323  *
2324  * Free all receive software resources
2325  **/
2326 void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
2327 {
2328         struct e1000_adapter *adapter = rx_ring->adapter;
2329         struct pci_dev *pdev = adapter->pdev;
2330         int i;
2331
2332         e1000_clean_rx_ring(rx_ring);
2333
2334         for (i = 0; i < rx_ring->count; i++)
2335                 kfree(rx_ring->buffer_info[i].ps_pages);
2336
2337         vfree(rx_ring->buffer_info);
2338         rx_ring->buffer_info = NULL;
2339
2340         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2341                           rx_ring->dma);
2342         rx_ring->desc = NULL;
2343 }
2344
2345 /**
2346  * e1000_update_itr - update the dynamic ITR value based on statistics
2347  * @adapter: pointer to adapter
2348  * @itr_setting: current adapter->itr
2349  * @packets: the number of packets during this measurement interval
2350  * @bytes: the number of bytes during this measurement interval
2351  *
2352  *      Stores a new ITR value based on packets and byte
2353  *      counts during the last interrupt.  The advantage of per interrupt
2354  *      computation is faster updates and more accurate ITR for the current
2355  *      traffic pattern.  Constants in this function were computed
2356  *      based on theoretical maximum wire speed and thresholds were set based
2357  *      on testing data as well as attempting to minimize response time
2358  *      while increasing bulk throughput.  This functionality is controlled
2359  *      by the InterruptThrottleRate module parameter.
2360  **/
2361 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2362                                      u16 itr_setting, int packets,
2363                                      int bytes)
2364 {
2365         unsigned int retval = itr_setting;
2366
2367         if (packets == 0)
2368                 return itr_setting;
2369
2370         switch (itr_setting) {
2371         case lowest_latency:
2372                 /* handle TSO and jumbo frames */
2373                 if (bytes/packets > 8000)
2374                         retval = bulk_latency;
2375                 else if ((packets < 5) && (bytes > 512))
2376                         retval = low_latency;
2377                 break;
2378         case low_latency:  /* 50 usec aka 20000 ints/s */
2379                 if (bytes > 10000) {
2380                         /* this if handles the TSO accounting */
2381                         if (bytes/packets > 8000)
2382                                 retval = bulk_latency;
2383                         else if ((packets < 10) || ((bytes/packets) > 1200))
2384                                 retval = bulk_latency;
2385                         else if ((packets > 35))
2386                                 retval = lowest_latency;
2387                 } else if (bytes/packets > 2000) {
2388                         retval = bulk_latency;
2389                 } else if (packets <= 2 && bytes < 512) {
2390                         retval = lowest_latency;
2391                 }
2392                 break;
2393         case bulk_latency: /* 250 usec aka 4000 ints/s */
2394                 if (bytes > 25000) {
2395                         if (packets > 35)
2396                                 retval = low_latency;
2397                 } else if (bytes < 6000) {
2398                         retval = low_latency;
2399                 }
2400                 break;
2401         }
2402
2403         return retval;
2404 }
2405
2406 static void e1000_set_itr(struct e1000_adapter *adapter)
2407 {
2408         struct e1000_hw *hw = &adapter->hw;
2409         u16 current_itr;
2410         u32 new_itr = adapter->itr;
2411
2412         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2413         if (adapter->link_speed != SPEED_1000) {
2414                 current_itr = 0;
2415                 new_itr = 4000;
2416                 goto set_itr_now;
2417         }
2418
2419         if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2420                 new_itr = 0;
2421                 goto set_itr_now;
2422         }
2423
2424         adapter->tx_itr = e1000_update_itr(adapter,
2425                                     adapter->tx_itr,
2426                                     adapter->total_tx_packets,
2427                                     adapter->total_tx_bytes);
2428         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2429         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2430                 adapter->tx_itr = low_latency;
2431
2432         adapter->rx_itr = e1000_update_itr(adapter,
2433                                     adapter->rx_itr,
2434                                     adapter->total_rx_packets,
2435                                     adapter->total_rx_bytes);
2436         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2437         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2438                 adapter->rx_itr = low_latency;
2439
2440         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2441
2442         switch (current_itr) {
2443         /* counts and packets in update_itr are dependent on these numbers */
2444         case lowest_latency:
2445                 new_itr = 70000;
2446                 break;
2447         case low_latency:
2448                 new_itr = 20000; /* aka hwitr = ~200 */
2449                 break;
2450         case bulk_latency:
2451                 new_itr = 4000;
2452                 break;
2453         default:
2454                 break;
2455         }
2456
2457 set_itr_now:
2458         if (new_itr != adapter->itr) {
2459                 /* this attempts to bias the interrupt rate towards Bulk
2460                  * by adding intermediate steps when interrupt rate is
2461                  * increasing
2462                  */
2463                 new_itr = new_itr > adapter->itr ?
2464                              min(adapter->itr + (new_itr >> 2), new_itr) :
2465                              new_itr;
2466                 adapter->itr = new_itr;
2467                 adapter->rx_ring->itr_val = new_itr;
2468                 if (adapter->msix_entries)
2469                         adapter->rx_ring->set_itr = 1;
2470                 else
2471                         if (new_itr)
2472                                 ew32(ITR, 1000000000 / (new_itr * 256));
2473                         else
2474                                 ew32(ITR, 0);
2475         }
2476 }
2477
2478 /**
2479  * e1000e_write_itr - write the ITR value to the appropriate registers
2480  * @adapter: address of board private structure
2481  * @itr: new ITR value to program
2482  *
2483  * e1000e_write_itr determines if the adapter is in MSI-X mode
2484  * and, if so, writes the EITR registers with the ITR value.
2485  * Otherwise, it writes the ITR value into the ITR register.
2486  **/
2487 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2488 {
2489         struct e1000_hw *hw = &adapter->hw;
2490         u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2491
2492         if (adapter->msix_entries) {
2493                 int vector;
2494
2495                 for (vector = 0; vector < adapter->num_vectors; vector++)
2496                         writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2497         } else {
2498                 ew32(ITR, new_itr);
2499         }
2500 }
2501
2502 /**
2503  * e1000_alloc_queues - Allocate memory for all rings
2504  * @adapter: board private structure to initialize
2505  **/
2506 static int e1000_alloc_queues(struct e1000_adapter *adapter)
2507 {
2508         int size = sizeof(struct e1000_ring);
2509
2510         adapter->tx_ring = kzalloc(size, GFP_KERNEL);
2511         if (!adapter->tx_ring)
2512                 goto err;
2513         adapter->tx_ring->count = adapter->tx_ring_count;
2514         adapter->tx_ring->adapter = adapter;
2515
2516         adapter->rx_ring = kzalloc(size, GFP_KERNEL);
2517         if (!adapter->rx_ring)
2518                 goto err;
2519         adapter->rx_ring->count = adapter->rx_ring_count;
2520         adapter->rx_ring->adapter = adapter;
2521
2522         return 0;
2523 err:
2524         e_err("Unable to allocate memory for queues\n");
2525         kfree(adapter->rx_ring);
2526         kfree(adapter->tx_ring);
2527         return -ENOMEM;
2528 }
2529
2530 /**
2531  * e1000e_poll - NAPI Rx polling callback
2532  * @napi: struct associated with this polling callback
2533  * @weight: number of packets driver is allowed to process this poll
2534  **/
2535 static int e1000e_poll(struct napi_struct *napi, int weight)
2536 {
2537         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2538                                                      napi);
2539         struct e1000_hw *hw = &adapter->hw;
2540         struct net_device *poll_dev = adapter->netdev;
2541         int tx_cleaned = 1, work_done = 0;
2542
2543         adapter = netdev_priv(poll_dev);
2544
2545         if (!adapter->msix_entries ||
2546             (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2547                 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
2548
2549         adapter->clean_rx(adapter->rx_ring, &work_done, weight);
2550
2551         if (!tx_cleaned)
2552                 work_done = weight;
2553
2554         /* If weight not fully consumed, exit the polling mode */
2555         if (work_done < weight) {
2556                 if (adapter->itr_setting & 3)
2557                         e1000_set_itr(adapter);
2558                 napi_complete(napi);
2559                 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2560                         if (adapter->msix_entries)
2561                                 ew32(IMS, adapter->rx_ring->ims_val);
2562                         else
2563                                 e1000_irq_enable(adapter);
2564                 }
2565         }
2566
2567         return work_done;
2568 }
2569
2570 static int e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
2571 {
2572         struct e1000_adapter *adapter = netdev_priv(netdev);
2573         struct e1000_hw *hw = &adapter->hw;
2574         u32 vfta, index;
2575
2576         /* don't update vlan cookie if already programmed */
2577         if ((adapter->hw.mng_cookie.status &
2578              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2579             (vid == adapter->mng_vlan_id))
2580                 return 0;
2581
2582         /* add VID to filter table */
2583         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2584                 index = (vid >> 5) & 0x7F;
2585                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2586                 vfta |= (1 << (vid & 0x1F));
2587                 hw->mac.ops.write_vfta(hw, index, vfta);
2588         }
2589
2590         set_bit(vid, adapter->active_vlans);
2591
2592         return 0;
2593 }
2594
2595 static int e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
2596 {
2597         struct e1000_adapter *adapter = netdev_priv(netdev);
2598         struct e1000_hw *hw = &adapter->hw;
2599         u32 vfta, index;
2600
2601         if ((adapter->hw.mng_cookie.status &
2602              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2603             (vid == adapter->mng_vlan_id)) {
2604                 /* release control to f/w */
2605                 e1000e_release_hw_control(adapter);
2606                 return 0;
2607         }
2608
2609         /* remove VID from filter table */
2610         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2611                 index = (vid >> 5) & 0x7F;
2612                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2613                 vfta &= ~(1 << (vid & 0x1F));
2614                 hw->mac.ops.write_vfta(hw, index, vfta);
2615         }
2616
2617         clear_bit(vid, adapter->active_vlans);
2618
2619         return 0;
2620 }
2621
2622 /**
2623  * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2624  * @adapter: board private structure to initialize
2625  **/
2626 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
2627 {
2628         struct net_device *netdev = adapter->netdev;
2629         struct e1000_hw *hw = &adapter->hw;
2630         u32 rctl;
2631
2632         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2633                 /* disable VLAN receive filtering */
2634                 rctl = er32(RCTL);
2635                 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2636                 ew32(RCTL, rctl);
2637
2638                 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2639                         e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2640                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
2641                 }
2642         }
2643 }
2644
2645 /**
2646  * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2647  * @adapter: board private structure to initialize
2648  **/
2649 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2650 {
2651         struct e1000_hw *hw = &adapter->hw;
2652         u32 rctl;
2653
2654         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2655                 /* enable VLAN receive filtering */
2656                 rctl = er32(RCTL);
2657                 rctl |= E1000_RCTL_VFE;
2658                 rctl &= ~E1000_RCTL_CFIEN;
2659                 ew32(RCTL, rctl);
2660         }
2661 }
2662
2663 /**
2664  * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2665  * @adapter: board private structure to initialize
2666  **/
2667 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
2668 {
2669         struct e1000_hw *hw = &adapter->hw;
2670         u32 ctrl;
2671
2672         /* disable VLAN tag insert/strip */
2673         ctrl = er32(CTRL);
2674         ctrl &= ~E1000_CTRL_VME;
2675         ew32(CTRL, ctrl);
2676 }
2677
2678 /**
2679  * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2680  * @adapter: board private structure to initialize
2681  **/
2682 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2683 {
2684         struct e1000_hw *hw = &adapter->hw;
2685         u32 ctrl;
2686
2687         /* enable VLAN tag insert/strip */
2688         ctrl = er32(CTRL);
2689         ctrl |= E1000_CTRL_VME;
2690         ew32(CTRL, ctrl);
2691 }
2692
2693 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2694 {
2695         struct net_device *netdev = adapter->netdev;
2696         u16 vid = adapter->hw.mng_cookie.vlan_id;
2697         u16 old_vid = adapter->mng_vlan_id;
2698
2699         if (adapter->hw.mng_cookie.status &
2700             E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2701                 e1000_vlan_rx_add_vid(netdev, vid);
2702                 adapter->mng_vlan_id = vid;
2703         }
2704
2705         if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2706                 e1000_vlan_rx_kill_vid(netdev, old_vid);
2707 }
2708
2709 static void e1000_restore_vlan(struct e1000_adapter *adapter)
2710 {
2711         u16 vid;
2712
2713         e1000_vlan_rx_add_vid(adapter->netdev, 0);
2714
2715         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2716                 e1000_vlan_rx_add_vid(adapter->netdev, vid);
2717 }
2718
2719 static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
2720 {
2721         struct e1000_hw *hw = &adapter->hw;
2722         u32 manc, manc2h, mdef, i, j;
2723
2724         if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2725                 return;
2726
2727         manc = er32(MANC);
2728
2729         /* enable receiving management packets to the host. this will probably
2730          * generate destination unreachable messages from the host OS, but
2731          * the packets will be handled on SMBUS
2732          */
2733         manc |= E1000_MANC_EN_MNG2HOST;
2734         manc2h = er32(MANC2H);
2735
2736         switch (hw->mac.type) {
2737         default:
2738                 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2739                 break;
2740         case e1000_82574:
2741         case e1000_82583:
2742                 /* Check if IPMI pass-through decision filter already exists;
2743                  * if so, enable it.
2744                  */
2745                 for (i = 0, j = 0; i < 8; i++) {
2746                         mdef = er32(MDEF(i));
2747
2748                         /* Ignore filters with anything other than IPMI ports */
2749                         if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2750                                 continue;
2751
2752                         /* Enable this decision filter in MANC2H */
2753                         if (mdef)
2754                                 manc2h |= (1 << i);
2755
2756                         j |= mdef;
2757                 }
2758
2759                 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2760                         break;
2761
2762                 /* Create new decision filter in an empty filter */
2763                 for (i = 0, j = 0; i < 8; i++)
2764                         if (er32(MDEF(i)) == 0) {
2765                                 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2766                                                E1000_MDEF_PORT_664));
2767                                 manc2h |= (1 << 1);
2768                                 j++;
2769                                 break;
2770                         }
2771
2772                 if (!j)
2773                         e_warn("Unable to create IPMI pass-through filter\n");
2774                 break;
2775         }
2776
2777         ew32(MANC2H, manc2h);
2778         ew32(MANC, manc);
2779 }
2780
2781 /**
2782  * e1000_configure_tx - Configure Transmit Unit after Reset
2783  * @adapter: board private structure
2784  *
2785  * Configure the Tx unit of the MAC after a reset.
2786  **/
2787 static void e1000_configure_tx(struct e1000_adapter *adapter)
2788 {
2789         struct e1000_hw *hw = &adapter->hw;
2790         struct e1000_ring *tx_ring = adapter->tx_ring;
2791         u64 tdba;
2792         u32 tdlen, tarc;
2793
2794         /* Setup the HW Tx Head and Tail descriptor pointers */
2795         tdba = tx_ring->dma;
2796         tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
2797         ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2798         ew32(TDBAH(0), (tdba >> 32));
2799         ew32(TDLEN(0), tdlen);
2800         ew32(TDH(0), 0);
2801         ew32(TDT(0), 0);
2802         tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);
2803         tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);
2804
2805         /* Set the Tx Interrupt Delay register */
2806         ew32(TIDV, adapter->tx_int_delay);
2807         /* Tx irq moderation */
2808         ew32(TADV, adapter->tx_abs_int_delay);
2809
2810         if (adapter->flags2 & FLAG2_DMA_BURST) {
2811                 u32 txdctl = er32(TXDCTL(0));
2812                 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2813                             E1000_TXDCTL_WTHRESH);
2814                 /* set up some performance related parameters to encourage the
2815                  * hardware to use the bus more efficiently in bursts, depends
2816                  * on the tx_int_delay to be enabled,
2817                  * wthresh = 1 ==> burst write is disabled to avoid Tx stalls
2818                  * hthresh = 1 ==> prefetch when one or more available
2819                  * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2820                  * BEWARE: this seems to work but should be considered first if
2821                  * there are Tx hangs or other Tx related bugs
2822                  */
2823                 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2824                 ew32(TXDCTL(0), txdctl);
2825         }
2826         /* erratum work around: set txdctl the same for both queues */
2827         ew32(TXDCTL(1), er32(TXDCTL(0)));
2828
2829         if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
2830                 tarc = er32(TARC(0));
2831                 /* set the speed mode bit, we'll clear it if we're not at
2832                  * gigabit link later
2833                  */
2834 #define SPEED_MODE_BIT (1 << 21)
2835                 tarc |= SPEED_MODE_BIT;
2836                 ew32(TARC(0), tarc);
2837         }
2838
2839         /* errata: program both queues to unweighted RR */
2840         if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
2841                 tarc = er32(TARC(0));
2842                 tarc |= 1;
2843                 ew32(TARC(0), tarc);
2844                 tarc = er32(TARC(1));
2845                 tarc |= 1;
2846                 ew32(TARC(1), tarc);
2847         }
2848
2849         /* Setup Transmit Descriptor Settings for eop descriptor */
2850         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2851
2852         /* only set IDE if we are delaying interrupts using the timers */
2853         if (adapter->tx_int_delay)
2854                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2855
2856         /* enable Report Status bit */
2857         adapter->txd_cmd |= E1000_TXD_CMD_RS;
2858
2859         hw->mac.ops.config_collision_dist(hw);
2860 }
2861
2862 /**
2863  * e1000_setup_rctl - configure the receive control registers
2864  * @adapter: Board private structure
2865  **/
2866 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2867                            (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2868 static void e1000_setup_rctl(struct e1000_adapter *adapter)
2869 {
2870         struct e1000_hw *hw = &adapter->hw;
2871         u32 rctl, rfctl;
2872         u32 pages = 0;
2873
2874         /* Workaround Si errata on PCHx - configure jumbo frame flow */
2875         if (hw->mac.type >= e1000_pch2lan) {
2876                 s32 ret_val;
2877
2878                 if (adapter->netdev->mtu > ETH_DATA_LEN)
2879                         ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2880                 else
2881                         ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
2882
2883                 if (ret_val)
2884                         e_dbg("failed to enable jumbo frame workaround mode\n");
2885         }
2886
2887         /* Program MC offset vector base */
2888         rctl = er32(RCTL);
2889         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2890         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
2891                 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
2892                 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2893
2894         /* Do not Store bad packets */
2895         rctl &= ~E1000_RCTL_SBP;
2896
2897         /* Enable Long Packet receive */
2898         if (adapter->netdev->mtu <= ETH_DATA_LEN)
2899                 rctl &= ~E1000_RCTL_LPE;
2900         else
2901                 rctl |= E1000_RCTL_LPE;
2902
2903         /* Some systems expect that the CRC is included in SMBUS traffic. The
2904          * hardware strips the CRC before sending to both SMBUS (BMC) and to
2905          * host memory when this is enabled
2906          */
2907         if (adapter->flags2 & FLAG2_CRC_STRIPPING)
2908                 rctl |= E1000_RCTL_SECRC;
2909
2910         /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
2911         if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
2912                 u16 phy_data;
2913
2914                 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
2915                 phy_data &= 0xfff8;
2916                 phy_data |= (1 << 2);
2917                 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
2918
2919                 e1e_rphy(hw, 22, &phy_data);
2920                 phy_data &= 0x0fff;
2921                 phy_data |= (1 << 14);
2922                 e1e_wphy(hw, 0x10, 0x2823);
2923                 e1e_wphy(hw, 0x11, 0x0003);
2924                 e1e_wphy(hw, 22, phy_data);
2925         }
2926
2927         /* Setup buffer sizes */
2928         rctl &= ~E1000_RCTL_SZ_4096;
2929         rctl |= E1000_RCTL_BSEX;
2930         switch (adapter->rx_buffer_len) {
2931         case 2048:
2932         default:
2933                 rctl |= E1000_RCTL_SZ_2048;
2934                 rctl &= ~E1000_RCTL_BSEX;
2935                 break;
2936         case 4096:
2937                 rctl |= E1000_RCTL_SZ_4096;
2938                 break;
2939         case 8192:
2940                 rctl |= E1000_RCTL_SZ_8192;
2941                 break;
2942         case 16384:
2943                 rctl |= E1000_RCTL_SZ_16384;
2944                 break;
2945         }
2946
2947         /* Enable Extended Status in all Receive Descriptors */
2948         rfctl = er32(RFCTL);
2949         rfctl |= E1000_RFCTL_EXTEN;
2950         ew32(RFCTL, rfctl);
2951
2952         /* 82571 and greater support packet-split where the protocol
2953          * header is placed in skb->data and the packet data is
2954          * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
2955          * In the case of a non-split, skb->data is linearly filled,
2956          * followed by the page buffers.  Therefore, skb->data is
2957          * sized to hold the largest protocol header.
2958          *
2959          * allocations using alloc_page take too long for regular MTU
2960          * so only enable packet split for jumbo frames
2961          *
2962          * Using pages when the page size is greater than 16k wastes
2963          * a lot of memory, since we allocate 3 pages at all times
2964          * per packet.
2965          */
2966         pages = PAGE_USE_COUNT(adapter->netdev->mtu);
2967         if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
2968                 adapter->rx_ps_pages = pages;
2969         else
2970                 adapter->rx_ps_pages = 0;
2971
2972         if (adapter->rx_ps_pages) {
2973                 u32 psrctl = 0;
2974
2975                 /* Enable Packet split descriptors */
2976                 rctl |= E1000_RCTL_DTYP_PS;
2977
2978                 psrctl |= adapter->rx_ps_bsize0 >>
2979                         E1000_PSRCTL_BSIZE0_SHIFT;
2980
2981                 switch (adapter->rx_ps_pages) {
2982                 case 3:
2983                         psrctl |= PAGE_SIZE <<
2984                                 E1000_PSRCTL_BSIZE3_SHIFT;
2985                 case 2:
2986                         psrctl |= PAGE_SIZE <<
2987                                 E1000_PSRCTL_BSIZE2_SHIFT;
2988                 case 1:
2989                         psrctl |= PAGE_SIZE >>
2990                                 E1000_PSRCTL_BSIZE1_SHIFT;
2991                         break;
2992                 }
2993
2994                 ew32(PSRCTL, psrctl);
2995         }
2996
2997         /* This is useful for sniffing bad packets. */
2998         if (adapter->netdev->features & NETIF_F_RXALL) {
2999                 /* UPE and MPE will be handled by normal PROMISC logic
3000                  * in e1000e_set_rx_mode
3001                  */
3002                 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
3003                          E1000_RCTL_BAM | /* RX All Bcast Pkts */
3004                          E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
3005
3006                 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
3007                           E1000_RCTL_DPF | /* Allow filtered pause */
3008                           E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
3009                 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
3010                  * and that breaks VLANs.
3011                  */
3012         }
3013
3014         ew32(RCTL, rctl);
3015         /* just started the receive unit, no need to restart */
3016         adapter->flags &= ~FLAG_RX_RESTART_NOW;
3017 }
3018
3019 /**
3020  * e1000_configure_rx - Configure Receive Unit after Reset
3021  * @adapter: board private structure
3022  *
3023  * Configure the Rx unit of the MAC after a reset.
3024  **/
3025 static void e1000_configure_rx(struct e1000_adapter *adapter)
3026 {
3027         struct e1000_hw *hw = &adapter->hw;
3028         struct e1000_ring *rx_ring = adapter->rx_ring;
3029         u64 rdba;
3030         u32 rdlen, rctl, rxcsum, ctrl_ext;
3031
3032         if (adapter->rx_ps_pages) {
3033                 /* this is a 32 byte descriptor */
3034                 rdlen = rx_ring->count *
3035                     sizeof(union e1000_rx_desc_packet_split);
3036                 adapter->clean_rx = e1000_clean_rx_irq_ps;
3037                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
3038         } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
3039                 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
3040                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
3041                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
3042         } else {
3043                 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);
3044                 adapter->clean_rx = e1000_clean_rx_irq;
3045                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
3046         }
3047
3048         /* disable receives while setting up the descriptors */
3049         rctl = er32(RCTL);
3050         if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3051                 ew32(RCTL, rctl & ~E1000_RCTL_EN);
3052         e1e_flush();
3053         usleep_range(10000, 20000);
3054
3055         if (adapter->flags2 & FLAG2_DMA_BURST) {
3056                 /* set the writeback threshold (only takes effect if the RDTR
3057                  * is set). set GRAN=1 and write back up to 0x4 worth, and
3058                  * enable prefetching of 0x20 Rx descriptors
3059                  * granularity = 01
3060                  * wthresh = 04,
3061                  * hthresh = 04,
3062                  * pthresh = 0x20
3063                  */
3064                 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
3065                 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
3066
3067                 /* override the delay timers for enabling bursting, only if
3068                  * the value was not set by the user via module options
3069                  */
3070                 if (adapter->rx_int_delay == DEFAULT_RDTR)
3071                         adapter->rx_int_delay = BURST_RDTR;
3072                 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
3073                         adapter->rx_abs_int_delay = BURST_RADV;
3074         }
3075
3076         /* set the Receive Delay Timer Register */
3077         ew32(RDTR, adapter->rx_int_delay);
3078
3079         /* irq moderation */
3080         ew32(RADV, adapter->rx_abs_int_delay);
3081         if ((adapter->itr_setting != 0) && (adapter->itr != 0))
3082                 e1000e_write_itr(adapter, adapter->itr);
3083
3084         ctrl_ext = er32(CTRL_EXT);
3085         /* Auto-Mask interrupts upon ICR access */
3086         ctrl_ext |= E1000_CTRL_EXT_IAME;
3087         ew32(IAM, 0xffffffff);
3088         ew32(CTRL_EXT, ctrl_ext);
3089         e1e_flush();
3090
3091         /* Setup the HW Rx Head and Tail Descriptor Pointers and
3092          * the Base and Length of the Rx Descriptor Ring
3093          */
3094         rdba = rx_ring->dma;
3095         ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
3096         ew32(RDBAH(0), (rdba >> 32));
3097         ew32(RDLEN(0), rdlen);
3098         ew32(RDH(0), 0);
3099         ew32(RDT(0), 0);
3100         rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);
3101         rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);
3102
3103         /* Enable Receive Checksum Offload for TCP and UDP */
3104         rxcsum = er32(RXCSUM);
3105         if (adapter->netdev->features & NETIF_F_RXCSUM)
3106                 rxcsum |= E1000_RXCSUM_TUOFL;
3107         else
3108                 rxcsum &= ~E1000_RXCSUM_TUOFL;
3109         ew32(RXCSUM, rxcsum);
3110
3111         if (adapter->hw.mac.type == e1000_pch2lan) {
3112                 /* With jumbo frames, excessive C-state transition
3113                  * latencies result in dropped transactions.
3114                  */
3115                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3116                         u32 rxdctl = er32(RXDCTL(0));
3117                         ew32(RXDCTL(0), rxdctl | 0x3);
3118                         pm_qos_update_request(&adapter->netdev->pm_qos_req, 55);
3119                 } else {
3120                         pm_qos_update_request(&adapter->netdev->pm_qos_req,
3121                                               PM_QOS_DEFAULT_VALUE);
3122                 }
3123         }
3124
3125         /* Enable Receives */
3126         ew32(RCTL, rctl);
3127 }
3128
3129 /**
3130  * e1000e_write_mc_addr_list - write multicast addresses to MTA
3131  * @netdev: network interface device structure
3132  *
3133  * Writes multicast address list to the MTA hash table.
3134  * Returns: -ENOMEM on failure
3135  *                0 on no addresses written
3136  *                X on writing X addresses to MTA
3137  */
3138 static int e1000e_write_mc_addr_list(struct net_device *netdev)
3139 {
3140         struct e1000_adapter *adapter = netdev_priv(netdev);
3141         struct e1000_hw *hw = &adapter->hw;
3142         struct netdev_hw_addr *ha;
3143         u8 *mta_list;
3144         int i;
3145
3146         if (netdev_mc_empty(netdev)) {
3147                 /* nothing to program, so clear mc list */
3148                 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);
3149                 return 0;
3150         }
3151
3152         mta_list = kzalloc(netdev_mc_count(netdev) * ETH_ALEN, GFP_ATOMIC);
3153         if (!mta_list)
3154                 return -ENOMEM;
3155
3156         /* update_mc_addr_list expects a packed array of only addresses. */
3157         i = 0;
3158         netdev_for_each_mc_addr(ha, netdev)
3159                 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3160
3161         hw->mac.ops.update_mc_addr_list(hw, mta_list, i);
3162         kfree(mta_list);
3163
3164         return netdev_mc_count(netdev);
3165 }
3166
3167 /**
3168  * e1000e_write_uc_addr_list - write unicast addresses to RAR table
3169  * @netdev: network interface device structure
3170  *
3171  * Writes unicast address list to the RAR table.
3172  * Returns: -ENOMEM on failure/insufficient address space
3173  *                0 on no addresses written
3174  *                X on writing X addresses to the RAR table
3175  **/
3176 static int e1000e_write_uc_addr_list(struct net_device *netdev)
3177 {
3178         struct e1000_adapter *adapter = netdev_priv(netdev);
3179         struct e1000_hw *hw = &adapter->hw;
3180         unsigned int rar_entries = hw->mac.rar_entry_count;
3181         int count = 0;
3182
3183         /* save a rar entry for our hardware address */
3184         rar_entries--;
3185
3186         /* save a rar entry for the LAA workaround */
3187         if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)
3188                 rar_entries--;
3189
3190         /* return ENOMEM indicating insufficient memory for addresses */
3191         if (netdev_uc_count(netdev) > rar_entries)
3192                 return -ENOMEM;
3193
3194         if (!netdev_uc_empty(netdev) && rar_entries) {
3195                 struct netdev_hw_addr *ha;
3196
3197                 /* write the addresses in reverse order to avoid write
3198                  * combining
3199                  */
3200                 netdev_for_each_uc_addr(ha, netdev) {
3201                         if (!rar_entries)
3202                                 break;
3203                         hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);
3204                         count++;
3205                 }
3206         }
3207
3208         /* zero out the remaining RAR entries not used above */
3209         for (; rar_entries > 0; rar_entries--) {
3210                 ew32(RAH(rar_entries), 0);
3211                 ew32(RAL(rar_entries), 0);
3212         }
3213         e1e_flush();
3214
3215         return count;
3216 }
3217
3218 /**
3219  * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set
3220  * @netdev: network interface device structure
3221  *
3222  * The ndo_set_rx_mode entry point is called whenever the unicast or multicast
3223  * address list or the network interface flags are updated.  This routine is
3224  * responsible for configuring the hardware for proper unicast, multicast,
3225  * promiscuous mode, and all-multi behavior.
3226  **/
3227 static void e1000e_set_rx_mode(struct net_device *netdev)
3228 {
3229         struct e1000_adapter *adapter = netdev_priv(netdev);
3230         struct e1000_hw *hw = &adapter->hw;
3231         u32 rctl;
3232
3233         /* Check for Promiscuous and All Multicast modes */
3234         rctl = er32(RCTL);
3235
3236         /* clear the affected bits */
3237         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3238
3239         if (netdev->flags & IFF_PROMISC) {
3240                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3241                 /* Do not hardware filter VLANs in promisc mode */
3242                 e1000e_vlan_filter_disable(adapter);
3243         } else {
3244                 int count;
3245
3246                 if (netdev->flags & IFF_ALLMULTI) {
3247                         rctl |= E1000_RCTL_MPE;
3248                 } else {
3249                         /* Write addresses to the MTA, if the attempt fails
3250                          * then we should just turn on promiscuous mode so
3251                          * that we can at least receive multicast traffic
3252                          */
3253                         count = e1000e_write_mc_addr_list(netdev);
3254                         if (count < 0)
3255                                 rctl |= E1000_RCTL_MPE;
3256                 }
3257                 e1000e_vlan_filter_enable(adapter);
3258                 /* Write addresses to available RAR registers, if there is not
3259                  * sufficient space to store all the addresses then enable
3260                  * unicast promiscuous mode
3261                  */
3262                 count = e1000e_write_uc_addr_list(netdev);
3263                 if (count < 0)
3264                         rctl |= E1000_RCTL_UPE;
3265         }
3266
3267         ew32(RCTL, rctl);
3268
3269         if (netdev->features & NETIF_F_HW_VLAN_RX)
3270                 e1000e_vlan_strip_enable(adapter);
3271         else
3272                 e1000e_vlan_strip_disable(adapter);
3273 }
3274
3275 static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)
3276 {
3277         struct e1000_hw *hw = &adapter->hw;
3278         u32 mrqc, rxcsum;
3279         int i;
3280         static const u32 rsskey[10] = {
3281                 0xda565a6d, 0xc20e5b25, 0x3d256741, 0xb08fa343, 0xcb2bcad0,
3282                 0xb4307bae, 0xa32dcb77, 0x0cf23080, 0x3bb7426a, 0xfa01acbe
3283         };
3284
3285         /* Fill out hash function seed */
3286         for (i = 0; i < 10; i++)
3287                 ew32(RSSRK(i), rsskey[i]);
3288
3289         /* Direct all traffic to queue 0 */
3290         for (i = 0; i < 32; i++)
3291                 ew32(RETA(i), 0);
3292
3293         /* Disable raw packet checksumming so that RSS hash is placed in
3294          * descriptor on writeback.
3295          */
3296         rxcsum = er32(RXCSUM);
3297         rxcsum |= E1000_RXCSUM_PCSD;
3298
3299         ew32(RXCSUM, rxcsum);
3300
3301         mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |
3302                 E1000_MRQC_RSS_FIELD_IPV4_TCP |
3303                 E1000_MRQC_RSS_FIELD_IPV6 |
3304                 E1000_MRQC_RSS_FIELD_IPV6_TCP |
3305                 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);
3306
3307         ew32(MRQC, mrqc);
3308 }
3309
3310 /**
3311  * e1000_configure - configure the hardware for Rx and Tx
3312  * @adapter: private board structure
3313  **/
3314 static void e1000_configure(struct e1000_adapter *adapter)
3315 {
3316         struct e1000_ring *rx_ring = adapter->rx_ring;
3317
3318         e1000e_set_rx_mode(adapter->netdev);
3319
3320         e1000_restore_vlan(adapter);
3321         e1000_init_manageability_pt(adapter);
3322
3323         e1000_configure_tx(adapter);
3324
3325         if (adapter->netdev->features & NETIF_F_RXHASH)
3326                 e1000e_setup_rss_hash(adapter);
3327         e1000_setup_rctl(adapter);
3328         e1000_configure_rx(adapter);
3329         adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);
3330 }
3331
3332 /**
3333  * e1000e_power_up_phy - restore link in case the phy was powered down
3334  * @adapter: address of board private structure
3335  *
3336  * The phy may be powered down to save power and turn off link when the
3337  * driver is unloaded and wake on lan is not enabled (among others)
3338  * *** this routine MUST be followed by a call to e1000e_reset ***
3339  **/
3340 void e1000e_power_up_phy(struct e1000_adapter *adapter)
3341 {
3342         if (adapter->hw.phy.ops.power_up)
3343                 adapter->hw.phy.ops.power_up(&adapter->hw);
3344
3345         adapter->hw.mac.ops.setup_link(&adapter->hw);
3346 }
3347
3348 /**
3349  * e1000_power_down_phy - Power down the PHY
3350  *
3351  * Power down the PHY so no link is implied when interface is down.
3352  * The PHY cannot be powered down if management or WoL is active.
3353  */
3354 static void e1000_power_down_phy(struct e1000_adapter *adapter)
3355 {
3356         /* WoL is enabled */
3357         if (adapter->wol)
3358                 return;
3359
3360         if (adapter->hw.phy.ops.power_down)
3361                 adapter->hw.phy.ops.power_down(&adapter->hw);
3362 }
3363
3364 /**
3365  * e1000e_reset - bring the hardware into a known good state
3366  *
3367  * This function boots the hardware and enables some settings that
3368  * require a configuration cycle of the hardware - those cannot be
3369  * set/changed during runtime. After reset the device needs to be
3370  * properly configured for Rx, Tx etc.
3371  */
3372 void e1000e_reset(struct e1000_adapter *adapter)
3373 {
3374         struct e1000_mac_info *mac = &adapter->hw.mac;
3375         struct e1000_fc_info *fc = &adapter->hw.fc;
3376         struct e1000_hw *hw = &adapter->hw;
3377         u32 tx_space, min_tx_space, min_rx_space;
3378         u32 pba = adapter->pba;
3379         u16 hwm;
3380
3381         /* reset Packet Buffer Allocation to default */
3382         ew32(PBA, pba);
3383
3384         if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
3385                 /* To maintain wire speed transmits, the Tx FIFO should be
3386                  * large enough to accommodate two full transmit packets,
3387                  * rounded up to the next 1KB and expressed in KB.  Likewise,
3388                  * the Rx FIFO should be large enough to accommodate at least
3389                  * one full receive packet and is similarly rounded up and
3390                  * expressed in KB.
3391                  */
3392                 pba = er32(PBA);
3393                 /* upper 16 bits has Tx packet buffer allocation size in KB */
3394                 tx_space = pba >> 16;
3395                 /* lower 16 bits has Rx packet buffer allocation size in KB */
3396                 pba &= 0xffff;
3397                 /* the Tx fifo also stores 16 bytes of information about the Tx
3398                  * but don't include ethernet FCS because hardware appends it
3399                  */
3400                 min_tx_space = (adapter->max_frame_size +
3401                                 sizeof(struct e1000_tx_desc) -
3402                                 ETH_FCS_LEN) * 2;
3403                 min_tx_space = ALIGN(min_tx_space, 1024);
3404                 min_tx_space >>= 10;
3405                 /* software strips receive CRC, so leave room for it */
3406                 min_rx_space = adapter->max_frame_size;
3407                 min_rx_space = ALIGN(min_rx_space, 1024);
3408                 min_rx_space >>= 10;
3409
3410                 /* If current Tx allocation is less than the min Tx FIFO size,
3411                  * and the min Tx FIFO size is less than the current Rx FIFO
3412                  * allocation, take space away from current Rx allocation
3413                  */
3414                 if ((tx_space < min_tx_space) &&
3415                     ((min_tx_space - tx_space) < pba)) {
3416                         pba -= min_tx_space - tx_space;
3417
3418                         /* if short on Rx space, Rx wins and must trump Tx
3419                          * adjustment
3420                          */
3421                         if (pba < min_rx_space)
3422                                 pba = min_rx_space;
3423                 }
3424
3425                 ew32(PBA, pba);
3426         }
3427
3428         /* flow control settings
3429          *
3430          * The high water mark must be low enough to fit one full frame
3431          * (or the size used for early receive) above it in the Rx FIFO.
3432          * Set it to the lower of:
3433          * - 90% of the Rx FIFO size, and
3434          * - the full Rx FIFO size minus one full frame
3435          */
3436         if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3437                 fc->pause_time = 0xFFFF;
3438         else
3439                 fc->pause_time = E1000_FC_PAUSE_TIME;
3440         fc->send_xon = true;
3441         fc->current_mode = fc->requested_mode;
3442
3443         switch (hw->mac.type) {
3444         case e1000_ich9lan:
3445         case e1000_ich10lan:
3446                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3447                         pba = 14;
3448                         ew32(PBA, pba);
3449                         fc->high_water = 0x2800;
3450                         fc->low_water = fc->high_water - 8;
3451                         break;
3452                 }
3453                 /* fall-through */
3454         default:
3455                 hwm = min(((pba << 10) * 9 / 10),
3456                           ((pba << 10) - adapter->max_frame_size));
3457
3458                 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3459                 fc->low_water = fc->high_water - 8;
3460                 break;
3461         case e1000_pchlan:
3462                 /* Workaround PCH LOM adapter hangs with certain network
3463                  * loads.  If hangs persist, try disabling Tx flow control.
3464                  */
3465                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3466                         fc->high_water = 0x3500;
3467                         fc->low_water  = 0x1500;
3468                 } else {
3469                         fc->high_water = 0x5000;
3470                         fc->low_water  = 0x3000;
3471                 }
3472                 fc->refresh_time = 0x1000;
3473                 break;
3474         case e1000_pch2lan:
3475         case e1000_pch_lpt:
3476                 fc->high_water = 0x05C20;
3477                 fc->low_water = 0x05048;
3478                 fc->pause_time = 0x0650;
3479                 fc->refresh_time = 0x0400;
3480                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3481                         pba = 14;
3482                         ew32(PBA, pba);
3483                 }
3484                 break;
3485         }
3486
3487         /* Alignment of Tx data is on an arbitrary byte boundary with the
3488          * maximum size per Tx descriptor limited only to the transmit
3489          * allocation of the packet buffer minus 96 bytes with an upper
3490          * limit of 24KB due to receive synchronization limitations.
3491          */
3492         adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,
3493                                        24 << 10);
3494
3495         /* Disable Adaptive Interrupt Moderation if 2 full packets cannot
3496          * fit in receive buffer.
3497          */
3498         if (adapter->itr_setting & 0x3) {
3499                 if ((adapter->max_frame_size * 2) > (pba << 10)) {
3500                         if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3501                                 dev_info(&adapter->pdev->dev,
3502                                         "Interrupt Throttle Rate turned off\n");
3503                                 adapter->flags2 |= FLAG2_DISABLE_AIM;
3504                                 e1000e_write_itr(adapter, 0);
3505                         }
3506                 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3507                         dev_info(&adapter->pdev->dev,
3508                                  "Interrupt Throttle Rate turned on\n");
3509                         adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3510                         adapter->itr = 20000;
3511                         e1000e_write_itr(adapter, adapter->itr);
3512                 }
3513         }
3514
3515         /* Allow time for pending master requests to run */
3516         mac->ops.reset_hw(hw);
3517
3518         /* For parts with AMT enabled, let the firmware know
3519          * that the network interface is in control
3520          */
3521         if (adapter->flags & FLAG_HAS_AMT)
3522                 e1000e_get_hw_control(adapter);
3523
3524         ew32(WUC, 0);
3525
3526         if (mac->ops.init_hw(hw))
3527                 e_err("Hardware Error\n");
3528
3529         e1000_update_mng_vlan(adapter);
3530
3531         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3532         ew32(VET, ETH_P_8021Q);
3533
3534         e1000e_reset_adaptive(hw);
3535
3536         if (!netif_running(adapter->netdev) &&
3537             !test_bit(__E1000_TESTING, &adapter->state)) {
3538                 e1000_power_down_phy(adapter);
3539                 return;
3540         }
3541
3542         e1000_get_phy_info(hw);
3543
3544         if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3545             !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
3546                 u16 phy_data = 0;
3547                 /* speed up time to link by disabling smart power down, ignore
3548                  * the return value of this function because there is nothing
3549                  * different we would do if it failed
3550                  */
3551                 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3552                 phy_data &= ~IGP02E1000_PM_SPD;
3553                 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3554         }
3555 }
3556
3557 int e1000e_up(struct e1000_adapter *adapter)
3558 {
3559         struct e1000_hw *hw = &adapter->hw;
3560
3561         /* hardware has been reset, we need to reload some things */
3562         e1000_configure(adapter);
3563
3564         clear_bit(__E1000_DOWN, &adapter->state);
3565
3566         if (adapter->msix_entries)
3567                 e1000_configure_msix(adapter);
3568         e1000_irq_enable(adapter);
3569
3570         netif_start_queue(adapter->netdev);
3571
3572         /* fire a link change interrupt to start the watchdog */
3573         if (adapter->msix_entries)
3574                 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3575         else
3576                 ew32(ICS, E1000_ICS_LSC);
3577
3578         return 0;
3579 }
3580
3581 static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3582 {
3583         struct e1000_hw *hw = &adapter->hw;
3584
3585         if (!(adapter->flags2 & FLAG2_DMA_BURST))
3586                 return;
3587
3588         /* flush pending descriptor writebacks to memory */
3589         ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3590         ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3591
3592         /* execute the writes immediately */
3593         e1e_flush();
3594
3595         /* due to rare timing issues, write to TIDV/RDTR again to ensure the
3596          * write is successful
3597          */
3598         ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3599         ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3600
3601         /* execute the writes immediately */
3602         e1e_flush();
3603 }
3604
3605 static void e1000e_update_stats(struct e1000_adapter *adapter);
3606
3607 void e1000e_down(struct e1000_adapter *adapter)
3608 {
3609         struct net_device *netdev = adapter->netdev;
3610         struct e1000_hw *hw = &adapter->hw;
3611         u32 tctl, rctl;
3612
3613         /* signal that we're down so the interrupt handler does not
3614          * reschedule our watchdog timer
3615          */
3616         set_bit(__E1000_DOWN, &adapter->state);
3617
3618         /* disable receives in the hardware */
3619         rctl = er32(RCTL);
3620         if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))
3621                 ew32(RCTL, rctl & ~E1000_RCTL_EN);
3622         /* flush and sleep below */
3623
3624         netif_stop_queue(netdev);
3625
3626         /* disable transmits in the hardware */
3627         tctl = er32(TCTL);
3628         tctl &= ~E1000_TCTL_EN;
3629         ew32(TCTL, tctl);
3630
3631         /* flush both disables and wait for them to finish */
3632         e1e_flush();
3633         usleep_range(10000, 20000);
3634
3635         e1000_irq_disable(adapter);
3636
3637         del_timer_sync(&adapter->watchdog_timer);
3638         del_timer_sync(&adapter->phy_info_timer);
3639
3640         netif_carrier_off(netdev);
3641
3642         spin_lock(&adapter->stats64_lock);
3643         e1000e_update_stats(adapter);
3644         spin_unlock(&adapter->stats64_lock);
3645
3646         e1000e_flush_descriptors(adapter);
3647         e1000_clean_tx_ring(adapter->tx_ring);
3648         e1000_clean_rx_ring(adapter->rx_ring);
3649
3650         adapter->link_speed = 0;
3651         adapter->link_duplex = 0;
3652
3653         if (!pci_channel_offline(adapter->pdev))
3654                 e1000e_reset(adapter);
3655
3656         /* TODO: for power management, we could drop the link and
3657          * pci_disable_device here.
3658          */
3659 }
3660
3661 void e1000e_reinit_locked(struct e1000_adapter *adapter)
3662 {
3663         might_sleep();
3664         while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
3665                 usleep_range(1000, 2000);
3666         e1000e_down(adapter);
3667         e1000e_up(adapter);
3668         clear_bit(__E1000_RESETTING, &adapter->state);
3669 }
3670
3671 /**
3672  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
3673  * @adapter: board private structure to initialize
3674  *
3675  * e1000_sw_init initializes the Adapter private data structure.
3676  * Fields are initialized based on PCI device information and
3677  * OS network device settings (MTU size).
3678  **/
3679 static int e1000_sw_init(struct e1000_adapter *adapter)
3680 {
3681         struct net_device *netdev = adapter->netdev;
3682
3683         adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
3684         adapter->rx_ps_bsize0 = 128;
3685         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3686         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3687         adapter->tx_ring_count = E1000_DEFAULT_TXD;
3688         adapter->rx_ring_count = E1000_DEFAULT_RXD;
3689
3690         spin_lock_init(&adapter->stats64_lock);
3691
3692         e1000e_set_interrupt_capability(adapter);
3693
3694         if (e1000_alloc_queues(adapter))
3695                 return -ENOMEM;
3696
3697         /* Explicitly disable IRQ since the NIC can be in any state. */
3698         e1000_irq_disable(adapter);
3699
3700         set_bit(__E1000_DOWN, &adapter->state);
3701         return 0;
3702 }
3703
3704 /**
3705  * e1000_intr_msi_test - Interrupt Handler
3706  * @irq: interrupt number
3707  * @data: pointer to a network interface device structure
3708  **/
3709 static irqreturn_t e1000_intr_msi_test(int irq, void *data)
3710 {
3711         struct net_device *netdev = data;
3712         struct e1000_adapter *adapter = netdev_priv(netdev);
3713         struct e1000_hw *hw = &adapter->hw;
3714         u32 icr = er32(ICR);
3715
3716         e_dbg("icr is %08X\n", icr);
3717         if (icr & E1000_ICR_RXSEQ) {
3718                 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
3719                 /* Force memory writes to complete before acknowledging the
3720                  * interrupt is handled.
3721                  */
3722                 wmb();
3723         }
3724
3725         return IRQ_HANDLED;
3726 }
3727
3728 /**
3729  * e1000_test_msi_interrupt - Returns 0 for successful test
3730  * @adapter: board private struct
3731  *
3732  * code flow taken from tg3.c
3733  **/
3734 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
3735 {
3736         struct net_device *netdev = adapter->netdev;
3737         struct e1000_hw *hw = &adapter->hw;
3738         int err;
3739
3740         /* poll_enable hasn't been called yet, so don't need disable */
3741         /* clear any pending events */
3742         er32(ICR);
3743
3744         /* free the real vector and request a test handler */
3745         e1000_free_irq(adapter);
3746         e1000e_reset_interrupt_capability(adapter);
3747
3748         /* Assume that the test fails, if it succeeds then the test
3749          * MSI irq handler will unset this flag
3750          */
3751         adapter->flags |= FLAG_MSI_TEST_FAILED;
3752
3753         err = pci_enable_msi(adapter->pdev);
3754         if (err)
3755                 goto msi_test_failed;
3756
3757         err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
3758                           netdev->name, netdev);
3759         if (err) {
3760                 pci_disable_msi(adapter->pdev);
3761                 goto msi_test_failed;
3762         }
3763
3764         /* Force memory writes to complete before enabling and firing an
3765          * interrupt.
3766          */
3767         wmb();
3768
3769         e1000_irq_enable(adapter);
3770
3771         /* fire an unusual interrupt on the test handler */
3772         ew32(ICS, E1000_ICS_RXSEQ);
3773         e1e_flush();
3774         msleep(100);
3775
3776         e1000_irq_disable(adapter);
3777
3778         rmb();                  /* read flags after interrupt has been fired */
3779
3780         if (adapter->flags & FLAG_MSI_TEST_FAILED) {
3781                 adapter->int_mode = E1000E_INT_MODE_LEGACY;
3782                 e_info("MSI interrupt test failed, using legacy interrupt.\n");
3783         } else {
3784                 e_dbg("MSI interrupt test succeeded!\n");
3785         }
3786
3787         free_irq(adapter->pdev->irq, netdev);
3788         pci_disable_msi(adapter->pdev);
3789
3790 msi_test_failed:
3791         e1000e_set_interrupt_capability(adapter);
3792         return e1000_request_irq(adapter);
3793 }
3794
3795 /**
3796  * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
3797  * @adapter: board private struct
3798  *
3799  * code flow taken from tg3.c, called with e1000 interrupts disabled.
3800  **/
3801 static int e1000_test_msi(struct e1000_adapter *adapter)
3802 {
3803         int err;
3804         u16 pci_cmd;
3805
3806         if (!(adapter->flags & FLAG_MSI_ENABLED))
3807                 return 0;
3808
3809         /* disable SERR in case the MSI write causes a master abort */
3810         pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
3811         if (pci_cmd & PCI_COMMAND_SERR)
3812                 pci_write_config_word(adapter->pdev, PCI_COMMAND,
3813                                       pci_cmd & ~PCI_COMMAND_SERR);
3814
3815         err = e1000_test_msi_interrupt(adapter);
3816
3817         /* re-enable SERR */
3818         if (pci_cmd & PCI_COMMAND_SERR) {
3819                 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
3820                 pci_cmd |= PCI_COMMAND_SERR;
3821                 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
3822         }
3823
3824         return err;
3825 }
3826
3827 /**
3828  * e1000_open - Called when a network interface is made active
3829  * @netdev: network interface device structure
3830  *
3831  * Returns 0 on success, negative value on failure
3832  *
3833  * The open entry point is called when a network interface is made
3834  * active by the system (IFF_UP).  At this point all resources needed
3835  * for transmit and receive operations are allocated, the interrupt
3836  * handler is registered with the OS, the watchdog timer is started,
3837  * and the stack is notified that the interface is ready.
3838  **/
3839 static int e1000_open(struct net_device *netdev)
3840 {
3841         struct e1000_adapter *adapter = netdev_priv(netdev);
3842         struct e1000_hw *hw = &adapter->hw;
3843         struct pci_dev *pdev = adapter->pdev;
3844         int err;
3845
3846         /* disallow open during test */
3847         if (test_bit(__E1000_TESTING, &adapter->state))
3848                 return -EBUSY;
3849
3850         pm_runtime_get_sync(&pdev->dev);
3851
3852         netif_carrier_off(netdev);
3853
3854         /* allocate transmit descriptors */
3855         err = e1000e_setup_tx_resources(adapter->tx_ring);
3856         if (err)
3857                 goto err_setup_tx;
3858
3859         /* allocate receive descriptors */
3860         err = e1000e_setup_rx_resources(adapter->rx_ring);
3861         if (err)
3862                 goto err_setup_rx;
3863
3864         /* If AMT is enabled, let the firmware know that the network
3865          * interface is now open and reset the part to a known state.
3866          */
3867         if (adapter->flags & FLAG_HAS_AMT) {
3868                 e1000e_get_hw_control(adapter);
3869                 e1000e_reset(adapter);
3870         }
3871
3872         e1000e_power_up_phy(adapter);
3873
3874         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
3875         if ((adapter->hw.mng_cookie.status &
3876              E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
3877                 e1000_update_mng_vlan(adapter);
3878
3879         /* DMA latency requirement to workaround jumbo issue */
3880         if (adapter->hw.mac.type == e1000_pch2lan)
3881                 pm_qos_add_request(&adapter->netdev->pm_qos_req,
3882                                    PM_QOS_CPU_DMA_LATENCY,
3883                                    PM_QOS_DEFAULT_VALUE);
3884
3885         /* before we allocate an interrupt, we must be ready to handle it.
3886          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3887          * as soon as we call pci_request_irq, so we have to setup our
3888          * clean_rx handler before we do so.
3889          */
3890         e1000_configure(adapter);
3891
3892         err = e1000_request_irq(adapter);
3893         if (err)
3894                 goto err_req_irq;
3895
3896         /* Work around PCIe errata with MSI interrupts causing some chipsets to
3897          * ignore e1000e MSI messages, which means we need to test our MSI
3898          * interrupt now
3899          */
3900         if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
3901                 err = e1000_test_msi(adapter);
3902                 if (err) {
3903                         e_err("Interrupt allocation failed\n");
3904                         goto err_req_irq;
3905                 }
3906         }
3907
3908         /* From here on the code is the same as e1000e_up() */
3909         clear_bit(__E1000_DOWN, &adapter->state);
3910
3911         napi_enable(&adapter->napi);
3912
3913         e1000_irq_enable(adapter);
3914
3915         adapter->tx_hang_recheck = false;
3916         netif_start_queue(netdev);
3917
3918         adapter->idle_check = true;
3919         pm_runtime_put(&pdev->dev);
3920
3921         /* fire a link status change interrupt to start the watchdog */
3922         if (adapter->msix_entries)
3923                 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3924         else
3925                 ew32(ICS, E1000_ICS_LSC);
3926
3927         return 0;
3928
3929 err_req_irq:
3930         e1000e_release_hw_control(adapter);
3931         e1000_power_down_phy(adapter);
3932         e1000e_free_rx_resources(adapter->rx_ring);
3933 err_setup_rx:
3934         e1000e_free_tx_resources(adapter->tx_ring);
3935 err_setup_tx:
3936         e1000e_reset(adapter);
3937         pm_runtime_put_sync(&pdev->dev);
3938
3939         return err;
3940 }
3941
3942 /**
3943  * e1000_close - Disables a network interface
3944  * @netdev: network interface device structure
3945  *
3946  * Returns 0, this is not allowed to fail
3947  *
3948  * The close entry point is called when an interface is de-activated
3949  * by the OS.  The hardware is still under the drivers control, but
3950  * needs to be disabled.  A global MAC reset is issued to stop the
3951  * hardware, and all transmit and receive resources are freed.
3952  **/
3953 static int e1000_close(struct net_device *netdev)
3954 {
3955         struct e1000_adapter *adapter = netdev_priv(netdev);
3956         struct pci_dev *pdev = adapter->pdev;
3957         int count = E1000_CHECK_RESET_COUNT;
3958
3959         while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
3960                 usleep_range(10000, 20000);
3961
3962         WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
3963
3964         pm_runtime_get_sync(&pdev->dev);
3965
3966         napi_disable(&adapter->napi);
3967
3968         if (!test_bit(__E1000_DOWN, &adapter->state)) {
3969                 e1000e_down(adapter);
3970                 e1000_free_irq(adapter);
3971         }
3972         e1000_power_down_phy(adapter);
3973
3974         e1000e_free_tx_resources(adapter->tx_ring);
3975         e1000e_free_rx_resources(adapter->rx_ring);
3976
3977         /* kill manageability vlan ID if supported, but not if a vlan with
3978          * the same ID is registered on the host OS (let 8021q kill it)
3979          */
3980         if (adapter->hw.mng_cookie.status &
3981             E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
3982                 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
3983
3984         /* If AMT is enabled, let the firmware know that the network
3985          * interface is now closed
3986          */
3987         if ((adapter->flags & FLAG_HAS_AMT) &&
3988             !test_bit(__E1000_TESTING, &adapter->state))
3989                 e1000e_release_hw_control(adapter);
3990
3991         if (adapter->hw.mac.type == e1000_pch2lan)
3992                 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
3993
3994         pm_runtime_put_sync(&pdev->dev);
3995
3996         return 0;
3997 }
3998 /**
3999  * e1000_set_mac - Change the Ethernet Address of the NIC
4000  * @netdev: network interface device structure
4001  * @p: pointer to an address structure
4002  *
4003  * Returns 0 on success, negative on failure
4004  **/
4005 static int e1000_set_mac(struct net_device *netdev, void *p)
4006 {
4007         struct e1000_adapter *adapter = netdev_priv(netdev);
4008         struct e1000_hw *hw = &adapter->hw;
4009         struct sockaddr *addr = p;
4010
4011         if (!is_valid_ether_addr(addr->sa_data))
4012                 return -EADDRNOTAVAIL;
4013
4014         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4015         memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
4016
4017         hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
4018
4019         if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
4020                 /* activate the work around */
4021                 e1000e_set_laa_state_82571(&adapter->hw, 1);
4022
4023                 /* Hold a copy of the LAA in RAR[14] This is done so that
4024                  * between the time RAR[0] gets clobbered  and the time it
4025                  * gets fixed (in e1000_watchdog), the actual LAA is in one
4026                  * of the RARs and no incoming packets directed to this port
4027                  * are dropped. Eventually the LAA will be in RAR[0] and
4028                  * RAR[14]
4029                  */
4030                 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,
4031                                     adapter->hw.mac.rar_entry_count - 1);
4032         }
4033
4034         return 0;
4035 }
4036
4037 /**
4038  * e1000e_update_phy_task - work thread to update phy
4039  * @work: pointer to our work struct
4040  *
4041  * this worker thread exists because we must acquire a
4042  * semaphore to read the phy, which we could msleep while
4043  * waiting for it, and we can't msleep in a timer.
4044  **/
4045 static void e1000e_update_phy_task(struct work_struct *work)
4046 {
4047         struct e1000_adapter *adapter = container_of(work,
4048                                         struct e1000_adapter, update_phy_task);
4049
4050         if (test_bit(__E1000_DOWN, &adapter->state))
4051                 return;
4052
4053         e1000_get_phy_info(&adapter->hw);
4054 }
4055
4056 /**
4057  * e1000_update_phy_info - timre call-back to update PHY info
4058  * @data: pointer to adapter cast into an unsigned long
4059  *
4060  * Need to wait a few seconds after link up to get diagnostic information from
4061  * the phy
4062  **/
4063 static void e1000_update_phy_info(unsigned long data)
4064 {
4065         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4066
4067         if (test_bit(__E1000_DOWN, &adapter->state))
4068                 return;
4069
4070         schedule_work(&adapter->update_phy_task);
4071 }
4072
4073 /**
4074  * e1000e_update_phy_stats - Update the PHY statistics counters
4075  * @adapter: board private structure
4076  *
4077  * Read/clear the upper 16-bit PHY registers and read/accumulate lower
4078  **/
4079 static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
4080 {
4081         struct e1000_hw *hw = &adapter->hw;
4082         s32 ret_val;
4083         u16 phy_data;
4084
4085         ret_val = hw->phy.ops.acquire(hw);
4086         if (ret_val)
4087                 return;
4088
4089         /* A page set is expensive so check if already on desired page.
4090          * If not, set to the page with the PHY status registers.
4091          */
4092         hw->phy.addr = 1;
4093         ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
4094                                            &phy_data);
4095         if (ret_val)
4096                 goto release;
4097         if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {
4098                 ret_val = hw->phy.ops.set_page(hw,
4099                                                HV_STATS_PAGE << IGP_PAGE_SHIFT);
4100                 if (ret_val)
4101                         goto release;
4102         }
4103
4104         /* Single Collision Count */
4105         hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);
4106         ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);
4107         if (!ret_val)
4108                 adapter->stats.scc += phy_data;
4109
4110         /* Excessive Collision Count */
4111         hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);
4112         ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);
4113         if (!ret_val)
4114                 adapter->stats.ecol += phy_data;
4115
4116         /* Multiple Collision Count */
4117         hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);
4118         ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);
4119         if (!ret_val)
4120                 adapter->stats.mcc += phy_data;
4121
4122         /* Late Collision Count */
4123         hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);
4124         ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);
4125         if (!ret_val)
4126                 adapter->stats.latecol += phy_data;
4127
4128         /* Collision Count - also used for adaptive IFS */
4129         hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);
4130         ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);
4131         if (!ret_val)
4132                 hw->mac.collision_delta = phy_data;
4133
4134         /* Defer Count */
4135         hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);
4136         ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);
4137         if (!ret_val)
4138                 adapter->stats.dc += phy_data;
4139
4140         /* Transmit with no CRS */
4141         hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);
4142         ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);
4143         if (!ret_val)
4144                 adapter->stats.tncrs += phy_data;
4145
4146 release:
4147         hw->phy.ops.release(hw);
4148 }
4149
4150 /**
4151  * e1000e_update_stats - Update the board statistics counters
4152  * @adapter: board private structure
4153  **/
4154 static void e1000e_update_stats(struct e1000_adapter *adapter)
4155 {
4156         struct net_device *netdev = adapter->netdev;
4157         struct e1000_hw *hw = &adapter->hw;
4158         struct pci_dev *pdev = adapter->pdev;
4159
4160         /* Prevent stats update while adapter is being reset, or if the pci
4161          * connection is down.
4162          */
4163         if (adapter->link_speed == 0)
4164                 return;
4165         if (pci_channel_offline(pdev))
4166                 return;
4167
4168         adapter->stats.crcerrs += er32(CRCERRS);
4169         adapter->stats.gprc += er32(GPRC);
4170         adapter->stats.gorc += er32(GORCL);
4171         er32(GORCH); /* Clear gorc */
4172         adapter->stats.bprc += er32(BPRC);
4173         adapter->stats.mprc += er32(MPRC);
4174         adapter->stats.roc += er32(ROC);
4175
4176         adapter->stats.mpc += er32(MPC);
4177
4178         /* Half-duplex statistics */
4179         if (adapter->link_duplex == HALF_DUPLEX) {
4180                 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
4181                         e1000e_update_phy_stats(adapter);
4182                 } else {
4183                         adapter->stats.scc += er32(SCC);
4184                         adapter->stats.ecol += er32(ECOL);
4185                         adapter->stats.mcc += er32(MCC);
4186                         adapter->stats.latecol += er32(LATECOL);
4187                         adapter->stats.dc += er32(DC);
4188
4189                         hw->mac.collision_delta = er32(COLC);
4190
4191                         if ((hw->mac.type != e1000_82574) &&
4192                             (hw->mac.type != e1000_82583))
4193                                 adapter->stats.tncrs += er32(TNCRS);
4194                 }
4195                 adapter->stats.colc += hw->mac.collision_delta;
4196         }
4197
4198         adapter->stats.xonrxc += er32(XONRXC);
4199         adapter->stats.xontxc += er32(XONTXC);
4200         adapter->stats.xoffrxc += er32(XOFFRXC);
4201         adapter->stats.xofftxc += er32(XOFFTXC);
4202         adapter->stats.gptc += er32(GPTC);
4203         adapter->stats.gotc += er32(GOTCL);
4204         er32(GOTCH); /* Clear gotc */
4205         adapter->stats.rnbc += er32(RNBC);
4206         adapter->stats.ruc += er32(RUC);
4207
4208         adapter->stats.mptc += er32(MPTC);
4209         adapter->stats.bptc += er32(BPTC);
4210
4211         /* used for adaptive IFS */
4212
4213         hw->mac.tx_packet_delta = er32(TPT);
4214         adapter->stats.tpt += hw->mac.tx_packet_delta;
4215
4216         adapter->stats.algnerrc += er32(ALGNERRC);
4217         adapter->stats.rxerrc += er32(RXERRC);
4218         adapter->stats.cexterr += er32(CEXTERR);
4219         adapter->stats.tsctc += er32(TSCTC);
4220         adapter->stats.tsctfc += er32(TSCTFC);
4221
4222         /* Fill out the OS statistics structure */
4223         netdev->stats.multicast = adapter->stats.mprc;
4224         netdev->stats.collisions = adapter->stats.colc;
4225
4226         /* Rx Errors */
4227
4228         /* RLEC on some newer hardware can be incorrect so build
4229          * our own version based on RUC and ROC
4230          */
4231         netdev->stats.rx_errors = adapter->stats.rxerrc +
4232                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4233                 adapter->stats.ruc + adapter->stats.roc +
4234                 adapter->stats.cexterr;
4235         netdev->stats.rx_length_errors = adapter->stats.ruc +
4236                                               adapter->stats.roc;
4237         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4238         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4239         netdev->stats.rx_missed_errors = adapter->stats.mpc;
4240
4241         /* Tx Errors */
4242         netdev->stats.tx_errors = adapter->stats.ecol +
4243                                        adapter->stats.latecol;
4244         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4245         netdev->stats.tx_window_errors = adapter->stats.latecol;
4246         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
4247
4248         /* Tx Dropped needs to be maintained elsewhere */
4249
4250         /* Management Stats */
4251         adapter->stats.mgptc += er32(MGTPTC);
4252         adapter->stats.mgprc += er32(MGTPRC);
4253         adapter->stats.mgpdc += er32(MGTPDC);
4254 }
4255
4256 /**
4257  * e1000_phy_read_status - Update the PHY register status snapshot
4258  * @adapter: board private structure
4259  **/
4260 static void e1000_phy_read_status(struct e1000_adapter *adapter)
4261 {
4262         struct e1000_hw *hw = &adapter->hw;
4263         struct e1000_phy_regs *phy = &adapter->phy_regs;
4264
4265         if ((er32(STATUS) & E1000_STATUS_LU) &&
4266             (adapter->hw.phy.media_type == e1000_media_type_copper)) {
4267                 int ret_val;
4268
4269                 ret_val  = e1e_rphy(hw, PHY_CONTROL, &phy->bmcr);
4270                 ret_val |= e1e_rphy(hw, PHY_STATUS, &phy->bmsr);
4271                 ret_val |= e1e_rphy(hw, PHY_AUTONEG_ADV, &phy->advertise);
4272                 ret_val |= e1e_rphy(hw, PHY_LP_ABILITY, &phy->lpa);
4273                 ret_val |= e1e_rphy(hw, PHY_AUTONEG_EXP, &phy->expansion);
4274                 ret_val |= e1e_rphy(hw, PHY_1000T_CTRL, &phy->ctrl1000);
4275                 ret_val |= e1e_rphy(hw, PHY_1000T_STATUS, &phy->stat1000);
4276                 ret_val |= e1e_rphy(hw, PHY_EXT_STATUS, &phy->estatus);
4277                 if (ret_val)
4278                         e_warn("Error reading PHY register\n");
4279         } else {
4280                 /* Do not read PHY registers if link is not up
4281                  * Set values to typical power-on defaults
4282                  */
4283                 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4284                 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4285                              BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4286                              BMSR_ERCAP);
4287                 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4288                                   ADVERTISE_ALL | ADVERTISE_CSMA);
4289                 phy->lpa = 0;
4290                 phy->expansion = EXPANSION_ENABLENPAGE;
4291                 phy->ctrl1000 = ADVERTISE_1000FULL;
4292                 phy->stat1000 = 0;
4293                 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4294         }
4295 }
4296
4297 static void e1000_print_link_info(struct e1000_adapter *adapter)
4298 {
4299         struct e1000_hw *hw = &adapter->hw;
4300         u32 ctrl = er32(CTRL);
4301
4302         /* Link status message must follow this format for user tools */
4303         printk(KERN_INFO "e1000e: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
4304                 adapter->netdev->name,
4305                 adapter->link_speed,
4306                 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",
4307                 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :
4308                 (ctrl & E1000_CTRL_RFCE) ? "Rx" :
4309                 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");
4310 }
4311
4312 static bool e1000e_has_link(struct e1000_adapter *adapter)
4313 {
4314         struct e1000_hw *hw = &adapter->hw;
4315         bool link_active = false;
4316         s32 ret_val = 0;
4317
4318         /* get_link_status is set on LSC (link status) interrupt or
4319          * Rx sequence error interrupt.  get_link_status will stay
4320          * false until the check_for_link establishes link
4321          * for copper adapters ONLY
4322          */
4323         switch (hw->phy.media_type) {
4324         case e1000_media_type_copper:
4325                 if (hw->mac.get_link_status) {
4326                         ret_val = hw->mac.ops.check_for_link(hw);
4327                         link_active = !hw->mac.get_link_status;
4328                 } else {
4329                         link_active = true;
4330                 }
4331                 break;
4332         case e1000_media_type_fiber:
4333                 ret_val = hw->mac.ops.check_for_link(hw);
4334                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4335                 break;
4336         case e1000_media_type_internal_serdes:
4337                 ret_val = hw->mac.ops.check_for_link(hw);
4338                 link_active = adapter->hw.mac.serdes_has_link;
4339                 break;
4340         default:
4341         case e1000_media_type_unknown:
4342                 break;
4343         }
4344
4345         if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4346             (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4347                 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
4348                 e_info("Gigabit has been disabled, downgrading speed\n");
4349         }
4350
4351         return link_active;
4352 }
4353
4354 static void e1000e_enable_receives(struct e1000_adapter *adapter)
4355 {
4356         /* make sure the receive unit is started */
4357         if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
4358             (adapter->flags & FLAG_RX_RESTART_NOW)) {
4359                 struct e1000_hw *hw = &adapter->hw;
4360                 u32 rctl = er32(RCTL);
4361                 ew32(RCTL, rctl | E1000_RCTL_EN);
4362                 adapter->flags &= ~FLAG_RX_RESTART_NOW;
4363         }
4364 }
4365
4366 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4367 {
4368         struct e1000_hw *hw = &adapter->hw;
4369
4370         /* With 82574 controllers, PHY needs to be checked periodically
4371          * for hung state and reset, if two calls return true
4372          */
4373         if (e1000_check_phy_82574(hw))
4374                 adapter->phy_hang_count++;
4375         else
4376                 adapter->phy_hang_count = 0;
4377
4378         if (adapter->phy_hang_count > 1) {
4379                 adapter->phy_hang_count = 0;
4380                 schedule_work(&adapter->reset_task);
4381         }
4382 }
4383
4384 /**
4385  * e1000_watchdog - Timer Call-back
4386  * @data: pointer to adapter cast into an unsigned long
4387  **/
4388 static void e1000_watchdog(unsigned long data)
4389 {
4390         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4391
4392         /* Do the rest outside of interrupt context */
4393         schedule_work(&adapter->watchdog_task);
4394
4395         /* TODO: make this use queue_delayed_work() */
4396 }
4397
4398 static void e1000_watchdog_task(struct work_struct *work)
4399 {
4400         struct e1000_adapter *adapter = container_of(work,
4401                                         struct e1000_adapter, watchdog_task);
4402         struct net_device *netdev = adapter->netdev;
4403         struct e1000_mac_info *mac = &adapter->hw.mac;
4404         struct e1000_phy_info *phy = &adapter->hw.phy;
4405         struct e1000_ring *tx_ring = adapter->tx_ring;
4406         struct e1000_hw *hw = &adapter->hw;
4407         u32 link, tctl;
4408
4409         if (test_bit(__E1000_DOWN, &adapter->state))
4410                 return;
4411
4412         link = e1000e_has_link(adapter);
4413         if ((netif_carrier_ok(netdev)) && link) {
4414                 /* Cancel scheduled suspend requests. */
4415                 pm_runtime_resume(netdev->dev.parent);
4416
4417                 e1000e_enable_receives(adapter);
4418                 goto link_up;
4419         }
4420
4421         if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4422             (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4423                 e1000_update_mng_vlan(adapter);
4424
4425         if (link) {
4426                 if (!netif_carrier_ok(netdev)) {
4427                         bool txb2b = true;
4428
4429                         /* Cancel scheduled suspend requests. */
4430                         pm_runtime_resume(netdev->dev.parent);
4431
4432                         /* update snapshot of PHY registers on LSC */
4433                         e1000_phy_read_status(adapter);
4434                         mac->ops.get_link_up_info(&adapter->hw,
4435                                                    &adapter->link_speed,
4436                                                    &adapter->link_duplex);
4437                         e1000_print_link_info(adapter);
4438                         /* On supported PHYs, check for duplex mismatch only
4439                          * if link has autonegotiated at 10/100 half
4440                          */
4441                         if ((hw->phy.type == e1000_phy_igp_3 ||
4442                              hw->phy.type == e1000_phy_bm) &&
4443                             (hw->mac.autoneg == true) &&
4444                             (adapter->link_speed == SPEED_10 ||
4445                              adapter->link_speed == SPEED_100) &&
4446                             (adapter->link_duplex == HALF_DUPLEX)) {
4447                                 u16 autoneg_exp;
4448
4449                                 e1e_rphy(hw, PHY_AUTONEG_EXP, &autoneg_exp);
4450
4451                                 if (!(autoneg_exp & NWAY_ER_LP_NWAY_CAPS))
4452                                         e_info("Autonegotiated half duplex but link partner cannot autoneg.  Try forcing full duplex if link gets many collisions.\n");
4453                         }
4454
4455                         /* adjust timeout factor according to speed/duplex */
4456                         adapter->tx_timeout_factor = 1;
4457                         switch (adapter->link_speed) {
4458                         case SPEED_10:
4459                                 txb2b = false;
4460                                 adapter->tx_timeout_factor = 16;
4461                                 break;
4462                         case SPEED_100:
4463                                 txb2b = false;
4464                                 adapter->tx_timeout_factor = 10;
4465                                 break;
4466                         }
4467
4468                         /* workaround: re-program speed mode bit after
4469                          * link-up event
4470                          */
4471                         if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4472                             !txb2b) {
4473                                 u32 tarc0;
4474                                 tarc0 = er32(TARC(0));
4475                                 tarc0 &= ~SPEED_MODE_BIT;
4476                                 ew32(TARC(0), tarc0);
4477                         }
4478
4479                         /* disable TSO for pcie and 10/100 speeds, to avoid
4480                          * some hardware issues
4481                          */
4482                         if (!(adapter->flags & FLAG_TSO_FORCE)) {
4483                                 switch (adapter->link_speed) {
4484                                 case SPEED_10:
4485                                 case SPEED_100:
4486                                         e_info("10/100 speed: disabling TSO\n");
4487                                         netdev->features &= ~NETIF_F_TSO;
4488                                         netdev->features &= ~NETIF_F_TSO6;
4489                                         break;
4490                                 case SPEED_1000:
4491                                         netdev->features |= NETIF_F_TSO;
4492                                         netdev->features |= NETIF_F_TSO6;
4493                                         break;
4494                                 default:
4495                                         /* oops */
4496                                         break;
4497                                 }
4498                         }
4499
4500                         /* enable transmits in the hardware, need to do this
4501                          * after setting TARC(0)
4502                          */
4503                         tctl = er32(TCTL);
4504                         tctl |= E1000_TCTL_EN;
4505                         ew32(TCTL, tctl);
4506
4507                         /* Perform any post-link-up configuration before
4508                          * reporting link up.
4509                          */
4510                         if (phy->ops.cfg_on_link_up)
4511                                 phy->ops.cfg_on_link_up(hw);
4512
4513                         netif_carrier_on(netdev);
4514
4515                         if (!test_bit(__E1000_DOWN, &adapter->state))
4516                                 mod_timer(&adapter->phy_info_timer,
4517                                           round_jiffies(jiffies + 2 * HZ));
4518                 }
4519         } else {
4520                 if (netif_carrier_ok(netdev)) {
4521                         adapter->link_speed = 0;
4522                         adapter->link_duplex = 0;
4523                         /* Link status message must follow this format */
4524                         printk(KERN_INFO "e1000e: %s NIC Link is Down\n",
4525                                adapter->netdev->name);
4526                         netif_carrier_off(netdev);
4527                         if (!test_bit(__E1000_DOWN, &adapter->state))
4528                                 mod_timer(&adapter->phy_info_timer,
4529                                           round_jiffies(jiffies + 2 * HZ));
4530
4531                         if (adapter->flags & FLAG_RX_NEEDS_RESTART)
4532                                 schedule_work(&adapter->reset_task);
4533                         else
4534                                 pm_schedule_suspend(netdev->dev.parent,
4535                                                         LINK_TIMEOUT);
4536                 }
4537         }
4538
4539 link_up:
4540         spin_lock(&adapter->stats64_lock);
4541         e1000e_update_stats(adapter);
4542
4543         mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4544         adapter->tpt_old = adapter->stats.tpt;
4545         mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4546         adapter->colc_old = adapter->stats.colc;
4547
4548         adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4549         adapter->gorc_old = adapter->stats.gorc;
4550         adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4551         adapter->gotc_old = adapter->stats.gotc;
4552         spin_unlock(&adapter->stats64_lock);
4553
4554         e1000e_update_adaptive(&adapter->hw);
4555
4556         if (!netif_carrier_ok(netdev) &&
4557             (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) {
4558                 /* We've lost link, so the controller stops DMA,
4559                  * but we've got queued Tx work that's never going
4560                  * to get done, so reset controller to flush Tx.
4561                  * (Do the reset outside of interrupt context).
4562                  */
4563                 schedule_work(&adapter->reset_task);
4564                 /* return immediately since reset is imminent */
4565                 return;
4566         }
4567
4568         /* Simple mode for Interrupt Throttle Rate (ITR) */
4569         if (adapter->itr_setting == 4) {
4570                 /* Symmetric Tx/Rx gets a reduced ITR=2000;
4571                  * Total asymmetrical Tx or Rx gets ITR=8000;
4572                  * everyone else is between 2000-8000.
4573                  */
4574                 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4575                 u32 dif = (adapter->gotc > adapter->gorc ?
4576                             adapter->gotc - adapter->gorc :
4577                             adapter->gorc - adapter->gotc) / 10000;
4578                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4579
4580                 e1000e_write_itr(adapter, itr);
4581         }
4582
4583         /* Cause software interrupt to ensure Rx ring is cleaned */
4584         if (adapter->msix_entries)
4585                 ew32(ICS, adapter->rx_ring->ims_val);
4586         else
4587                 ew32(ICS, E1000_ICS_RXDMT0);
4588
4589         /* flush pending descriptors to memory before detecting Tx hang */
4590         e1000e_flush_descriptors(adapter);
4591
4592         /* Force detection of hung controller every watchdog period */
4593         adapter->detect_tx_hung = true;
4594
4595         /* With 82571 controllers, LAA may be overwritten due to controller
4596          * reset from the other port. Set the appropriate LAA in RAR[0]
4597          */
4598         if (e1000e_get_laa_state_82571(hw))
4599                 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);
4600
4601         if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
4602                 e1000e_check_82574_phy_workaround(adapter);
4603
4604         /* Reset the timer */
4605         if (!test_bit(__E1000_DOWN, &adapter->state))
4606                 mod_timer(&adapter->watchdog_timer,
4607                           round_jiffies(jiffies + 2 * HZ));
4608 }
4609
4610 #define E1000_TX_FLAGS_CSUM             0x00000001
4611 #define E1000_TX_FLAGS_VLAN             0x00000002
4612 #define E1000_TX_FLAGS_TSO              0x00000004
4613 #define E1000_TX_FLAGS_IPV4             0x00000008
4614 #define E1000_TX_FLAGS_NO_FCS           0x00000010
4615 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
4616 #define E1000_TX_FLAGS_VLAN_SHIFT       16
4617
4618 static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb)
4619 {
4620         struct e1000_context_desc *context_desc;
4621         struct e1000_buffer *buffer_info;
4622         unsigned int i;
4623         u32 cmd_length = 0;
4624         u16 ipcse = 0, mss;
4625         u8 ipcss, ipcso, tucss, tucso, hdr_len;
4626
4627         if (!skb_is_gso(skb))
4628                 return 0;
4629
4630         if (skb_header_cloned(skb)) {
4631                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4632
4633                 if (err)
4634                         return err;
4635         }
4636
4637         hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
4638         mss = skb_shinfo(skb)->gso_size;
4639         if (skb->protocol == htons(ETH_P_IP)) {
4640                 struct iphdr *iph = ip_hdr(skb);
4641                 iph->tot_len = 0;
4642                 iph->check = 0;
4643                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
4644                                                          0, IPPROTO_TCP, 0);
4645                 cmd_length = E1000_TXD_CMD_IP;
4646                 ipcse = skb_transport_offset(skb) - 1;
4647         } else if (skb_is_gso_v6(skb)) {
4648                 ipv6_hdr(skb)->payload_len = 0;
4649                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4650                                                        &ipv6_hdr(skb)->daddr,
4651                                                        0, IPPROTO_TCP, 0);
4652                 ipcse = 0;
4653         }
4654         ipcss = skb_network_offset(skb);
4655         ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
4656         tucss = skb_transport_offset(skb);
4657         tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
4658
4659         cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
4660                        E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
4661
4662         i = tx_ring->next_to_use;
4663         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
4664         buffer_info = &tx_ring->buffer_info[i];
4665
4666         context_desc->lower_setup.ip_fields.ipcss  = ipcss;
4667         context_desc->lower_setup.ip_fields.ipcso  = ipcso;
4668         context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
4669         context_desc->upper_setup.tcp_fields.tucss = tucss;
4670         context_desc->upper_setup.tcp_fields.tucso = tucso;
4671         context_desc->upper_setup.tcp_fields.tucse = 0;
4672         context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
4673         context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
4674         context_desc->cmd_and_length = cpu_to_le32(cmd_length);
4675
4676         buffer_info->time_stamp = jiffies;
4677         buffer_info->next_to_watch = i;
4678
4679         i++;
4680         if (i == tx_ring->count)
4681                 i = 0;
4682         tx_ring->next_to_use = i;
4683
4684         return 1;
4685 }
4686
4687 static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb)
4688 {
4689         struct e1000_adapter *adapter = tx_ring->adapter;
4690         struct e1000_context_desc *context_desc;
4691         struct e1000_buffer *buffer_info;
4692         unsigned int i;
4693         u8 css;
4694         u32 cmd_len = E1000_TXD_CMD_DEXT;
4695         __be16 protocol;
4696
4697         if (skb->ip_summed != CHECKSUM_PARTIAL)
4698                 return 0;
4699
4700         if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
4701                 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
4702         else
4703                 protocol = skb->protocol;
4704
4705         switch (protocol) {
4706         case cpu_to_be16(ETH_P_IP):
4707                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4708                         cmd_len |= E1000_TXD_CMD_TCP;
4709                 break;
4710         case cpu_to_be16(ETH_P_IPV6):
4711                 /* XXX not handling all IPV6 headers */
4712                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4713                         cmd_len |= E1000_TXD_CMD_TCP;
4714                 break;
4715         default:
4716                 if (unlikely(net_ratelimit()))
4717                         e_warn("checksum_partial proto=%x!\n",
4718                                be16_to_cpu(protocol));
4719                 break;
4720         }
4721
4722         css = skb_checksum_start_offset(skb);
4723
4724         i = tx_ring->next_to_use;
4725         buffer_info = &tx_ring->buffer_info[i];
4726         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
4727
4728         context_desc->lower_setup.ip_config = 0;
4729         context_desc->upper_setup.tcp_fields.tucss = css;
4730         context_desc->upper_setup.tcp_fields.tucso =
4731                                 css + skb->csum_offset;
4732         context_desc->upper_setup.tcp_fields.tucse = 0;
4733         context_desc->tcp_seg_setup.data = 0;
4734         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
4735
4736         buffer_info->time_stamp = jiffies;
4737         buffer_info->next_to_watch = i;
4738
4739         i++;
4740         if (i == tx_ring->count)
4741                 i = 0;
4742         tx_ring->next_to_use = i;
4743
4744         return 1;
4745 }
4746
4747 static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,
4748                         unsigned int first, unsigned int max_per_txd,
4749                         unsigned int nr_frags)
4750 {
4751         struct e1000_adapter *adapter = tx_ring->adapter;
4752         struct pci_dev *pdev = adapter->pdev;
4753         struct e1000_buffer *buffer_info;
4754         unsigned int len = skb_headlen(skb);
4755         unsigned int offset = 0, size, count = 0, i;
4756         unsigned int f, bytecount, segs;
4757
4758         i = tx_ring->next_to_use;
4759
4760         while (len) {
4761                 buffer_info = &tx_ring->buffer_info[i];
4762                 size = min(len, max_per_txd);
4763
4764                 buffer_info->length = size;
4765                 buffer_info->time_stamp = jiffies;
4766                 buffer_info->next_to_watch = i;
4767                 buffer_info->dma = dma_map_single(&pdev->dev,
4768                                                   skb->data + offset,
4769                                                   size, DMA_TO_DEVICE);
4770                 buffer_info->mapped_as_page = false;
4771                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
4772                         goto dma_error;
4773
4774                 len -= size;
4775                 offset += size;
4776                 count++;
4777
4778                 if (len) {
4779                         i++;
4780                         if (i == tx_ring->count)
4781                                 i = 0;
4782                 }
4783         }
4784
4785         for (f = 0; f < nr_frags; f++) {
4786                 const struct skb_frag_struct *frag;
4787
4788                 frag = &skb_shinfo(skb)->frags[f];
4789                 len = skb_frag_size(frag);
4790                 offset = 0;
4791
4792                 while (len) {
4793                         i++;
4794                         if (i == tx_ring->count)
4795                                 i = 0;
4796
4797                         buffer_info = &tx_ring->buffer_info[i];
4798                         size = min(len, max_per_txd);
4799
4800                         buffer_info->length = size;
4801                         buffer_info->time_stamp = jiffies;
4802                         buffer_info->next_to_watch = i;
4803                         buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,
4804                                                 offset, size, DMA_TO_DEVICE);
4805                         buffer_info->mapped_as_page = true;
4806                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
4807                                 goto dma_error;
4808
4809                         len -= size;
4810                         offset += size;
4811                         count++;
4812                 }
4813         }
4814
4815         segs = skb_shinfo(skb)->gso_segs ? : 1;
4816         /* multiply data chunks by size of headers */
4817         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
4818
4819         tx_ring->buffer_info[i].skb = skb;
4820         tx_ring->buffer_info[i].segs = segs;
4821         tx_ring->buffer_info[i].bytecount = bytecount;
4822         tx_ring->buffer_info[first].next_to_watch = i;
4823
4824         return count;
4825
4826 dma_error:
4827         dev_err(&pdev->dev, "Tx DMA map failed\n");
4828         buffer_info->dma = 0;
4829         if (count)
4830                 count--;
4831
4832         while (count--) {
4833                 if (i == 0)
4834                         i += tx_ring->count;
4835                 i--;
4836                 buffer_info = &tx_ring->buffer_info[i];
4837                 e1000_put_txbuf(tx_ring, buffer_info);
4838         }
4839
4840         return 0;
4841 }
4842
4843 static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)
4844 {
4845         struct e1000_adapter *adapter = tx_ring->adapter;
4846         struct e1000_tx_desc *tx_desc = NULL;
4847         struct e1000_buffer *buffer_info;
4848         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
4849         unsigned int i;
4850
4851         if (tx_flags & E1000_TX_FLAGS_TSO) {
4852                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
4853                              E1000_TXD_CMD_TSE;
4854                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
4855
4856                 if (tx_flags & E1000_TX_FLAGS_IPV4)
4857                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
4858         }
4859
4860         if (tx_flags & E1000_TX_FLAGS_CSUM) {
4861                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
4862                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
4863         }
4864
4865         if (tx_flags & E1000_TX_FLAGS_VLAN) {
4866                 txd_lower |= E1000_TXD_CMD_VLE;
4867                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
4868         }
4869
4870         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
4871                 txd_lower &= ~(E1000_TXD_CMD_IFCS);
4872
4873         i = tx_ring->next_to_use;
4874
4875         do {
4876                 buffer_info = &tx_ring->buffer_info[i];
4877                 tx_desc = E1000_TX_DESC(*tx_ring, i);
4878                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4879                 tx_desc->lower.data =
4880                         cpu_to_le32(txd_lower | buffer_info->length);
4881                 tx_desc->upper.data = cpu_to_le32(txd_upper);
4882
4883                 i++;
4884                 if (i == tx_ring->count)
4885                         i = 0;
4886         } while (--count > 0);
4887
4888         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
4889
4890         /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */
4891         if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))
4892                 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));
4893
4894         /* Force memory writes to complete before letting h/w
4895          * know there are new descriptors to fetch.  (Only
4896          * applicable for weak-ordered memory model archs,
4897          * such as IA-64).
4898          */
4899         wmb();
4900
4901         tx_ring->next_to_use = i;
4902
4903         if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
4904                 e1000e_update_tdt_wa(tx_ring, i);
4905         else
4906                 writel(i, tx_ring->tail);
4907
4908         /* we need this if more than one processor can write to our tail
4909          * at a time, it synchronizes IO on IA64/Altix systems
4910          */
4911         mmiowb();
4912 }
4913
4914 #define MINIMUM_DHCP_PACKET_SIZE 282
4915 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
4916                                     struct sk_buff *skb)
4917 {
4918         struct e1000_hw *hw =  &adapter->hw;
4919         u16 length, offset;
4920
4921         if (vlan_tx_tag_present(skb)) {
4922                 if (!((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
4923                     (adapter->hw.mng_cookie.status &
4924                         E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
4925                         return 0;
4926         }
4927
4928         if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
4929                 return 0;
4930
4931         if (((struct ethhdr *) skb->data)->h_proto != htons(ETH_P_IP))
4932                 return 0;
4933
4934         {
4935                 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data+14);
4936                 struct udphdr *udp;
4937
4938                 if (ip->protocol != IPPROTO_UDP)
4939                         return 0;
4940
4941                 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
4942                 if (ntohs(udp->dest) != 67)
4943                         return 0;
4944
4945                 offset = (u8 *)udp + 8 - skb->data;
4946                 length = skb->len - offset;
4947                 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
4948         }
4949
4950         return 0;
4951 }
4952
4953 static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
4954 {
4955         struct e1000_adapter *adapter = tx_ring->adapter;
4956
4957         netif_stop_queue(adapter->netdev);
4958         /* Herbert's original patch had:
4959          *  smp_mb__after_netif_stop_queue();
4960          * but since that doesn't exist yet, just open code it.
4961          */
4962         smp_mb();
4963
4964         /* We need to check again in a case another CPU has just
4965          * made room available.
4966          */
4967         if (e1000_desc_unused(tx_ring) < size)
4968                 return -EBUSY;
4969
4970         /* A reprieve! */
4971         netif_start_queue(adapter->netdev);
4972         ++adapter->restart_queue;
4973         return 0;
4974 }
4975
4976 static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)
4977 {
4978         BUG_ON(size > tx_ring->count);
4979
4980         if (e1000_desc_unused(tx_ring) >= size)
4981                 return 0;
4982         return __e1000_maybe_stop_tx(tx_ring, size);
4983 }
4984
4985 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
4986                                     struct net_device *netdev)
4987 {
4988         struct e1000_adapter *adapter = netdev_priv(netdev);
4989         struct e1000_ring *tx_ring = adapter->tx_ring;
4990         unsigned int first;
4991         unsigned int tx_flags = 0;
4992         unsigned int len = skb_headlen(skb);
4993         unsigned int nr_frags;
4994         unsigned int mss;
4995         int count = 0;
4996         int tso;
4997         unsigned int f;
4998
4999         if (test_bit(__E1000_DOWN, &adapter->state)) {
5000                 dev_kfree_skb_any(skb);
5001                 return NETDEV_TX_OK;
5002         }
5003
5004         if (skb->len <= 0) {
5005                 dev_kfree_skb_any(skb);
5006                 return NETDEV_TX_OK;
5007         }
5008
5009         /* The minimum packet size with TCTL.PSP set is 17 bytes so
5010          * pad skb in order to meet this minimum size requirement
5011          */
5012         if (unlikely(skb->len < 17)) {
5013                 if (skb_pad(skb, 17 - skb->len))
5014                         return NETDEV_TX_OK;
5015                 skb->len = 17;
5016                 skb_set_tail_pointer(skb, 17);
5017         }
5018
5019         mss = skb_shinfo(skb)->gso_size;
5020         if (mss) {
5021                 u8 hdr_len;
5022
5023                 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data
5024                  * points to just header, pull a few bytes of payload from
5025                  * frags into skb->data
5026                  */
5027                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
5028                 /* we do this workaround for ES2LAN, but it is un-necessary,
5029                  * avoiding it could save a lot of cycles
5030                  */
5031                 if (skb->data_len && (hdr_len == len)) {
5032                         unsigned int pull_size;
5033
5034                         pull_size = min_t(unsigned int, 4, skb->data_len);
5035                         if (!__pskb_pull_tail(skb, pull_size)) {
5036                                 e_err("__pskb_pull_tail failed.\n");
5037                                 dev_kfree_skb_any(skb);
5038                                 return NETDEV_TX_OK;
5039                         }
5040                         len = skb_headlen(skb);
5041                 }
5042         }
5043
5044         /* reserve a descriptor for the offload context */
5045         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
5046                 count++;
5047         count++;
5048
5049         count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);
5050
5051         nr_frags = skb_shinfo(skb)->nr_frags;
5052         for (f = 0; f < nr_frags; f++)
5053                 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),
5054                                       adapter->tx_fifo_limit);
5055
5056         if (adapter->hw.mac.tx_pkt_filtering)
5057                 e1000_transfer_dhcp_info(adapter, skb);
5058
5059         /* need: count + 2 desc gap to keep tail from touching
5060          * head, otherwise try next time
5061          */
5062         if (e1000_maybe_stop_tx(tx_ring, count + 2))
5063                 return NETDEV_TX_BUSY;
5064
5065         if (vlan_tx_tag_present(skb)) {
5066                 tx_flags |= E1000_TX_FLAGS_VLAN;
5067                 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
5068         }
5069
5070         first = tx_ring->next_to_use;
5071
5072         tso = e1000_tso(tx_ring, skb);
5073         if (tso < 0) {
5074                 dev_kfree_skb_any(skb);
5075                 return NETDEV_TX_OK;
5076         }
5077
5078         if (tso)
5079                 tx_flags |= E1000_TX_FLAGS_TSO;
5080         else if (e1000_tx_csum(tx_ring, skb))
5081                 tx_flags |= E1000_TX_FLAGS_CSUM;
5082
5083         /* Old method was to assume IPv4 packet by default if TSO was enabled.
5084          * 82571 hardware supports TSO capabilities for IPv6 as well...
5085          * no longer assume, we must.
5086          */
5087         if (skb->protocol == htons(ETH_P_IP))
5088                 tx_flags |= E1000_TX_FLAGS_IPV4;
5089
5090         if (unlikely(skb->no_fcs))
5091                 tx_flags |= E1000_TX_FLAGS_NO_FCS;
5092
5093         /* if count is 0 then mapping error has occurred */
5094         count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,
5095                              nr_frags);
5096         if (count) {
5097                 skb_tx_timestamp(skb);
5098
5099                 netdev_sent_queue(netdev, skb->len);
5100                 e1000_tx_queue(tx_ring, tx_flags, count);
5101                 /* Make sure there is space in the ring for the next send. */
5102                 e1000_maybe_stop_tx(tx_ring,
5103                                     (MAX_SKB_FRAGS *
5104                                      DIV_ROUND_UP(PAGE_SIZE,
5105                                                   adapter->tx_fifo_limit) + 2));
5106         } else {
5107                 dev_kfree_skb_any(skb);
5108                 tx_ring->buffer_info[first].time_stamp = 0;
5109                 tx_ring->next_to_use = first;
5110         }
5111
5112         return NETDEV_TX_OK;
5113 }
5114
5115 /**
5116  * e1000_tx_timeout - Respond to a Tx Hang
5117  * @netdev: network interface device structure
5118  **/
5119 static void e1000_tx_timeout(struct net_device *netdev)
5120 {
5121         struct e1000_adapter *adapter = netdev_priv(netdev);
5122
5123         /* Do the reset outside of interrupt context */
5124         adapter->tx_timeout_count++;
5125         schedule_work(&adapter->reset_task);
5126 }
5127
5128 static void e1000_reset_task(struct work_struct *work)
5129 {
5130         struct e1000_adapter *adapter;
5131         adapter = container_of(work, struct e1000_adapter, reset_task);
5132
5133         /* don't run the task if already down */
5134         if (test_bit(__E1000_DOWN, &adapter->state))
5135                 return;
5136
5137         if (!((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
5138               (adapter->flags & FLAG_RX_RESTART_NOW))) {
5139                 e1000e_dump(adapter);
5140                 e_err("Reset adapter\n");
5141         }
5142         e1000e_reinit_locked(adapter);
5143 }
5144
5145 /**
5146  * e1000_get_stats64 - Get System Network Statistics
5147  * @netdev: network interface device structure
5148  * @stats: rtnl_link_stats64 pointer
5149  *
5150  * Returns the address of the device statistics structure.
5151  **/
5152 struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
5153                                              struct rtnl_link_stats64 *stats)
5154 {
5155         struct e1000_adapter *adapter = netdev_priv(netdev);
5156
5157         memset(stats, 0, sizeof(struct rtnl_link_stats64));
5158         spin_lock(&adapter->stats64_lock);
5159         e1000e_update_stats(adapter);
5160         /* Fill out the OS statistics structure */
5161         stats->rx_bytes = adapter->stats.gorc;
5162         stats->rx_packets = adapter->stats.gprc;
5163         stats->tx_bytes = adapter->stats.gotc;
5164         stats->tx_packets = adapter->stats.gptc;
5165         stats->multicast = adapter->stats.mprc;
5166         stats->collisions = adapter->stats.colc;
5167
5168         /* Rx Errors */
5169
5170         /* RLEC on some newer hardware can be incorrect so build
5171          * our own version based on RUC and ROC
5172          */
5173         stats->rx_errors = adapter->stats.rxerrc +
5174                 adapter->stats.crcerrs + adapter->stats.algnerrc +
5175                 adapter->stats.ruc + adapter->stats.roc +
5176                 adapter->stats.cexterr;
5177         stats->rx_length_errors = adapter->stats.ruc +
5178                                               adapter->stats.roc;
5179         stats->rx_crc_errors = adapter->stats.crcerrs;
5180         stats->rx_frame_errors = adapter->stats.algnerrc;
5181         stats->rx_missed_errors = adapter->stats.mpc;
5182
5183         /* Tx Errors */
5184         stats->tx_errors = adapter->stats.ecol +
5185                                        adapter->stats.latecol;
5186         stats->tx_aborted_errors = adapter->stats.ecol;
5187         stats->tx_window_errors = adapter->stats.latecol;
5188         stats->tx_carrier_errors = adapter->stats.tncrs;
5189
5190         /* Tx Dropped needs to be maintained elsewhere */
5191
5192         spin_unlock(&adapter->stats64_lock);
5193         return stats;
5194 }
5195
5196 /**
5197  * e1000_change_mtu - Change the Maximum Transfer Unit
5198  * @netdev: network interface device structure
5199  * @new_mtu: new value for maximum frame size
5200  *
5201  * Returns 0 on success, negative on failure
5202  **/
5203 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5204 {
5205         struct e1000_adapter *adapter = netdev_priv(netdev);
5206         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5207
5208         /* Jumbo frame support */
5209         if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5210             !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5211                 e_err("Jumbo Frames not supported.\n");
5212                 return -EINVAL;
5213         }
5214
5215         /* Supported frame sizes */
5216         if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5217             (max_frame > adapter->max_hw_frame_size)) {
5218                 e_err("Unsupported MTU setting\n");
5219                 return -EINVAL;
5220         }
5221
5222         /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */
5223         if ((adapter->hw.mac.type >= e1000_pch2lan) &&
5224             !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5225             (new_mtu > ETH_DATA_LEN)) {
5226                 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");
5227                 return -EINVAL;
5228         }
5229
5230         while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
5231                 usleep_range(1000, 2000);
5232         /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
5233         adapter->max_frame_size = max_frame;
5234         e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5235         netdev->mtu = new_mtu;
5236         if (netif_running(netdev))
5237                 e1000e_down(adapter);
5238
5239         /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
5240          * means we reserve 2 more, this pushes us to allocate from the next
5241          * larger slab size.
5242          * i.e. RXBUFFER_2048 --> size-4096 slab
5243          * However with the new *_jumbo_rx* routines, jumbo receives will use
5244          * fragmented skbs
5245          */
5246
5247         if (max_frame <= 2048)
5248                 adapter->rx_buffer_len = 2048;
5249         else
5250                 adapter->rx_buffer_len = 4096;
5251
5252         /* adjust allocation if LPE protects us, and we aren't using SBP */
5253         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
5254              (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
5255                 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
5256                                          + ETH_FCS_LEN;
5257
5258         if (netif_running(netdev))
5259                 e1000e_up(adapter);
5260         else
5261                 e1000e_reset(adapter);
5262
5263         clear_bit(__E1000_RESETTING, &adapter->state);
5264
5265         return 0;
5266 }
5267
5268 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5269                            int cmd)
5270 {
5271         struct e1000_adapter *adapter = netdev_priv(netdev);
5272         struct mii_ioctl_data *data = if_mii(ifr);
5273
5274         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5275                 return -EOPNOTSUPP;
5276
5277         switch (cmd) {
5278         case SIOCGMIIPHY:
5279                 data->phy_id = adapter->hw.phy.addr;
5280                 break;
5281         case SIOCGMIIREG:
5282                 e1000_phy_read_status(adapter);
5283
5284                 switch (data->reg_num & 0x1F) {
5285                 case MII_BMCR:
5286                         data->val_out = adapter->phy_regs.bmcr;
5287                         break;
5288                 case MII_BMSR:
5289                         data->val_out = adapter->phy_regs.bmsr;
5290                         break;
5291                 case MII_PHYSID1:
5292                         data->val_out = (adapter->hw.phy.id >> 16);
5293                         break;
5294                 case MII_PHYSID2:
5295                         data->val_out = (adapter->hw.phy.id & 0xFFFF);
5296                         break;
5297                 case MII_ADVERTISE:
5298                         data->val_out = adapter->phy_regs.advertise;
5299                         break;
5300                 case MII_LPA:
5301                         data->val_out = adapter->phy_regs.lpa;
5302                         break;
5303                 case MII_EXPANSION:
5304                         data->val_out = adapter->phy_regs.expansion;
5305                         break;
5306                 case MII_CTRL1000:
5307                         data->val_out = adapter->phy_regs.ctrl1000;
5308                         break;
5309                 case MII_STAT1000:
5310                         data->val_out = adapter->phy_regs.stat1000;
5311                         break;
5312                 case MII_ESTATUS:
5313                         data->val_out = adapter->phy_regs.estatus;
5314                         break;
5315                 default:
5316                         return -EIO;
5317                 }
5318                 break;
5319         case SIOCSMIIREG:
5320         default:
5321                 return -EOPNOTSUPP;
5322         }
5323         return 0;
5324 }
5325
5326 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5327 {
5328         switch (cmd) {
5329         case SIOCGMIIPHY:
5330         case SIOCGMIIREG:
5331         case SIOCSMIIREG:
5332                 return e1000_mii_ioctl(netdev, ifr, cmd);
5333         default:
5334                 return -EOPNOTSUPP;
5335         }
5336 }
5337
5338 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5339 {
5340         struct e1000_hw *hw = &adapter->hw;
5341         u32 i, mac_reg;
5342         u16 phy_reg, wuc_enable;
5343         int retval = 0;
5344
5345         /* copy MAC RARs to PHY RARs */
5346         e1000_copy_rx_addrs_to_phy_ich8lan(hw);
5347
5348         retval = hw->phy.ops.acquire(hw);
5349         if (retval) {
5350                 e_err("Could not acquire PHY\n");
5351                 return retval;
5352         }
5353
5354         /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */
5355         retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5356         if (retval)
5357                 goto release;
5358
5359         /* copy MAC MTA to PHY MTA - only needed for pchlan */
5360         for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5361                 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5362                 hw->phy.ops.write_reg_page(hw, BM_MTA(i),
5363                                            (u16)(mac_reg & 0xFFFF));
5364                 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,
5365                                            (u16)((mac_reg >> 16) & 0xFFFF));
5366         }
5367
5368         /* configure PHY Rx Control register */
5369         hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);
5370         mac_reg = er32(RCTL);
5371         if (mac_reg & E1000_RCTL_UPE)
5372                 phy_reg |= BM_RCTL_UPE;
5373         if (mac_reg & E1000_RCTL_MPE)
5374                 phy_reg |= BM_RCTL_MPE;
5375         phy_reg &= ~(BM_RCTL_MO_MASK);
5376         if (mac_reg & E1000_RCTL_MO_3)
5377                 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5378                                 << BM_RCTL_MO_SHIFT);
5379         if (mac_reg & E1000_RCTL_BAM)
5380                 phy_reg |= BM_RCTL_BAM;
5381         if (mac_reg & E1000_RCTL_PMCF)
5382                 phy_reg |= BM_RCTL_PMCF;
5383         mac_reg = er32(CTRL);
5384         if (mac_reg & E1000_CTRL_RFCE)
5385                 phy_reg |= BM_RCTL_RFCE;
5386         hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);
5387
5388         /* enable PHY wakeup in MAC register */
5389         ew32(WUFC, wufc);
5390         ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5391
5392         /* configure and enable PHY wakeup in PHY registers */
5393         hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);
5394         hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5395
5396         /* activate PHY wakeup */
5397         wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5398         retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);
5399         if (retval)
5400                 e_err("Could not set PHY Host Wakeup bit\n");
5401 release:
5402         hw->phy.ops.release(hw);
5403
5404         return retval;
5405 }
5406
5407 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5408                             bool runtime)
5409 {
5410         struct net_device *netdev = pci_get_drvdata(pdev);
5411         struct e1000_adapter *adapter = netdev_priv(netdev);
5412         struct e1000_hw *hw = &adapter->hw;
5413         u32 ctrl, ctrl_ext, rctl, status;
5414         /* Runtime suspend should only enable wakeup for link changes */
5415         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
5416         int retval = 0;
5417
5418         netif_device_detach(netdev);
5419
5420         if (netif_running(netdev)) {
5421                 int count = E1000_CHECK_RESET_COUNT;
5422
5423                 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)
5424                         usleep_range(10000, 20000);
5425
5426                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5427                 e1000e_down(adapter);
5428                 e1000_free_irq(adapter);
5429         }
5430         e1000e_reset_interrupt_capability(adapter);
5431
5432         retval = pci_save_state(pdev);
5433         if (retval)
5434                 return retval;
5435
5436         status = er32(STATUS);
5437         if (status & E1000_STATUS_LU)
5438                 wufc &= ~E1000_WUFC_LNKC;
5439
5440         if (wufc) {
5441                 e1000_setup_rctl(adapter);
5442                 e1000e_set_rx_mode(netdev);
5443
5444                 /* turn on all-multi mode if wake on multicast is enabled */
5445                 if (wufc & E1000_WUFC_MC) {
5446                         rctl = er32(RCTL);
5447                         rctl |= E1000_RCTL_MPE;
5448                         ew32(RCTL, rctl);
5449                 }
5450
5451                 ctrl = er32(CTRL);
5452                 /* advertise wake from D3Cold */
5453                 #define E1000_CTRL_ADVD3WUC 0x00100000
5454                 /* phy power management enable */
5455                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5456                 ctrl |= E1000_CTRL_ADVD3WUC;
5457                 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5458                         ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
5459                 ew32(CTRL, ctrl);
5460
5461                 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5462                     adapter->hw.phy.media_type ==
5463                     e1000_media_type_internal_serdes) {
5464                         /* keep the laser running in D3 */
5465                         ctrl_ext = er32(CTRL_EXT);
5466                         ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
5467                         ew32(CTRL_EXT, ctrl_ext);
5468                 }
5469
5470                 if (adapter->flags & FLAG_IS_ICH)
5471                         e1000_suspend_workarounds_ich8lan(&adapter->hw);
5472
5473                 /* Allow time for pending master requests to run */
5474                 e1000e_disable_pcie_master(&adapter->hw);
5475
5476                 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
5477                         /* enable wakeup by the PHY */
5478                         retval = e1000_init_phy_wakeup(adapter, wufc);
5479                         if (retval)
5480                                 return retval;
5481                 } else {
5482                         /* enable wakeup by the MAC */
5483                         ew32(WUFC, wufc);
5484                         ew32(WUC, E1000_WUC_PME_EN);
5485                 }
5486         } else {
5487                 ew32(WUC, 0);
5488                 ew32(WUFC, 0);
5489         }
5490
5491         *enable_wake = !!wufc;
5492
5493         /* make sure adapter isn't asleep if manageability is enabled */
5494         if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5495             (hw->mac.ops.check_mng_mode(hw)))
5496                 *enable_wake = true;
5497
5498         if (adapter->hw.phy.type == e1000_phy_igp_3)
5499                 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5500
5501         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
5502          * would have already happened in close and is redundant.
5503          */
5504         e1000e_release_hw_control(adapter);
5505
5506         pci_disable_device(pdev);
5507
5508         return 0;
5509 }
5510
5511 static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5512 {
5513         if (sleep && wake) {
5514                 pci_prepare_to_sleep(pdev);
5515                 return;
5516         }
5517
5518         pci_wake_from_d3(pdev, wake);
5519         pci_set_power_state(pdev, PCI_D3hot);
5520 }
5521
5522 static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep,
5523                                     bool wake)
5524 {
5525         struct net_device *netdev = pci_get_drvdata(pdev);
5526         struct e1000_adapter *adapter = netdev_priv(netdev);
5527
5528         /* The pci-e switch on some quad port adapters will report a
5529          * correctable error when the MAC transitions from D0 to D3.  To
5530          * prevent this we need to mask off the correctable errors on the
5531          * downstream port of the pci-e switch.
5532          */
5533         if (adapter->flags & FLAG_IS_QUAD_PORT) {
5534                 struct pci_dev *us_dev = pdev->bus->self;
5535                 u16 devctl;
5536
5537                 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);
5538                 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,
5539                                            (devctl & ~PCI_EXP_DEVCTL_CERE));
5540
5541                 e1000_power_off(pdev, sleep, wake);
5542
5543                 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);
5544         } else {
5545                 e1000_power_off(pdev, sleep, wake);
5546         }
5547 }
5548
5549 #ifdef CONFIG_PCIEASPM
5550 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5551 {
5552         pci_disable_link_state_locked(pdev, state);
5553 }
5554 #else
5555 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5556 {
5557         /* Both device and parent should have the same ASPM setting.
5558          * Disable ASPM in downstream component first and then upstream.
5559          */
5560         pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, state);
5561
5562         if (pdev->bus->self)
5563                 pcie_capability_clear_word(pdev->bus->self, PCI_EXP_LNKCTL,
5564                                            state);
5565 }
5566 #endif
5567 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5568 {
5569         dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
5570                  (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
5571                  (state & PCIE_LINK_STATE_L1) ? "L1" : "");
5572
5573         __e1000e_disable_aspm(pdev, state);
5574 }
5575
5576 #ifdef CONFIG_PM
5577 static bool e1000e_pm_ready(struct e1000_adapter *adapter)
5578 {
5579         return !!adapter->tx_ring->buffer_info;
5580 }
5581
5582 static int __e1000_resume(struct pci_dev *pdev)
5583 {
5584         struct net_device *netdev = pci_get_drvdata(pdev);
5585         struct e1000_adapter *adapter = netdev_priv(netdev);
5586         struct e1000_hw *hw = &adapter->hw;
5587         u16 aspm_disable_flag = 0;
5588         u32 err;
5589
5590         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
5591                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
5592         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
5593                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
5594         if (aspm_disable_flag)
5595                 e1000e_disable_aspm(pdev, aspm_disable_flag);
5596
5597         pci_set_power_state(pdev, PCI_D0);
5598         pci_restore_state(pdev);
5599         pci_save_state(pdev);
5600
5601         e1000e_set_interrupt_capability(adapter);
5602         if (netif_running(netdev)) {
5603                 err = e1000_request_irq(adapter);
5604                 if (err)
5605                         return err;
5606         }
5607
5608         if (hw->mac.type >= e1000_pch2lan)
5609                 e1000_resume_workarounds_pchlan(&adapter->hw);
5610
5611         e1000e_power_up_phy(adapter);
5612
5613         /* report the system wakeup cause from S3/S4 */
5614         if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
5615                 u16 phy_data;
5616
5617                 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
5618                 if (phy_data) {
5619                         e_info("PHY Wakeup cause - %s\n",
5620                                 phy_data & E1000_WUS_EX ? "Unicast Packet" :
5621                                 phy_data & E1000_WUS_MC ? "Multicast Packet" :
5622                                 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
5623                                 phy_data & E1000_WUS_MAG ? "Magic Packet" :
5624                                 phy_data & E1000_WUS_LNKC ?
5625                                 "Link Status Change" : "other");
5626                 }
5627                 e1e_wphy(&adapter->hw, BM_WUS, ~0);
5628         } else {
5629                 u32 wus = er32(WUS);
5630                 if (wus) {
5631                         e_info("MAC Wakeup cause - %s\n",
5632                                 wus & E1000_WUS_EX ? "Unicast Packet" :
5633                                 wus & E1000_WUS_MC ? "Multicast Packet" :
5634                                 wus & E1000_WUS_BC ? "Broadcast Packet" :
5635                                 wus & E1000_WUS_MAG ? "Magic Packet" :
5636                                 wus & E1000_WUS_LNKC ? "Link Status Change" :
5637                                 "other");
5638                 }
5639                 ew32(WUS, ~0);
5640         }
5641
5642         e1000e_reset(adapter);
5643
5644         e1000_init_manageability_pt(adapter);
5645
5646         if (netif_running(netdev))
5647                 e1000e_up(adapter);
5648
5649         netif_device_attach(netdev);
5650
5651         /* If the controller has AMT, do not set DRV_LOAD until the interface
5652          * is up.  For all other cases, let the f/w know that the h/w is now
5653          * under the control of the driver.
5654          */
5655         if (!(adapter->flags & FLAG_HAS_AMT))
5656                 e1000e_get_hw_control(adapter);
5657
5658         return 0;
5659 }
5660
5661 #ifdef CONFIG_PM_SLEEP
5662 static int e1000_suspend(struct device *dev)
5663 {
5664         struct pci_dev *pdev = to_pci_dev(dev);
5665         int retval;
5666         bool wake;
5667
5668         retval = __e1000_shutdown(pdev, &wake, false);
5669         if (!retval)
5670                 e1000_complete_shutdown(pdev, true, wake);
5671
5672         return retval;
5673 }
5674
5675 static int e1000_resume(struct device *dev)
5676 {
5677         struct pci_dev *pdev = to_pci_dev(dev);
5678         struct net_device *netdev = pci_get_drvdata(pdev);
5679         struct e1000_adapter *adapter = netdev_priv(netdev);
5680
5681         if (e1000e_pm_ready(adapter))
5682                 adapter->idle_check = true;
5683
5684         return __e1000_resume(pdev);
5685 }
5686 #endif /* CONFIG_PM_SLEEP */
5687
5688 #ifdef CONFIG_PM_RUNTIME
5689 static int e1000_runtime_suspend(struct device *dev)
5690 {
5691         struct pci_dev *pdev = to_pci_dev(dev);
5692         struct net_device *netdev = pci_get_drvdata(pdev);
5693         struct e1000_adapter *adapter = netdev_priv(netdev);
5694
5695         if (e1000e_pm_ready(adapter)) {
5696                 bool wake;
5697
5698                 __e1000_shutdown(pdev, &wake, true);
5699         }
5700
5701         return 0;
5702 }
5703
5704 static int e1000_idle(struct device *dev)
5705 {
5706         struct pci_dev *pdev = to_pci_dev(dev);
5707         struct net_device *netdev = pci_get_drvdata(pdev);
5708         struct e1000_adapter *adapter = netdev_priv(netdev);
5709
5710         if (!e1000e_pm_ready(adapter))
5711                 return 0;
5712
5713         if (adapter->idle_check) {
5714                 adapter->idle_check = false;
5715                 if (!e1000e_has_link(adapter))
5716                         pm_schedule_suspend(dev, MSEC_PER_SEC);
5717         }
5718
5719         return -EBUSY;
5720 }
5721
5722 static int e1000_runtime_resume(struct device *dev)
5723 {
5724         struct pci_dev *pdev = to_pci_dev(dev);
5725         struct net_device *netdev = pci_get_drvdata(pdev);
5726         struct e1000_adapter *adapter = netdev_priv(netdev);
5727
5728         if (!e1000e_pm_ready(adapter))
5729                 return 0;
5730
5731         adapter->idle_check = !dev->power.runtime_auto;
5732         return __e1000_resume(pdev);
5733 }
5734 #endif /* CONFIG_PM_RUNTIME */
5735 #endif /* CONFIG_PM */
5736
5737 static void e1000_shutdown(struct pci_dev *pdev)
5738 {
5739         bool wake = false;
5740
5741         __e1000_shutdown(pdev, &wake, false);
5742
5743         if (system_state == SYSTEM_POWER_OFF)
5744                 e1000_complete_shutdown(pdev, false, wake);
5745 }
5746
5747 #ifdef CONFIG_NET_POLL_CONTROLLER
5748
5749 static irqreturn_t e1000_intr_msix(int irq, void *data)
5750 {
5751         struct net_device *netdev = data;
5752         struct e1000_adapter *adapter = netdev_priv(netdev);
5753
5754         if (adapter->msix_entries) {
5755                 int vector, msix_irq;
5756
5757                 vector = 0;
5758                 msix_irq = adapter->msix_entries[vector].vector;
5759                 disable_irq(msix_irq);
5760                 e1000_intr_msix_rx(msix_irq, netdev);
5761                 enable_irq(msix_irq);
5762
5763                 vector++;
5764                 msix_irq = adapter->msix_entries[vector].vector;
5765                 disable_irq(msix_irq);
5766                 e1000_intr_msix_tx(msix_irq, netdev);
5767                 enable_irq(msix_irq);
5768
5769                 vector++;
5770                 msix_irq = adapter->msix_entries[vector].vector;
5771                 disable_irq(msix_irq);
5772                 e1000_msix_other(msix_irq, netdev);
5773                 enable_irq(msix_irq);
5774         }
5775
5776         return IRQ_HANDLED;
5777 }
5778
5779 /**
5780  * e1000_netpoll
5781  * @netdev: network interface device structure
5782  *
5783  * Polling 'interrupt' - used by things like netconsole to send skbs
5784  * without having to re-enable interrupts. It's not called while
5785  * the interrupt routine is executing.
5786  */
5787 static void e1000_netpoll(struct net_device *netdev)
5788 {
5789         struct e1000_adapter *adapter = netdev_priv(netdev);
5790
5791         switch (adapter->int_mode) {
5792         case E1000E_INT_MODE_MSIX:
5793                 e1000_intr_msix(adapter->pdev->irq, netdev);
5794                 break;
5795         case E1000E_INT_MODE_MSI:
5796                 disable_irq(adapter->pdev->irq);
5797                 e1000_intr_msi(adapter->pdev->irq, netdev);
5798                 enable_irq(adapter->pdev->irq);
5799                 break;
5800         default: /* E1000E_INT_MODE_LEGACY */
5801                 disable_irq(adapter->pdev->irq);
5802                 e1000_intr(adapter->pdev->irq, netdev);
5803                 enable_irq(adapter->pdev->irq);
5804                 break;
5805         }
5806 }
5807 #endif
5808
5809 /**
5810  * e1000_io_error_detected - called when PCI error is detected
5811  * @pdev: Pointer to PCI device
5812  * @state: The current pci connection state
5813  *
5814  * This function is called after a PCI bus error affecting
5815  * this device has been detected.
5816  */
5817 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5818                                                 pci_channel_state_t state)
5819 {
5820         struct net_device *netdev = pci_get_drvdata(pdev);
5821         struct e1000_adapter *adapter = netdev_priv(netdev);
5822
5823         netif_device_detach(netdev);
5824
5825         if (state == pci_channel_io_perm_failure)
5826                 return PCI_ERS_RESULT_DISCONNECT;
5827
5828         if (netif_running(netdev))
5829                 e1000e_down(adapter);
5830         pci_disable_device(pdev);
5831
5832         /* Request a slot slot reset. */
5833         return PCI_ERS_RESULT_NEED_RESET;
5834 }
5835
5836 /**
5837  * e1000_io_slot_reset - called after the pci bus has been reset.
5838  * @pdev: Pointer to PCI device
5839  *
5840  * Restart the card from scratch, as if from a cold-boot. Implementation
5841  * resembles the first-half of the e1000_resume routine.
5842  */
5843 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5844 {
5845         struct net_device *netdev = pci_get_drvdata(pdev);
5846         struct e1000_adapter *adapter = netdev_priv(netdev);
5847         struct e1000_hw *hw = &adapter->hw;
5848         u16 aspm_disable_flag = 0;
5849         int err;
5850         pci_ers_result_t result;
5851
5852         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
5853                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
5854         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
5855                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
5856         if (aspm_disable_flag)
5857                 e1000e_disable_aspm(pdev, aspm_disable_flag);
5858
5859         err = pci_enable_device_mem(pdev);
5860         if (err) {
5861                 dev_err(&pdev->dev,
5862                         "Cannot re-enable PCI device after reset.\n");
5863                 result = PCI_ERS_RESULT_DISCONNECT;
5864         } else {
5865                 pci_set_master(pdev);
5866                 pdev->state_saved = true;
5867                 pci_restore_state(pdev);
5868
5869                 pci_enable_wake(pdev, PCI_D3hot, 0);
5870                 pci_enable_wake(pdev, PCI_D3cold, 0);
5871
5872                 e1000e_reset(adapter);
5873                 ew32(WUS, ~0);
5874                 result = PCI_ERS_RESULT_RECOVERED;
5875         }
5876
5877         pci_cleanup_aer_uncorrect_error_status(pdev);
5878
5879         return result;
5880 }
5881
5882 /**
5883  * e1000_io_resume - called when traffic can start flowing again.
5884  * @pdev: Pointer to PCI device
5885  *
5886  * This callback is called when the error recovery driver tells us that
5887  * its OK to resume normal operation. Implementation resembles the
5888  * second-half of the e1000_resume routine.
5889  */
5890 static void e1000_io_resume(struct pci_dev *pdev)
5891 {
5892         struct net_device *netdev = pci_get_drvdata(pdev);
5893         struct e1000_adapter *adapter = netdev_priv(netdev);
5894
5895         e1000_init_manageability_pt(adapter);
5896
5897         if (netif_running(netdev)) {
5898                 if (e1000e_up(adapter)) {
5899                         dev_err(&pdev->dev,
5900                                 "can't bring device back up after reset\n");
5901                         return;
5902                 }
5903         }
5904
5905         netif_device_attach(netdev);
5906
5907         /* If the controller has AMT, do not set DRV_LOAD until the interface
5908          * is up.  For all other cases, let the f/w know that the h/w is now
5909          * under the control of the driver.
5910          */
5911         if (!(adapter->flags & FLAG_HAS_AMT))
5912                 e1000e_get_hw_control(adapter);
5913
5914 }
5915
5916 static void e1000_print_device_info(struct e1000_adapter *adapter)
5917 {
5918         struct e1000_hw *hw = &adapter->hw;
5919         struct net_device *netdev = adapter->netdev;
5920         u32 ret_val;
5921         u8 pba_str[E1000_PBANUM_LENGTH];
5922
5923         /* print bus type/speed/width info */
5924         e_info("(PCI Express:2.5GT/s:%s) %pM\n",
5925                /* bus width */
5926                ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
5927                 "Width x1"),
5928                /* MAC address */
5929                netdev->dev_addr);
5930         e_info("Intel(R) PRO/%s Network Connection\n",
5931                (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
5932         ret_val = e1000_read_pba_string_generic(hw, pba_str,
5933                                                 E1000_PBANUM_LENGTH);
5934         if (ret_val)
5935                 strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
5936         e_info("MAC: %d, PHY: %d, PBA No: %s\n",
5937                hw->mac.type, hw->phy.type, pba_str);
5938 }
5939
5940 static void e1000_eeprom_checks(struct e1000_adapter *adapter)
5941 {
5942         struct e1000_hw *hw = &adapter->hw;
5943         int ret_val;
5944         u16 buf = 0;
5945
5946         if (hw->mac.type != e1000_82573)
5947                 return;
5948
5949         ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
5950         le16_to_cpus(&buf);
5951         if (!ret_val && (!(buf & (1 << 0)))) {
5952                 /* Deep Smart Power Down (DSPD) */
5953                 dev_warn(&adapter->pdev->dev,
5954                          "Warning: detected DSPD enabled in EEPROM\n");
5955         }
5956 }
5957
5958 static int e1000_set_features(struct net_device *netdev,
5959                               netdev_features_t features)
5960 {
5961         struct e1000_adapter *adapter = netdev_priv(netdev);
5962         netdev_features_t changed = features ^ netdev->features;
5963
5964         if (changed & (NETIF_F_TSO | NETIF_F_TSO6))
5965                 adapter->flags |= FLAG_TSO_FORCE;
5966
5967         if (!(changed & (NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_TX |
5968                          NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |
5969                          NETIF_F_RXALL)))
5970                 return 0;
5971
5972         if (changed & NETIF_F_RXFCS) {
5973                 if (features & NETIF_F_RXFCS) {
5974                         adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
5975                 } else {
5976                         /* We need to take it back to defaults, which might mean
5977                          * stripping is still disabled at the adapter level.
5978                          */
5979                         if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)
5980                                 adapter->flags2 |= FLAG2_CRC_STRIPPING;
5981                         else
5982                                 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;
5983                 }
5984         }
5985
5986         netdev->features = features;
5987
5988         if (netif_running(netdev))
5989                 e1000e_reinit_locked(adapter);
5990         else
5991                 e1000e_reset(adapter);
5992
5993         return 0;
5994 }
5995
5996 static const struct net_device_ops e1000e_netdev_ops = {
5997         .ndo_open               = e1000_open,
5998         .ndo_stop               = e1000_close,
5999         .ndo_start_xmit         = e1000_xmit_frame,
6000         .ndo_get_stats64        = e1000e_get_stats64,
6001         .ndo_set_rx_mode        = e1000e_set_rx_mode,
6002         .ndo_set_mac_address    = e1000_set_mac,
6003         .ndo_change_mtu         = e1000_change_mtu,
6004         .ndo_do_ioctl           = e1000_ioctl,
6005         .ndo_tx_timeout         = e1000_tx_timeout,
6006         .ndo_validate_addr      = eth_validate_addr,
6007
6008         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
6009         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
6010 #ifdef CONFIG_NET_POLL_CONTROLLER
6011         .ndo_poll_controller    = e1000_netpoll,
6012 #endif
6013         .ndo_set_features = e1000_set_features,
6014 };
6015
6016 /**
6017  * e1000_probe - Device Initialization Routine
6018  * @pdev: PCI device information struct
6019  * @ent: entry in e1000_pci_tbl
6020  *
6021  * Returns 0 on success, negative on failure
6022  *
6023  * e1000_probe initializes an adapter identified by a pci_dev structure.
6024  * The OS initialization, configuring of the adapter private structure,
6025  * and a hardware reset occur.
6026  **/
6027 static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
6028 {
6029         struct net_device *netdev;
6030         struct e1000_adapter *adapter;
6031         struct e1000_hw *hw;
6032         const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
6033         resource_size_t mmio_start, mmio_len;
6034         resource_size_t flash_start, flash_len;
6035         static int cards_found;
6036         u16 aspm_disable_flag = 0;
6037         int i, err, pci_using_dac;
6038         u16 eeprom_data = 0;
6039         u16 eeprom_apme_mask = E1000_EEPROM_APME;
6040
6041         if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
6042                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
6043         if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
6044                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
6045         if (aspm_disable_flag)
6046                 e1000e_disable_aspm(pdev, aspm_disable_flag);
6047
6048         err = pci_enable_device_mem(pdev);
6049         if (err)
6050                 return err;
6051
6052         pci_using_dac = 0;
6053         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
6054         if (!err) {
6055                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
6056                 if (!err)
6057                         pci_using_dac = 1;
6058         } else {
6059                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
6060                 if (err) {
6061                         err = dma_set_coherent_mask(&pdev->dev,
6062                                                     DMA_BIT_MASK(32));
6063                         if (err) {
6064                                 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n");
6065                                 goto err_dma;
6066                         }
6067                 }
6068         }
6069
6070         err = pci_request_selected_regions_exclusive(pdev,
6071                                           pci_select_bars(pdev, IORESOURCE_MEM),
6072                                           e1000e_driver_name);
6073         if (err)
6074                 goto err_pci_reg;
6075
6076         /* AER (Advanced Error Reporting) hooks */
6077         pci_enable_pcie_error_reporting(pdev);
6078
6079         pci_set_master(pdev);
6080         /* PCI config space info */
6081         err = pci_save_state(pdev);
6082         if (err)
6083                 goto err_alloc_etherdev;
6084
6085         err = -ENOMEM;
6086         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
6087         if (!netdev)
6088                 goto err_alloc_etherdev;
6089
6090         SET_NETDEV_DEV(netdev, &pdev->dev);
6091
6092         netdev->irq = pdev->irq;
6093
6094         pci_set_drvdata(pdev, netdev);
6095         adapter = netdev_priv(netdev);
6096         hw = &adapter->hw;
6097         adapter->netdev = netdev;
6098         adapter->pdev = pdev;
6099         adapter->ei = ei;
6100         adapter->pba = ei->pba;
6101         adapter->flags = ei->flags;
6102         adapter->flags2 = ei->flags2;
6103         adapter->hw.adapter = adapter;
6104         adapter->hw.mac.type = ei->mac;
6105         adapter->max_hw_frame_size = ei->max_hw_frame_size;
6106         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6107
6108         mmio_start = pci_resource_start(pdev, 0);
6109         mmio_len = pci_resource_len(pdev, 0);
6110
6111         err = -EIO;
6112         adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
6113         if (!adapter->hw.hw_addr)
6114                 goto err_ioremap;
6115
6116         if ((adapter->flags & FLAG_HAS_FLASH) &&
6117             (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
6118                 flash_start = pci_resource_start(pdev, 1);
6119                 flash_len = pci_resource_len(pdev, 1);
6120                 adapter->hw.flash_address = ioremap(flash_start, flash_len);
6121                 if (!adapter->hw.flash_address)
6122                         goto err_flashmap;
6123         }
6124
6125         /* construct the net_device struct */
6126         netdev->netdev_ops              = &e1000e_netdev_ops;
6127         e1000e_set_ethtool_ops(netdev);
6128         netdev->watchdog_timeo          = 5 * HZ;
6129         netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
6130         strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
6131
6132         netdev->mem_start = mmio_start;
6133         netdev->mem_end = mmio_start + mmio_len;
6134
6135         adapter->bd_number = cards_found++;
6136
6137         e1000e_check_options(adapter);
6138
6139         /* setup adapter struct */
6140         err = e1000_sw_init(adapter);
6141         if (err)
6142                 goto err_sw_init;
6143
6144         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
6145         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
6146         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
6147
6148         err = ei->get_variants(adapter);
6149         if (err)
6150                 goto err_hw_init;
6151
6152         if ((adapter->flags & FLAG_IS_ICH) &&
6153             (adapter->flags & FLAG_READ_ONLY_NVM))
6154                 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
6155
6156         hw->mac.ops.get_bus_info(&adapter->hw);
6157
6158         adapter->hw.phy.autoneg_wait_to_complete = 0;
6159
6160         /* Copper options */
6161         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
6162                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
6163                 adapter->hw.phy.disable_polarity_correction = 0;
6164                 adapter->hw.phy.ms_type = e1000_ms_hw_default;
6165         }
6166
6167         if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))
6168                 dev_info(&pdev->dev,
6169                          "PHY reset is blocked due to SOL/IDER session.\n");
6170
6171         /* Set initial default active device features */
6172         netdev->features = (NETIF_F_SG |
6173                             NETIF_F_HW_VLAN_RX |
6174                             NETIF_F_HW_VLAN_TX |
6175                             NETIF_F_TSO |
6176                             NETIF_F_TSO6 |
6177                             NETIF_F_RXHASH |
6178                             NETIF_F_RXCSUM |
6179                             NETIF_F_HW_CSUM);
6180
6181         /* Set user-changeable features (subset of all device features) */
6182         netdev->hw_features = netdev->features;
6183         netdev->hw_features |= NETIF_F_RXFCS;
6184         netdev->priv_flags |= IFF_SUPP_NOFCS;
6185         netdev->hw_features |= NETIF_F_RXALL;
6186
6187         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
6188                 netdev->features |= NETIF_F_HW_VLAN_FILTER;
6189
6190         netdev->vlan_features |= (NETIF_F_SG |
6191                                   NETIF_F_TSO |
6192                                   NETIF_F_TSO6 |
6193                                   NETIF_F_HW_CSUM);
6194
6195         netdev->priv_flags |= IFF_UNICAST_FLT;
6196
6197         if (pci_using_dac) {
6198                 netdev->features |= NETIF_F_HIGHDMA;
6199                 netdev->vlan_features |= NETIF_F_HIGHDMA;
6200         }
6201
6202         if (e1000e_enable_mng_pass_thru(&adapter->hw))
6203                 adapter->flags |= FLAG_MNG_PT_ENABLED;
6204
6205         /* before reading the NVM, reset the controller to
6206          * put the device in a known good starting state
6207          */
6208         adapter->hw.mac.ops.reset_hw(&adapter->hw);
6209
6210         /* systems with ASPM and others may see the checksum fail on the first
6211          * attempt. Let's give it a few tries
6212          */
6213         for (i = 0;; i++) {
6214                 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
6215                         break;
6216                 if (i == 2) {
6217                         dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
6218                         err = -EIO;
6219                         goto err_eeprom;
6220                 }
6221         }
6222
6223         e1000_eeprom_checks(adapter);
6224
6225         /* copy the MAC address */
6226         if (e1000e_read_mac_addr(&adapter->hw))
6227                 dev_err(&pdev->dev,
6228                         "NVM Read Error while reading MAC address\n");
6229
6230         memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
6231
6232         if (!is_valid_ether_addr(netdev->dev_addr)) {
6233                 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",
6234                         netdev->dev_addr);
6235                 err = -EIO;
6236                 goto err_eeprom;
6237         }
6238
6239         init_timer(&adapter->watchdog_timer);
6240         adapter->watchdog_timer.function = e1000_watchdog;
6241         adapter->watchdog_timer.data = (unsigned long) adapter;
6242
6243         init_timer(&adapter->phy_info_timer);
6244         adapter->phy_info_timer.function = e1000_update_phy_info;
6245         adapter->phy_info_timer.data = (unsigned long) adapter;
6246
6247         INIT_WORK(&adapter->reset_task, e1000_reset_task);
6248         INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
6249         INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6250         INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
6251         INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
6252
6253         /* Initialize link parameters. User can change them with ethtool */
6254         adapter->hw.mac.autoneg = 1;
6255         adapter->fc_autoneg = true;
6256         adapter->hw.fc.requested_mode = e1000_fc_default;
6257         adapter->hw.fc.current_mode = e1000_fc_default;
6258         adapter->hw.phy.autoneg_advertised = 0x2f;
6259
6260         /* ring size defaults */
6261         adapter->rx_ring->count = E1000_DEFAULT_RXD;
6262         adapter->tx_ring->count = E1000_DEFAULT_TXD;
6263
6264         /* Initial Wake on LAN setting - If APM wake is enabled in
6265          * the EEPROM, enable the ACPI Magic Packet filter
6266          */
6267         if (adapter->flags & FLAG_APME_IN_WUC) {
6268                 /* APME bit in EEPROM is mapped to WUC.APME */
6269                 eeprom_data = er32(WUC);
6270                 eeprom_apme_mask = E1000_WUC_APME;
6271                 if ((hw->mac.type > e1000_ich10lan) &&
6272                     (eeprom_data & E1000_WUC_PHY_WAKE))
6273                         adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
6274         } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6275                 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6276                     (adapter->hw.bus.func == 1))
6277                         e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_B,
6278                                        1, &eeprom_data);
6279                 else
6280                         e1000_read_nvm(&adapter->hw, NVM_INIT_CONTROL3_PORT_A,
6281                                        1, &eeprom_data);
6282         }
6283
6284         /* fetch WoL from EEPROM */
6285         if (eeprom_data & eeprom_apme_mask)
6286                 adapter->eeprom_wol |= E1000_WUFC_MAG;
6287
6288         /* now that we have the eeprom settings, apply the special cases
6289          * where the eeprom may be wrong or the board simply won't support
6290          * wake on lan on a particular port
6291          */
6292         if (!(adapter->flags & FLAG_HAS_WOL))
6293                 adapter->eeprom_wol = 0;
6294
6295         /* initialize the wol settings based on the eeprom settings */
6296         adapter->wol = adapter->eeprom_wol;
6297         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
6298
6299         /* save off EEPROM version number */
6300         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6301
6302         /* reset the hardware with the new settings */
6303         e1000e_reset(adapter);
6304
6305         /* If the controller has AMT, do not set DRV_LOAD until the interface
6306          * is up.  For all other cases, let the f/w know that the h/w is now
6307          * under the control of the driver.
6308          */
6309         if (!(adapter->flags & FLAG_HAS_AMT))
6310                 e1000e_get_hw_control(adapter);
6311
6312         strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
6313         err = register_netdev(netdev);
6314         if (err)
6315                 goto err_register;
6316
6317         /* carrier off reporting is important to ethtool even BEFORE open */
6318         netif_carrier_off(netdev);
6319
6320         e1000_print_device_info(adapter);
6321
6322         if (pci_dev_run_wake(pdev))
6323                 pm_runtime_put_noidle(&pdev->dev);
6324
6325         return 0;
6326
6327 err_register:
6328         if (!(adapter->flags & FLAG_HAS_AMT))
6329                 e1000e_release_hw_control(adapter);
6330 err_eeprom:
6331         if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))
6332                 e1000_phy_hw_reset(&adapter->hw);
6333 err_hw_init:
6334         kfree(adapter->tx_ring);
6335         kfree(adapter->rx_ring);
6336 err_sw_init:
6337         if (adapter->hw.flash_address)
6338                 iounmap(adapter->hw.flash_address);
6339         e1000e_reset_interrupt_capability(adapter);
6340 err_flashmap:
6341         iounmap(adapter->hw.hw_addr);
6342 err_ioremap:
6343         free_netdev(netdev);
6344 err_alloc_etherdev:
6345         pci_release_selected_regions(pdev,
6346                                      pci_select_bars(pdev, IORESOURCE_MEM));
6347 err_pci_reg:
6348 err_dma:
6349         pci_disable_device(pdev);
6350         return err;
6351 }
6352
6353 /**
6354  * e1000_remove - Device Removal Routine
6355  * @pdev: PCI device information struct
6356  *
6357  * e1000_remove is called by the PCI subsystem to alert the driver
6358  * that it should release a PCI device.  The could be caused by a
6359  * Hot-Plug event, or because the driver is going to be removed from
6360  * memory.
6361  **/
6362 static void e1000_remove(struct pci_dev *pdev)
6363 {
6364         struct net_device *netdev = pci_get_drvdata(pdev);
6365         struct e1000_adapter *adapter = netdev_priv(netdev);
6366         bool down = test_bit(__E1000_DOWN, &adapter->state);
6367
6368         /* The timers may be rescheduled, so explicitly disable them
6369          * from being rescheduled.
6370          */
6371         if (!down)
6372                 set_bit(__E1000_DOWN, &adapter->state);
6373         del_timer_sync(&adapter->watchdog_timer);
6374         del_timer_sync(&adapter->phy_info_timer);
6375
6376         cancel_work_sync(&adapter->reset_task);
6377         cancel_work_sync(&adapter->watchdog_task);
6378         cancel_work_sync(&adapter->downshift_task);
6379         cancel_work_sync(&adapter->update_phy_task);
6380         cancel_work_sync(&adapter->print_hang_task);
6381
6382         if (!(netdev->flags & IFF_UP))
6383                 e1000_power_down_phy(adapter);
6384
6385         /* Don't lie to e1000_close() down the road. */
6386         if (!down)
6387                 clear_bit(__E1000_DOWN, &adapter->state);
6388         unregister_netdev(netdev);
6389
6390         if (pci_dev_run_wake(pdev))
6391                 pm_runtime_get_noresume(&pdev->dev);
6392
6393         /* Release control of h/w to f/w.  If f/w is AMT enabled, this
6394          * would have already happened in close and is redundant.
6395          */
6396         e1000e_release_hw_control(adapter);
6397
6398         e1000e_reset_interrupt_capability(adapter);
6399         kfree(adapter->tx_ring);
6400         kfree(adapter->rx_ring);
6401
6402         iounmap(adapter->hw.hw_addr);
6403         if (adapter->hw.flash_address)
6404                 iounmap(adapter->hw.flash_address);
6405         pci_release_selected_regions(pdev,
6406                                      pci_select_bars(pdev, IORESOURCE_MEM));
6407
6408         free_netdev(netdev);
6409
6410         /* AER disable */
6411         pci_disable_pcie_error_reporting(pdev);
6412
6413         pci_disable_device(pdev);
6414 }
6415
6416 /* PCI Error Recovery (ERS) */
6417 static const struct pci_error_handlers e1000_err_handler = {
6418         .error_detected = e1000_io_error_detected,
6419         .slot_reset = e1000_io_slot_reset,
6420         .resume = e1000_io_resume,
6421 };
6422
6423 static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
6424         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6425         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6426         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
6427         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), board_82571 },
6428         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6429         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
6430         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6431         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6432         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
6433
6434         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6435         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6436         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6437         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
6438
6439         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6440         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6441         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
6442
6443         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
6444         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
6445         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
6446
6447         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6448           board_80003es2lan },
6449         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6450           board_80003es2lan },
6451         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6452           board_80003es2lan },
6453         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6454           board_80003es2lan },
6455
6456         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6457         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6458         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6459         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6460         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6461         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6462         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
6463         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
6464
6465         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6466         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6467         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6468         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6469         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
6470         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
6471         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6472         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6473         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6474
6475         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6476         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6477         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
6478
6479         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6480         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
6481         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
6482
6483         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6484         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6485         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6486         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6487
6488         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6489         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6490
6491         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
6492         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
6493         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
6494         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
6495
6496         { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */
6497 };
6498 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
6499
6500 #ifdef CONFIG_PM
6501 static const struct dev_pm_ops e1000_pm_ops = {
6502         SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
6503         SET_RUNTIME_PM_OPS(e1000_runtime_suspend,
6504                                 e1000_runtime_resume, e1000_idle)
6505 };
6506 #endif
6507
6508 /* PCI Device API Driver */
6509 static struct pci_driver e1000_driver = {
6510         .name     = e1000e_driver_name,
6511         .id_table = e1000_pci_tbl,
6512         .probe    = e1000_probe,
6513         .remove   = e1000_remove,
6514 #ifdef CONFIG_PM
6515         .driver   = {
6516                 .pm = &e1000_pm_ops,
6517         },
6518 #endif
6519         .shutdown = e1000_shutdown,
6520         .err_handler = &e1000_err_handler
6521 };
6522
6523 /**
6524  * e1000_init_module - Driver Registration Routine
6525  *
6526  * e1000_init_module is the first routine called when the driver is
6527  * loaded. All it does is register with the PCI subsystem.
6528  **/
6529 static int __init e1000_init_module(void)
6530 {
6531         int ret;
6532         pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
6533                 e1000e_driver_version);
6534         pr_info("Copyright(c) 1999 - 2012 Intel Corporation.\n");
6535         ret = pci_register_driver(&e1000_driver);
6536
6537         return ret;
6538 }
6539 module_init(e1000_init_module);
6540
6541 /**
6542  * e1000_exit_module - Driver Exit Cleanup Routine
6543  *
6544  * e1000_exit_module is called just before the driver is removed
6545  * from memory.
6546  **/
6547 static void __exit e1000_exit_module(void)
6548 {
6549         pci_unregister_driver(&e1000_driver);
6550 }
6551 module_exit(e1000_exit_module);
6552
6553
6554 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
6555 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
6556 MODULE_LICENSE("GPL");
6557 MODULE_VERSION(DRV_VERSION);
6558
6559 /* netdev.c */