1 /*******************************************************************************
3 Intel 10 Gigabit PCI Express Linux driver
4 Copyright(c) 1999 - 2011 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/types.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/vmalloc.h>
33 #include <linux/string.h>
35 #include <linux/interrupt.h>
37 #include <linux/tcp.h>
38 #include <linux/sctp.h>
39 #include <linux/pkt_sched.h>
40 #include <linux/ipv6.h>
41 #include <linux/slab.h>
42 #include <net/checksum.h>
43 #include <net/ip6_checksum.h>
44 #include <linux/ethtool.h>
46 #include <linux/if_vlan.h>
47 #include <linux/prefetch.h>
48 #include <scsi/fc/fc_fcoe.h>
51 #include "ixgbe_common.h"
52 #include "ixgbe_dcb_82599.h"
53 #include "ixgbe_sriov.h"
55 char ixgbe_driver_name[] = "ixgbe";
56 static const char ixgbe_driver_string[] =
57 "Intel(R) 10 Gigabit PCI Express Network Driver";
61 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
62 __stringify(BUILD) "-k"
63 const char ixgbe_driver_version[] = DRV_VERSION;
64 static const char ixgbe_copyright[] =
65 "Copyright (c) 1999-2011 Intel Corporation.";
67 static const struct ixgbe_info *ixgbe_info_tbl[] = {
68 [board_82598] = &ixgbe_82598_info,
69 [board_82599] = &ixgbe_82599_info,
70 [board_X540] = &ixgbe_X540_info,
73 /* ixgbe_pci_tbl - PCI Device ID Table
75 * Wildcard entries (PCI_ANY_ID) should come last
76 * Last entry must be all 0s
78 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
79 * Class, Class Mask, private data (not used) }
81 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
82 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
83 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
84 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
85 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
86 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
87 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
88 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
111 /* required last entry */
114 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
116 #ifdef CONFIG_IXGBE_DCA
117 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
119 static struct notifier_block dca_notifier = {
120 .notifier_call = ixgbe_notify_dca,
126 #ifdef CONFIG_PCI_IOV
127 static unsigned int max_vfs;
128 module_param(max_vfs, uint, 0);
129 MODULE_PARM_DESC(max_vfs,
130 "Maximum number of virtual functions to allocate per physical function");
131 #endif /* CONFIG_PCI_IOV */
133 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
134 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
135 MODULE_LICENSE("GPL");
136 MODULE_VERSION(DRV_VERSION);
138 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
140 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
142 if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
143 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
144 schedule_work(&adapter->service_task);
147 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
149 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
151 /* flush memory to make sure state is correct before next watchog */
152 smp_mb__before_clear_bit();
153 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
156 struct ixgbe_reg_info {
161 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
163 /* General Registers */
164 {IXGBE_CTRL, "CTRL"},
165 {IXGBE_STATUS, "STATUS"},
166 {IXGBE_CTRL_EXT, "CTRL_EXT"},
168 /* Interrupt Registers */
169 {IXGBE_EICR, "EICR"},
172 {IXGBE_SRRCTL(0), "SRRCTL"},
173 {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
174 {IXGBE_RDLEN(0), "RDLEN"},
175 {IXGBE_RDH(0), "RDH"},
176 {IXGBE_RDT(0), "RDT"},
177 {IXGBE_RXDCTL(0), "RXDCTL"},
178 {IXGBE_RDBAL(0), "RDBAL"},
179 {IXGBE_RDBAH(0), "RDBAH"},
182 {IXGBE_TDBAL(0), "TDBAL"},
183 {IXGBE_TDBAH(0), "TDBAH"},
184 {IXGBE_TDLEN(0), "TDLEN"},
185 {IXGBE_TDH(0), "TDH"},
186 {IXGBE_TDT(0), "TDT"},
187 {IXGBE_TXDCTL(0), "TXDCTL"},
189 /* List Terminator */
195 * ixgbe_regdump - register printout routine
197 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
203 switch (reginfo->ofs) {
204 case IXGBE_SRRCTL(0):
205 for (i = 0; i < 64; i++)
206 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
208 case IXGBE_DCA_RXCTRL(0):
209 for (i = 0; i < 64; i++)
210 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
213 for (i = 0; i < 64; i++)
214 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
217 for (i = 0; i < 64; i++)
218 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
221 for (i = 0; i < 64; i++)
222 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
224 case IXGBE_RXDCTL(0):
225 for (i = 0; i < 64; i++)
226 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
229 for (i = 0; i < 64; i++)
230 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
233 for (i = 0; i < 64; i++)
234 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
237 for (i = 0; i < 64; i++)
238 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
241 for (i = 0; i < 64; i++)
242 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
245 for (i = 0; i < 64; i++)
246 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
249 for (i = 0; i < 64; i++)
250 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
253 for (i = 0; i < 64; i++)
254 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
256 case IXGBE_TXDCTL(0):
257 for (i = 0; i < 64; i++)
258 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
261 pr_info("%-15s %08x\n", reginfo->name,
262 IXGBE_READ_REG(hw, reginfo->ofs));
266 for (i = 0; i < 8; i++) {
267 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
268 pr_err("%-15s", rname);
269 for (j = 0; j < 8; j++)
270 pr_cont(" %08x", regs[i*8+j]);
277 * ixgbe_dump - Print registers, tx-rings and rx-rings
279 static void ixgbe_dump(struct ixgbe_adapter *adapter)
281 struct net_device *netdev = adapter->netdev;
282 struct ixgbe_hw *hw = &adapter->hw;
283 struct ixgbe_reg_info *reginfo;
285 struct ixgbe_ring *tx_ring;
286 struct ixgbe_tx_buffer *tx_buffer_info;
287 union ixgbe_adv_tx_desc *tx_desc;
288 struct my_u0 { u64 a; u64 b; } *u0;
289 struct ixgbe_ring *rx_ring;
290 union ixgbe_adv_rx_desc *rx_desc;
291 struct ixgbe_rx_buffer *rx_buffer_info;
295 if (!netif_msg_hw(adapter))
298 /* Print netdevice Info */
300 dev_info(&adapter->pdev->dev, "Net device Info\n");
301 pr_info("Device Name state "
302 "trans_start last_rx\n");
303 pr_info("%-15s %016lX %016lX %016lX\n",
310 /* Print Registers */
311 dev_info(&adapter->pdev->dev, "Register Dump\n");
312 pr_info(" Register Name Value\n");
313 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
314 reginfo->name; reginfo++) {
315 ixgbe_regdump(hw, reginfo);
318 /* Print TX Ring Summary */
319 if (!netdev || !netif_running(netdev))
322 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
323 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
324 for (n = 0; n < adapter->num_tx_queues; n++) {
325 tx_ring = adapter->tx_ring[n];
327 &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
328 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
329 n, tx_ring->next_to_use, tx_ring->next_to_clean,
330 (u64)tx_buffer_info->dma,
331 tx_buffer_info->length,
332 tx_buffer_info->next_to_watch,
333 (u64)tx_buffer_info->time_stamp);
337 if (!netif_msg_tx_done(adapter))
338 goto rx_ring_summary;
340 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
342 /* Transmit Descriptor Formats
344 * Advanced Transmit Descriptor
345 * +--------------------------------------------------------------+
346 * 0 | Buffer Address [63:0] |
347 * +--------------------------------------------------------------+
348 * 8 | PAYLEN | PORTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |
349 * +--------------------------------------------------------------+
350 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0
353 for (n = 0; n < adapter->num_tx_queues; n++) {
354 tx_ring = adapter->tx_ring[n];
355 pr_info("------------------------------------\n");
356 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
357 pr_info("------------------------------------\n");
358 pr_info("T [desc] [address 63:0 ] "
359 "[PlPOIdStDDt Ln] [bi->dma ] "
360 "leng ntw timestamp bi->skb\n");
362 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
363 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
364 tx_buffer_info = &tx_ring->tx_buffer_info[i];
365 u0 = (struct my_u0 *)tx_desc;
366 pr_info("T [0x%03X] %016llX %016llX %016llX"
367 " %04X %p %016llX %p", i,
370 (u64)tx_buffer_info->dma,
371 tx_buffer_info->length,
372 tx_buffer_info->next_to_watch,
373 (u64)tx_buffer_info->time_stamp,
374 tx_buffer_info->skb);
375 if (i == tx_ring->next_to_use &&
376 i == tx_ring->next_to_clean)
378 else if (i == tx_ring->next_to_use)
380 else if (i == tx_ring->next_to_clean)
385 if (netif_msg_pktdata(adapter) &&
386 tx_buffer_info->dma != 0)
387 print_hex_dump(KERN_INFO, "",
388 DUMP_PREFIX_ADDRESS, 16, 1,
389 phys_to_virt(tx_buffer_info->dma),
390 tx_buffer_info->length, true);
394 /* Print RX Rings Summary */
396 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
397 pr_info("Queue [NTU] [NTC]\n");
398 for (n = 0; n < adapter->num_rx_queues; n++) {
399 rx_ring = adapter->rx_ring[n];
400 pr_info("%5d %5X %5X\n",
401 n, rx_ring->next_to_use, rx_ring->next_to_clean);
405 if (!netif_msg_rx_status(adapter))
408 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
410 /* Advanced Receive Descriptor (Read) Format
412 * +-----------------------------------------------------+
413 * 0 | Packet Buffer Address [63:1] |A0/NSE|
414 * +----------------------------------------------+------+
415 * 8 | Header Buffer Address [63:1] | DD |
416 * +-----------------------------------------------------+
419 * Advanced Receive Descriptor (Write-Back) Format
421 * 63 48 47 32 31 30 21 20 16 15 4 3 0
422 * +------------------------------------------------------+
423 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
424 * | Checksum Ident | | | | Type | Type |
425 * +------------------------------------------------------+
426 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
427 * +------------------------------------------------------+
428 * 63 48 47 32 31 20 19 0
430 for (n = 0; n < adapter->num_rx_queues; n++) {
431 rx_ring = adapter->rx_ring[n];
432 pr_info("------------------------------------\n");
433 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("R [desc] [ PktBuf A0] "
436 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
437 "<-- Adv Rx Read format\n");
438 pr_info("RWB[desc] [PcsmIpSHl PtRs] "
439 "[vl er S cks ln] ---------------- [bi->skb] "
440 "<-- Adv Rx Write-Back format\n");
442 for (i = 0; i < rx_ring->count; i++) {
443 rx_buffer_info = &rx_ring->rx_buffer_info[i];
444 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
445 u0 = (struct my_u0 *)rx_desc;
446 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
447 if (staterr & IXGBE_RXD_STAT_DD) {
448 /* Descriptor Done */
449 pr_info("RWB[0x%03X] %016llX "
450 "%016llX ---------------- %p", i,
453 rx_buffer_info->skb);
455 pr_info("R [0x%03X] %016llX "
456 "%016llX %016llX %p", i,
459 (u64)rx_buffer_info->dma,
460 rx_buffer_info->skb);
462 if (netif_msg_pktdata(adapter)) {
463 print_hex_dump(KERN_INFO, "",
464 DUMP_PREFIX_ADDRESS, 16, 1,
465 phys_to_virt(rx_buffer_info->dma),
466 rx_ring->rx_buf_len, true);
468 if (rx_ring->rx_buf_len
470 print_hex_dump(KERN_INFO, "",
471 DUMP_PREFIX_ADDRESS, 16, 1,
473 rx_buffer_info->page_dma +
474 rx_buffer_info->page_offset
480 if (i == rx_ring->next_to_use)
482 else if (i == rx_ring->next_to_clean)
494 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
498 /* Let firmware take over control of h/w */
499 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
500 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
501 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
504 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
508 /* Let firmware know the driver has taken over */
509 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
510 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
511 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
515 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
516 * @adapter: pointer to adapter struct
517 * @direction: 0 for Rx, 1 for Tx, -1 for other causes
518 * @queue: queue to map the corresponding interrupt to
519 * @msix_vector: the vector to map to the corresponding queue
522 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
523 u8 queue, u8 msix_vector)
526 struct ixgbe_hw *hw = &adapter->hw;
527 switch (hw->mac.type) {
528 case ixgbe_mac_82598EB:
529 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
532 index = (((direction * 64) + queue) >> 2) & 0x1F;
533 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
534 ivar &= ~(0xFF << (8 * (queue & 0x3)));
535 ivar |= (msix_vector << (8 * (queue & 0x3)));
536 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
538 case ixgbe_mac_82599EB:
540 if (direction == -1) {
542 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
543 index = ((queue & 1) * 8);
544 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
545 ivar &= ~(0xFF << index);
546 ivar |= (msix_vector << index);
547 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
550 /* tx or rx causes */
551 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
552 index = ((16 * (queue & 1)) + (8 * direction));
553 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
554 ivar &= ~(0xFF << index);
555 ivar |= (msix_vector << index);
556 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
564 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
569 switch (adapter->hw.mac.type) {
570 case ixgbe_mac_82598EB:
571 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
572 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
574 case ixgbe_mac_82599EB:
576 mask = (qmask & 0xFFFFFFFF);
577 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
578 mask = (qmask >> 32);
579 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
586 static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
587 struct ixgbe_tx_buffer *tx_buffer)
589 if (tx_buffer->dma) {
590 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
591 dma_unmap_page(ring->dev,
596 dma_unmap_single(ring->dev,
604 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
605 struct ixgbe_tx_buffer *tx_buffer_info)
607 ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
608 if (tx_buffer_info->skb)
609 dev_kfree_skb_any(tx_buffer_info->skb);
610 tx_buffer_info->skb = NULL;
611 /* tx_buffer_info must be completely set up in the transmit path */
614 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
616 struct ixgbe_hw *hw = &adapter->hw;
617 struct ixgbe_hw_stats *hwstats = &adapter->stats;
622 if ((hw->fc.current_mode == ixgbe_fc_full) ||
623 (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
624 switch (hw->mac.type) {
625 case ixgbe_mac_82598EB:
626 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
629 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
631 hwstats->lxoffrxc += data;
633 /* refill credits (no tx hang) if we received xoff */
637 for (i = 0; i < adapter->num_tx_queues; i++)
638 clear_bit(__IXGBE_HANG_CHECK_ARMED,
639 &adapter->tx_ring[i]->state);
641 } else if (!(adapter->dcb_cfg.pfc_mode_enable))
644 /* update stats for each tc, only valid with PFC enabled */
645 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
646 switch (hw->mac.type) {
647 case ixgbe_mac_82598EB:
648 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
651 xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
653 hwstats->pxoffrxc[i] += xoff[i];
656 /* disarm tx queues that have received xoff frames */
657 for (i = 0; i < adapter->num_tx_queues; i++) {
658 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
659 u8 tc = tx_ring->dcb_tc;
662 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
666 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
668 return ring->tx_stats.completed;
671 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
673 struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
674 struct ixgbe_hw *hw = &adapter->hw;
676 u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
677 u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
680 return (head < tail) ?
681 tail - head : (tail + ring->count - head);
686 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
688 u32 tx_done = ixgbe_get_tx_completed(tx_ring);
689 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
690 u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
693 clear_check_for_tx_hang(tx_ring);
696 * Check for a hung queue, but be thorough. This verifies
697 * that a transmit has been completed since the previous
698 * check AND there is at least one packet pending. The
699 * ARMED bit is set to indicate a potential hang. The
700 * bit is cleared if a pause frame is received to remove
701 * false hang detection due to PFC or 802.3x frames. By
702 * requiring this to fail twice we avoid races with
703 * pfc clearing the ARMED bit and conditions where we
704 * run the check_tx_hang logic with a transmit completion
705 * pending but without time to complete it yet.
707 if ((tx_done_old == tx_done) && tx_pending) {
708 /* make sure it is true for two checks in a row */
709 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
712 /* update completed stats and continue */
713 tx_ring->tx_stats.tx_done_old = tx_done;
714 /* reset the countdown */
715 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
722 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
723 * @adapter: driver private struct
725 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
728 /* Do the reset outside of interrupt context */
729 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
730 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
731 ixgbe_service_event_schedule(adapter);
736 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
737 * @q_vector: structure containing interrupt and ring information
738 * @tx_ring: tx ring to clean
740 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
741 struct ixgbe_ring *tx_ring)
743 struct ixgbe_adapter *adapter = q_vector->adapter;
744 struct ixgbe_tx_buffer *tx_buffer;
745 union ixgbe_adv_tx_desc *tx_desc;
746 unsigned int total_bytes = 0, total_packets = 0;
747 unsigned int budget = q_vector->tx.work_limit;
748 u16 i = tx_ring->next_to_clean;
750 tx_buffer = &tx_ring->tx_buffer_info[i];
751 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
753 for (; budget; budget--) {
754 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
756 /* if next_to_watch is not set then there is no work pending */
760 /* if DD is not set pending work has not been completed */
761 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
764 /* count the packet as being completed */
765 tx_ring->tx_stats.completed++;
767 /* clear next_to_watch to prevent false hangs */
768 tx_buffer->next_to_watch = NULL;
770 /* prevent any other reads prior to eop_desc being verified */
774 ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
775 tx_desc->wb.status = 0;
776 if (likely(tx_desc == eop_desc)) {
778 dev_kfree_skb_any(tx_buffer->skb);
779 tx_buffer->skb = NULL;
781 total_bytes += tx_buffer->bytecount;
782 total_packets += tx_buffer->gso_segs;
788 if (unlikely(i == tx_ring->count)) {
791 tx_buffer = tx_ring->tx_buffer_info;
792 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
798 tx_ring->next_to_clean = i;
799 u64_stats_update_begin(&tx_ring->syncp);
800 tx_ring->stats.bytes += total_bytes;
801 tx_ring->stats.packets += total_packets;
802 u64_stats_update_end(&tx_ring->syncp);
803 q_vector->tx.total_bytes += total_bytes;
804 q_vector->tx.total_packets += total_packets;
806 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
807 /* schedule immediate reset if we believe we hung */
808 struct ixgbe_hw *hw = &adapter->hw;
809 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
810 e_err(drv, "Detected Tx Unit Hang\n"
812 " TDH, TDT <%x>, <%x>\n"
813 " next_to_use <%x>\n"
814 " next_to_clean <%x>\n"
815 "tx_buffer_info[next_to_clean]\n"
816 " time_stamp <%lx>\n"
818 tx_ring->queue_index,
819 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
820 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
821 tx_ring->next_to_use, i,
822 tx_ring->tx_buffer_info[i].time_stamp, jiffies);
824 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
827 "tx hang %d detected on queue %d, resetting adapter\n",
828 adapter->tx_timeout_count + 1, tx_ring->queue_index);
830 /* schedule immediate reset if we believe we hung */
831 ixgbe_tx_timeout_reset(adapter);
833 /* the adapter is about to reset, no point in enabling stuff */
837 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
838 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
839 (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
840 /* Make sure that anybody stopping the queue after this
841 * sees the new next_to_clean.
844 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
845 !test_bit(__IXGBE_DOWN, &adapter->state)) {
846 netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
847 ++tx_ring->tx_stats.restart_queue;
854 #ifdef CONFIG_IXGBE_DCA
855 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
856 struct ixgbe_ring *rx_ring,
859 struct ixgbe_hw *hw = &adapter->hw;
861 u8 reg_idx = rx_ring->reg_idx;
863 rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
864 switch (hw->mac.type) {
865 case ixgbe_mac_82598EB:
866 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
867 rxctrl |= dca3_get_tag(rx_ring->dev, cpu);
869 case ixgbe_mac_82599EB:
871 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
872 rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) <<
873 IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
878 rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
879 rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
880 rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
881 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
884 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
885 struct ixgbe_ring *tx_ring,
888 struct ixgbe_hw *hw = &adapter->hw;
890 u8 reg_idx = tx_ring->reg_idx;
892 switch (hw->mac.type) {
893 case ixgbe_mac_82598EB:
894 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
895 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
896 txctrl |= dca3_get_tag(tx_ring->dev, cpu);
897 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
898 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
900 case ixgbe_mac_82599EB:
902 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
903 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
904 txctrl |= (dca3_get_tag(tx_ring->dev, cpu) <<
905 IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
906 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
907 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
914 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
916 struct ixgbe_adapter *adapter = q_vector->adapter;
917 struct ixgbe_ring *ring;
920 if (q_vector->cpu == cpu)
923 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
924 ixgbe_update_tx_dca(adapter, ring, cpu);
926 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
927 ixgbe_update_rx_dca(adapter, ring, cpu);
934 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
939 if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
942 /* always use CB2 mode, difference is masked in the CB driver */
943 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
945 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
946 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
950 for (i = 0; i < num_q_vectors; i++) {
951 adapter->q_vector[i]->cpu = -1;
952 ixgbe_update_dca(adapter->q_vector[i]);
956 static int __ixgbe_notify_dca(struct device *dev, void *data)
958 struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
959 unsigned long event = *(unsigned long *)data;
961 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
965 case DCA_PROVIDER_ADD:
966 /* if we're already enabled, don't do it again */
967 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
969 if (dca_add_requester(dev) == 0) {
970 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
971 ixgbe_setup_dca(adapter);
974 /* Fall Through since DCA is disabled. */
975 case DCA_PROVIDER_REMOVE:
976 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
977 dca_remove_requester(dev);
978 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
979 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
986 #endif /* CONFIG_IXGBE_DCA */
988 static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
991 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
995 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
996 * @adapter: address of board private structure
997 * @rx_desc: advanced rx descriptor
999 * Returns : true if it is FCoE pkt
1001 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
1002 union ixgbe_adv_rx_desc *rx_desc)
1004 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1006 return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
1007 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1008 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1009 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1013 * ixgbe_receive_skb - Send a completed packet up the stack
1014 * @adapter: board private structure
1015 * @skb: packet to send up
1016 * @status: hardware indication of status of receive
1017 * @rx_ring: rx descriptor ring (for a specific queue) to setup
1018 * @rx_desc: rx descriptor
1020 static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1021 struct sk_buff *skb, u8 status,
1022 struct ixgbe_ring *ring,
1023 union ixgbe_adv_rx_desc *rx_desc)
1025 struct ixgbe_adapter *adapter = q_vector->adapter;
1026 struct napi_struct *napi = &q_vector->napi;
1027 bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1028 u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1030 if (is_vlan && (tag & VLAN_VID_MASK))
1031 __vlan_hwaccel_put_tag(skb, tag);
1033 if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1034 napi_gro_receive(napi, skb);
1040 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1041 * @adapter: address of board private structure
1042 * @status_err: hardware indication of status of receive
1043 * @skb: skb currently being received and modified
1044 * @status_err: status error value of last descriptor in packet
1046 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1047 union ixgbe_adv_rx_desc *rx_desc,
1048 struct sk_buff *skb,
1051 skb->ip_summed = CHECKSUM_NONE;
1053 /* Rx csum disabled */
1054 if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1057 /* if IP and error */
1058 if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1059 (status_err & IXGBE_RXDADV_ERR_IPE)) {
1060 adapter->hw_csum_rx_error++;
1064 if (!(status_err & IXGBE_RXD_STAT_L4CS))
1067 if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1068 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1071 * 82599 errata, UDP frames with a 0 checksum can be marked as
1074 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1075 (adapter->hw.mac.type == ixgbe_mac_82599EB))
1078 adapter->hw_csum_rx_error++;
1082 /* It must be a TCP or UDP packet with a valid checksum */
1083 skb->ip_summed = CHECKSUM_UNNECESSARY;
1086 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1089 * Force memory writes to complete before letting h/w
1090 * know there are new descriptors to fetch. (Only
1091 * applicable for weak-ordered memory model archs,
1095 writel(val, rx_ring->tail);
1099 * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1100 * @rx_ring: ring to place buffers on
1101 * @cleaned_count: number of buffers to replace
1103 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1105 union ixgbe_adv_rx_desc *rx_desc;
1106 struct ixgbe_rx_buffer *bi;
1107 struct sk_buff *skb;
1108 u16 i = rx_ring->next_to_use;
1110 /* do nothing if no valid netdev defined */
1111 if (!rx_ring->netdev)
1114 while (cleaned_count--) {
1115 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1116 bi = &rx_ring->rx_buffer_info[i];
1120 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1121 rx_ring->rx_buf_len);
1123 rx_ring->rx_stats.alloc_rx_buff_failed++;
1126 /* initialize queue mapping */
1127 skb_record_rx_queue(skb, rx_ring->queue_index);
1132 bi->dma = dma_map_single(rx_ring->dev,
1134 rx_ring->rx_buf_len,
1136 if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1137 rx_ring->rx_stats.alloc_rx_buff_failed++;
1143 if (ring_is_ps_enabled(rx_ring)) {
1145 bi->page = netdev_alloc_page(rx_ring->netdev);
1147 rx_ring->rx_stats.alloc_rx_page_failed++;
1152 if (!bi->page_dma) {
1153 /* use a half page if we're re-using */
1154 bi->page_offset ^= PAGE_SIZE / 2;
1155 bi->page_dma = dma_map_page(rx_ring->dev,
1160 if (dma_mapping_error(rx_ring->dev,
1162 rx_ring->rx_stats.alloc_rx_page_failed++;
1168 /* Refresh the desc even if buffer_addrs didn't change
1169 * because each write-back erases this info. */
1170 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1171 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1173 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1174 rx_desc->read.hdr_addr = 0;
1178 if (i == rx_ring->count)
1183 if (rx_ring->next_to_use != i) {
1184 rx_ring->next_to_use = i;
1185 ixgbe_release_rx_desc(rx_ring, i);
1189 static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1191 /* HW will not DMA in data larger than the given buffer, even if it
1192 * parses the (NFS, of course) header to be larger. In that case, it
1193 * fills the header buffer and spills the rest into the page.
1195 u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1196 u16 hlen = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1197 IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1198 if (hlen > IXGBE_RX_HDR_SIZE)
1199 hlen = IXGBE_RX_HDR_SIZE;
1204 * ixgbe_transform_rsc_queue - change rsc queue into a full packet
1205 * @skb: pointer to the last skb in the rsc queue
1207 * This function changes a queue full of hw rsc buffers into a completed
1208 * packet. It uses the ->prev pointers to find the first packet and then
1209 * turns it into the frag list owner.
1211 static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1213 unsigned int frag_list_size = 0;
1214 unsigned int skb_cnt = 1;
1217 struct sk_buff *prev = skb->prev;
1218 frag_list_size += skb->len;
1224 skb_shinfo(skb)->frag_list = skb->next;
1226 skb->len += frag_list_size;
1227 skb->data_len += frag_list_size;
1228 skb->truesize += frag_list_size;
1229 IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
1234 static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
1236 return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
1237 IXGBE_RXDADV_RSCCNT_MASK);
1240 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1241 struct ixgbe_ring *rx_ring,
1244 struct ixgbe_adapter *adapter = q_vector->adapter;
1245 union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
1246 struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
1247 struct sk_buff *skb;
1248 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1249 const int current_node = numa_node_id();
1252 #endif /* IXGBE_FCOE */
1255 u16 cleaned_count = 0;
1256 bool pkt_is_rsc = false;
1258 i = rx_ring->next_to_clean;
1259 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1260 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1262 while (staterr & IXGBE_RXD_STAT_DD) {
1265 rmb(); /* read descriptor and rx_buffer_info after status DD */
1267 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1269 skb = rx_buffer_info->skb;
1270 rx_buffer_info->skb = NULL;
1271 prefetch(skb->data);
1273 if (ring_is_rsc_enabled(rx_ring))
1274 pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1276 /* linear means we are building an skb from multiple pages */
1277 if (!skb_is_nonlinear(skb)) {
1280 !(staterr & IXGBE_RXD_STAT_EOP) &&
1283 * When HWRSC is enabled, delay unmapping
1284 * of the first packet. It carries the
1285 * header information, HW may still
1286 * access the header after the writeback.
1287 * Only unmap it when EOP is reached
1289 IXGBE_RSC_CB(skb)->delay_unmap = true;
1290 IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1292 dma_unmap_single(rx_ring->dev,
1293 rx_buffer_info->dma,
1294 rx_ring->rx_buf_len,
1297 rx_buffer_info->dma = 0;
1299 if (ring_is_ps_enabled(rx_ring)) {
1300 hlen = ixgbe_get_hlen(rx_desc);
1301 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1303 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1308 /* assume packet split since header is unmapped */
1309 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1313 dma_unmap_page(rx_ring->dev,
1314 rx_buffer_info->page_dma,
1317 rx_buffer_info->page_dma = 0;
1318 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1319 rx_buffer_info->page,
1320 rx_buffer_info->page_offset,
1323 if ((page_count(rx_buffer_info->page) == 1) &&
1324 (page_to_nid(rx_buffer_info->page) == current_node))
1325 get_page(rx_buffer_info->page);
1327 rx_buffer_info->page = NULL;
1329 skb->len += upper_len;
1330 skb->data_len += upper_len;
1331 skb->truesize += PAGE_SIZE / 2;
1335 if (i == rx_ring->count)
1338 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1343 u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1344 IXGBE_RXDADV_NEXTP_SHIFT;
1345 next_buffer = &rx_ring->rx_buffer_info[nextp];
1347 next_buffer = &rx_ring->rx_buffer_info[i];
1350 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1351 if (ring_is_ps_enabled(rx_ring)) {
1352 rx_buffer_info->skb = next_buffer->skb;
1353 rx_buffer_info->dma = next_buffer->dma;
1354 next_buffer->skb = skb;
1355 next_buffer->dma = 0;
1357 skb->next = next_buffer->skb;
1358 skb->next->prev = skb;
1360 rx_ring->rx_stats.non_eop_descs++;
1365 skb = ixgbe_transform_rsc_queue(skb);
1366 /* if we got here without RSC the packet is invalid */
1368 __pskb_trim(skb, 0);
1369 rx_buffer_info->skb = skb;
1374 if (ring_is_rsc_enabled(rx_ring)) {
1375 if (IXGBE_RSC_CB(skb)->delay_unmap) {
1376 dma_unmap_single(rx_ring->dev,
1377 IXGBE_RSC_CB(skb)->dma,
1378 rx_ring->rx_buf_len,
1380 IXGBE_RSC_CB(skb)->dma = 0;
1381 IXGBE_RSC_CB(skb)->delay_unmap = false;
1385 if (ring_is_ps_enabled(rx_ring))
1386 rx_ring->rx_stats.rsc_count +=
1387 skb_shinfo(skb)->nr_frags;
1389 rx_ring->rx_stats.rsc_count +=
1390 IXGBE_RSC_CB(skb)->skb_cnt;
1391 rx_ring->rx_stats.rsc_flush++;
1394 /* ERR_MASK will only have valid bits if EOP set */
1395 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
1396 dev_kfree_skb_any(skb);
1400 ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
1401 if (adapter->netdev->features & NETIF_F_RXHASH)
1402 ixgbe_rx_hash(rx_desc, skb);
1404 /* probably a little skewed due to removing CRC */
1405 total_rx_bytes += skb->len;
1408 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1410 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1411 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
1412 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
1415 dev_kfree_skb_any(skb);
1419 #endif /* IXGBE_FCOE */
1420 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1424 rx_desc->wb.upper.status_error = 0;
1429 /* return some buffers to hardware, one at a time is too slow */
1430 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1431 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1435 /* use prefetched values */
1437 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1440 rx_ring->next_to_clean = i;
1441 cleaned_count = ixgbe_desc_unused(rx_ring);
1444 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1447 /* include DDPed FCoE data */
1448 if (ddp_bytes > 0) {
1451 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1452 sizeof(struct fc_frame_header) -
1453 sizeof(struct fcoe_crc_eof);
1456 total_rx_bytes += ddp_bytes;
1457 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1459 #endif /* IXGBE_FCOE */
1461 u64_stats_update_begin(&rx_ring->syncp);
1462 rx_ring->stats.packets += total_rx_packets;
1463 rx_ring->stats.bytes += total_rx_bytes;
1464 u64_stats_update_end(&rx_ring->syncp);
1465 q_vector->rx.total_packets += total_rx_packets;
1466 q_vector->rx.total_bytes += total_rx_bytes;
1472 * ixgbe_configure_msix - Configure MSI-X hardware
1473 * @adapter: board private structure
1475 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1478 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1480 struct ixgbe_q_vector *q_vector;
1481 int q_vectors, v_idx;
1484 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1486 /* Populate MSIX to EITR Select */
1487 if (adapter->num_vfs > 32) {
1488 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1489 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1493 * Populate the IVAR table and set the ITR values to the
1494 * corresponding register.
1496 for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1497 struct ixgbe_ring *ring;
1498 q_vector = adapter->q_vector[v_idx];
1500 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
1501 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1503 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
1504 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1506 if (q_vector->tx.ring && !q_vector->rx.ring) {
1507 /* tx only vector */
1508 if (adapter->tx_itr_setting == 1)
1509 q_vector->itr = IXGBE_10K_ITR;
1511 q_vector->itr = adapter->tx_itr_setting;
1513 /* rx or rx/tx vector */
1514 if (adapter->rx_itr_setting == 1)
1515 q_vector->itr = IXGBE_20K_ITR;
1517 q_vector->itr = adapter->rx_itr_setting;
1520 ixgbe_write_eitr(q_vector);
1523 switch (adapter->hw.mac.type) {
1524 case ixgbe_mac_82598EB:
1525 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1528 case ixgbe_mac_82599EB:
1529 case ixgbe_mac_X540:
1530 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1535 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1537 /* set up to autoclear timer, and the vectors */
1538 mask = IXGBE_EIMS_ENABLE_MASK;
1539 mask &= ~(IXGBE_EIMS_OTHER |
1540 IXGBE_EIMS_MAILBOX |
1543 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1546 enum latency_range {
1550 latency_invalid = 255
1554 * ixgbe_update_itr - update the dynamic ITR value based on statistics
1555 * @q_vector: structure containing interrupt and ring information
1556 * @ring_container: structure containing ring performance data
1558 * Stores a new ITR value based on packets and byte
1559 * counts during the last interrupt. The advantage of per interrupt
1560 * computation is faster updates and more accurate ITR for the current
1561 * traffic pattern. Constants in this function were computed
1562 * based on theoretical maximum wire speed and thresholds were set based
1563 * on testing data as well as attempting to minimize response time
1564 * while increasing bulk throughput.
1565 * this functionality is controlled by the InterruptThrottleRate module
1566 * parameter (see ixgbe_param.c)
1568 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1569 struct ixgbe_ring_container *ring_container)
1572 struct ixgbe_adapter *adapter = q_vector->adapter;
1573 int bytes = ring_container->total_bytes;
1574 int packets = ring_container->total_packets;
1576 u8 itr_setting = ring_container->itr;
1581 /* simple throttlerate management
1582 * 0-20MB/s lowest (100000 ints/s)
1583 * 20-100MB/s low (20000 ints/s)
1584 * 100-1249MB/s bulk (8000 ints/s)
1586 /* what was last interrupt timeslice? */
1587 timepassed_us = q_vector->itr >> 2;
1588 bytes_perint = bytes / timepassed_us; /* bytes/usec */
1590 switch (itr_setting) {
1591 case lowest_latency:
1592 if (bytes_perint > adapter->eitr_low)
1593 itr_setting = low_latency;
1596 if (bytes_perint > adapter->eitr_high)
1597 itr_setting = bulk_latency;
1598 else if (bytes_perint <= adapter->eitr_low)
1599 itr_setting = lowest_latency;
1602 if (bytes_perint <= adapter->eitr_high)
1603 itr_setting = low_latency;
1607 /* clear work counters since we have the values we need */
1608 ring_container->total_bytes = 0;
1609 ring_container->total_packets = 0;
1611 /* write updated itr to ring container */
1612 ring_container->itr = itr_setting;
1616 * ixgbe_write_eitr - write EITR register in hardware specific way
1617 * @q_vector: structure containing interrupt and ring information
1619 * This function is made to be called by ethtool and by the driver
1620 * when it needs to update EITR registers at runtime. Hardware
1621 * specific quirks/differences are taken care of here.
1623 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1625 struct ixgbe_adapter *adapter = q_vector->adapter;
1626 struct ixgbe_hw *hw = &adapter->hw;
1627 int v_idx = q_vector->v_idx;
1628 u32 itr_reg = q_vector->itr;
1630 switch (adapter->hw.mac.type) {
1631 case ixgbe_mac_82598EB:
1632 /* must write high and low 16 bits to reset counter */
1633 itr_reg |= (itr_reg << 16);
1635 case ixgbe_mac_82599EB:
1636 case ixgbe_mac_X540:
1638 * set the WDIS bit to not clear the timer bits and cause an
1639 * immediate assertion of the interrupt
1641 itr_reg |= IXGBE_EITR_CNT_WDIS;
1646 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1649 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
1651 u32 new_itr = q_vector->itr;
1654 ixgbe_update_itr(q_vector, &q_vector->tx);
1655 ixgbe_update_itr(q_vector, &q_vector->rx);
1657 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1659 switch (current_itr) {
1660 /* counts and packets in update_itr are dependent on these numbers */
1661 case lowest_latency:
1662 new_itr = IXGBE_100K_ITR;
1665 new_itr = IXGBE_20K_ITR;
1668 new_itr = IXGBE_8K_ITR;
1674 if (new_itr != q_vector->itr) {
1675 /* do an exponential smoothing */
1676 new_itr = (10 * new_itr * q_vector->itr) /
1677 ((9 * new_itr) + q_vector->itr);
1679 /* save the algorithm value here */
1680 q_vector->itr = new_itr & IXGBE_MAX_EITR;
1682 ixgbe_write_eitr(q_vector);
1687 * ixgbe_check_overtemp_subtask - check for over tempurature
1688 * @adapter: pointer to adapter
1690 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
1692 struct ixgbe_hw *hw = &adapter->hw;
1693 u32 eicr = adapter->interrupt_event;
1695 if (test_bit(__IXGBE_DOWN, &adapter->state))
1698 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1699 !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
1702 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1704 switch (hw->device_id) {
1705 case IXGBE_DEV_ID_82599_T3_LOM:
1707 * Since the warning interrupt is for both ports
1708 * we don't have to check if:
1709 * - This interrupt wasn't for our port.
1710 * - We may have missed the interrupt so always have to
1711 * check if we got a LSC
1713 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
1714 !(eicr & IXGBE_EICR_LSC))
1717 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
1719 bool link_up = false;
1721 hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1727 /* Check if this is not due to overtemp */
1728 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
1733 if (!(eicr & IXGBE_EICR_GPI_SDP0))
1738 "Network adapter has been stopped because it has over heated. "
1739 "Restart the computer. If the problem persists, "
1740 "power off the system and replace the adapter\n");
1742 adapter->interrupt_event = 0;
1745 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1747 struct ixgbe_hw *hw = &adapter->hw;
1749 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1750 (eicr & IXGBE_EICR_GPI_SDP1)) {
1751 e_crit(probe, "Fan has stopped, replace the adapter\n");
1752 /* write to clear the interrupt */
1753 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1757 static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)
1759 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))
1762 switch (adapter->hw.mac.type) {
1763 case ixgbe_mac_82599EB:
1765 * Need to check link state so complete overtemp check
1768 if (((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC)) &&
1769 (!test_bit(__IXGBE_DOWN, &adapter->state))) {
1770 adapter->interrupt_event = eicr;
1771 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1772 ixgbe_service_event_schedule(adapter);
1776 case ixgbe_mac_X540:
1777 if (!(eicr & IXGBE_EICR_TS))
1785 "Network adapter has been stopped because it has over heated. "
1786 "Restart the computer. If the problem persists, "
1787 "power off the system and replace the adapter\n");
1790 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1792 struct ixgbe_hw *hw = &adapter->hw;
1794 if (eicr & IXGBE_EICR_GPI_SDP2) {
1795 /* Clear the interrupt */
1796 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1797 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1798 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
1799 ixgbe_service_event_schedule(adapter);
1803 if (eicr & IXGBE_EICR_GPI_SDP1) {
1804 /* Clear the interrupt */
1805 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1806 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1807 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
1808 ixgbe_service_event_schedule(adapter);
1813 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1815 struct ixgbe_hw *hw = &adapter->hw;
1818 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1819 adapter->link_check_timeout = jiffies;
1820 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1821 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1822 IXGBE_WRITE_FLUSH(hw);
1823 ixgbe_service_event_schedule(adapter);
1827 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
1831 struct ixgbe_hw *hw = &adapter->hw;
1833 switch (hw->mac.type) {
1834 case ixgbe_mac_82598EB:
1835 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1836 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1838 case ixgbe_mac_82599EB:
1839 case ixgbe_mac_X540:
1840 mask = (qmask & 0xFFFFFFFF);
1842 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1843 mask = (qmask >> 32);
1845 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1850 /* skip the flush */
1853 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1857 struct ixgbe_hw *hw = &adapter->hw;
1859 switch (hw->mac.type) {
1860 case ixgbe_mac_82598EB:
1861 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1862 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1864 case ixgbe_mac_82599EB:
1865 case ixgbe_mac_X540:
1866 mask = (qmask & 0xFFFFFFFF);
1868 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1869 mask = (qmask >> 32);
1871 IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1876 /* skip the flush */
1880 * ixgbe_irq_enable - Enable default interrupt generation settings
1881 * @adapter: board private structure
1883 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
1886 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1888 /* don't reenable LSC while waiting for link */
1889 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
1890 mask &= ~IXGBE_EIMS_LSC;
1892 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
1893 switch (adapter->hw.mac.type) {
1894 case ixgbe_mac_82599EB:
1895 mask |= IXGBE_EIMS_GPI_SDP0;
1897 case ixgbe_mac_X540:
1898 mask |= IXGBE_EIMS_TS;
1903 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
1904 mask |= IXGBE_EIMS_GPI_SDP1;
1905 switch (adapter->hw.mac.type) {
1906 case ixgbe_mac_82599EB:
1907 mask |= IXGBE_EIMS_GPI_SDP1;
1908 mask |= IXGBE_EIMS_GPI_SDP2;
1909 case ixgbe_mac_X540:
1910 mask |= IXGBE_EIMS_ECC;
1911 mask |= IXGBE_EIMS_MAILBOX;
1916 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
1917 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
1918 mask |= IXGBE_EIMS_FLOW_DIR;
1920 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
1922 ixgbe_irq_enable_queues(adapter, ~0);
1924 IXGBE_WRITE_FLUSH(&adapter->hw);
1927 static irqreturn_t ixgbe_msix_other(int irq, void *data)
1929 struct ixgbe_adapter *adapter = data;
1930 struct ixgbe_hw *hw = &adapter->hw;
1934 * Workaround for Silicon errata. Use clear-by-write instead
1935 * of clear-by-read. Reading with EICS will return the
1936 * interrupt causes without clearing, which later be done
1937 * with the write to EICR.
1939 eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1940 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1942 if (eicr & IXGBE_EICR_LSC)
1943 ixgbe_check_lsc(adapter);
1945 if (eicr & IXGBE_EICR_MAILBOX)
1946 ixgbe_msg_task(adapter);
1948 switch (hw->mac.type) {
1949 case ixgbe_mac_82599EB:
1950 case ixgbe_mac_X540:
1951 if (eicr & IXGBE_EICR_ECC)
1952 e_info(link, "Received unrecoverable ECC Err, please "
1954 /* Handle Flow Director Full threshold interrupt */
1955 if (eicr & IXGBE_EICR_FLOW_DIR) {
1956 int reinit_count = 0;
1958 for (i = 0; i < adapter->num_tx_queues; i++) {
1959 struct ixgbe_ring *ring = adapter->tx_ring[i];
1960 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1965 /* no more flow director interrupts until after init */
1966 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
1967 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
1968 ixgbe_service_event_schedule(adapter);
1971 ixgbe_check_sfp_event(adapter, eicr);
1972 ixgbe_check_overtemp_event(adapter, eicr);
1978 ixgbe_check_fan_failure(adapter, eicr);
1980 /* re-enable the original interrupt state, no lsc, no queues */
1981 if (!test_bit(__IXGBE_DOWN, &adapter->state))
1982 ixgbe_irq_enable(adapter, false, false);
1987 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
1989 struct ixgbe_q_vector *q_vector = data;
1991 /* EIAM disabled interrupts (on this vector) for us */
1993 if (q_vector->rx.ring || q_vector->tx.ring)
1994 napi_schedule(&q_vector->napi);
1999 static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
2002 struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2003 struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
2005 rx_ring->q_vector = q_vector;
2006 rx_ring->next = q_vector->rx.ring;
2007 q_vector->rx.ring = rx_ring;
2008 q_vector->rx.count++;
2011 static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
2014 struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
2015 struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
2017 tx_ring->q_vector = q_vector;
2018 tx_ring->next = q_vector->tx.ring;
2019 q_vector->tx.ring = tx_ring;
2020 q_vector->tx.count++;
2021 q_vector->tx.work_limit = a->tx_work_limit;
2025 * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
2026 * @adapter: board private structure to initialize
2028 * This function maps descriptor rings to the queue-specific vectors
2029 * we were allotted through the MSI-X enabling code. Ideally, we'd have
2030 * one vector per ring/queue, but on a constrained vector budget, we
2031 * group the rings as "efficiently" as possible. You would add new
2032 * mapping configurations in here.
2034 static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2036 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2037 int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0;
2038 int txr_remaining = adapter->num_tx_queues, txr_idx = 0;
2041 /* only one q_vector if MSI-X is disabled. */
2042 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2046 * If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2047 * group them so there are multiple queues per vector.
2049 * Re-adjusting *qpv takes care of the remainder.
2051 for (; v_start < q_vectors && rxr_remaining; v_start++) {
2052 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start);
2053 for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--)
2054 map_vector_to_rxq(adapter, v_start, rxr_idx);
2058 * If there are not enough q_vectors for each ring to have it's own
2059 * vector then we must pair up Rx/Tx on a each vector
2061 if ((v_start + txr_remaining) > q_vectors)
2064 for (; v_start < q_vectors && txr_remaining; v_start++) {
2065 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start);
2066 for (; tqpv; tqpv--, txr_idx++, txr_remaining--)
2067 map_vector_to_txq(adapter, v_start, txr_idx);
2072 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2073 * @adapter: board private structure
2075 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2076 * interrupts from the kernel.
2078 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2080 struct net_device *netdev = adapter->netdev;
2081 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2085 for (vector = 0; vector < q_vectors; vector++) {
2086 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2087 struct msix_entry *entry = &adapter->msix_entries[vector];
2089 if (q_vector->tx.ring && q_vector->rx.ring) {
2090 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2091 "%s-%s-%d", netdev->name, "TxRx", ri++);
2093 } else if (q_vector->rx.ring) {
2094 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2095 "%s-%s-%d", netdev->name, "rx", ri++);
2096 } else if (q_vector->tx.ring) {
2097 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2098 "%s-%s-%d", netdev->name, "tx", ti++);
2100 /* skip this unused q_vector */
2103 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2104 q_vector->name, q_vector);
2106 e_err(probe, "request_irq failed for MSIX interrupt "
2107 "Error: %d\n", err);
2108 goto free_queue_irqs;
2110 /* If Flow Director is enabled, set interrupt affinity */
2111 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2112 /* assign the mask for this irq */
2113 irq_set_affinity_hint(entry->vector,
2114 q_vector->affinity_mask);
2118 err = request_irq(adapter->msix_entries[vector].vector,
2119 ixgbe_msix_other, 0, netdev->name, adapter);
2121 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2122 goto free_queue_irqs;
2130 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2132 free_irq(adapter->msix_entries[vector].vector,
2133 adapter->q_vector[vector]);
2135 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2136 pci_disable_msix(adapter->pdev);
2137 kfree(adapter->msix_entries);
2138 adapter->msix_entries = NULL;
2143 * ixgbe_intr - legacy mode Interrupt Handler
2144 * @irq: interrupt number
2145 * @data: pointer to a network interface device structure
2147 static irqreturn_t ixgbe_intr(int irq, void *data)
2149 struct ixgbe_adapter *adapter = data;
2150 struct ixgbe_hw *hw = &adapter->hw;
2151 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2155 * Workaround for silicon errata on 82598. Mask the interrupts
2156 * before the read of EICR.
2158 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2160 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2161 * therefore no explict interrupt disable is necessary */
2162 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2165 * shared interrupt alert!
2166 * make sure interrupts are enabled because the read will
2167 * have disabled interrupts due to EIAM
2168 * finish the workaround of silicon errata on 82598. Unmask
2169 * the interrupt that we masked before the EICR read.
2171 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2172 ixgbe_irq_enable(adapter, true, true);
2173 return IRQ_NONE; /* Not our interrupt */
2176 if (eicr & IXGBE_EICR_LSC)
2177 ixgbe_check_lsc(adapter);
2179 switch (hw->mac.type) {
2180 case ixgbe_mac_82599EB:
2181 ixgbe_check_sfp_event(adapter, eicr);
2183 case ixgbe_mac_X540:
2184 if (eicr & IXGBE_EICR_ECC)
2185 e_info(link, "Received unrecoverable ECC err, please "
2187 ixgbe_check_overtemp_event(adapter, eicr);
2193 ixgbe_check_fan_failure(adapter, eicr);
2195 if (napi_schedule_prep(&(q_vector->napi))) {
2196 /* would disable interrupts here but EIAM disabled it */
2197 __napi_schedule(&(q_vector->napi));
2201 * re-enable link(maybe) and non-queue interrupts, no flush.
2202 * ixgbe_poll will re-enable the queue interrupts
2205 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2206 ixgbe_irq_enable(adapter, false, false);
2211 static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2213 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2216 /* legacy and MSI only use one vector */
2217 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2220 for (i = 0; i < adapter->num_rx_queues; i++) {
2221 adapter->rx_ring[i]->q_vector = NULL;
2222 adapter->rx_ring[i]->next = NULL;
2224 for (i = 0; i < adapter->num_tx_queues; i++) {
2225 adapter->tx_ring[i]->q_vector = NULL;
2226 adapter->tx_ring[i]->next = NULL;
2229 for (i = 0; i < q_vectors; i++) {
2230 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2231 memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container));
2232 memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container));
2237 * ixgbe_request_irq - initialize interrupts
2238 * @adapter: board private structure
2240 * Attempts to configure interrupts using the best available
2241 * capabilities of the hardware and kernel.
2243 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2245 struct net_device *netdev = adapter->netdev;
2248 /* map all of the rings to the q_vectors */
2249 ixgbe_map_rings_to_vectors(adapter);
2251 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
2252 err = ixgbe_request_msix_irqs(adapter);
2253 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
2254 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2255 netdev->name, adapter);
2257 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2258 netdev->name, adapter);
2261 e_err(probe, "request_irq failed, Error %d\n", err);
2263 /* place q_vectors and rings back into a known good state */
2264 ixgbe_reset_q_vectors(adapter);
2270 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2272 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2275 q_vectors = adapter->num_msix_vectors;
2277 free_irq(adapter->msix_entries[i].vector, adapter);
2280 for (; i >= 0; i--) {
2281 /* free only the irqs that were actually requested */
2282 if (!adapter->q_vector[i]->rx.ring &&
2283 !adapter->q_vector[i]->tx.ring)
2286 /* clear the affinity_mask in the IRQ descriptor */
2287 irq_set_affinity_hint(adapter->msix_entries[i].vector,
2290 free_irq(adapter->msix_entries[i].vector,
2291 adapter->q_vector[i]);
2294 free_irq(adapter->pdev->irq, adapter);
2297 /* clear q_vector state information */
2298 ixgbe_reset_q_vectors(adapter);
2302 * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2303 * @adapter: board private structure
2305 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2307 switch (adapter->hw.mac.type) {
2308 case ixgbe_mac_82598EB:
2309 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2311 case ixgbe_mac_82599EB:
2312 case ixgbe_mac_X540:
2313 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2314 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2315 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2320 IXGBE_WRITE_FLUSH(&adapter->hw);
2321 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2323 for (i = 0; i < adapter->num_msix_vectors; i++)
2324 synchronize_irq(adapter->msix_entries[i].vector);
2326 synchronize_irq(adapter->pdev->irq);
2331 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2334 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2336 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2339 if (adapter->rx_itr_setting == 1)
2340 q_vector->itr = IXGBE_20K_ITR;
2342 q_vector->itr = adapter->rx_itr_setting;
2344 ixgbe_write_eitr(q_vector);
2346 ixgbe_set_ivar(adapter, 0, 0, 0);
2347 ixgbe_set_ivar(adapter, 1, 0, 0);
2349 e_info(hw, "Legacy interrupt IVAR setup done\n");
2353 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2354 * @adapter: board private structure
2355 * @ring: structure containing ring specific data
2357 * Configure the Tx descriptor ring after a reset.
2359 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2360 struct ixgbe_ring *ring)
2362 struct ixgbe_hw *hw = &adapter->hw;
2363 u64 tdba = ring->dma;
2365 u32 txdctl = IXGBE_TXDCTL_ENABLE;
2366 u8 reg_idx = ring->reg_idx;
2368 /* disable queue to avoid issues while updating state */
2369 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2370 IXGBE_WRITE_FLUSH(hw);
2372 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2373 (tdba & DMA_BIT_MASK(32)));
2374 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2375 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2376 ring->count * sizeof(union ixgbe_adv_tx_desc));
2377 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2378 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2379 ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2382 * set WTHRESH to encourage burst writeback, it should not be set
2383 * higher than 1 when ITR is 0 as it could cause false TX hangs
2385 * In order to avoid issues WTHRESH + PTHRESH should always be equal
2386 * to or less than the number of on chip descriptors, which is
2389 if (!adapter->tx_itr_setting || !adapter->rx_itr_setting)
2390 txdctl |= (1 << 16); /* WTHRESH = 1 */
2392 txdctl |= (8 << 16); /* WTHRESH = 8 */
2394 /* PTHRESH=32 is needed to avoid a Tx hang with DFP enabled. */
2395 txdctl |= (1 << 8) | /* HTHRESH = 1 */
2396 32; /* PTHRESH = 32 */
2398 /* reinitialize flowdirector state */
2399 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2400 adapter->atr_sample_rate) {
2401 ring->atr_sample_rate = adapter->atr_sample_rate;
2402 ring->atr_count = 0;
2403 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2405 ring->atr_sample_rate = 0;
2408 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2411 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2413 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2414 if (hw->mac.type == ixgbe_mac_82598EB &&
2415 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2418 /* poll to verify queue is enabled */
2420 usleep_range(1000, 2000);
2421 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2422 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2424 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2427 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2429 struct ixgbe_hw *hw = &adapter->hw;
2432 u8 tcs = netdev_get_num_tc(adapter->netdev);
2434 if (hw->mac.type == ixgbe_mac_82598EB)
2437 /* disable the arbiter while setting MTQC */
2438 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2439 rttdcs |= IXGBE_RTTDCS_ARBDIS;
2440 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2442 /* set transmit pool layout */
2443 switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2444 case (IXGBE_FLAG_SRIOV_ENABLED):
2445 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2446 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2450 reg = IXGBE_MTQC_64Q_1PB;
2452 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2454 reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2456 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2458 /* Enable Security TX Buffer IFG for multiple pb */
2460 reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2461 reg |= IXGBE_SECTX_DCB;
2462 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2467 /* re-enable the arbiter */
2468 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2469 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2473 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2474 * @adapter: board private structure
2476 * Configure the Tx unit of the MAC after a reset.
2478 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2480 struct ixgbe_hw *hw = &adapter->hw;
2484 ixgbe_setup_mtqc(adapter);
2486 if (hw->mac.type != ixgbe_mac_82598EB) {
2487 /* DMATXCTL.EN must be before Tx queues are enabled */
2488 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2489 dmatxctl |= IXGBE_DMATXCTL_TE;
2490 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2493 /* Setup the HW Tx Head and Tail descriptor pointers */
2494 for (i = 0; i < adapter->num_tx_queues; i++)
2495 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2498 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2500 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2501 struct ixgbe_ring *rx_ring)
2504 u8 reg_idx = rx_ring->reg_idx;
2506 switch (adapter->hw.mac.type) {
2507 case ixgbe_mac_82598EB: {
2508 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2509 const int mask = feature[RING_F_RSS].mask;
2510 reg_idx = reg_idx & mask;
2513 case ixgbe_mac_82599EB:
2514 case ixgbe_mac_X540:
2519 srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2521 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2522 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2523 if (adapter->num_vfs)
2524 srrctl |= IXGBE_SRRCTL_DROP_EN;
2526 srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2527 IXGBE_SRRCTL_BSIZEHDR_MASK;
2529 if (ring_is_ps_enabled(rx_ring)) {
2530 #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2531 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2533 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2535 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2537 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2538 IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2539 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2542 IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2545 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2547 struct ixgbe_hw *hw = &adapter->hw;
2548 static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2549 0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2550 0x6A3E67EA, 0x14364D17, 0x3BED200D};
2551 u32 mrqc = 0, reta = 0;
2554 u8 tcs = netdev_get_num_tc(adapter->netdev);
2555 int maxq = adapter->ring_feature[RING_F_RSS].indices;
2558 maxq = min(maxq, adapter->num_tx_queues / tcs);
2560 /* Fill out hash function seeds */
2561 for (i = 0; i < 10; i++)
2562 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2564 /* Fill out redirection table */
2565 for (i = 0, j = 0; i < 128; i++, j++) {
2568 /* reta = 4-byte sliding window of
2569 * 0x00..(indices-1)(indices-1)00..etc. */
2570 reta = (reta << 8) | (j * 0x11);
2572 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2575 /* Disable indicating checksum in descriptor, enables RSS hash */
2576 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2577 rxcsum |= IXGBE_RXCSUM_PCSD;
2578 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2580 if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
2581 (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
2582 mrqc = IXGBE_MRQC_RSSEN;
2584 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2585 | IXGBE_FLAG_SRIOV_ENABLED);
2588 case (IXGBE_FLAG_RSS_ENABLED):
2590 mrqc = IXGBE_MRQC_RSSEN;
2592 mrqc = IXGBE_MRQC_RTRSS4TCEN;
2594 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2596 case (IXGBE_FLAG_SRIOV_ENABLED):
2597 mrqc = IXGBE_MRQC_VMDQEN;
2604 /* Perform hash on these packet types */
2605 mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2606 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2607 | IXGBE_MRQC_RSS_FIELD_IPV6
2608 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2610 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2614 * ixgbe_configure_rscctl - enable RSC for the indicated ring
2615 * @adapter: address of board private structure
2616 * @index: index of ring to set
2618 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2619 struct ixgbe_ring *ring)
2621 struct ixgbe_hw *hw = &adapter->hw;
2624 u8 reg_idx = ring->reg_idx;
2626 if (!ring_is_rsc_enabled(ring))
2629 rx_buf_len = ring->rx_buf_len;
2630 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2631 rscctrl |= IXGBE_RSCCTL_RSCEN;
2633 * we must limit the number of descriptors so that the
2634 * total size of max desc * buf_len is not greater
2637 if (ring_is_ps_enabled(ring)) {
2638 #if (MAX_SKB_FRAGS > 16)
2639 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2640 #elif (MAX_SKB_FRAGS > 8)
2641 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2642 #elif (MAX_SKB_FRAGS > 4)
2643 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2645 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2648 if (rx_buf_len < IXGBE_RXBUFFER_4K)
2649 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2650 else if (rx_buf_len < IXGBE_RXBUFFER_8K)
2651 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2653 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2655 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2659 * ixgbe_set_uta - Set unicast filter table address
2660 * @adapter: board private structure
2662 * The unicast table address is a register array of 32-bit registers.
2663 * The table is meant to be used in a way similar to how the MTA is used
2664 * however due to certain limitations in the hardware it is necessary to
2665 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2666 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
2668 static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2670 struct ixgbe_hw *hw = &adapter->hw;
2673 /* The UTA table only exists on 82599 hardware and newer */
2674 if (hw->mac.type < ixgbe_mac_82599EB)
2677 /* we only need to do this if VMDq is enabled */
2678 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2681 for (i = 0; i < 128; i++)
2682 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2685 #define IXGBE_MAX_RX_DESC_POLL 10
2686 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
2687 struct ixgbe_ring *ring)
2689 struct ixgbe_hw *hw = &adapter->hw;
2690 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2692 u8 reg_idx = ring->reg_idx;
2694 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2695 if (hw->mac.type == ixgbe_mac_82598EB &&
2696 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2700 usleep_range(1000, 2000);
2701 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2702 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
2705 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
2706 "the polling period\n", reg_idx);
2710 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
2711 struct ixgbe_ring *ring)
2713 struct ixgbe_hw *hw = &adapter->hw;
2714 int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2716 u8 reg_idx = ring->reg_idx;
2718 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2719 rxdctl &= ~IXGBE_RXDCTL_ENABLE;
2721 /* write value back with RXDCTL.ENABLE bit cleared */
2722 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2724 if (hw->mac.type == ixgbe_mac_82598EB &&
2725 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2728 /* the hardware may take up to 100us to really disable the rx queue */
2731 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2732 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
2735 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
2736 "the polling period\n", reg_idx);
2740 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
2741 struct ixgbe_ring *ring)
2743 struct ixgbe_hw *hw = &adapter->hw;
2744 u64 rdba = ring->dma;
2746 u8 reg_idx = ring->reg_idx;
2748 /* disable queue to avoid issues while updating state */
2749 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2750 ixgbe_disable_rx_queue(adapter, ring);
2752 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
2753 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
2754 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
2755 ring->count * sizeof(union ixgbe_adv_rx_desc));
2756 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
2757 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
2758 ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
2760 ixgbe_configure_srrctl(adapter, ring);
2761 ixgbe_configure_rscctl(adapter, ring);
2763 /* If operating in IOV mode set RLPML for X540 */
2764 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
2765 hw->mac.type == ixgbe_mac_X540) {
2766 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
2767 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
2768 ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
2771 if (hw->mac.type == ixgbe_mac_82598EB) {
2773 * enable cache line friendly hardware writes:
2774 * PTHRESH=32 descriptors (half the internal cache),
2775 * this also removes ugly rx_no_buffer_count increment
2776 * HTHRESH=4 descriptors (to minimize latency on fetch)
2777 * WTHRESH=8 burst writeback up to two cache lines
2779 rxdctl &= ~0x3FFFFF;
2783 /* enable receive descriptor ring */
2784 rxdctl |= IXGBE_RXDCTL_ENABLE;
2785 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2787 ixgbe_rx_desc_queue_enable(adapter, ring);
2788 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
2791 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
2793 struct ixgbe_hw *hw = &adapter->hw;
2796 /* PSRTYPE must be initialized in non 82598 adapters */
2797 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
2798 IXGBE_PSRTYPE_UDPHDR |
2799 IXGBE_PSRTYPE_IPV4HDR |
2800 IXGBE_PSRTYPE_L2HDR |
2801 IXGBE_PSRTYPE_IPV6HDR;
2803 if (hw->mac.type == ixgbe_mac_82598EB)
2806 if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
2807 psrtype |= (adapter->num_rx_queues_per_pool << 29);
2809 for (p = 0; p < adapter->num_rx_pools; p++)
2810 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
2814 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
2816 struct ixgbe_hw *hw = &adapter->hw;
2819 u32 reg_offset, vf_shift;
2823 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2826 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2827 vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
2828 vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
2829 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
2831 vf_shift = adapter->num_vfs % 32;
2832 reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
2834 /* Enable only the PF's pool for Tx/Rx */
2835 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
2836 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
2837 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
2838 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
2839 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2841 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
2842 hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
2845 * Set up VF register offsets for selected VT Mode,
2846 * i.e. 32 or 64 VFs for SR-IOV
2848 gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
2849 gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
2850 gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
2851 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
2853 /* enable Tx loopback for VF/PF communication */
2854 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2855 /* Enable MAC Anti-Spoofing */
2856 hw->mac.ops.set_mac_anti_spoofing(hw,
2857 (adapter->num_vfs != 0),
2859 /* For VFs that have spoof checking turned off */
2860 for (i = 0; i < adapter->num_vfs; i++) {
2861 if (!adapter->vfinfo[i].spoofchk_enabled)
2862 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i, false);