drivers/net/ethernet/intel/i40evf/i40evf_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * The full GNU General Public License is included in this distribution in
  16  * the file called "COPYING".
  17  *
  18  * Contact Information:
  19  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  20  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  21  *
  22  ******************************************************************************/
  23
  24 #include "i40evf.h"
  25 #include "i40e_prototype.h"
  26 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter);
  27 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter);
  28 static int i40evf_close(struct net_device *netdev);
  29
  30 char i40evf_driver_name[] = "i40evf";
  31 static const char i40evf_driver_string[] =
  32         "Intel(R) XL710 X710 Virtual Function Network Driver";
  33
  34 #define DRV_VERSION "0.9.14"
  35 const char i40evf_driver_version[] = DRV_VERSION;
  36 static const char i40evf_copyright[] =
  37         "Copyright (c) 2013 - 2014 Intel Corporation.";
  38
  39 /* i40evf_pci_tbl - PCI Device ID Table
  40  *
  41  * Wildcard entries (PCI_ANY_ID) should come last
  42  * Last entry must be all 0s
  43  *
  44  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  45  *   Class, Class Mask, private data (not used) }
  46  */
  47 static DEFINE_PCI_DEVICE_TABLE(i40evf_pci_tbl) = {
  48         {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF), 0},
  49         /* required last entry */
  50         {0, }
  51 };
  52
  53 MODULE_DEVICE_TABLE(pci, i40evf_pci_tbl);
  54
  55 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  56 MODULE_DESCRIPTION("Intel(R) XL710 X710 Virtual Function Network Driver");
  57 MODULE_LICENSE("GPL");
  58 MODULE_VERSION(DRV_VERSION);
  59
  60 /**
  61  * i40evf_allocate_dma_mem_d - OS specific memory alloc for shared code
  62  * @hw:   pointer to the HW structure
  63  * @mem:  ptr to mem struct to fill out
  64  * @size: size of memory requested
  65  * @alignment: what to align the allocation to
  66  **/
  67 i40e_status i40evf_allocate_dma_mem_d(struct i40e_hw *hw,
  68                                       struct i40e_dma_mem *mem,
  69                                       u64 size, u32 alignment)
  70 {
  71         struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
  72
  73         if (!mem)
  74                 return I40E_ERR_PARAM;
  75
  76         mem->size = ALIGN(size, alignment);
  77         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
  78                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
  79         if (mem->va)
  80                 return 0;
  81         else
  82                 return I40E_ERR_NO_MEMORY;
  83 }
  84
  85 /**
  86  * i40evf_free_dma_mem_d - OS specific memory free for shared code
  87  * @hw:   pointer to the HW structure
  88  * @mem:  ptr to mem struct to free
  89  **/
  90 i40e_status i40evf_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
  91 {
  92         struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
  93
  94         if (!mem || !mem->va)
  95                 return I40E_ERR_PARAM;
  96         dma_free_coherent(&adapter->pdev->dev, mem->size,
  97                           mem->va, (dma_addr_t)mem->pa);
  98         return 0;
  99 }
 100
 101 /**
 102  * i40evf_allocate_virt_mem_d - OS specific memory alloc for shared code
 103  * @hw:   pointer to the HW structure
 104  * @mem:  ptr to mem struct to fill out
 105  * @size: size of memory requested
 106  **/
 107 i40e_status i40evf_allocate_virt_mem_d(struct i40e_hw *hw,
 108                                        struct i40e_virt_mem *mem, u32 size)
 109 {
 110         if (!mem)
 111                 return I40E_ERR_PARAM;
 112
 113         mem->size = size;
 114         mem->va = kzalloc(size, GFP_KERNEL);
 115
 116         if (mem->va)
 117                 return 0;
 118         else
 119                 return I40E_ERR_NO_MEMORY;
 120 }
 121
 122 /**
 123  * i40evf_free_virt_mem_d - OS specific memory free for shared code
 124  * @hw:   pointer to the HW structure
 125  * @mem:  ptr to mem struct to free
 126  **/
 127 i40e_status i40evf_free_virt_mem_d(struct i40e_hw *hw,
 128                                    struct i40e_virt_mem *mem)
 129 {
 130         if (!mem)
 131                 return I40E_ERR_PARAM;
 132
 133         /* it's ok to kfree a NULL pointer */
 134         kfree(mem->va);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * i40evf_debug_d - OS dependent version of debug printing
 141  * @hw:  pointer to the HW structure
 142  * @mask: debug level mask
 143  * @fmt_str: printf-type format description
 144  **/
 145 void i40evf_debug_d(void *hw, u32 mask, char *fmt_str, ...)
 146 {
 147         char buf[512];
 148         va_list argptr;
 149
 150         if (!(mask & ((struct i40e_hw *)hw)->debug_mask))
 151                 return;
 152
 153         va_start(argptr, fmt_str);
 154         vsnprintf(buf, sizeof(buf), fmt_str, argptr);
 155         va_end(argptr);
 156
 157         /* the debug string is already formatted with a newline */
 158         pr_info("%s", buf);
 159 }
 160
 161 /**
 162  * i40evf_tx_timeout - Respond to a Tx Hang
 163  * @netdev: network interface device structure
 164  **/
 165 static void i40evf_tx_timeout(struct net_device *netdev)
 166 {
 167         struct i40evf_adapter *adapter = netdev_priv(netdev);
 168
 169         adapter->tx_timeout_count++;
 170         dev_info(&adapter->pdev->dev, "TX timeout detected.\n");
 171         if (!(adapter->flags & I40EVF_FLAG_RESET_PENDING)) {
 172                 dev_info(&adapter->pdev->dev, "Requesting reset from PF\n");
 173                 i40evf_request_reset(adapter);
 174                 adapter->flags |= I40EVF_FLAG_RESET_PENDING;
 175                 schedule_work(&adapter->reset_task);
 176         }
 177 }
 178
 179 /**
 180  * i40evf_misc_irq_disable - Mask off interrupt generation on the NIC
 181  * @adapter: board private structure
 182  **/
 183 static void i40evf_misc_irq_disable(struct i40evf_adapter *adapter)
 184 {
 185         struct i40e_hw *hw = &adapter->hw;
 186         wr32(hw, I40E_VFINT_DYN_CTL01, 0);
 187
 188         /* read flush */
 189         rd32(hw, I40E_VFGEN_RSTAT);
 190
 191         synchronize_irq(adapter->msix_entries[0].vector);
 192 }
 193
 194 /**
 195  * i40evf_misc_irq_enable - Enable default interrupt generation settings
 196  * @adapter: board private structure
 197  **/
 198 static void i40evf_misc_irq_enable(struct i40evf_adapter *adapter)
 199 {
 200         struct i40e_hw *hw = &adapter->hw;
 201         wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK |
 202                                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 203         wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA_ADMINQ_MASK);
 204
 205         /* read flush */
 206         rd32(hw, I40E_VFGEN_RSTAT);
 207 }
 208
 209 /**
 210  * i40evf_irq_disable - Mask off interrupt generation on the NIC
 211  * @adapter: board private structure
 212  **/
 213 static void i40evf_irq_disable(struct i40evf_adapter *adapter)
 214 {
 215         int i;
 216         struct i40e_hw *hw = &adapter->hw;
 217
 218         if (!adapter->msix_entries)
 219                 return;
 220
 221         for (i = 1; i < adapter->num_msix_vectors; i++) {
 222                 wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), 0);
 223                 synchronize_irq(adapter->msix_entries[i].vector);
 224         }
 225         /* read flush */
 226         rd32(hw, I40E_VFGEN_RSTAT);
 227
 228 }
 229
 230 /**
 231  * i40evf_irq_enable_queues - Enable interrupt for specified queues
 232  * @adapter: board private structure
 233  * @mask: bitmap of queues to enable
 234  **/
 235 void i40evf_irq_enable_queues(struct i40evf_adapter *adapter, u32 mask)
 236 {
 237         struct i40e_hw *hw = &adapter->hw;
 238         int i;
 239
 240         for (i = 1; i < adapter->num_msix_vectors; i++) {
 241                 if (mask & (1 << (i - 1))) {
 242                         wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1),
 243                              I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 244                              I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
 245                 }
 246         }
 247 }
 248
 249 /**
 250  * i40evf_fire_sw_int - Generate SW interrupt for specified vectors
 251  * @adapter: board private structure
 252  * @mask: bitmap of vectors to trigger
 253  **/
 254 static void i40evf_fire_sw_int(struct i40evf_adapter *adapter,
 255                                             u32 mask)
 256 {
 257         struct i40e_hw *hw = &adapter->hw;
 258         int i;
 259         uint32_t dyn_ctl;
 260
 261         for (i = 1; i < adapter->num_msix_vectors; i++) {
 262                 if (mask & (1 << i)) {
 263                         dyn_ctl = rd32(hw, I40E_VFINT_DYN_CTLN1(i - 1));
 264                         dyn_ctl |= I40E_VFINT_DYN_CTLN_SWINT_TRIG_MASK |
 265                                    I40E_VFINT_DYN_CTLN_CLEARPBA_MASK;
 266                         wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), dyn_ctl);
 267                 }
 268         }
 269 }
 270
 271 /**
 272  * i40evf_irq_enable - Enable default interrupt generation settings
 273  * @adapter: board private structure
 274  **/
 275 void i40evf_irq_enable(struct i40evf_adapter *adapter, bool flush)
 276 {
 277         struct i40e_hw *hw = &adapter->hw;
 278
 279         i40evf_irq_enable_queues(adapter, ~0);
 280
 281         if (flush)
 282                 rd32(hw, I40E_VFGEN_RSTAT);
 283 }
 284
 285 /**
 286  * i40evf_msix_aq - Interrupt handler for vector 0
 287  * @irq: interrupt number
 288  * @data: pointer to netdev
 289  **/
 290 static irqreturn_t i40evf_msix_aq(int irq, void *data)
 291 {
 292         struct net_device *netdev = data;
 293         struct i40evf_adapter *adapter = netdev_priv(netdev);
 294         struct i40e_hw *hw = &adapter->hw;
 295         u32 val;
 296         u32 ena_mask;
 297
 298         /* handle non-queue interrupts */
 299         val = rd32(hw, I40E_VFINT_ICR01);
 300         ena_mask = rd32(hw, I40E_VFINT_ICR0_ENA1);
 301
 302
 303         val = rd32(hw, I40E_VFINT_DYN_CTL01);
 304         val = val | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
 305         wr32(hw, I40E_VFINT_DYN_CTL01, val);
 306
 307         /* re-enable interrupt causes */
 308         wr32(hw, I40E_VFINT_ICR0_ENA1, ena_mask);
 309         wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK);
 310
 311         /* schedule work on the private workqueue */
 312         schedule_work(&adapter->adminq_task);
 313
 314         return IRQ_HANDLED;
 315 }
 316
 317 /**
 318  * i40evf_msix_clean_rings - MSIX mode Interrupt Handler
 319  * @irq: interrupt number
 320  * @data: pointer to a q_vector
 321  **/
 322 static irqreturn_t i40evf_msix_clean_rings(int irq, void *data)
 323 {
 324         struct i40e_q_vector *q_vector = data;
 325
 326         if (!q_vector->tx.ring && !q_vector->rx.ring)
 327                 return IRQ_HANDLED;
 328
 329         napi_schedule(&q_vector->napi);
 330
 331         return IRQ_HANDLED;
 332 }
 333
 334 /**
 335  * i40evf_map_vector_to_rxq - associate irqs with rx queues
 336  * @adapter: board private structure
 337  * @v_idx: interrupt number
 338  * @r_idx: queue number
 339  **/
 340 static void
 341 i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
 342 {
 343         struct i40e_q_vector *q_vector = adapter->q_vector[v_idx];
 344         struct i40e_ring *rx_ring = adapter->rx_rings[r_idx];
 345
 346         rx_ring->q_vector = q_vector;
 347         rx_ring->next = q_vector->rx.ring;
 348         rx_ring->vsi = &adapter->vsi;
 349         q_vector->rx.ring = rx_ring;
 350         q_vector->rx.count++;
 351         q_vector->rx.latency_range = I40E_LOW_LATENCY;
 352 }
 353
 354 /**
 355  * i40evf_map_vector_to_txq - associate irqs with tx queues
 356  * @adapter: board private structure
 357  * @v_idx: interrupt number
 358  * @t_idx: queue number
 359  **/
 360 static void
 361 i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
 362 {
 363         struct i40e_q_vector *q_vector = adapter->q_vector[v_idx];
 364         struct i40e_ring *tx_ring = adapter->tx_rings[t_idx];
 365
 366         tx_ring->q_vector = q_vector;
 367         tx_ring->next = q_vector->tx.ring;
 368         tx_ring->vsi = &adapter->vsi;
 369         q_vector->tx.ring = tx_ring;
 370         q_vector->tx.count++;
 371         q_vector->tx.latency_range = I40E_LOW_LATENCY;
 372         q_vector->num_ringpairs++;
 373         q_vector->ring_mask |= (1 << t_idx);
 374 }
 375
 376 /**
 377  * i40evf_map_rings_to_vectors - Maps descriptor rings to vectors
 378  * @adapter: board private structure to initialize
 379  *
 380  * This function maps descriptor rings to the queue-specific vectors
 381  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 382  * one vector per ring/queue, but on a constrained vector budget, we
 383  * group the rings as "efficiently" as possible.  You would add new
 384  * mapping configurations in here.
 385  **/
 386 static int i40evf_map_rings_to_vectors(struct i40evf_adapter *adapter)
 387 {
 388         int q_vectors;
 389         int v_start = 0;
 390         int rxr_idx = 0, txr_idx = 0;
 391         int rxr_remaining = adapter->vsi_res->num_queue_pairs;
 392         int txr_remaining = adapter->vsi_res->num_queue_pairs;
 393         int i, j;
 394         int rqpv, tqpv;
 395         int err = 0;
 396
 397         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 398
 399         /* The ideal configuration...
 400          * We have enough vectors to map one per queue.
 401          */
 402         if (q_vectors == (rxr_remaining * 2)) {
 403                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 404                         i40evf_map_vector_to_rxq(adapter, v_start, rxr_idx);
 405
 406                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 407                         i40evf_map_vector_to_txq(adapter, v_start, txr_idx);
 408                 goto out;
 409         }
 410
 411         /* If we don't have enough vectors for a 1-to-1
 412          * mapping, we'll have to group them so there are
 413          * multiple queues per vector.
 414          * Re-adjusting *qpv takes care of the remainder.
 415          */
 416         for (i = v_start; i < q_vectors; i++) {
 417                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 418                 for (j = 0; j < rqpv; j++) {
 419                         i40evf_map_vector_to_rxq(adapter, i, rxr_idx);
 420                         rxr_idx++;
 421                         rxr_remaining--;
 422                 }
 423         }
 424         for (i = v_start; i < q_vectors; i++) {
 425                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 426                 for (j = 0; j < tqpv; j++) {
 427                         i40evf_map_vector_to_txq(adapter, i, txr_idx);
 428                         txr_idx++;
 429                         txr_remaining--;
 430                 }
 431         }
 432
 433 out:
 434         adapter->aq_required |= I40EVF_FLAG_AQ_MAP_VECTORS;
 435
 436         return err;
 437 }
 438
 439 /**
 440  * i40evf_request_traffic_irqs - Initialize MSI-X interrupts
 441  * @adapter: board private structure
 442  *
 443  * Allocates MSI-X vectors for tx and rx handling, and requests
 444  * interrupts from the kernel.
 445  **/
 446 static int
 447 i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
 448 {
 449         int vector, err, q_vectors;
 450         int rx_int_idx = 0, tx_int_idx = 0;
 451
 452         i40evf_irq_disable(adapter);
 453         /* Decrement for Other and TCP Timer vectors */
 454         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 455
 456         for (vector = 0; vector < q_vectors; vector++) {
 457                 struct i40e_q_vector *q_vector = adapter->q_vector[vector];
 458
 459                 if (q_vector->tx.ring && q_vector->rx.ring) {
 460                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 461                                  "i40evf-%s-%s-%d", basename,
 462                                  "TxRx", rx_int_idx++);
 463                         tx_int_idx++;
 464                 } else if (q_vector->rx.ring) {
 465                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 466                                  "i40evf-%s-%s-%d", basename,
 467                                  "rx", rx_int_idx++);
 468                 } else if (q_vector->tx.ring) {
 469                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 470                                  "i40evf-%s-%s-%d", basename,
 471                                  "tx", tx_int_idx++);
 472                 } else {
 473                         /* skip this unused q_vector */
 474                         continue;
 475                 }
 476                 err = request_irq(
 477                         adapter->msix_entries[vector + NONQ_VECS].vector,
 478                         i40evf_msix_clean_rings,
 479                         0,
 480                         q_vector->name,
 481                         q_vector);
 482                 if (err) {
 483                         dev_info(&adapter->pdev->dev,
 484                                  "%s: request_irq failed, error: %d\n",
 485                                 __func__, err);
 486                         goto free_queue_irqs;
 487                 }
 488                 /* assign the mask for this irq */
 489                 irq_set_affinity_hint(
 490                         adapter->msix_entries[vector + NONQ_VECS].vector,
 491                         q_vector->affinity_mask);
 492         }
 493
 494         return 0;
 495
 496 free_queue_irqs:
 497         while (vector) {
 498                 vector--;
 499                 irq_set_affinity_hint(
 500                         adapter->msix_entries[vector + NONQ_VECS].vector,
 501                         NULL);
 502                 free_irq(adapter->msix_entries[vector + NONQ_VECS].vector,
 503                          adapter->q_vector[vector]);
 504         }
 505         return err;
 506 }
 507
 508 /**
 509  * i40evf_request_misc_irq - Initialize MSI-X interrupts
 510  * @adapter: board private structure
 511  *
 512  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 513  * vector is only for the admin queue, and stays active even when the netdev
 514  * is closed.
 515  **/
 516 static int i40evf_request_misc_irq(struct i40evf_adapter *adapter)
 517 {
 518         struct net_device *netdev = adapter->netdev;
 519         int err;
 520
 521         sprintf(adapter->misc_vector_name, "i40evf:mbx");
 522         err = request_irq(adapter->msix_entries[0].vector,
 523                           &i40evf_msix_aq, 0,
 524                           adapter->misc_vector_name, netdev);
 525         if (err) {
 526                 dev_err(&adapter->pdev->dev,
 527                         "request_irq for %s failed: %d\n",
 528                         adapter->misc_vector_name, err);
 529                 free_irq(adapter->msix_entries[0].vector, netdev);
 530         }
 531         return err;
 532 }
 533
 534 /**
 535  * i40evf_free_traffic_irqs - Free MSI-X interrupts
 536  * @adapter: board private structure
 537  *
 538  * Frees all MSI-X vectors other than 0.
 539  **/
 540 static void i40evf_free_traffic_irqs(struct i40evf_adapter *adapter)
 541 {
 542         int i;
 543         int q_vectors;
 544         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 545
 546         for (i = 0; i < q_vectors; i++) {
 547                 irq_set_affinity_hint(adapter->msix_entries[i+1].vector,
 548                                       NULL);
 549                 free_irq(adapter->msix_entries[i+1].vector,
 550                          adapter->q_vector[i]);
 551         }
 552 }
 553
 554 /**
 555  * i40evf_free_misc_irq - Free MSI-X miscellaneous vector
 556  * @adapter: board private structure
 557  *
 558  * Frees MSI-X vector 0.
 559  **/
 560 static void i40evf_free_misc_irq(struct i40evf_adapter *adapter)
 561 {
 562         struct net_device *netdev = adapter->netdev;
 563
 564         free_irq(adapter->msix_entries[0].vector, netdev);
 565 }
 566
 567 /**
 568  * i40evf_configure_tx - Configure Transmit Unit after Reset
 569  * @adapter: board private structure
 570  *
 571  * Configure the Tx unit of the MAC after a reset.
 572  **/
 573 static void i40evf_configure_tx(struct i40evf_adapter *adapter)
 574 {
 575         struct i40e_hw *hw = &adapter->hw;
 576         int i;
 577         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 578                 adapter->tx_rings[i]->tail = hw->hw_addr + I40E_QTX_TAIL1(i);
 579 }
 580
 581 /**
 582  * i40evf_configure_rx - Configure Receive Unit after Reset
 583  * @adapter: board private structure
 584  *
 585  * Configure the Rx unit of the MAC after a reset.
 586  **/
 587 static void i40evf_configure_rx(struct i40evf_adapter *adapter)
 588 {
 589         struct i40e_hw *hw = &adapter->hw;
 590         struct net_device *netdev = adapter->netdev;
 591         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
 592         int i;
 593         int rx_buf_len;
 594
 595
 596         adapter->flags &= ~I40EVF_FLAG_RX_PS_CAPABLE;
 597         adapter->flags |= I40EVF_FLAG_RX_1BUF_CAPABLE;
 598
 599         /* Decide whether to use packet split mode or not */
 600         if (netdev->mtu > ETH_DATA_LEN) {
 601                 if (adapter->flags & I40EVF_FLAG_RX_PS_CAPABLE)
 602                         adapter->flags |= I40EVF_FLAG_RX_PS_ENABLED;
 603                 else
 604                         adapter->flags &= ~I40EVF_FLAG_RX_PS_ENABLED;
 605         } else {
 606                 if (adapter->flags & I40EVF_FLAG_RX_1BUF_CAPABLE)
 607                         adapter->flags &= ~I40EVF_FLAG_RX_PS_ENABLED;
 608                 else
 609                         adapter->flags |= I40EVF_FLAG_RX_PS_ENABLED;
 610         }
 611
 612         /* Set the RX buffer length according to the mode */
 613         if (adapter->flags & I40EVF_FLAG_RX_PS_ENABLED) {
 614                 rx_buf_len = I40E_RX_HDR_SIZE;
 615         } else {
 616                 if (netdev->mtu <= ETH_DATA_LEN)
 617                         rx_buf_len = I40EVF_RXBUFFER_2048;
 618                 else
 619                         rx_buf_len = ALIGN(max_frame, 1024);
 620         }
 621
 622         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
 623                 adapter->rx_rings[i]->tail = hw->hw_addr + I40E_QRX_TAIL1(i);
 624                 adapter->rx_rings[i]->rx_buf_len = rx_buf_len;
 625         }
 626 }
 627
 628 /**
 629  * i40evf_find_vlan - Search filter list for specific vlan filter
 630  * @adapter: board private structure
 631  * @vlan: vlan tag
 632  *
 633  * Returns ptr to the filter object or NULL
 634  **/
 635 static struct
 636 i40evf_vlan_filter *i40evf_find_vlan(struct i40evf_adapter *adapter, u16 vlan)
 637 {
 638         struct i40evf_vlan_filter *f;
 639
 640         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 641                 if (vlan == f->vlan)
 642                         return f;
 643         }
 644         return NULL;
 645 }
 646
 647 /**
 648  * i40evf_add_vlan - Add a vlan filter to the list
 649  * @adapter: board private structure
 650  * @vlan: VLAN tag
 651  *
 652  * Returns ptr to the filter object or NULL when no memory available.
 653  **/
 654 static struct
 655 i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 656 {
 657         struct i40evf_vlan_filter *f;
 658
 659         f = i40evf_find_vlan(adapter, vlan);
 660         if (NULL == f) {
 661                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 662                 if (NULL == f) {
 663                         dev_info(&adapter->pdev->dev,
 664                                  "%s: no memory for new VLAN filter\n",
 665                                  __func__);
 666                         return NULL;
 667                 }
 668                 f->vlan = vlan;
 669
 670                 INIT_LIST_HEAD(&f->list);
 671                 list_add(&f->list, &adapter->vlan_filter_list);
 672                 f->add = true;
 673                 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
 674         }
 675
 676         return f;
 677 }
 678
 679 /**
 680  * i40evf_del_vlan - Remove a vlan filter from the list
 681  * @adapter: board private structure
 682  * @vlan: VLAN tag
 683  **/
 684 static void i40evf_del_vlan(struct i40evf_adapter *adapter, u16 vlan)
 685 {
 686         struct i40evf_vlan_filter *f;
 687
 688         f = i40evf_find_vlan(adapter, vlan);
 689         if (f) {
 690                 f->remove = true;
 691                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
 692         }
 693         return;
 694 }
 695
 696 /**
 697  * i40evf_vlan_rx_add_vid - Add a VLAN filter to a device
 698  * @netdev: network device struct
 699  * @vid: VLAN tag
 700  **/
 701 static int i40evf_vlan_rx_add_vid(struct net_device *netdev,
 702                          __always_unused __be16 proto, u16 vid)
 703 {
 704         struct i40evf_adapter *adapter = netdev_priv(netdev);
 705
 706         if (i40evf_add_vlan(adapter, vid) == NULL)
 707                 return -ENOMEM;
 708         return 0;
 709 }
 710
 711 /**
 712  * i40evf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 713  * @netdev: network device struct
 714  * @vid: VLAN tag
 715  **/
 716 static int i40evf_vlan_rx_kill_vid(struct net_device *netdev,
 717                           __always_unused __be16 proto, u16 vid)
 718 {
 719         struct i40evf_adapter *adapter = netdev_priv(netdev);
 720
 721         i40evf_del_vlan(adapter, vid);
 722         return 0;
 723 }
 724
 725 /**
 726  * i40evf_find_filter - Search filter list for specific mac filter
 727  * @adapter: board private structure
 728  * @macaddr: the MAC address
 729  *
 730  * Returns ptr to the filter object or NULL
 731  **/
 732 static struct
 733 i40evf_mac_filter *i40evf_find_filter(struct i40evf_adapter *adapter,
 734                                       u8 *macaddr)
 735 {
 736         struct i40evf_mac_filter *f;
 737
 738         if (!macaddr)
 739                 return NULL;
 740
 741         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 742                 if (ether_addr_equal(macaddr, f->macaddr))
 743                         return f;
 744         }
 745         return NULL;
 746 }
 747
 748 /**
 749  * i40e_add_filter - Add a mac filter to the filter list
 750  * @adapter: board private structure
 751  * @macaddr: the MAC address
 752  *
 753  * Returns ptr to the filter object or NULL when no memory available.
 754  **/
 755 static struct
 756 i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
 757                                      u8 *macaddr)
 758 {
 759         struct i40evf_mac_filter *f;
 760
 761         if (!macaddr)
 762                 return NULL;
 763
 764         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
 765                                 &adapter->crit_section))
 766                 mdelay(1);
 767
 768         f = i40evf_find_filter(adapter, macaddr);
 769         if (NULL == f) {
 770                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 771                 if (NULL == f) {
 772                         dev_info(&adapter->pdev->dev,
 773                                  "%s: no memory for new filter\n", __func__);
 774                         clear_bit(__I40EVF_IN_CRITICAL_TASK,
 775                                   &adapter->crit_section);
 776                         return NULL;
 777                 }
 778
 779                 memcpy(f->macaddr, macaddr, ETH_ALEN);
 780
 781                 list_add(&f->list, &adapter->mac_filter_list);
 782                 f->add = true;
 783                 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
 784         }
 785
 786         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 787         return f;
 788 }
 789
 790 /**
 791  * i40evf_set_mac - NDO callback to set port mac address
 792  * @netdev: network interface device structure
 793  * @p: pointer to an address structure
 794  *
 795  * Returns 0 on success, negative on failure
 796  **/
 797 static int i40evf_set_mac(struct net_device *netdev, void *p)
 798 {
 799         struct i40evf_adapter *adapter = netdev_priv(netdev);
 800         struct i40e_hw *hw = &adapter->hw;
 801         struct i40evf_mac_filter *f;
 802         struct sockaddr *addr = p;
 803
 804         if (!is_valid_ether_addr(addr->sa_data))
 805                 return -EADDRNOTAVAIL;
 806
 807         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
 808                 return 0;
 809
 810         f = i40evf_add_filter(adapter, addr->sa_data);
 811         if (f) {
 812                 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 813                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
 814                        netdev->addr_len);
 815         }
 816
 817         return (f == NULL) ? -ENOMEM : 0;
 818 }
 819
 820 /**
 821  * i40evf_set_rx_mode - NDO callback to set the netdev filters
 822  * @netdev: network interface device structure
 823  **/
 824 static void i40evf_set_rx_mode(struct net_device *netdev)
 825 {
 826         struct i40evf_adapter *adapter = netdev_priv(netdev);
 827         struct i40evf_mac_filter *f, *ftmp;
 828         struct netdev_hw_addr *uca;
 829         struct netdev_hw_addr *mca;
 830
 831         /* add addr if not already in the filter list */
 832         netdev_for_each_uc_addr(uca, netdev) {
 833                 i40evf_add_filter(adapter, uca->addr);
 834         }
 835         netdev_for_each_mc_addr(mca, netdev) {
 836                 i40evf_add_filter(adapter, mca->addr);
 837         }
 838
 839         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
 840                                 &adapter->crit_section))
 841                 mdelay(1);
 842         /* remove filter if not in netdev list */
 843         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
 844                 bool found = false;
 845
 846                 if (f->macaddr[0] & 0x01) {
 847                         netdev_for_each_mc_addr(mca, netdev) {
 848                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
 849                                         found = true;
 850                                         break;
 851                                 }
 852                         }
 853                 } else {
 854                         netdev_for_each_uc_addr(uca, netdev) {
 855                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
 856                                         found = true;
 857                                         break;
 858                                 }
 859                         }
 860                 }
 861                 if (found) {
 862                         f->remove = true;
 863                         adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
 864                 }
 865         }
 866         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 867 }
 868
 869 /**
 870  * i40evf_napi_enable_all - enable NAPI on all queue vectors
 871  * @adapter: board private structure
 872  **/
 873 static void i40evf_napi_enable_all(struct i40evf_adapter *adapter)
 874 {
 875         int q_idx;
 876         struct i40e_q_vector *q_vector;
 877         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 878
 879         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 880                 struct napi_struct *napi;
 881                 q_vector = adapter->q_vector[q_idx];
 882                 napi = &q_vector->napi;
 883                 napi_enable(napi);
 884         }
 885 }
 886
 887 /**
 888  * i40evf_napi_disable_all - disable NAPI on all queue vectors
 889  * @adapter: board private structure
 890  **/
 891 static void i40evf_napi_disable_all(struct i40evf_adapter *adapter)
 892 {
 893         int q_idx;
 894         struct i40e_q_vector *q_vector;
 895         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 896
 897         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 898                 q_vector = adapter->q_vector[q_idx];
 899                 napi_disable(&q_vector->napi);
 900         }
 901 }
 902
 903 /**
 904  * i40evf_configure - set up transmit and receive data structures
 905  * @adapter: board private structure
 906  **/
 907 static void i40evf_configure(struct i40evf_adapter *adapter)
 908 {
 909         struct net_device *netdev = adapter->netdev;
 910         int i;
 911
 912         i40evf_set_rx_mode(netdev);
 913
 914         i40evf_configure_tx(adapter);
 915         i40evf_configure_rx(adapter);
 916         adapter->aq_required |= I40EVF_FLAG_AQ_CONFIGURE_QUEUES;
 917
 918         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
 919                 struct i40e_ring *ring = adapter->rx_rings[i];
 920                 i40evf_alloc_rx_buffers(ring, ring->count);
 921                 ring->next_to_use = ring->count - 1;
 922                 writel(ring->next_to_use, ring->tail);
 923         }
 924 }
 925
 926 /**
 927  * i40evf_up_complete - Finish the last steps of bringing up a connection
 928  * @adapter: board private structure
 929  **/
 930 static int i40evf_up_complete(struct i40evf_adapter *adapter)
 931 {
 932         adapter->state = __I40EVF_RUNNING;
 933         clear_bit(__I40E_DOWN, &adapter->vsi.state);
 934
 935         i40evf_napi_enable_all(adapter);
 936
 937         adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_QUEUES;
 938         mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 939         return 0;
 940 }
 941
 942 /**
 943  * i40evf_clean_all_rx_rings - Free Rx Buffers for all queues
 944  * @adapter: board private structure
 945  **/
 946 static void i40evf_clean_all_rx_rings(struct i40evf_adapter *adapter)
 947 {
 948         int i;
 949
 950         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 951                 i40evf_clean_rx_ring(adapter->rx_rings[i]);
 952 }
 953
 954 /**
 955  * i40evf_clean_all_tx_rings - Free Tx Buffers for all queues
 956  * @adapter: board private structure
 957  **/
 958 static void i40evf_clean_all_tx_rings(struct i40evf_adapter *adapter)
 959 {
 960         int i;
 961
 962         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 963                 i40evf_clean_tx_ring(adapter->tx_rings[i]);
 964 }
 965
 966 /**
 967  * i40e_down - Shutdown the connection processing
 968  * @adapter: board private structure
 969  **/
 970 void i40evf_down(struct i40evf_adapter *adapter)
 971 {
 972         struct net_device *netdev = adapter->netdev;
 973         struct i40evf_mac_filter *f;
 974
 975         /* remove all MAC filters */
 976         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 977                 f->remove = true;
 978         }
 979         /* remove all VLAN filters */
 980         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 981                 f->remove = true;
 982         }
 983         if (!(adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) &&
 984             adapter->state != __I40EVF_RESETTING) {
 985                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
 986                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
 987                 /* disable receives */
 988                 adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_QUEUES;
 989                 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 990                 msleep(20);
 991         }
 992         netif_tx_disable(netdev);
 993
 994         netif_tx_stop_all_queues(netdev);
 995
 996         i40evf_irq_disable(adapter);
 997
 998         i40evf_napi_disable_all(adapter);
 999
1000         netif_carrier_off(netdev);
1001
1002         i40evf_clean_all_tx_rings(adapter);
1003         i40evf_clean_all_rx_rings(adapter);
1004 }
1005
1006 /**
1007  * i40evf_acquire_msix_vectors - Setup the MSIX capability
1008  * @adapter: board private structure
1009  * @vectors: number of vectors to request
1010  *
1011  * Work with the OS to set up the MSIX vectors needed.
1012  *
1013  * Returns 0 on success, negative on failure
1014  **/
1015 static int
1016 i40evf_acquire_msix_vectors(struct i40evf_adapter *adapter, int vectors)
1017 {
1018         int err, vector_threshold;
1019
1020         /* We'll want at least 3 (vector_threshold):
1021          * 0) Other (Admin Queue and link, mostly)
1022          * 1) TxQ[0] Cleanup
1023          * 2) RxQ[0] Cleanup
1024          */
1025         vector_threshold = MIN_MSIX_COUNT;
1026
1027         /* The more we get, the more we will assign to Tx/Rx Cleanup
1028          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1029          * Right now, we simply care about how many we'll get; we'll
1030          * set them up later while requesting irq's.
1031          */
1032         while (vectors >= vector_threshold) {
1033                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1034                                       vectors);
1035                 if (!err) /* Success in acquiring all requested vectors. */
1036                         break;
1037                 else if (err < 0)
1038                         vectors = 0; /* Nasty failure, quit now */
1039                 else /* err == number of vectors we should try again with */
1040                         vectors = err;
1041         }
1042
1043         if (vectors < vector_threshold) {
1044                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts.\n");
1045                 kfree(adapter->msix_entries);
1046                 adapter->msix_entries = NULL;
1047                 err = -EIO;
1048         } else {
1049                 /* Adjust for only the vectors we'll use, which is minimum
1050                  * of max_msix_q_vectors + NONQ_VECS, or the number of
1051                  * vectors we were allocated.
1052                  */
1053                 adapter->num_msix_vectors = vectors;
1054         }
1055         return err;
1056 }
1057
1058 /**
1059  * i40evf_free_queues - Free memory for all rings
1060  * @adapter: board private structure to initialize
1061  *
1062  * Free all of the memory associated with queue pairs.
1063  **/
1064 static void i40evf_free_queues(struct i40evf_adapter *adapter)
1065 {
1066         int i;
1067
1068         if (!adapter->vsi_res)
1069                 return;
1070         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1071                 if (adapter->tx_rings[i])
1072                         kfree_rcu(adapter->tx_rings[i], rcu);
1073                 adapter->tx_rings[i] = NULL;
1074                 adapter->rx_rings[i] = NULL;
1075         }
1076 }
1077
1078 /**
1079  * i40evf_alloc_queues - Allocate memory for all rings
1080  * @adapter: board private structure to initialize
1081  *
1082  * We allocate one ring per queue at run-time since we don't know the
1083  * number of queues at compile-time.  The polling_netdev array is
1084  * intended for Multiqueue, but should work fine with a single queue.
1085  **/
1086 static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
1087 {
1088         int i;
1089
1090         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1091                 struct i40e_ring *tx_ring;
1092                 struct i40e_ring *rx_ring;
1093
1094                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
1095                 if (!tx_ring)
1096                         goto err_out;
1097
1098                 tx_ring->queue_index = i;
1099                 tx_ring->netdev = adapter->netdev;
1100                 tx_ring->dev = &adapter->pdev->dev;
1101                 tx_ring->count = I40EVF_DEFAULT_TXD;
1102                 adapter->tx_rings[i] = tx_ring;
1103
1104                 rx_ring = &tx_ring[1];
1105                 rx_ring->queue_index = i;
1106                 rx_ring->netdev = adapter->netdev;
1107                 rx_ring->dev = &adapter->pdev->dev;
1108                 rx_ring->count = I40EVF_DEFAULT_RXD;
1109                 adapter->rx_rings[i] = rx_ring;
1110         }
1111
1112         return 0;
1113
1114 err_out:
1115         i40evf_free_queues(adapter);
1116         return -ENOMEM;
1117 }
1118
1119 /**
1120  * i40evf_set_interrupt_capability - set MSI-X or FAIL if not supported
1121  * @adapter: board private structure to initialize
1122  *
1123  * Attempt to configure the interrupts using the best available
1124  * capabilities of the hardware and the kernel.
1125  **/
1126 static int i40evf_set_interrupt_capability(struct i40evf_adapter *adapter)
1127 {
1128         int vector, v_budget;
1129         int pairs = 0;
1130         int err = 0;
1131
1132         if (!adapter->vsi_res) {
1133                 err = -EIO;
1134                 goto out;
1135         }
1136         pairs = adapter->vsi_res->num_queue_pairs;
1137
1138         /* It's easy to be greedy for MSI-X vectors, but it really
1139          * doesn't do us much good if we have a lot more vectors
1140          * than CPU's.  So let's be conservative and only ask for
1141          * (roughly) twice the number of vectors as there are CPU's.
1142          */
1143         v_budget = min(pairs, (int)(num_online_cpus() * 2)) + NONQ_VECS;
1144         v_budget = min(v_budget, (int)adapter->vf_res->max_vectors);
1145
1146         /* A failure in MSI-X entry allocation isn't fatal, but it does
1147          * mean we disable MSI-X capabilities of the adapter.
1148          */
1149         adapter->msix_entries = kcalloc(v_budget,
1150                                         sizeof(struct msix_entry), GFP_KERNEL);
1151         if (!adapter->msix_entries) {
1152                 err = -ENOMEM;
1153                 goto out;
1154         }
1155
1156         for (vector = 0; vector < v_budget; vector++)
1157                 adapter->msix_entries[vector].entry = vector;
1158
1159         i40evf_acquire_msix_vectors(adapter, v_budget);
1160
1161 out:
1162         adapter->netdev->real_num_tx_queues = pairs;
1163         return err;
1164 }
1165
1166 /**
1167  * i40evf_alloc_q_vectors - Allocate memory for interrupt vectors
1168  * @adapter: board private structure to initialize
1169  *
1170  * We allocate one q_vector per queue interrupt.  If allocation fails we
1171  * return -ENOMEM.
1172  **/
1173 static int i40evf_alloc_q_vectors(struct i40evf_adapter *adapter)
1174 {
1175         int q_idx, num_q_vectors;
1176         struct i40e_q_vector *q_vector;
1177
1178         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1179
1180         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1181                 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
1182                 if (!q_vector)
1183                         goto err_out;
1184                 q_vector->adapter = adapter;
1185                 q_vector->vsi = &adapter->vsi;
1186                 q_vector->v_idx = q_idx;
1187                 netif_napi_add(adapter->netdev, &q_vector->napi,
1188                                        i40evf_napi_poll, 64);
1189                 adapter->q_vector[q_idx] = q_vector;
1190         }
1191
1192         return 0;
1193
1194 err_out:
1195         while (q_idx) {
1196                 q_idx--;
1197                 q_vector = adapter->q_vector[q_idx];
1198                 netif_napi_del(&q_vector->napi);
1199                 kfree(q_vector);
1200                 adapter->q_vector[q_idx] = NULL;
1201         }
1202         return -ENOMEM;
1203 }
1204
1205 /**
1206  * i40evf_free_q_vectors - Free memory allocated for interrupt vectors
1207  * @adapter: board private structure to initialize
1208  *
1209  * This function frees the memory allocated to the q_vectors.  In addition if
1210  * NAPI is enabled it will delete any references to the NAPI struct prior
1211  * to freeing the q_vector.
1212  **/
1213 static void i40evf_free_q_vectors(struct i40evf_adapter *adapter)
1214 {
1215         int q_idx, num_q_vectors;
1216         int napi_vectors;
1217
1218         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1219         napi_vectors = adapter->vsi_res->num_queue_pairs;
1220
1221         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1222                 struct i40e_q_vector *q_vector = adapter->q_vector[q_idx];
1223
1224                 adapter->q_vector[q_idx] = NULL;
1225                 if (q_idx < napi_vectors)
1226                         netif_napi_del(&q_vector->napi);
1227                 kfree(q_vector);
1228         }
1229 }
1230
1231 /**
1232  * i40evf_reset_interrupt_capability - Reset MSIX setup
1233  * @adapter: board private structure
1234  *
1235  **/
1236 void i40evf_reset_interrupt_capability(struct i40evf_adapter *adapter)
1237 {
1238         pci_disable_msix(adapter->pdev);
1239         kfree(adapter->msix_entries);
1240         adapter->msix_entries = NULL;
1241
1242         return;
1243 }
1244
1245 /**
1246  * i40evf_init_interrupt_scheme - Determine if MSIX is supported and init
1247  * @adapter: board private structure to initialize
1248  *
1249  **/
1250 int i40evf_init_interrupt_scheme(struct i40evf_adapter *adapter)
1251 {
1252         int err;
1253
1254         err = i40evf_set_interrupt_capability(adapter);
1255         if (err) {
1256                 dev_err(&adapter->pdev->dev,
1257                         "Unable to setup interrupt capabilities\n");
1258                 goto err_set_interrupt;
1259         }
1260
1261         err = i40evf_alloc_q_vectors(adapter);
1262         if (err) {
1263                 dev_err(&adapter->pdev->dev,
1264                         "Unable to allocate memory for queue vectors\n");
1265                 goto err_alloc_q_vectors;
1266         }
1267
1268         err = i40evf_alloc_queues(adapter);
1269         if (err) {
1270                 dev_err(&adapter->pdev->dev,
1271                         "Unable to allocate memory for queues\n");
1272                 goto err_alloc_queues;
1273         }
1274
1275         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1276                 (adapter->vsi_res->num_queue_pairs > 1) ? "Enabled" :
1277                 "Disabled", adapter->vsi_res->num_queue_pairs);
1278
1279         return 0;
1280 err_alloc_queues:
1281         i40evf_free_q_vectors(adapter);
1282 err_alloc_q_vectors:
1283         i40evf_reset_interrupt_capability(adapter);
1284 err_set_interrupt:
1285         return err;
1286 }
1287
1288 /**
1289  * i40evf_watchdog_timer - Periodic call-back timer
1290  * @data: pointer to adapter disguised as unsigned long
1291  **/
1292 static void i40evf_watchdog_timer(unsigned long data)
1293 {
1294         struct i40evf_adapter *adapter = (struct i40evf_adapter *)data;
1295         schedule_work(&adapter->watchdog_task);
1296         /* timer will be rescheduled in watchdog task */
1297 }
1298
1299 /**
1300  * i40evf_watchdog_task - Periodic call-back task
1301  * @work: pointer to work_struct
1302  **/
1303 static void i40evf_watchdog_task(struct work_struct *work)
1304 {
1305         struct i40evf_adapter *adapter = container_of(work,
1306                                           struct i40evf_adapter,
1307                                           watchdog_task);
1308         struct i40e_hw *hw = &adapter->hw;
1309
1310         if (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section))
1311                 goto restart_watchdog;
1312
1313         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
1314                 dev_info(&adapter->pdev->dev, "Checking for redemption\n");
1315                 if ((rd32(hw, I40E_VFGEN_RSTAT) & 0x3) == I40E_VFR_VFACTIVE) {
1316                         /* A chance for redemption! */
1317                         dev_err(&adapter->pdev->dev, "Hardware came out of reset. Attempting reinit.\n");
1318                         adapter->state = __I40EVF_STARTUP;
1319                         adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
1320                         schedule_delayed_work(&adapter->init_task, 10);
1321                         clear_bit(__I40EVF_IN_CRITICAL_TASK,
1322                                   &adapter->crit_section);
1323                         /* Don't reschedule the watchdog, since we've restarted
1324                          * the init task. When init_task contacts the PF and
1325                          * gets everything set up again, it'll restart the
1326                          * watchdog for us. Down, boy. Sit. Stay. Woof.
1327                          */
1328                         return;
1329                 }
1330                 adapter->aq_pending = 0;
1331                 adapter->aq_required = 0;
1332                 adapter->current_op = I40E_VIRTCHNL_OP_UNKNOWN;
1333                 goto watchdog_done;
1334         }
1335
1336         if ((adapter->state < __I40EVF_DOWN) ||
1337             (adapter->flags & I40EVF_FLAG_RESET_PENDING))
1338                 goto watchdog_done;
1339
1340         /* check for reset */
1341         if (!(adapter->flags & I40EVF_FLAG_RESET_PENDING) &&
1342             (rd32(hw, I40E_VFGEN_RSTAT) & 0x3) != I40E_VFR_VFACTIVE) {
1343                 adapter->state = __I40EVF_RESETTING;
1344                 adapter->flags |= I40EVF_FLAG_RESET_PENDING;
1345                 dev_err(&adapter->pdev->dev, "Hardware reset detected.\n");
1346                 dev_info(&adapter->pdev->dev, "Scheduling reset task\n");
1347                 schedule_work(&adapter->reset_task);
1348                 adapter->aq_pending = 0;
1349                 adapter->aq_required = 0;
1350                 adapter->current_op = I40E_VIRTCHNL_OP_UNKNOWN;
1351                 goto watchdog_done;
1352         }
1353
1354         /* Process admin queue tasks. After init, everything gets done
1355          * here so we don't race on the admin queue.
1356          */
1357         if (adapter->aq_pending)
1358                 goto watchdog_done;
1359
1360         if (adapter->aq_required & I40EVF_FLAG_AQ_MAP_VECTORS) {
1361                 i40evf_map_queues(adapter);
1362                 goto watchdog_done;
1363         }
1364
1365         if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_MAC_FILTER) {
1366                 i40evf_add_ether_addrs(adapter);
1367                 goto watchdog_done;
1368         }
1369
1370         if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_VLAN_FILTER) {
1371                 i40evf_add_vlans(adapter);
1372                 goto watchdog_done;
1373         }
1374
1375         if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_MAC_FILTER) {
1376                 i40evf_del_ether_addrs(adapter);
1377                 goto watchdog_done;
1378         }
1379
1380         if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_VLAN_FILTER) {
1381                 i40evf_del_vlans(adapter);
1382                 goto watchdog_done;
1383         }
1384
1385         if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_QUEUES) {
1386                 i40evf_disable_queues(adapter);
1387                 goto watchdog_done;
1388         }
1389
1390         if (adapter->aq_required & I40EVF_FLAG_AQ_CONFIGURE_QUEUES) {
1391                 i40evf_configure_queues(adapter);
1392                 goto watchdog_done;
1393         }
1394
1395         if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_QUEUES) {
1396                 i40evf_enable_queues(adapter);
1397                 goto watchdog_done;
1398         }
1399
1400         if (adapter->state == __I40EVF_RUNNING)
1401                 i40evf_request_stats(adapter);
1402
1403         i40evf_irq_enable(adapter, true);
1404         i40evf_fire_sw_int(adapter, 0xFF);
1405
1406 watchdog_done:
1407         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1408 restart_watchdog:
1409         if (adapter->aq_required)
1410                 mod_timer(&adapter->watchdog_timer,
1411                           jiffies + msecs_to_jiffies(20));
1412         else
1413                 mod_timer(&adapter->watchdog_timer, jiffies + (HZ * 2));
1414         schedule_work(&adapter->adminq_task);
1415 }
1416
1417 /**
1418  * i40evf_configure_rss - Prepare for RSS if used
1419  * @adapter: board private structure
1420  **/
1421 static void i40evf_configure_rss(struct i40evf_adapter *adapter)
1422 {
1423         struct i40e_hw *hw = &adapter->hw;
1424         u32 lut = 0;
1425         int i, j;
1426         u64 hena;
1427
1428         /* Set of random keys generated using kernel random number generator */
1429         static const u32 seed[I40E_VFQF_HKEY_MAX_INDEX + 1] = {
1430                         0x794221b4, 0xbca0c5ab, 0x6cd5ebd9, 0x1ada6127,
1431                         0x983b3aa1, 0x1c4e71eb, 0x7f6328b2, 0xfcdc0da0,
1432                         0xc135cafa, 0x7a6f7e2d, 0xe7102d28, 0x163cd12e,
1433                         0x4954b126 };
1434
1435         /* Hash type is configured by the PF - we just supply the key */
1436
1437         /* Fill out hash function seed */
1438         for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
1439                 wr32(hw, I40E_VFQF_HKEY(i), seed[i]);
1440
1441         /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1442         hena = I40E_DEFAULT_RSS_HENA;
1443         wr32(hw, I40E_VFQF_HENA(0), (u32)hena);
1444         wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32));
1445
1446         /* Populate the LUT with max no. of queues in round robin fashion */
1447         for (i = 0, j = 0; i < I40E_VFQF_HLUT_MAX_INDEX; i++, j++) {
1448                 if (j == adapter->vsi_res->num_queue_pairs)
1449                         j = 0;
1450                 /* lut = 4-byte sliding window of 4 lut entries */
1451                 lut = (lut << 8) | (j &
1452                          ((0x1 << 8) - 1));
1453                 /* On i = 3, we have 4 entries in lut; write to the register */
1454                 if ((i & 3) == 3)
1455                         wr32(hw, I40E_VFQF_HLUT(i >> 2), lut);
1456         }
1457         i40e_flush(hw);
1458 }
1459
1460 #define I40EVF_RESET_WAIT_MS 100
1461 #define I40EVF_RESET_WAIT_COUNT 200
1462 /**
1463  * i40evf_reset_task - Call-back task to handle hardware reset
1464  * @work: pointer to work_struct
1465  *
1466  * During reset we need to shut down and reinitialize the admin queue
1467  * before we can use it to communicate with the PF again. We also clear
1468  * and reinit the rings because that context is lost as well.
1469  **/
1470 static void i40evf_reset_task(struct work_struct *work)
1471 {
1472         struct i40evf_adapter *adapter = container_of(work,
1473                                                       struct i40evf_adapter,
1474                                                       reset_task);
1475         struct i40e_hw *hw = &adapter->hw;
1476         int i = 0, err;
1477         uint32_t rstat_val;
1478
1479         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
1480                                 &adapter->crit_section))
1481                 udelay(500);
1482         /* poll until we see the reset actually happen */
1483         for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1484                 rstat_val = rd32(hw, I40E_VFGEN_RSTAT) &
1485                             I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1486                 if (rstat_val != I40E_VFR_VFACTIVE) {
1487                         dev_info(&adapter->pdev->dev, "Reset now occurring\n");
1488                         break;
1489                 } else {
1490                         msleep(I40EVF_RESET_WAIT_MS);
1491                 }
1492         }
1493         if (i == I40EVF_RESET_WAIT_COUNT) {
1494                 dev_err(&adapter->pdev->dev, "Reset was not detected\n");
1495                 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1496                 goto continue_reset; /* act like the reset happened */
1497         }
1498
1499         /* wait until the reset is complete and the PF is responding to us */
1500         for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1501                 rstat_val = rd32(hw, I40E_VFGEN_RSTAT) &
1502                             I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1503                 if (rstat_val == I40E_VFR_VFACTIVE) {
1504                         dev_info(&adapter->pdev->dev, "Reset is complete. Reinitializing.\n");
1505                         break;
1506                 } else {
1507                         msleep(I40EVF_RESET_WAIT_MS);
1508                 }
1509         }
1510         if (i == I40EVF_RESET_WAIT_COUNT) {
1511                 /* reset never finished */
1512                 dev_err(&adapter->pdev->dev, "Reset never finished (%x). PF driver is dead, and so am I.\n",
1513                         rstat_val);
1514                 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
1515
1516                 if (netif_running(adapter->netdev))
1517                         i40evf_close(adapter->netdev);
1518
1519                 i40evf_free_misc_irq(adapter);
1520                 i40evf_reset_interrupt_capability(adapter);
1521                 i40evf_free_queues(adapter);
1522                 kfree(adapter->vf_res);
1523                 i40evf_shutdown_adminq(hw);
1524                 adapter->netdev->flags &= ~IFF_UP;
1525                 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1526                 return; /* Do not attempt to reinit. It's dead, Jim. */
1527         }
1528
1529 continue_reset:
1530         adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1531
1532         i40evf_down(adapter);
1533         adapter->state = __I40EVF_RESETTING;
1534
1535         /* kill and reinit the admin queue */
1536         if (i40evf_shutdown_adminq(hw))
1537                 dev_warn(&adapter->pdev->dev,
1538                         "%s: Failed to destroy the Admin Queue resources\n",
1539                         __func__);
1540         err = i40evf_init_adminq(hw);
1541         if (err)
1542                 dev_info(&adapter->pdev->dev, "%s: init_adminq failed: %d\n",
1543                         __func__, err);
1544
1545         adapter->aq_pending = 0;
1546         adapter->aq_required = 0;
1547         i40evf_map_queues(adapter);
1548         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1549
1550         mod_timer(&adapter->watchdog_timer, jiffies + 2);
1551
1552         if (netif_running(adapter->netdev)) {
1553                 /* allocate transmit descriptors */
1554                 err = i40evf_setup_all_tx_resources(adapter);
1555                 if (err)
1556                         goto reset_err;
1557
1558                 /* allocate receive descriptors */
1559                 err = i40evf_setup_all_rx_resources(adapter);
1560                 if (err)
1561                         goto reset_err;
1562
1563                 i40evf_configure(adapter);
1564
1565                 err = i40evf_up_complete(adapter);
1566                 if (err)
1567                         goto reset_err;
1568
1569                 i40evf_irq_enable(adapter, true);
1570         }
1571         return;
1572 reset_err:
1573         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit.\n");
1574         i40evf_close(adapter->netdev);
1575 }
1576
1577 /**
1578  * i40evf_adminq_task - worker thread to clean the admin queue
1579  * @work: pointer to work_struct containing our data
1580  **/
1581 static void i40evf_adminq_task(struct work_struct *work)
1582 {
1583         struct i40evf_adapter *adapter =
1584                 container_of(work, struct i40evf_adapter, adminq_task);
1585         struct i40e_hw *hw = &adapter->hw;
1586         struct i40e_arq_event_info event;
1587         struct i40e_virtchnl_msg *v_msg;
1588         i40e_status ret;
1589         u16 pending;
1590
1591         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED)
1592                 return;
1593
1594         event.msg_size = I40EVF_MAX_AQ_BUF_SIZE;
1595         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
1596         if (!event.msg_buf) {
1597                 dev_info(&adapter->pdev->dev, "%s: no memory for ARQ clean\n",
1598                                  __func__);
1599                 return;
1600         }
1601         v_msg = (struct i40e_virtchnl_msg *)&event.desc;
1602         do {
1603                 ret = i40evf_clean_arq_element(hw, &event, &pending);
1604                 if (ret)
1605                         break; /* No event to process or error cleaning ARQ */
1606
1607                 i40evf_virtchnl_completion(adapter, v_msg->v_opcode,
1608                                            v_msg->v_retval, event.msg_buf,
1609                                            event.msg_size);
1610                 if (pending != 0) {
1611                         dev_info(&adapter->pdev->dev,
1612                                  "%s: ARQ: Pending events %d\n",
1613                                  __func__, pending);
1614                         memset(event.msg_buf, 0, I40EVF_MAX_AQ_BUF_SIZE);
1615                 }
1616         } while (pending);
1617
1618         /* re-enable Admin queue interrupt cause */
1619         i40evf_misc_irq_enable(adapter);
1620
1621         kfree(event.msg_buf);
1622 }
1623
1624 /**
1625  * i40evf_free_all_tx_resources - Free Tx Resources for All Queues
1626  * @adapter: board private structure
1627  *
1628  * Free all transmit software resources
1629  **/
1630 static void i40evf_free_all_tx_resources(struct i40evf_adapter *adapter)
1631 {
1632         int i;
1633
1634         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
1635                 if (adapter->tx_rings[i]->desc)
1636                         i40evf_free_tx_resources(adapter->tx_rings[i]);
1637
1638 }
1639
1640 /**
1641  * i40evf_setup_all_tx_resources - allocate all queues Tx resources
1642  * @adapter: board private structure
1643  *
1644  * If this function returns with an error, then it's possible one or
1645  * more of the rings is populated (while the rest are not).  It is the
1646  * callers duty to clean those orphaned rings.
1647  *
1648  * Return 0 on success, negative on failure
1649  **/
1650 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter)
1651 {
1652         int i, err = 0;
1653
1654         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1655                 err = i40evf_setup_tx_descriptors(adapter->tx_rings[i]);
1656                 if (!err)
1657                         continue;
1658                 dev_err(&adapter->pdev->dev,
1659                         "%s: Allocation for Tx Queue %u failed\n",
1660                         __func__, i);
1661                 break;
1662         }
1663
1664         return err;
1665 }
1666
1667 /**
1668  * i40evf_setup_all_rx_resources - allocate all queues Rx resources
1669  * @adapter: board private structure
1670  *
1671  * If this function returns with an error, then it's possible one or
1672  * more of the rings is populated (while the rest are not).  It is the
1673  * callers duty to clean those orphaned rings.
1674  *
1675  * Return 0 on success, negative on failure
1676  **/
1677 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter)
1678 {
1679         int i, err = 0;
1680
1681         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1682                 err = i40evf_setup_rx_descriptors(adapter->rx_rings[i]);
1683                 if (!err)
1684                         continue;
1685                 dev_err(&adapter->pdev->dev,
1686                         "%s: Allocation for Rx Queue %u failed\n",
1687                         __func__, i);
1688                 break;
1689         }
1690         return err;
1691 }
1692
1693 /**
1694  * i40evf_free_all_rx_resources - Free Rx Resources for All Queues
1695  * @adapter: board private structure
1696  *
1697  * Free all receive software resources
1698  **/
1699 static void i40evf_free_all_rx_resources(struct i40evf_adapter *adapter)
1700 {
1701         int i;
1702
1703         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
1704                 if (adapter->rx_rings[i]->desc)
1705                         i40evf_free_rx_resources(adapter->rx_rings[i]);
1706 }
1707
1708 /**
1709  * i40evf_open - Called when a network interface is made active
1710  * @netdev: network interface device structure
1711  *
1712  * Returns 0 on success, negative value on failure
1713  *
1714  * The open entry point is called when a network interface is made
1715  * active by the system (IFF_UP).  At this point all resources needed
1716  * for transmit and receive operations are allocated, the interrupt
1717  * handler is registered with the OS, the watchdog timer is started,
1718  * and the stack is notified that the interface is ready.
1719  **/
1720 static int i40evf_open(struct net_device *netdev)
1721 {
1722         struct i40evf_adapter *adapter = netdev_priv(netdev);
1723         int err;
1724
1725         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
1726                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
1727                 return -EIO;
1728         }
1729         if (adapter->state != __I40EVF_DOWN)
1730                 return -EBUSY;
1731
1732         /* allocate transmit descriptors */
1733         err = i40evf_setup_all_tx_resources(adapter);
1734         if (err)
1735                 goto err_setup_tx;
1736
1737         /* allocate receive descriptors */
1738         err = i40evf_setup_all_rx_resources(adapter);
1739         if (err)
1740                 goto err_setup_rx;
1741
1742         /* clear any pending interrupts, may auto mask */
1743         err = i40evf_request_traffic_irqs(adapter, netdev->name);
1744         if (err)
1745                 goto err_req_irq;
1746
1747         i40evf_configure(adapter);
1748
1749         err = i40evf_up_complete(adapter);
1750         if (err)
1751                 goto err_req_irq;
1752
1753         i40evf_irq_enable(adapter, true);
1754
1755         return 0;
1756
1757 err_req_irq:
1758         i40evf_down(adapter);
1759         i40evf_free_traffic_irqs(adapter);
1760 err_setup_rx:
1761         i40evf_free_all_rx_resources(adapter);
1762 err_setup_tx:
1763         i40evf_free_all_tx_resources(adapter);
1764
1765         return err;
1766 }
1767
1768 /**
1769  * i40evf_close - Disables a network interface
1770  * @netdev: network interface device structure
1771  *
1772  * Returns 0, this is not allowed to fail
1773  *
1774  * The close entry point is called when an interface is de-activated
1775  * by the OS.  The hardware is still under the drivers control, but
1776  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
1777  * are freed, along with all transmit and receive resources.
1778  **/
1779 static int i40evf_close(struct net_device *netdev)
1780 {
1781         struct i40evf_adapter *adapter = netdev_priv(netdev);
1782
1783         if (adapter->state <= __I40EVF_DOWN)
1784                 return 0;
1785
1786         /* signal that we are down to the interrupt handler */
1787         adapter->state = __I40EVF_DOWN;
1788
1789         set_bit(__I40E_DOWN, &adapter->vsi.state);
1790
1791         i40evf_down(adapter);
1792         i40evf_free_traffic_irqs(adapter);
1793
1794         i40evf_free_all_tx_resources(adapter);
1795         i40evf_free_all_rx_resources(adapter);
1796
1797         return 0;
1798 }
1799
1800 /**
1801  * i40evf_get_stats - Get System Network Statistics
1802  * @netdev: network interface device structure
1803  *
1804  * Returns the address of the device statistics structure.
1805  * The statistics are actually updated from the timer callback.
1806  **/
1807 static struct net_device_stats *i40evf_get_stats(struct net_device *netdev)
1808 {
1809         struct i40evf_adapter *adapter = netdev_priv(netdev);
1810
1811         /* only return the current stats */
1812         return &adapter->net_stats;
1813 }
1814
1815 /**
1816  * i40evf_reinit_locked - Software reinit
1817  * @adapter: board private structure
1818  *
1819  * Reinititalizes the ring structures in response to a software configuration
1820  * change. Roughly the same as close followed by open, but skips releasing
1821  * and reallocating the interrupts.
1822  **/
1823 void i40evf_reinit_locked(struct i40evf_adapter *adapter)
1824 {
1825         struct net_device *netdev = adapter->netdev;
1826         int err;
1827
1828         WARN_ON(in_interrupt());
1829
1830         adapter->state = __I40EVF_RESETTING;
1831
1832         i40evf_down(adapter);
1833
1834         /* allocate transmit descriptors */
1835         err = i40evf_setup_all_tx_resources(adapter);
1836         if (err)
1837                 goto err_reinit;
1838
1839         /* allocate receive descriptors */
1840         err = i40evf_setup_all_rx_resources(adapter);
1841         if (err)
1842                 goto err_reinit;
1843
1844         i40evf_configure(adapter);
1845
1846         err = i40evf_up_complete(adapter);
1847         if (err)
1848                 goto err_reinit;
1849
1850         i40evf_irq_enable(adapter, true);
1851         return;
1852
1853 err_reinit:
1854         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit.\n");
1855         i40evf_close(netdev);
1856 }
1857
1858 /**
1859  * i40evf_change_mtu - Change the Maximum Transfer Unit
1860  * @netdev: network interface device structure
1861  * @new_mtu: new value for maximum frame size
1862  *
1863  * Returns 0 on success, negative on failure
1864  **/
1865 static int i40evf_change_mtu(struct net_device *netdev, int new_mtu)
1866 {
1867         struct i40evf_adapter *adapter = netdev_priv(netdev);
1868         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
1869
1870         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1871                 return -EINVAL;
1872
1873         /* must set new MTU before calling down or up */
1874         netdev->mtu = new_mtu;
1875         i40evf_reinit_locked(adapter);
1876         return 0;
1877 }
1878
1879 static const struct net_device_ops i40evf_netdev_ops = {
1880         .ndo_open               = i40evf_open,
1881         .ndo_stop               = i40evf_close,
1882         .ndo_start_xmit         = i40evf_xmit_frame,
1883         .ndo_get_stats          = i40evf_get_stats,
1884         .ndo_set_rx_mode        = i40evf_set_rx_mode,
1885         .ndo_validate_addr      = eth_validate_addr,
1886         .ndo_set_mac_address    = i40evf_set_mac,
1887         .ndo_change_mtu         = i40evf_change_mtu,
1888         .ndo_tx_timeout         = i40evf_tx_timeout,
1889         .ndo_vlan_rx_add_vid    = i40evf_vlan_rx_add_vid,
1890         .ndo_vlan_rx_kill_vid   = i40evf_vlan_rx_kill_vid,
1891 };
1892
1893 /**
1894  * i40evf_check_reset_complete - check that VF reset is complete
1895  * @hw: pointer to hw struct
1896  *
1897  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
1898  **/
1899 static int i40evf_check_reset_complete(struct i40e_hw *hw)
1900 {
1901         u32 rstat;
1902         int i;
1903
1904         for (i = 0; i < 100; i++) {
1905                 rstat = rd32(hw, I40E_VFGEN_RSTAT);
1906                 if (rstat == I40E_VFR_VFACTIVE)
1907                         return 0;
1908                 udelay(10);
1909         }
1910         return -EBUSY;
1911 }
1912
1913 /**
1914  * i40evf_init_task - worker thread to perform delayed initialization
1915  * @work: pointer to work_struct containing our data
1916  *
1917  * This task completes the work that was begun in probe. Due to the nature
1918  * of VF-PF communications, we may need to wait tens of milliseconds to get
1919  * reponses back from the PF. Rather than busy-wait in probe and bog down the
1920  * whole system, we'll do it in a task so we can sleep.
1921  * This task only runs during driver init. Once we've established
1922  * communications with the PF driver and set up our netdev, the watchdog
1923  * takes over.
1924  **/
1925 static void i40evf_init_task(struct work_struct *work)
1926 {
1927         struct i40evf_adapter *adapter = container_of(work,
1928                                                       struct i40evf_adapter,
1929                                                       init_task.work);
1930         struct net_device *netdev = adapter->netdev;
1931         struct i40evf_mac_filter *f;
1932         struct i40e_hw *hw = &adapter->hw;
1933         struct pci_dev *pdev = adapter->pdev;
1934         int i, err, bufsz;
1935
1936         switch (adapter->state) {
1937         case __I40EVF_STARTUP:
1938                 /* driver loaded, probe complete */
1939                 adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
1940                 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1941                 err = i40e_set_mac_type(hw);
1942                 if (err) {
1943                         dev_err(&pdev->dev, "Failed to set MAC type (%d)\n",
1944                                 err);
1945                 goto err;
1946                 }
1947                 err = i40evf_check_reset_complete(hw);
1948                 if (err) {
1949                         dev_err(&pdev->dev, "Device is still in reset (%d)\n",
1950                                 err);
1951                         goto err;
1952                 }
1953                 hw->aq.num_arq_entries = I40EVF_AQ_LEN;
1954                 hw->aq.num_asq_entries = I40EVF_AQ_LEN;
1955                 hw->aq.arq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
1956                 hw->aq.asq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
1957
1958                 err = i40evf_init_adminq(hw);
1959                 if (err) {
1960                         dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
1961                                 err);
1962                         goto err;
1963                 }
1964                 err = i40evf_send_api_ver(adapter);
1965                 if (err) {
1966                         dev_err(&pdev->dev, "Unable to send to PF (%d)\n",
1967                                 err);
1968                         i40evf_shutdown_adminq(hw);
1969                         goto err;
1970                 }
1971                 adapter->state = __I40EVF_INIT_VERSION_CHECK;
1972                 goto restart;
1973                 break;
1974         case __I40EVF_INIT_VERSION_CHECK:
1975                 if (!i40evf_asq_done(hw))
1976                         goto err;
1977
1978                 /* aq msg sent, awaiting reply */
1979                 err = i40evf_verify_api_ver(adapter);
1980                 if (err) {
1981                         dev_err(&pdev->dev, "Unable to verify API version (%d)\n",
1982                                 err);
1983                         goto err;
1984                 }
1985                 err = i40evf_send_vf_config_msg(adapter);
1986                 if (err) {
1987                         dev_err(&pdev->dev, "Unable send config request (%d)\n",
1988                                 err);
1989                         goto err;
1990                 }
1991                 adapter->state = __I40EVF_INIT_GET_RESOURCES;
1992                 goto restart;
1993                 break;
1994         case __I40EVF_INIT_GET_RESOURCES:
1995                 /* aq msg sent, awaiting reply */
1996                 if (!adapter->vf_res) {
1997                         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
1998                                 (I40E_MAX_VF_VSI *
1999                                  sizeof(struct i40e_virtchnl_vsi_resource));
2000                         adapter->vf_res = kzalloc(bufsz, GFP_KERNEL);
2001                         if (!adapter->vf_res)
2002                                 goto err;
2003                 }
2004                 err = i40evf_get_vf_config(adapter);
2005                 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK)
2006                         goto restart;
2007                 if (err) {
2008                         dev_err(&pdev->dev, "Unable to get VF config (%d)\n",
2009                                 err);
2010                         goto err_alloc;
2011                 }
2012                 adapter->state = __I40EVF_INIT_SW;
2013                 break;
2014         default:
2015                 goto err_alloc;
2016         }
2017         /* got VF config message back from PF, now we can parse it */
2018         for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2019                 if (adapter->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
2020                         adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2021         }
2022         if (!adapter->vsi_res) {
2023                 dev_err(&pdev->dev, "No LAN VSI found\n");
2024                 goto err_alloc;
2025         }
2026
2027         adapter->flags |= I40EVF_FLAG_RX_CSUM_ENABLED;
2028
2029         netdev->netdev_ops = &i40evf_netdev_ops;
2030         i40evf_set_ethtool_ops(netdev);
2031         netdev->watchdog_timeo = 5 * HZ;
2032         netdev->features |= NETIF_F_HIGHDMA |
2033                             NETIF_F_SG |
2034                             NETIF_F_IP_CSUM |
2035                             NETIF_F_SCTP_CSUM |
2036                             NETIF_F_IPV6_CSUM |
2037                             NETIF_F_TSO |
2038                             NETIF_F_TSO6 |
2039                             NETIF_F_RXCSUM |
2040                             NETIF_F_GRO;
2041
2042         if (adapter->vf_res->vf_offload_flags
2043             & I40E_VIRTCHNL_VF_OFFLOAD_VLAN) {
2044                 netdev->vlan_features = netdev->features;
2045                 netdev->features |= NETIF_F_HW_VLAN_CTAG_TX |
2046                                     NETIF_F_HW_VLAN_CTAG_RX |
2047                                     NETIF_F_HW_VLAN_CTAG_FILTER;
2048         }
2049
2050         /* copy netdev features into list of user selectable features */
2051         netdev->hw_features |= netdev->features;
2052         netdev->hw_features &= ~NETIF_F_RXCSUM;
2053
2054         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2055                 dev_info(&pdev->dev, "Invalid MAC address %pMAC, using random\n",
2056                          adapter->hw.mac.addr);
2057                 random_ether_addr(adapter->hw.mac.addr);
2058         }
2059         memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
2060         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
2061
2062         INIT_LIST_HEAD(&adapter->mac_filter_list);
2063         INIT_LIST_HEAD(&adapter->vlan_filter_list);
2064         f = kzalloc(sizeof(*f), GFP_ATOMIC);
2065         if (NULL == f)
2066                 goto err_sw_init;
2067
2068         memcpy(f->macaddr, adapter->hw.mac.addr, ETH_ALEN);
2069         f->add = true;
2070         adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
2071
2072         list_add(&f->list, &adapter->mac_filter_list);
2073
2074         init_timer(&adapter->watchdog_timer);
2075         adapter->watchdog_timer.function = &i40evf_watchdog_timer;
2076         adapter->watchdog_timer.data = (unsigned long)adapter;
2077         mod_timer(&adapter->watchdog_timer, jiffies + 1);
2078
2079         err = i40evf_init_interrupt_scheme(adapter);
2080         if (err)
2081                 goto err_sw_init;
2082         i40evf_map_rings_to_vectors(adapter);
2083         i40evf_configure_rss(adapter);
2084         err = i40evf_request_misc_irq(adapter);
2085         if (err)
2086                 goto err_sw_init;
2087
2088         netif_carrier_off(netdev);
2089
2090         adapter->vsi.id = adapter->vsi_res->vsi_id;
2091         adapter->vsi.seid = adapter->vsi_res->vsi_id; /* dummy */
2092         adapter->vsi.back = adapter;
2093         adapter->vsi.base_vector = 1;
2094         adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
2095         adapter->vsi.rx_itr_setting = I40E_ITR_DYNAMIC;
2096         adapter->vsi.tx_itr_setting = I40E_ITR_DYNAMIC;
2097         adapter->vsi.netdev = adapter->netdev;
2098
2099         if (!adapter->netdev_registered) {
2100                 err = register_netdev(netdev);
2101                 if (err)
2102                         goto err_register;
2103         }
2104
2105         adapter->netdev_registered = true;
2106
2107         netif_tx_stop_all_queues(netdev);
2108
2109         dev_info(&pdev->dev, "MAC address: %pMAC\n", adapter->hw.mac.addr);
2110         if (netdev->features & NETIF_F_GRO)
2111                 dev_info(&pdev->dev, "GRO is enabled\n");
2112
2113         dev_info(&pdev->dev, "%s\n", i40evf_driver_string);
2114         adapter->state = __I40EVF_DOWN;
2115         set_bit(__I40E_DOWN, &adapter->vsi.state);
2116         i40evf_misc_irq_enable(adapter);
2117         return;
2118 restart:
2119         schedule_delayed_work(&adapter->init_task,
2120                               msecs_to_jiffies(50));
2121         return;
2122
2123 err_register:
2124         i40evf_free_misc_irq(adapter);
2125 err_sw_init:
2126         i40evf_reset_interrupt_capability(adapter);
2127 err_alloc:
2128         kfree(adapter->vf_res);
2129         adapter->vf_res = NULL;
2130 err:
2131         if (hw->aq.asq.count)
2132                 i40evf_shutdown_adminq(hw); /* ignore error */
2133         /* Things went into the weeds, so try again later */
2134         if (++adapter->aq_wait_count > I40EVF_AQ_MAX_ERR) {
2135                 dev_err(&pdev->dev, "Failed to communicate with PF; giving up.\n");
2136                 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
2137                 return; /* do not reschedule */
2138         }
2139         schedule_delayed_work(&adapter->init_task, HZ * 3);
2140         return;
2141 }
2142
2143 /**
2144  * i40evf_shutdown - Shutdown the device in preparation for a reboot
2145  * @pdev: pci device structure
2146  **/
2147 static void i40evf_shutdown(struct pci_dev *pdev)
2148 {
2149         struct net_device *netdev = pci_get_drvdata(pdev);
2150
2151         netif_device_detach(netdev);
2152
2153         if (netif_running(netdev))
2154                 i40evf_close(netdev);
2155
2156 #ifdef CONFIG_PM
2157         pci_save_state(pdev);
2158
2159 #endif
2160         pci_disable_device(pdev);
2161 }
2162
2163 /**
2164  * i40evf_probe - Device Initialization Routine
2165  * @pdev: PCI device information struct
2166  * @ent: entry in i40evf_pci_tbl
2167  *
2168  * Returns 0 on success, negative on failure
2169  *
2170  * i40evf_probe initializes an adapter identified by a pci_dev structure.
2171  * The OS initialization, configuring of the adapter private structure,
2172  * and a hardware reset occur.
2173  **/
2174 static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2175 {
2176         struct net_device *netdev;
2177         struct i40evf_adapter *adapter = NULL;
2178         struct i40e_hw *hw = NULL;
2179         int err;
2180
2181         err = pci_enable_device(pdev);
2182         if (err)
2183                 return err;
2184
2185         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2186         if (err)
2187                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2188         if (err) {
2189                 dev_err(&pdev->dev,
2190                         "DMA configuration failed: 0x%x\n", err);
2191                 goto err_dma;
2192         }
2193
2194         err = pci_request_regions(pdev, i40evf_driver_name);
2195         if (err) {
2196                 dev_err(&pdev->dev,
2197                         "pci_request_regions failed 0x%x\n", err);
2198                 goto err_pci_reg;
2199         }
2200
2201         pci_enable_pcie_error_reporting(pdev);
2202
2203         pci_set_master(pdev);
2204
2205         netdev = alloc_etherdev_mq(sizeof(struct i40evf_adapter),
2206                                    MAX_TX_QUEUES);
2207         if (!netdev) {
2208                 err = -ENOMEM;
2209                 goto err_alloc_etherdev;
2210         }
2211
2212         SET_NETDEV_DEV(netdev, &pdev->dev);
2213
2214         pci_set_drvdata(pdev, netdev);
2215         adapter = netdev_priv(netdev);
2216
2217         adapter->netdev = netdev;
2218         adapter->pdev = pdev;
2219
2220         hw = &adapter->hw;
2221         hw->back = adapter;
2222
2223         adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
2224         adapter->state = __I40EVF_STARTUP;
2225
2226         /* Call save state here because it relies on the adapter struct. */
2227         pci_save_state(pdev);
2228
2229         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2230                               pci_resource_len(pdev, 0));
2231         if (!hw->hw_addr) {
2232                 err = -EIO;
2233                 goto err_ioremap;
2234         }
2235         hw->vendor_id = pdev->vendor;
2236         hw->device_id = pdev->device;
2237         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2238         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2239         hw->subsystem_device_id = pdev->subsystem_device;
2240         hw->bus.device = PCI_SLOT(pdev->devfn);
2241         hw->bus.func = PCI_FUNC(pdev->devfn);
2242
2243         INIT_WORK(&adapter->reset_task, i40evf_reset_task);
2244         INIT_WORK(&adapter->adminq_task, i40evf_adminq_task);
2245         INIT_WORK(&adapter->watchdog_task, i40evf_watchdog_task);
2246         INIT_DELAYED_WORK(&adapter->init_task, i40evf_init_task);
2247         schedule_delayed_work(&adapter->init_task, 10);
2248
2249         return 0;
2250
2251 err_ioremap:
2252         free_netdev(netdev);
2253 err_alloc_etherdev:
2254         pci_release_regions(pdev);
2255 err_pci_reg:
2256 err_dma:
2257         pci_disable_device(pdev);
2258         return err;
2259 }
2260
2261 #ifdef CONFIG_PM
2262 /**
2263  * i40evf_suspend - Power management suspend routine
2264  * @pdev: PCI device information struct
2265  * @state: unused
2266  *
2267  * Called when the system (VM) is entering sleep/suspend.
2268  **/
2269 static int i40evf_suspend(struct pci_dev *pdev, pm_message_t state)
2270 {
2271         struct net_device *netdev = pci_get_drvdata(pdev);
2272         struct i40evf_adapter *adapter = netdev_priv(netdev);
2273         int retval = 0;
2274
2275         netif_device_detach(netdev);
2276
2277         if (netif_running(netdev)) {
2278                 rtnl_lock();
2279                 i40evf_down(adapter);
2280                 rtnl_unlock();
2281         }
2282         i40evf_free_misc_irq(adapter);
2283         i40evf_reset_interrupt_capability(adapter);
2284
2285         retval = pci_save_state(pdev);
2286         if (retval)
2287                 return retval;
2288
2289         pci_disable_device(pdev);
2290
2291         return 0;
2292 }
2293
2294 /**
2295  * i40evf_resume - Power managment resume routine
2296  * @pdev: PCI device information struct
2297  *
2298  * Called when the system (VM) is resumed from sleep/suspend.
2299  **/
2300 static int i40evf_resume(struct pci_dev *pdev)
2301 {
2302         struct i40evf_adapter *adapter = pci_get_drvdata(pdev);
2303         struct net_device *netdev = adapter->netdev;
2304         u32 err;
2305
2306         pci_set_power_state(pdev, PCI_D0);
2307         pci_restore_state(pdev);
2308         /* pci_restore_state clears dev->state_saved so call
2309          * pci_save_state to restore it.
2310          */
2311         pci_save_state(pdev);
2312
2313         err = pci_enable_device_mem(pdev);
2314         if (err) {
2315                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend.\n");
2316                 return err;
2317         }
2318         pci_set_master(pdev);
2319
2320         rtnl_lock();
2321         err = i40evf_set_interrupt_capability(adapter);
2322         if (err) {
2323                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
2324                 return err;
2325         }
2326         err = i40evf_request_misc_irq(adapter);
2327         rtnl_unlock();
2328         if (err) {
2329                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
2330                 return err;
2331         }
2332
2333         schedule_work(&adapter->reset_task);
2334
2335         netif_device_attach(netdev);
2336
2337         return err;
2338 }
2339
2340 #endif /* CONFIG_PM */
2341 /**
2342  * i40evf_remove - Device Removal Routine
2343  * @pdev: PCI device information struct
2344  *
2345  * i40evf_remove is called by the PCI subsystem to alert the driver
2346  * that it should release a PCI device.  The could be caused by a
2347  * Hot-Plug event, or because the driver is going to be removed from
2348  * memory.
2349  **/
2350 static void i40evf_remove(struct pci_dev *pdev)
2351 {
2352         struct net_device *netdev = pci_get_drvdata(pdev);
2353         struct i40evf_adapter *adapter = netdev_priv(netdev);
2354         struct i40e_hw *hw = &adapter->hw;
2355
2356         cancel_delayed_work_sync(&adapter->init_task);
2357         cancel_work_sync(&adapter->reset_task);
2358
2359         if (adapter->netdev_registered) {
2360                 unregister_netdev(netdev);
2361                 adapter->netdev_registered = false;
2362         }
2363         adapter->state = __I40EVF_REMOVE;
2364
2365         if (adapter->msix_entries) {
2366                 i40evf_misc_irq_disable(adapter);
2367                 i40evf_free_misc_irq(adapter);
2368                 i40evf_reset_interrupt_capability(adapter);
2369         }
2370
2371         del_timer_sync(&adapter->watchdog_timer);
2372         flush_scheduled_work();
2373
2374         if (hw->aq.asq.count)
2375                 i40evf_shutdown_adminq(hw);
2376
2377         iounmap(hw->hw_addr);
2378         pci_release_regions(pdev);
2379
2380         i40evf_free_queues(adapter);
2381         kfree(adapter->vf_res);
2382
2383         free_netdev(netdev);
2384
2385         pci_disable_pcie_error_reporting(pdev);
2386
2387         pci_disable_device(pdev);
2388 }
2389
2390 static struct pci_driver i40evf_driver = {
2391         .name     = i40evf_driver_name,
2392         .id_table = i40evf_pci_tbl,
2393         .probe    = i40evf_probe,
2394         .remove   = i40evf_remove,
2395 #ifdef CONFIG_PM
2396         .suspend  = i40evf_suspend,
2397         .resume   = i40evf_resume,
2398 #endif
2399         .shutdown = i40evf_shutdown,
2400 };
2401
2402 /**
2403  * i40e_init_module - Driver Registration Routine
2404  *
2405  * i40e_init_module is the first routine called when the driver is
2406  * loaded. All it does is register with the PCI subsystem.
2407  **/
2408 static int __init i40evf_init_module(void)
2409 {
2410         int ret;
2411         pr_info("i40evf: %s - version %s\n", i40evf_driver_string,
2412                i40evf_driver_version);
2413
2414         pr_info("%s\n", i40evf_copyright);
2415
2416         ret = pci_register_driver(&i40evf_driver);
2417         return ret;
2418 }
2419
2420 module_init(i40evf_init_module);
2421
2422 /**
2423  * i40e_exit_module - Driver Exit Cleanup Routine
2424  *
2425  * i40e_exit_module is called just before the driver is removed
2426  * from memory.
2427  **/
2428 static void __exit i40evf_exit_module(void)
2429 {
2430         pci_unregister_driver(&i40evf_driver);
2431 }
2432
2433 module_exit(i40evf_exit_module);
2434
2435 /* i40evf_main.c */