drivers/net/ethernet/intel/i40evf/i40evf_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
   4  * Copyright(c) 2013 - 2014 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * The full GNU General Public License is included in this distribution in
  16  * the file called "COPYING".
  17  *
  18  * Contact Information:
  19  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  20  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  21  *
  22  ******************************************************************************/
  23
  24 #include "i40evf.h"
  25 #include "i40e_prototype.h"
  26 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter);
  27 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter);
  28 static int i40evf_close(struct net_device *netdev);
  29
  30 char i40evf_driver_name[] = "i40evf";
  31 static const char i40evf_driver_string[] =
  32         "Intel(R) XL710 X710 Virtual Function Network Driver";
  33
  34 #define DRV_VERSION "0.9.16"
  35 const char i40evf_driver_version[] = DRV_VERSION;
  36 static const char i40evf_copyright[] =
  37         "Copyright (c) 2013 - 2014 Intel Corporation.";
  38
  39 /* i40evf_pci_tbl - PCI Device ID Table
  40  *
  41  * Wildcard entries (PCI_ANY_ID) should come last
  42  * Last entry must be all 0s
  43  *
  44  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  45  *   Class, Class Mask, private data (not used) }
  46  */
  47 static DEFINE_PCI_DEVICE_TABLE(i40evf_pci_tbl) = {
  48         {PCI_VDEVICE(INTEL, I40E_DEV_ID_VF), 0},
  49         /* required last entry */
  50         {0, }
  51 };
  52
  53 MODULE_DEVICE_TABLE(pci, i40evf_pci_tbl);
  54
  55 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  56 MODULE_DESCRIPTION("Intel(R) XL710 X710 Virtual Function Network Driver");
  57 MODULE_LICENSE("GPL");
  58 MODULE_VERSION(DRV_VERSION);
  59
  60 /**
  61  * i40evf_allocate_dma_mem_d - OS specific memory alloc for shared code
  62  * @hw:   pointer to the HW structure
  63  * @mem:  ptr to mem struct to fill out
  64  * @size: size of memory requested
  65  * @alignment: what to align the allocation to
  66  **/
  67 i40e_status i40evf_allocate_dma_mem_d(struct i40e_hw *hw,
  68                                       struct i40e_dma_mem *mem,
  69                                       u64 size, u32 alignment)
  70 {
  71         struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
  72
  73         if (!mem)
  74                 return I40E_ERR_PARAM;
  75
  76         mem->size = ALIGN(size, alignment);
  77         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
  78                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
  79         if (mem->va)
  80                 return 0;
  81         else
  82                 return I40E_ERR_NO_MEMORY;
  83 }
  84
  85 /**
  86  * i40evf_free_dma_mem_d - OS specific memory free for shared code
  87  * @hw:   pointer to the HW structure
  88  * @mem:  ptr to mem struct to free
  89  **/
  90 i40e_status i40evf_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
  91 {
  92         struct i40evf_adapter *adapter = (struct i40evf_adapter *)hw->back;
  93
  94         if (!mem || !mem->va)
  95                 return I40E_ERR_PARAM;
  96         dma_free_coherent(&adapter->pdev->dev, mem->size,
  97                           mem->va, (dma_addr_t)mem->pa);
  98         return 0;
  99 }
 100
 101 /**
 102  * i40evf_allocate_virt_mem_d - OS specific memory alloc for shared code
 103  * @hw:   pointer to the HW structure
 104  * @mem:  ptr to mem struct to fill out
 105  * @size: size of memory requested
 106  **/
 107 i40e_status i40evf_allocate_virt_mem_d(struct i40e_hw *hw,
 108                                        struct i40e_virt_mem *mem, u32 size)
 109 {
 110         if (!mem)
 111                 return I40E_ERR_PARAM;
 112
 113         mem->size = size;
 114         mem->va = kzalloc(size, GFP_KERNEL);
 115
 116         if (mem->va)
 117                 return 0;
 118         else
 119                 return I40E_ERR_NO_MEMORY;
 120 }
 121
 122 /**
 123  * i40evf_free_virt_mem_d - OS specific memory free for shared code
 124  * @hw:   pointer to the HW structure
 125  * @mem:  ptr to mem struct to free
 126  **/
 127 i40e_status i40evf_free_virt_mem_d(struct i40e_hw *hw,
 128                                    struct i40e_virt_mem *mem)
 129 {
 130         if (!mem)
 131                 return I40E_ERR_PARAM;
 132
 133         /* it's ok to kfree a NULL pointer */
 134         kfree(mem->va);
 135
 136         return 0;
 137 }
 138
 139 /**
 140  * i40evf_debug_d - OS dependent version of debug printing
 141  * @hw:  pointer to the HW structure
 142  * @mask: debug level mask
 143  * @fmt_str: printf-type format description
 144  **/
 145 void i40evf_debug_d(void *hw, u32 mask, char *fmt_str, ...)
 146 {
 147         char buf[512];
 148         va_list argptr;
 149
 150         if (!(mask & ((struct i40e_hw *)hw)->debug_mask))
 151                 return;
 152
 153         va_start(argptr, fmt_str);
 154         vsnprintf(buf, sizeof(buf), fmt_str, argptr);
 155         va_end(argptr);
 156
 157         /* the debug string is already formatted with a newline */
 158         pr_info("%s", buf);
 159 }
 160
 161 /**
 162  * i40evf_tx_timeout - Respond to a Tx Hang
 163  * @netdev: network interface device structure
 164  **/
 165 static void i40evf_tx_timeout(struct net_device *netdev)
 166 {
 167         struct i40evf_adapter *adapter = netdev_priv(netdev);
 168
 169         adapter->tx_timeout_count++;
 170         dev_info(&adapter->pdev->dev, "TX timeout detected.\n");
 171         if (!(adapter->flags & I40EVF_FLAG_RESET_PENDING)) {
 172                 adapter->flags |= I40EVF_FLAG_RESET_NEEDED;
 173                 schedule_work(&adapter->reset_task);
 174         }
 175 }
 176
 177 /**
 178  * i40evf_misc_irq_disable - Mask off interrupt generation on the NIC
 179  * @adapter: board private structure
 180  **/
 181 static void i40evf_misc_irq_disable(struct i40evf_adapter *adapter)
 182 {
 183         struct i40e_hw *hw = &adapter->hw;
 184         wr32(hw, I40E_VFINT_DYN_CTL01, 0);
 185
 186         /* read flush */
 187         rd32(hw, I40E_VFGEN_RSTAT);
 188
 189         synchronize_irq(adapter->msix_entries[0].vector);
 190 }
 191
 192 /**
 193  * i40evf_misc_irq_enable - Enable default interrupt generation settings
 194  * @adapter: board private structure
 195  **/
 196 static void i40evf_misc_irq_enable(struct i40evf_adapter *adapter)
 197 {
 198         struct i40e_hw *hw = &adapter->hw;
 199         wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK |
 200                                        I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
 201         wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA_ADMINQ_MASK);
 202
 203         /* read flush */
 204         rd32(hw, I40E_VFGEN_RSTAT);
 205 }
 206
 207 /**
 208  * i40evf_irq_disable - Mask off interrupt generation on the NIC
 209  * @adapter: board private structure
 210  **/
 211 static void i40evf_irq_disable(struct i40evf_adapter *adapter)
 212 {
 213         int i;
 214         struct i40e_hw *hw = &adapter->hw;
 215
 216         if (!adapter->msix_entries)
 217                 return;
 218
 219         for (i = 1; i < adapter->num_msix_vectors; i++) {
 220                 wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), 0);
 221                 synchronize_irq(adapter->msix_entries[i].vector);
 222         }
 223         /* read flush */
 224         rd32(hw, I40E_VFGEN_RSTAT);
 225
 226 }
 227
 228 /**
 229  * i40evf_irq_enable_queues - Enable interrupt for specified queues
 230  * @adapter: board private structure
 231  * @mask: bitmap of queues to enable
 232  **/
 233 void i40evf_irq_enable_queues(struct i40evf_adapter *adapter, u32 mask)
 234 {
 235         struct i40e_hw *hw = &adapter->hw;
 236         int i;
 237
 238         for (i = 1; i < adapter->num_msix_vectors; i++) {
 239                 if (mask & (1 << (i - 1))) {
 240                         wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1),
 241                              I40E_VFINT_DYN_CTLN1_INTENA_MASK |
 242                              I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
 243                 }
 244         }
 245 }
 246
 247 /**
 248  * i40evf_fire_sw_int - Generate SW interrupt for specified vectors
 249  * @adapter: board private structure
 250  * @mask: bitmap of vectors to trigger
 251  **/
 252 static void i40evf_fire_sw_int(struct i40evf_adapter *adapter,
 253                                             u32 mask)
 254 {
 255         struct i40e_hw *hw = &adapter->hw;
 256         int i;
 257         uint32_t dyn_ctl;
 258
 259         for (i = 1; i < adapter->num_msix_vectors; i++) {
 260                 if (mask & (1 << i)) {
 261                         dyn_ctl = rd32(hw, I40E_VFINT_DYN_CTLN1(i - 1));
 262                         dyn_ctl |= I40E_VFINT_DYN_CTLN_SWINT_TRIG_MASK |
 263                                    I40E_VFINT_DYN_CTLN_CLEARPBA_MASK;
 264                         wr32(hw, I40E_VFINT_DYN_CTLN1(i - 1), dyn_ctl);
 265                 }
 266         }
 267 }
 268
 269 /**
 270  * i40evf_irq_enable - Enable default interrupt generation settings
 271  * @adapter: board private structure
 272  **/
 273 void i40evf_irq_enable(struct i40evf_adapter *adapter, bool flush)
 274 {
 275         struct i40e_hw *hw = &adapter->hw;
 276
 277         i40evf_irq_enable_queues(adapter, ~0);
 278
 279         if (flush)
 280                 rd32(hw, I40E_VFGEN_RSTAT);
 281 }
 282
 283 /**
 284  * i40evf_msix_aq - Interrupt handler for vector 0
 285  * @irq: interrupt number
 286  * @data: pointer to netdev
 287  **/
 288 static irqreturn_t i40evf_msix_aq(int irq, void *data)
 289 {
 290         struct net_device *netdev = data;
 291         struct i40evf_adapter *adapter = netdev_priv(netdev);
 292         struct i40e_hw *hw = &adapter->hw;
 293         u32 val;
 294         u32 ena_mask;
 295
 296         /* handle non-queue interrupts */
 297         val = rd32(hw, I40E_VFINT_ICR01);
 298         ena_mask = rd32(hw, I40E_VFINT_ICR0_ENA1);
 299
 300
 301         val = rd32(hw, I40E_VFINT_DYN_CTL01);
 302         val = val | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
 303         wr32(hw, I40E_VFINT_DYN_CTL01, val);
 304
 305         /* re-enable interrupt causes */
 306         wr32(hw, I40E_VFINT_ICR0_ENA1, ena_mask);
 307         wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK);
 308
 309         /* schedule work on the private workqueue */
 310         schedule_work(&adapter->adminq_task);
 311
 312         return IRQ_HANDLED;
 313 }
 314
 315 /**
 316  * i40evf_msix_clean_rings - MSIX mode Interrupt Handler
 317  * @irq: interrupt number
 318  * @data: pointer to a q_vector
 319  **/
 320 static irqreturn_t i40evf_msix_clean_rings(int irq, void *data)
 321 {
 322         struct i40e_q_vector *q_vector = data;
 323
 324         if (!q_vector->tx.ring && !q_vector->rx.ring)
 325                 return IRQ_HANDLED;
 326
 327         napi_schedule(&q_vector->napi);
 328
 329         return IRQ_HANDLED;
 330 }
 331
 332 /**
 333  * i40evf_map_vector_to_rxq - associate irqs with rx queues
 334  * @adapter: board private structure
 335  * @v_idx: interrupt number
 336  * @r_idx: queue number
 337  **/
 338 static void
 339 i40evf_map_vector_to_rxq(struct i40evf_adapter *adapter, int v_idx, int r_idx)
 340 {
 341         struct i40e_q_vector *q_vector = adapter->q_vector[v_idx];
 342         struct i40e_ring *rx_ring = adapter->rx_rings[r_idx];
 343
 344         rx_ring->q_vector = q_vector;
 345         rx_ring->next = q_vector->rx.ring;
 346         rx_ring->vsi = &adapter->vsi;
 347         q_vector->rx.ring = rx_ring;
 348         q_vector->rx.count++;
 349         q_vector->rx.latency_range = I40E_LOW_LATENCY;
 350 }
 351
 352 /**
 353  * i40evf_map_vector_to_txq - associate irqs with tx queues
 354  * @adapter: board private structure
 355  * @v_idx: interrupt number
 356  * @t_idx: queue number
 357  **/
 358 static void
 359 i40evf_map_vector_to_txq(struct i40evf_adapter *adapter, int v_idx, int t_idx)
 360 {
 361         struct i40e_q_vector *q_vector = adapter->q_vector[v_idx];
 362         struct i40e_ring *tx_ring = adapter->tx_rings[t_idx];
 363
 364         tx_ring->q_vector = q_vector;
 365         tx_ring->next = q_vector->tx.ring;
 366         tx_ring->vsi = &adapter->vsi;
 367         q_vector->tx.ring = tx_ring;
 368         q_vector->tx.count++;
 369         q_vector->tx.latency_range = I40E_LOW_LATENCY;
 370         q_vector->num_ringpairs++;
 371         q_vector->ring_mask |= (1 << t_idx);
 372 }
 373
 374 /**
 375  * i40evf_map_rings_to_vectors - Maps descriptor rings to vectors
 376  * @adapter: board private structure to initialize
 377  *
 378  * This function maps descriptor rings to the queue-specific vectors
 379  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 380  * one vector per ring/queue, but on a constrained vector budget, we
 381  * group the rings as "efficiently" as possible.  You would add new
 382  * mapping configurations in here.
 383  **/
 384 static int i40evf_map_rings_to_vectors(struct i40evf_adapter *adapter)
 385 {
 386         int q_vectors;
 387         int v_start = 0;
 388         int rxr_idx = 0, txr_idx = 0;
 389         int rxr_remaining = adapter->vsi_res->num_queue_pairs;
 390         int txr_remaining = adapter->vsi_res->num_queue_pairs;
 391         int i, j;
 392         int rqpv, tqpv;
 393         int err = 0;
 394
 395         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 396
 397         /* The ideal configuration...
 398          * We have enough vectors to map one per queue.
 399          */
 400         if (q_vectors == (rxr_remaining * 2)) {
 401                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
 402                         i40evf_map_vector_to_rxq(adapter, v_start, rxr_idx);
 403
 404                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
 405                         i40evf_map_vector_to_txq(adapter, v_start, txr_idx);
 406                 goto out;
 407         }
 408
 409         /* If we don't have enough vectors for a 1-to-1
 410          * mapping, we'll have to group them so there are
 411          * multiple queues per vector.
 412          * Re-adjusting *qpv takes care of the remainder.
 413          */
 414         for (i = v_start; i < q_vectors; i++) {
 415                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
 416                 for (j = 0; j < rqpv; j++) {
 417                         i40evf_map_vector_to_rxq(adapter, i, rxr_idx);
 418                         rxr_idx++;
 419                         rxr_remaining--;
 420                 }
 421         }
 422         for (i = v_start; i < q_vectors; i++) {
 423                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
 424                 for (j = 0; j < tqpv; j++) {
 425                         i40evf_map_vector_to_txq(adapter, i, txr_idx);
 426                         txr_idx++;
 427                         txr_remaining--;
 428                 }
 429         }
 430
 431 out:
 432         adapter->aq_required |= I40EVF_FLAG_AQ_MAP_VECTORS;
 433
 434         return err;
 435 }
 436
 437 /**
 438  * i40evf_request_traffic_irqs - Initialize MSI-X interrupts
 439  * @adapter: board private structure
 440  *
 441  * Allocates MSI-X vectors for tx and rx handling, and requests
 442  * interrupts from the kernel.
 443  **/
 444 static int
 445 i40evf_request_traffic_irqs(struct i40evf_adapter *adapter, char *basename)
 446 {
 447         int vector, err, q_vectors;
 448         int rx_int_idx = 0, tx_int_idx = 0;
 449
 450         i40evf_irq_disable(adapter);
 451         /* Decrement for Other and TCP Timer vectors */
 452         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 453
 454         for (vector = 0; vector < q_vectors; vector++) {
 455                 struct i40e_q_vector *q_vector = adapter->q_vector[vector];
 456
 457                 if (q_vector->tx.ring && q_vector->rx.ring) {
 458                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 459                                  "i40evf-%s-%s-%d", basename,
 460                                  "TxRx", rx_int_idx++);
 461                         tx_int_idx++;
 462                 } else if (q_vector->rx.ring) {
 463                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 464                                  "i40evf-%s-%s-%d", basename,
 465                                  "rx", rx_int_idx++);
 466                 } else if (q_vector->tx.ring) {
 467                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
 468                                  "i40evf-%s-%s-%d", basename,
 469                                  "tx", tx_int_idx++);
 470                 } else {
 471                         /* skip this unused q_vector */
 472                         continue;
 473                 }
 474                 err = request_irq(
 475                         adapter->msix_entries[vector + NONQ_VECS].vector,
 476                         i40evf_msix_clean_rings,
 477                         0,
 478                         q_vector->name,
 479                         q_vector);
 480                 if (err) {
 481                         dev_info(&adapter->pdev->dev,
 482                                  "%s: request_irq failed, error: %d\n",
 483                                 __func__, err);
 484                         goto free_queue_irqs;
 485                 }
 486                 /* assign the mask for this irq */
 487                 irq_set_affinity_hint(
 488                         adapter->msix_entries[vector + NONQ_VECS].vector,
 489                         q_vector->affinity_mask);
 490         }
 491
 492         return 0;
 493
 494 free_queue_irqs:
 495         while (vector) {
 496                 vector--;
 497                 irq_set_affinity_hint(
 498                         adapter->msix_entries[vector + NONQ_VECS].vector,
 499                         NULL);
 500                 free_irq(adapter->msix_entries[vector + NONQ_VECS].vector,
 501                          adapter->q_vector[vector]);
 502         }
 503         return err;
 504 }
 505
 506 /**
 507  * i40evf_request_misc_irq - Initialize MSI-X interrupts
 508  * @adapter: board private structure
 509  *
 510  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 511  * vector is only for the admin queue, and stays active even when the netdev
 512  * is closed.
 513  **/
 514 static int i40evf_request_misc_irq(struct i40evf_adapter *adapter)
 515 {
 516         struct net_device *netdev = adapter->netdev;
 517         int err;
 518
 519         sprintf(adapter->misc_vector_name, "i40evf:mbx");
 520         err = request_irq(adapter->msix_entries[0].vector,
 521                           &i40evf_msix_aq, 0,
 522                           adapter->misc_vector_name, netdev);
 523         if (err) {
 524                 dev_err(&adapter->pdev->dev,
 525                         "request_irq for %s failed: %d\n",
 526                         adapter->misc_vector_name, err);
 527                 free_irq(adapter->msix_entries[0].vector, netdev);
 528         }
 529         return err;
 530 }
 531
 532 /**
 533  * i40evf_free_traffic_irqs - Free MSI-X interrupts
 534  * @adapter: board private structure
 535  *
 536  * Frees all MSI-X vectors other than 0.
 537  **/
 538 static void i40evf_free_traffic_irqs(struct i40evf_adapter *adapter)
 539 {
 540         int i;
 541         int q_vectors;
 542         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 543
 544         for (i = 0; i < q_vectors; i++) {
 545                 irq_set_affinity_hint(adapter->msix_entries[i+1].vector,
 546                                       NULL);
 547                 free_irq(adapter->msix_entries[i+1].vector,
 548                          adapter->q_vector[i]);
 549         }
 550 }
 551
 552 /**
 553  * i40evf_free_misc_irq - Free MSI-X miscellaneous vector
 554  * @adapter: board private structure
 555  *
 556  * Frees MSI-X vector 0.
 557  **/
 558 static void i40evf_free_misc_irq(struct i40evf_adapter *adapter)
 559 {
 560         struct net_device *netdev = adapter->netdev;
 561
 562         free_irq(adapter->msix_entries[0].vector, netdev);
 563 }
 564
 565 /**
 566  * i40evf_configure_tx - Configure Transmit Unit after Reset
 567  * @adapter: board private structure
 568  *
 569  * Configure the Tx unit of the MAC after a reset.
 570  **/
 571 static void i40evf_configure_tx(struct i40evf_adapter *adapter)
 572 {
 573         struct i40e_hw *hw = &adapter->hw;
 574         int i;
 575         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 576                 adapter->tx_rings[i]->tail = hw->hw_addr + I40E_QTX_TAIL1(i);
 577 }
 578
 579 /**
 580  * i40evf_configure_rx - Configure Receive Unit after Reset
 581  * @adapter: board private structure
 582  *
 583  * Configure the Rx unit of the MAC after a reset.
 584  **/
 585 static void i40evf_configure_rx(struct i40evf_adapter *adapter)
 586 {
 587         struct i40e_hw *hw = &adapter->hw;
 588         struct net_device *netdev = adapter->netdev;
 589         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
 590         int i;
 591         int rx_buf_len;
 592
 593
 594         adapter->flags &= ~I40EVF_FLAG_RX_PS_CAPABLE;
 595         adapter->flags |= I40EVF_FLAG_RX_1BUF_CAPABLE;
 596
 597         /* Decide whether to use packet split mode or not */
 598         if (netdev->mtu > ETH_DATA_LEN) {
 599                 if (adapter->flags & I40EVF_FLAG_RX_PS_CAPABLE)
 600                         adapter->flags |= I40EVF_FLAG_RX_PS_ENABLED;
 601                 else
 602                         adapter->flags &= ~I40EVF_FLAG_RX_PS_ENABLED;
 603         } else {
 604                 if (adapter->flags & I40EVF_FLAG_RX_1BUF_CAPABLE)
 605                         adapter->flags &= ~I40EVF_FLAG_RX_PS_ENABLED;
 606                 else
 607                         adapter->flags |= I40EVF_FLAG_RX_PS_ENABLED;
 608         }
 609
 610         /* Set the RX buffer length according to the mode */
 611         if (adapter->flags & I40EVF_FLAG_RX_PS_ENABLED) {
 612                 rx_buf_len = I40E_RX_HDR_SIZE;
 613         } else {
 614                 if (netdev->mtu <= ETH_DATA_LEN)
 615                         rx_buf_len = I40EVF_RXBUFFER_2048;
 616                 else
 617                         rx_buf_len = ALIGN(max_frame, 1024);
 618         }
 619
 620         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
 621                 adapter->rx_rings[i]->tail = hw->hw_addr + I40E_QRX_TAIL1(i);
 622                 adapter->rx_rings[i]->rx_buf_len = rx_buf_len;
 623         }
 624 }
 625
 626 /**
 627  * i40evf_find_vlan - Search filter list for specific vlan filter
 628  * @adapter: board private structure
 629  * @vlan: vlan tag
 630  *
 631  * Returns ptr to the filter object or NULL
 632  **/
 633 static struct
 634 i40evf_vlan_filter *i40evf_find_vlan(struct i40evf_adapter *adapter, u16 vlan)
 635 {
 636         struct i40evf_vlan_filter *f;
 637
 638         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 639                 if (vlan == f->vlan)
 640                         return f;
 641         }
 642         return NULL;
 643 }
 644
 645 /**
 646  * i40evf_add_vlan - Add a vlan filter to the list
 647  * @adapter: board private structure
 648  * @vlan: VLAN tag
 649  *
 650  * Returns ptr to the filter object or NULL when no memory available.
 651  **/
 652 static struct
 653 i40evf_vlan_filter *i40evf_add_vlan(struct i40evf_adapter *adapter, u16 vlan)
 654 {
 655         struct i40evf_vlan_filter *f;
 656
 657         f = i40evf_find_vlan(adapter, vlan);
 658         if (NULL == f) {
 659                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 660                 if (NULL == f) {
 661                         dev_info(&adapter->pdev->dev,
 662                                  "%s: no memory for new VLAN filter\n",
 663                                  __func__);
 664                         return NULL;
 665                 }
 666                 f->vlan = vlan;
 667
 668                 INIT_LIST_HEAD(&f->list);
 669                 list_add(&f->list, &adapter->vlan_filter_list);
 670                 f->add = true;
 671                 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_VLAN_FILTER;
 672         }
 673
 674         return f;
 675 }
 676
 677 /**
 678  * i40evf_del_vlan - Remove a vlan filter from the list
 679  * @adapter: board private structure
 680  * @vlan: VLAN tag
 681  **/
 682 static void i40evf_del_vlan(struct i40evf_adapter *adapter, u16 vlan)
 683 {
 684         struct i40evf_vlan_filter *f;
 685
 686         f = i40evf_find_vlan(adapter, vlan);
 687         if (f) {
 688                 f->remove = true;
 689                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
 690         }
 691         return;
 692 }
 693
 694 /**
 695  * i40evf_vlan_rx_add_vid - Add a VLAN filter to a device
 696  * @netdev: network device struct
 697  * @vid: VLAN tag
 698  **/
 699 static int i40evf_vlan_rx_add_vid(struct net_device *netdev,
 700                          __always_unused __be16 proto, u16 vid)
 701 {
 702         struct i40evf_adapter *adapter = netdev_priv(netdev);
 703
 704         if (i40evf_add_vlan(adapter, vid) == NULL)
 705                 return -ENOMEM;
 706         return 0;
 707 }
 708
 709 /**
 710  * i40evf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 711  * @netdev: network device struct
 712  * @vid: VLAN tag
 713  **/
 714 static int i40evf_vlan_rx_kill_vid(struct net_device *netdev,
 715                           __always_unused __be16 proto, u16 vid)
 716 {
 717         struct i40evf_adapter *adapter = netdev_priv(netdev);
 718
 719         i40evf_del_vlan(adapter, vid);
 720         return 0;
 721 }
 722
 723 /**
 724  * i40evf_find_filter - Search filter list for specific mac filter
 725  * @adapter: board private structure
 726  * @macaddr: the MAC address
 727  *
 728  * Returns ptr to the filter object or NULL
 729  **/
 730 static struct
 731 i40evf_mac_filter *i40evf_find_filter(struct i40evf_adapter *adapter,
 732                                       u8 *macaddr)
 733 {
 734         struct i40evf_mac_filter *f;
 735
 736         if (!macaddr)
 737                 return NULL;
 738
 739         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 740                 if (ether_addr_equal(macaddr, f->macaddr))
 741                         return f;
 742         }
 743         return NULL;
 744 }
 745
 746 /**
 747  * i40e_add_filter - Add a mac filter to the filter list
 748  * @adapter: board private structure
 749  * @macaddr: the MAC address
 750  *
 751  * Returns ptr to the filter object or NULL when no memory available.
 752  **/
 753 static struct
 754 i40evf_mac_filter *i40evf_add_filter(struct i40evf_adapter *adapter,
 755                                      u8 *macaddr)
 756 {
 757         struct i40evf_mac_filter *f;
 758
 759         if (!macaddr)
 760                 return NULL;
 761
 762         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
 763                                 &adapter->crit_section))
 764                 mdelay(1);
 765
 766         f = i40evf_find_filter(adapter, macaddr);
 767         if (NULL == f) {
 768                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 769                 if (NULL == f) {
 770                         dev_info(&adapter->pdev->dev,
 771                                  "%s: no memory for new filter\n", __func__);
 772                         clear_bit(__I40EVF_IN_CRITICAL_TASK,
 773                                   &adapter->crit_section);
 774                         return NULL;
 775                 }
 776
 777                 memcpy(f->macaddr, macaddr, ETH_ALEN);
 778
 779                 list_add(&f->list, &adapter->mac_filter_list);
 780                 f->add = true;
 781                 adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
 782         }
 783
 784         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 785         return f;
 786 }
 787
 788 /**
 789  * i40evf_set_mac - NDO callback to set port mac address
 790  * @netdev: network interface device structure
 791  * @p: pointer to an address structure
 792  *
 793  * Returns 0 on success, negative on failure
 794  **/
 795 static int i40evf_set_mac(struct net_device *netdev, void *p)
 796 {
 797         struct i40evf_adapter *adapter = netdev_priv(netdev);
 798         struct i40e_hw *hw = &adapter->hw;
 799         struct i40evf_mac_filter *f;
 800         struct sockaddr *addr = p;
 801
 802         if (!is_valid_ether_addr(addr->sa_data))
 803                 return -EADDRNOTAVAIL;
 804
 805         if (ether_addr_equal(netdev->dev_addr, addr->sa_data))
 806                 return 0;
 807
 808         f = i40evf_add_filter(adapter, addr->sa_data);
 809         if (f) {
 810                 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 811                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
 812                        netdev->addr_len);
 813         }
 814
 815         return (f == NULL) ? -ENOMEM : 0;
 816 }
 817
 818 /**
 819  * i40evf_set_rx_mode - NDO callback to set the netdev filters
 820  * @netdev: network interface device structure
 821  **/
 822 static void i40evf_set_rx_mode(struct net_device *netdev)
 823 {
 824         struct i40evf_adapter *adapter = netdev_priv(netdev);
 825         struct i40evf_mac_filter *f, *ftmp;
 826         struct netdev_hw_addr *uca;
 827         struct netdev_hw_addr *mca;
 828
 829         /* add addr if not already in the filter list */
 830         netdev_for_each_uc_addr(uca, netdev) {
 831                 i40evf_add_filter(adapter, uca->addr);
 832         }
 833         netdev_for_each_mc_addr(mca, netdev) {
 834                 i40evf_add_filter(adapter, mca->addr);
 835         }
 836
 837         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
 838                                 &adapter->crit_section))
 839                 mdelay(1);
 840         /* remove filter if not in netdev list */
 841         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
 842                 bool found = false;
 843
 844                 if (f->macaddr[0] & 0x01) {
 845                         netdev_for_each_mc_addr(mca, netdev) {
 846                                 if (ether_addr_equal(mca->addr, f->macaddr)) {
 847                                         found = true;
 848                                         break;
 849                                 }
 850                         }
 851                 } else {
 852                         netdev_for_each_uc_addr(uca, netdev) {
 853                                 if (ether_addr_equal(uca->addr, f->macaddr)) {
 854                                         found = true;
 855                                         break;
 856                                 }
 857                         }
 858                 }
 859                 if (found) {
 860                         f->remove = true;
 861                         adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
 862                 }
 863         }
 864         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
 865 }
 866
 867 /**
 868  * i40evf_napi_enable_all - enable NAPI on all queue vectors
 869  * @adapter: board private structure
 870  **/
 871 static void i40evf_napi_enable_all(struct i40evf_adapter *adapter)
 872 {
 873         int q_idx;
 874         struct i40e_q_vector *q_vector;
 875         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 876
 877         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 878                 struct napi_struct *napi;
 879                 q_vector = adapter->q_vector[q_idx];
 880                 napi = &q_vector->napi;
 881                 napi_enable(napi);
 882         }
 883 }
 884
 885 /**
 886  * i40evf_napi_disable_all - disable NAPI on all queue vectors
 887  * @adapter: board private structure
 888  **/
 889 static void i40evf_napi_disable_all(struct i40evf_adapter *adapter)
 890 {
 891         int q_idx;
 892         struct i40e_q_vector *q_vector;
 893         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 894
 895         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
 896                 q_vector = adapter->q_vector[q_idx];
 897                 napi_disable(&q_vector->napi);
 898         }
 899 }
 900
 901 /**
 902  * i40evf_configure - set up transmit and receive data structures
 903  * @adapter: board private structure
 904  **/
 905 static void i40evf_configure(struct i40evf_adapter *adapter)
 906 {
 907         struct net_device *netdev = adapter->netdev;
 908         int i;
 909
 910         i40evf_set_rx_mode(netdev);
 911
 912         i40evf_configure_tx(adapter);
 913         i40evf_configure_rx(adapter);
 914         adapter->aq_required |= I40EVF_FLAG_AQ_CONFIGURE_QUEUES;
 915
 916         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
 917                 struct i40e_ring *ring = adapter->rx_rings[i];
 918                 i40evf_alloc_rx_buffers(ring, ring->count);
 919                 ring->next_to_use = ring->count - 1;
 920                 writel(ring->next_to_use, ring->tail);
 921         }
 922 }
 923
 924 /**
 925  * i40evf_up_complete - Finish the last steps of bringing up a connection
 926  * @adapter: board private structure
 927  **/
 928 static int i40evf_up_complete(struct i40evf_adapter *adapter)
 929 {
 930         adapter->state = __I40EVF_RUNNING;
 931         clear_bit(__I40E_DOWN, &adapter->vsi.state);
 932
 933         i40evf_napi_enable_all(adapter);
 934
 935         adapter->aq_required |= I40EVF_FLAG_AQ_ENABLE_QUEUES;
 936         mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 937         return 0;
 938 }
 939
 940 /**
 941  * i40evf_clean_all_rx_rings - Free Rx Buffers for all queues
 942  * @adapter: board private structure
 943  **/
 944 static void i40evf_clean_all_rx_rings(struct i40evf_adapter *adapter)
 945 {
 946         int i;
 947
 948         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 949                 i40evf_clean_rx_ring(adapter->rx_rings[i]);
 950 }
 951
 952 /**
 953  * i40evf_clean_all_tx_rings - Free Tx Buffers for all queues
 954  * @adapter: board private structure
 955  **/
 956 static void i40evf_clean_all_tx_rings(struct i40evf_adapter *adapter)
 957 {
 958         int i;
 959
 960         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
 961                 i40evf_clean_tx_ring(adapter->tx_rings[i]);
 962 }
 963
 964 /**
 965  * i40e_down - Shutdown the connection processing
 966  * @adapter: board private structure
 967  **/
 968 void i40evf_down(struct i40evf_adapter *adapter)
 969 {
 970         struct net_device *netdev = adapter->netdev;
 971         struct i40evf_mac_filter *f;
 972
 973         /* remove all MAC filters */
 974         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 975                 f->remove = true;
 976         }
 977         /* remove all VLAN filters */
 978         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 979                 f->remove = true;
 980         }
 981         if (!(adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) &&
 982             adapter->state != __I40EVF_RESETTING) {
 983                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_MAC_FILTER;
 984                 adapter->aq_required |= I40EVF_FLAG_AQ_DEL_VLAN_FILTER;
 985                 /* disable receives */
 986                 adapter->aq_required |= I40EVF_FLAG_AQ_DISABLE_QUEUES;
 987                 mod_timer_pending(&adapter->watchdog_timer, jiffies + 1);
 988                 msleep(20);
 989         }
 990         netif_tx_disable(netdev);
 991
 992         netif_tx_stop_all_queues(netdev);
 993
 994         i40evf_irq_disable(adapter);
 995
 996         i40evf_napi_disable_all(adapter);
 997
 998         netif_carrier_off(netdev);
 999
1000         i40evf_clean_all_tx_rings(adapter);
1001         i40evf_clean_all_rx_rings(adapter);
1002 }
1003
1004 /**
1005  * i40evf_acquire_msix_vectors - Setup the MSIX capability
1006  * @adapter: board private structure
1007  * @vectors: number of vectors to request
1008  *
1009  * Work with the OS to set up the MSIX vectors needed.
1010  *
1011  * Returns 0 on success, negative on failure
1012  **/
1013 static int
1014 i40evf_acquire_msix_vectors(struct i40evf_adapter *adapter, int vectors)
1015 {
1016         int err, vector_threshold;
1017
1018         /* We'll want at least 3 (vector_threshold):
1019          * 0) Other (Admin Queue and link, mostly)
1020          * 1) TxQ[0] Cleanup
1021          * 2) RxQ[0] Cleanup
1022          */
1023         vector_threshold = MIN_MSIX_COUNT;
1024
1025         /* The more we get, the more we will assign to Tx/Rx Cleanup
1026          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1027          * Right now, we simply care about how many we'll get; we'll
1028          * set them up later while requesting irq's.
1029          */
1030         while (vectors >= vector_threshold) {
1031                 err = pci_enable_msix(adapter->pdev, adapter->msix_entries,
1032                                       vectors);
1033                 if (!err) /* Success in acquiring all requested vectors. */
1034                         break;
1035                 else if (err < 0)
1036                         vectors = 0; /* Nasty failure, quit now */
1037                 else /* err == number of vectors we should try again with */
1038                         vectors = err;
1039         }
1040
1041         if (vectors < vector_threshold) {
1042                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts.\n");
1043                 kfree(adapter->msix_entries);
1044                 adapter->msix_entries = NULL;
1045                 err = -EIO;
1046         } else {
1047                 /* Adjust for only the vectors we'll use, which is minimum
1048                  * of max_msix_q_vectors + NONQ_VECS, or the number of
1049                  * vectors we were allocated.
1050                  */
1051                 adapter->num_msix_vectors = vectors;
1052         }
1053         return err;
1054 }
1055
1056 /**
1057  * i40evf_free_queues - Free memory for all rings
1058  * @adapter: board private structure to initialize
1059  *
1060  * Free all of the memory associated with queue pairs.
1061  **/
1062 static void i40evf_free_queues(struct i40evf_adapter *adapter)
1063 {
1064         int i;
1065
1066         if (!adapter->vsi_res)
1067                 return;
1068         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1069                 if (adapter->tx_rings[i])
1070                         kfree_rcu(adapter->tx_rings[i], rcu);
1071                 adapter->tx_rings[i] = NULL;
1072                 adapter->rx_rings[i] = NULL;
1073         }
1074 }
1075
1076 /**
1077  * i40evf_alloc_queues - Allocate memory for all rings
1078  * @adapter: board private structure to initialize
1079  *
1080  * We allocate one ring per queue at run-time since we don't know the
1081  * number of queues at compile-time.  The polling_netdev array is
1082  * intended for Multiqueue, but should work fine with a single queue.
1083  **/
1084 static int i40evf_alloc_queues(struct i40evf_adapter *adapter)
1085 {
1086         int i;
1087
1088         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1089                 struct i40e_ring *tx_ring;
1090                 struct i40e_ring *rx_ring;
1091
1092                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
1093                 if (!tx_ring)
1094                         goto err_out;
1095
1096                 tx_ring->queue_index = i;
1097                 tx_ring->netdev = adapter->netdev;
1098                 tx_ring->dev = &adapter->pdev->dev;
1099                 tx_ring->count = I40EVF_DEFAULT_TXD;
1100                 adapter->tx_rings[i] = tx_ring;
1101
1102                 rx_ring = &tx_ring[1];
1103                 rx_ring->queue_index = i;
1104                 rx_ring->netdev = adapter->netdev;
1105                 rx_ring->dev = &adapter->pdev->dev;
1106                 rx_ring->count = I40EVF_DEFAULT_RXD;
1107                 adapter->rx_rings[i] = rx_ring;
1108         }
1109
1110         return 0;
1111
1112 err_out:
1113         i40evf_free_queues(adapter);
1114         return -ENOMEM;
1115 }
1116
1117 /**
1118  * i40evf_set_interrupt_capability - set MSI-X or FAIL if not supported
1119  * @adapter: board private structure to initialize
1120  *
1121  * Attempt to configure the interrupts using the best available
1122  * capabilities of the hardware and the kernel.
1123  **/
1124 static int i40evf_set_interrupt_capability(struct i40evf_adapter *adapter)
1125 {
1126         int vector, v_budget;
1127         int pairs = 0;
1128         int err = 0;
1129
1130         if (!adapter->vsi_res) {
1131                 err = -EIO;
1132                 goto out;
1133         }
1134         pairs = adapter->vsi_res->num_queue_pairs;
1135
1136         /* It's easy to be greedy for MSI-X vectors, but it really
1137          * doesn't do us much good if we have a lot more vectors
1138          * than CPU's.  So let's be conservative and only ask for
1139          * (roughly) twice the number of vectors as there are CPU's.
1140          */
1141         v_budget = min_t(int, pairs, (int)(num_online_cpus() * 2)) + NONQ_VECS;
1142         v_budget = min_t(int, v_budget, (int)adapter->vf_res->max_vectors);
1143
1144         /* A failure in MSI-X entry allocation isn't fatal, but it does
1145          * mean we disable MSI-X capabilities of the adapter.
1146          */
1147         adapter->msix_entries = kcalloc(v_budget,
1148                                         sizeof(struct msix_entry), GFP_KERNEL);
1149         if (!adapter->msix_entries) {
1150                 err = -ENOMEM;
1151                 goto out;
1152         }
1153
1154         for (vector = 0; vector < v_budget; vector++)
1155                 adapter->msix_entries[vector].entry = vector;
1156
1157         i40evf_acquire_msix_vectors(adapter, v_budget);
1158
1159 out:
1160         adapter->netdev->real_num_tx_queues = pairs;
1161         return err;
1162 }
1163
1164 /**
1165  * i40evf_alloc_q_vectors - Allocate memory for interrupt vectors
1166  * @adapter: board private structure to initialize
1167  *
1168  * We allocate one q_vector per queue interrupt.  If allocation fails we
1169  * return -ENOMEM.
1170  **/
1171 static int i40evf_alloc_q_vectors(struct i40evf_adapter *adapter)
1172 {
1173         int q_idx, num_q_vectors;
1174         struct i40e_q_vector *q_vector;
1175
1176         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1177
1178         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1179                 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
1180                 if (!q_vector)
1181                         goto err_out;
1182                 q_vector->adapter = adapter;
1183                 q_vector->vsi = &adapter->vsi;
1184                 q_vector->v_idx = q_idx;
1185                 netif_napi_add(adapter->netdev, &q_vector->napi,
1186                                        i40evf_napi_poll, 64);
1187                 adapter->q_vector[q_idx] = q_vector;
1188         }
1189
1190         return 0;
1191
1192 err_out:
1193         while (q_idx) {
1194                 q_idx--;
1195                 q_vector = adapter->q_vector[q_idx];
1196                 netif_napi_del(&q_vector->napi);
1197                 kfree(q_vector);
1198                 adapter->q_vector[q_idx] = NULL;
1199         }
1200         return -ENOMEM;
1201 }
1202
1203 /**
1204  * i40evf_free_q_vectors - Free memory allocated for interrupt vectors
1205  * @adapter: board private structure to initialize
1206  *
1207  * This function frees the memory allocated to the q_vectors.  In addition if
1208  * NAPI is enabled it will delete any references to the NAPI struct prior
1209  * to freeing the q_vector.
1210  **/
1211 static void i40evf_free_q_vectors(struct i40evf_adapter *adapter)
1212 {
1213         int q_idx, num_q_vectors;
1214         int napi_vectors;
1215
1216         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1217         napi_vectors = adapter->vsi_res->num_queue_pairs;
1218
1219         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1220                 struct i40e_q_vector *q_vector = adapter->q_vector[q_idx];
1221
1222                 adapter->q_vector[q_idx] = NULL;
1223                 if (q_idx < napi_vectors)
1224                         netif_napi_del(&q_vector->napi);
1225                 kfree(q_vector);
1226         }
1227 }
1228
1229 /**
1230  * i40evf_reset_interrupt_capability - Reset MSIX setup
1231  * @adapter: board private structure
1232  *
1233  **/
1234 void i40evf_reset_interrupt_capability(struct i40evf_adapter *adapter)
1235 {
1236         pci_disable_msix(adapter->pdev);
1237         kfree(adapter->msix_entries);
1238         adapter->msix_entries = NULL;
1239
1240         return;
1241 }
1242
1243 /**
1244  * i40evf_init_interrupt_scheme - Determine if MSIX is supported and init
1245  * @adapter: board private structure to initialize
1246  *
1247  **/
1248 int i40evf_init_interrupt_scheme(struct i40evf_adapter *adapter)
1249 {
1250         int err;
1251
1252         err = i40evf_set_interrupt_capability(adapter);
1253         if (err) {
1254                 dev_err(&adapter->pdev->dev,
1255                         "Unable to setup interrupt capabilities\n");
1256                 goto err_set_interrupt;
1257         }
1258
1259         err = i40evf_alloc_q_vectors(adapter);
1260         if (err) {
1261                 dev_err(&adapter->pdev->dev,
1262                         "Unable to allocate memory for queue vectors\n");
1263                 goto err_alloc_q_vectors;
1264         }
1265
1266         err = i40evf_alloc_queues(adapter);
1267         if (err) {
1268                 dev_err(&adapter->pdev->dev,
1269                         "Unable to allocate memory for queues\n");
1270                 goto err_alloc_queues;
1271         }
1272
1273         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1274                 (adapter->vsi_res->num_queue_pairs > 1) ? "Enabled" :
1275                 "Disabled", adapter->vsi_res->num_queue_pairs);
1276
1277         return 0;
1278 err_alloc_queues:
1279         i40evf_free_q_vectors(adapter);
1280 err_alloc_q_vectors:
1281         i40evf_reset_interrupt_capability(adapter);
1282 err_set_interrupt:
1283         return err;
1284 }
1285
1286 /**
1287  * i40evf_watchdog_timer - Periodic call-back timer
1288  * @data: pointer to adapter disguised as unsigned long
1289  **/
1290 static void i40evf_watchdog_timer(unsigned long data)
1291 {
1292         struct i40evf_adapter *adapter = (struct i40evf_adapter *)data;
1293         schedule_work(&adapter->watchdog_task);
1294         /* timer will be rescheduled in watchdog task */
1295 }
1296
1297 /**
1298  * i40evf_watchdog_task - Periodic call-back task
1299  * @work: pointer to work_struct
1300  **/
1301 static void i40evf_watchdog_task(struct work_struct *work)
1302 {
1303         struct i40evf_adapter *adapter = container_of(work,
1304                                           struct i40evf_adapter,
1305                                           watchdog_task);
1306         struct i40e_hw *hw = &adapter->hw;
1307
1308         if (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section))
1309                 goto restart_watchdog;
1310
1311         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
1312                 dev_info(&adapter->pdev->dev, "Checking for redemption\n");
1313                 if ((rd32(hw, I40E_VFGEN_RSTAT) & 0x3) == I40E_VFR_VFACTIVE) {
1314                         /* A chance for redemption! */
1315                         dev_err(&adapter->pdev->dev, "Hardware came out of reset. Attempting reinit.\n");
1316                         adapter->state = __I40EVF_STARTUP;
1317                         adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
1318                         schedule_delayed_work(&adapter->init_task, 10);
1319                         clear_bit(__I40EVF_IN_CRITICAL_TASK,
1320                                   &adapter->crit_section);
1321                         /* Don't reschedule the watchdog, since we've restarted
1322                          * the init task. When init_task contacts the PF and
1323                          * gets everything set up again, it'll restart the
1324                          * watchdog for us. Down, boy. Sit. Stay. Woof.
1325                          */
1326                         return;
1327                 }
1328                 adapter->aq_pending = 0;
1329                 adapter->aq_required = 0;
1330                 adapter->current_op = I40E_VIRTCHNL_OP_UNKNOWN;
1331                 goto watchdog_done;
1332         }
1333
1334         if ((adapter->state < __I40EVF_DOWN) ||
1335             (adapter->flags & I40EVF_FLAG_RESET_PENDING))
1336                 goto watchdog_done;
1337
1338         /* check for reset */
1339         if (!(adapter->flags & I40EVF_FLAG_RESET_PENDING) &&
1340             (rd32(hw, I40E_VFGEN_RSTAT) & 0x3) != I40E_VFR_VFACTIVE) {
1341                 adapter->state = __I40EVF_RESETTING;
1342                 adapter->flags |= I40EVF_FLAG_RESET_PENDING;
1343                 dev_err(&adapter->pdev->dev, "Hardware reset detected.\n");
1344                 dev_info(&adapter->pdev->dev, "Scheduling reset task\n");
1345                 schedule_work(&adapter->reset_task);
1346                 adapter->aq_pending = 0;
1347                 adapter->aq_required = 0;
1348                 adapter->current_op = I40E_VIRTCHNL_OP_UNKNOWN;
1349                 goto watchdog_done;
1350         }
1351
1352         /* Process admin queue tasks. After init, everything gets done
1353          * here so we don't race on the admin queue.
1354          */
1355         if (adapter->aq_pending)
1356                 goto watchdog_done;
1357
1358         if (adapter->aq_required & I40EVF_FLAG_AQ_MAP_VECTORS) {
1359                 i40evf_map_queues(adapter);
1360                 goto watchdog_done;
1361         }
1362
1363         if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_MAC_FILTER) {
1364                 i40evf_add_ether_addrs(adapter);
1365                 goto watchdog_done;
1366         }
1367
1368         if (adapter->aq_required & I40EVF_FLAG_AQ_ADD_VLAN_FILTER) {
1369                 i40evf_add_vlans(adapter);
1370                 goto watchdog_done;
1371         }
1372
1373         if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_MAC_FILTER) {
1374                 i40evf_del_ether_addrs(adapter);
1375                 goto watchdog_done;
1376         }
1377
1378         if (adapter->aq_required & I40EVF_FLAG_AQ_DEL_VLAN_FILTER) {
1379                 i40evf_del_vlans(adapter);
1380                 goto watchdog_done;
1381         }
1382
1383         if (adapter->aq_required & I40EVF_FLAG_AQ_DISABLE_QUEUES) {
1384                 i40evf_disable_queues(adapter);
1385                 goto watchdog_done;
1386         }
1387
1388         if (adapter->aq_required & I40EVF_FLAG_AQ_CONFIGURE_QUEUES) {
1389                 i40evf_configure_queues(adapter);
1390                 goto watchdog_done;
1391         }
1392
1393         if (adapter->aq_required & I40EVF_FLAG_AQ_ENABLE_QUEUES) {
1394                 i40evf_enable_queues(adapter);
1395                 goto watchdog_done;
1396         }
1397
1398         if (adapter->state == __I40EVF_RUNNING)
1399                 i40evf_request_stats(adapter);
1400
1401         i40evf_irq_enable(adapter, true);
1402         i40evf_fire_sw_int(adapter, 0xFF);
1403
1404 watchdog_done:
1405         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1406 restart_watchdog:
1407         if (adapter->aq_required)
1408                 mod_timer(&adapter->watchdog_timer,
1409                           jiffies + msecs_to_jiffies(20));
1410         else
1411                 mod_timer(&adapter->watchdog_timer, jiffies + (HZ * 2));
1412         schedule_work(&adapter->adminq_task);
1413 }
1414
1415 /**
1416  * i40evf_configure_rss - increment to next available tx queue
1417  * @adapter: board private structure
1418  * @j: queue counter
1419  *
1420  * Helper function for RSS programming to increment through available
1421  * queus. Returns the next queue value.
1422  **/
1423 static int next_queue(struct i40evf_adapter *adapter, int j)
1424 {
1425         j += 1;
1426
1427         return j >= adapter->vsi_res->num_queue_pairs ? 0 : j;
1428 }
1429
1430 /**
1431  * i40evf_configure_rss - Prepare for RSS if used
1432  * @adapter: board private structure
1433  **/
1434 static void i40evf_configure_rss(struct i40evf_adapter *adapter)
1435 {
1436         struct i40e_hw *hw = &adapter->hw;
1437         u32 lut = 0;
1438         int i, j;
1439         u64 hena;
1440
1441         /* Set of random keys generated using kernel random number generator */
1442         static const u32 seed[I40E_VFQF_HKEY_MAX_INDEX + 1] = {
1443                         0x794221b4, 0xbca0c5ab, 0x6cd5ebd9, 0x1ada6127,
1444                         0x983b3aa1, 0x1c4e71eb, 0x7f6328b2, 0xfcdc0da0,
1445                         0xc135cafa, 0x7a6f7e2d, 0xe7102d28, 0x163cd12e,
1446                         0x4954b126 };
1447
1448         /* Hash type is configured by the PF - we just supply the key */
1449
1450         /* Fill out hash function seed */
1451         for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
1452                 wr32(hw, I40E_VFQF_HKEY(i), seed[i]);
1453
1454         /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1455         hena = I40E_DEFAULT_RSS_HENA;
1456         wr32(hw, I40E_VFQF_HENA(0), (u32)hena);
1457         wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32));
1458
1459         /* Populate the LUT with max no. of queues in round robin fashion */
1460         j = adapter->vsi_res->num_queue_pairs;
1461         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) {
1462                 j = next_queue(adapter, j);
1463                 lut = j;
1464                 j = next_queue(adapter, j);
1465                 lut |= j << 8;
1466                 j = next_queue(adapter, j);
1467                 lut |= j << 16;
1468                 j = next_queue(adapter, j);
1469                 lut |= j << 24;
1470                 wr32(hw, I40E_VFQF_HLUT(i), lut);
1471         }
1472         i40e_flush(hw);
1473 }
1474
1475 #define I40EVF_RESET_WAIT_MS 100
1476 #define I40EVF_RESET_WAIT_COUNT 200
1477 /**
1478  * i40evf_reset_task - Call-back task to handle hardware reset
1479  * @work: pointer to work_struct
1480  *
1481  * During reset we need to shut down and reinitialize the admin queue
1482  * before we can use it to communicate with the PF again. We also clear
1483  * and reinit the rings because that context is lost as well.
1484  **/
1485 static void i40evf_reset_task(struct work_struct *work)
1486 {
1487         struct i40evf_adapter *adapter = container_of(work,
1488                                                       struct i40evf_adapter,
1489                                                       reset_task);
1490         struct i40e_hw *hw = &adapter->hw;
1491         int i = 0, err;
1492         uint32_t rstat_val;
1493
1494         while (test_and_set_bit(__I40EVF_IN_CRITICAL_TASK,
1495                                 &adapter->crit_section))
1496                 udelay(500);
1497
1498         if (adapter->flags & I40EVF_FLAG_RESET_NEEDED) {
1499                 dev_info(&adapter->pdev->dev, "Requesting reset from PF\n");
1500                 i40evf_request_reset(adapter);
1501         }
1502
1503         /* poll until we see the reset actually happen */
1504         for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1505                 rstat_val = rd32(hw, I40E_VFGEN_RSTAT) &
1506                             I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1507                 if (rstat_val != I40E_VFR_VFACTIVE) {
1508                         dev_info(&adapter->pdev->dev, "Reset now occurring\n");
1509                         break;
1510                 } else {
1511                         msleep(I40EVF_RESET_WAIT_MS);
1512                 }
1513         }
1514         if (i == I40EVF_RESET_WAIT_COUNT) {
1515                 dev_err(&adapter->pdev->dev, "Reset was not detected\n");
1516                 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1517                 goto continue_reset; /* act like the reset happened */
1518         }
1519
1520         /* wait until the reset is complete and the PF is responding to us */
1521         for (i = 0; i < I40EVF_RESET_WAIT_COUNT; i++) {
1522                 rstat_val = rd32(hw, I40E_VFGEN_RSTAT) &
1523                             I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1524                 if (rstat_val == I40E_VFR_VFACTIVE) {
1525                         dev_info(&adapter->pdev->dev, "Reset is complete. Reinitializing.\n");
1526                         break;
1527                 } else {
1528                         msleep(I40EVF_RESET_WAIT_MS);
1529                 }
1530         }
1531         if (i == I40EVF_RESET_WAIT_COUNT) {
1532                 /* reset never finished */
1533                 dev_err(&adapter->pdev->dev, "Reset never finished (%x). PF driver is dead, and so am I.\n",
1534                         rstat_val);
1535                 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
1536
1537                 if (netif_running(adapter->netdev))
1538                         i40evf_close(adapter->netdev);
1539
1540                 i40evf_free_misc_irq(adapter);
1541                 i40evf_reset_interrupt_capability(adapter);
1542                 i40evf_free_queues(adapter);
1543                 kfree(adapter->vf_res);
1544                 i40evf_shutdown_adminq(hw);
1545                 adapter->netdev->flags &= ~IFF_UP;
1546                 clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1547                 return; /* Do not attempt to reinit. It's dead, Jim. */
1548         }
1549
1550 continue_reset:
1551         adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1552
1553         i40evf_down(adapter);
1554         adapter->state = __I40EVF_RESETTING;
1555
1556         /* kill and reinit the admin queue */
1557         if (i40evf_shutdown_adminq(hw))
1558                 dev_warn(&adapter->pdev->dev,
1559                         "%s: Failed to destroy the Admin Queue resources\n",
1560                         __func__);
1561         err = i40evf_init_adminq(hw);
1562         if (err)
1563                 dev_info(&adapter->pdev->dev, "%s: init_adminq failed: %d\n",
1564                         __func__, err);
1565
1566         adapter->aq_pending = 0;
1567         adapter->aq_required = 0;
1568         i40evf_map_queues(adapter);
1569         clear_bit(__I40EVF_IN_CRITICAL_TASK, &adapter->crit_section);
1570
1571         mod_timer(&adapter->watchdog_timer, jiffies + 2);
1572
1573         if (netif_running(adapter->netdev)) {
1574                 /* allocate transmit descriptors */
1575                 err = i40evf_setup_all_tx_resources(adapter);
1576                 if (err)
1577                         goto reset_err;
1578
1579                 /* allocate receive descriptors */
1580                 err = i40evf_setup_all_rx_resources(adapter);
1581                 if (err)
1582                         goto reset_err;
1583
1584                 i40evf_configure(adapter);
1585
1586                 err = i40evf_up_complete(adapter);
1587                 if (err)
1588                         goto reset_err;
1589
1590                 i40evf_irq_enable(adapter, true);
1591         }
1592         return;
1593 reset_err:
1594         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit.\n");
1595         i40evf_close(adapter->netdev);
1596 }
1597
1598 /**
1599  * i40evf_adminq_task - worker thread to clean the admin queue
1600  * @work: pointer to work_struct containing our data
1601  **/
1602 static void i40evf_adminq_task(struct work_struct *work)
1603 {
1604         struct i40evf_adapter *adapter =
1605                 container_of(work, struct i40evf_adapter, adminq_task);
1606         struct i40e_hw *hw = &adapter->hw;
1607         struct i40e_arq_event_info event;
1608         struct i40e_virtchnl_msg *v_msg;
1609         i40e_status ret;
1610         u16 pending;
1611
1612         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED)
1613                 return;
1614
1615         event.msg_size = I40EVF_MAX_AQ_BUF_SIZE;
1616         event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL);
1617         if (!event.msg_buf) {
1618                 dev_info(&adapter->pdev->dev, "%s: no memory for ARQ clean\n",
1619                                  __func__);
1620                 return;
1621         }
1622         v_msg = (struct i40e_virtchnl_msg *)&event.desc;
1623         do {
1624                 ret = i40evf_clean_arq_element(hw, &event, &pending);
1625                 if (ret)
1626                         break; /* No event to process or error cleaning ARQ */
1627
1628                 i40evf_virtchnl_completion(adapter, v_msg->v_opcode,
1629                                            v_msg->v_retval, event.msg_buf,
1630                                            event.msg_size);
1631                 if (pending != 0) {
1632                         dev_info(&adapter->pdev->dev,
1633                                  "%s: ARQ: Pending events %d\n",
1634                                  __func__, pending);
1635                         memset(event.msg_buf, 0, I40EVF_MAX_AQ_BUF_SIZE);
1636                 }
1637         } while (pending);
1638
1639         /* re-enable Admin queue interrupt cause */
1640         i40evf_misc_irq_enable(adapter);
1641
1642         kfree(event.msg_buf);
1643 }
1644
1645 /**
1646  * i40evf_free_all_tx_resources - Free Tx Resources for All Queues
1647  * @adapter: board private structure
1648  *
1649  * Free all transmit software resources
1650  **/
1651 static void i40evf_free_all_tx_resources(struct i40evf_adapter *adapter)
1652 {
1653         int i;
1654
1655         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
1656                 if (adapter->tx_rings[i]->desc)
1657                         i40evf_free_tx_resources(adapter->tx_rings[i]);
1658
1659 }
1660
1661 /**
1662  * i40evf_setup_all_tx_resources - allocate all queues Tx resources
1663  * @adapter: board private structure
1664  *
1665  * If this function returns with an error, then it's possible one or
1666  * more of the rings is populated (while the rest are not).  It is the
1667  * callers duty to clean those orphaned rings.
1668  *
1669  * Return 0 on success, negative on failure
1670  **/
1671 static int i40evf_setup_all_tx_resources(struct i40evf_adapter *adapter)
1672 {
1673         int i, err = 0;
1674
1675         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1676                 err = i40evf_setup_tx_descriptors(adapter->tx_rings[i]);
1677                 if (!err)
1678                         continue;
1679                 dev_err(&adapter->pdev->dev,
1680                         "%s: Allocation for Tx Queue %u failed\n",
1681                         __func__, i);
1682                 break;
1683         }
1684
1685         return err;
1686 }
1687
1688 /**
1689  * i40evf_setup_all_rx_resources - allocate all queues Rx resources
1690  * @adapter: board private structure
1691  *
1692  * If this function returns with an error, then it's possible one or
1693  * more of the rings is populated (while the rest are not).  It is the
1694  * callers duty to clean those orphaned rings.
1695  *
1696  * Return 0 on success, negative on failure
1697  **/
1698 static int i40evf_setup_all_rx_resources(struct i40evf_adapter *adapter)
1699 {
1700         int i, err = 0;
1701
1702         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++) {
1703                 err = i40evf_setup_rx_descriptors(adapter->rx_rings[i]);
1704                 if (!err)
1705                         continue;
1706                 dev_err(&adapter->pdev->dev,
1707                         "%s: Allocation for Rx Queue %u failed\n",
1708                         __func__, i);
1709                 break;
1710         }
1711         return err;
1712 }
1713
1714 /**
1715  * i40evf_free_all_rx_resources - Free Rx Resources for All Queues
1716  * @adapter: board private structure
1717  *
1718  * Free all receive software resources
1719  **/
1720 static void i40evf_free_all_rx_resources(struct i40evf_adapter *adapter)
1721 {
1722         int i;
1723
1724         for (i = 0; i < adapter->vsi_res->num_queue_pairs; i++)
1725                 if (adapter->rx_rings[i]->desc)
1726                         i40evf_free_rx_resources(adapter->rx_rings[i]);
1727 }
1728
1729 /**
1730  * i40evf_open - Called when a network interface is made active
1731  * @netdev: network interface device structure
1732  *
1733  * Returns 0 on success, negative value on failure
1734  *
1735  * The open entry point is called when a network interface is made
1736  * active by the system (IFF_UP).  At this point all resources needed
1737  * for transmit and receive operations are allocated, the interrupt
1738  * handler is registered with the OS, the watchdog timer is started,
1739  * and the stack is notified that the interface is ready.
1740  **/
1741 static int i40evf_open(struct net_device *netdev)
1742 {
1743         struct i40evf_adapter *adapter = netdev_priv(netdev);
1744         int err;
1745
1746         if (adapter->flags & I40EVF_FLAG_PF_COMMS_FAILED) {
1747                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
1748                 return -EIO;
1749         }
1750         if (adapter->state != __I40EVF_DOWN)
1751                 return -EBUSY;
1752
1753         /* allocate transmit descriptors */
1754         err = i40evf_setup_all_tx_resources(adapter);
1755         if (err)
1756                 goto err_setup_tx;
1757
1758         /* allocate receive descriptors */
1759         err = i40evf_setup_all_rx_resources(adapter);
1760         if (err)
1761                 goto err_setup_rx;
1762
1763         /* clear any pending interrupts, may auto mask */
1764         err = i40evf_request_traffic_irqs(adapter, netdev->name);
1765         if (err)
1766                 goto err_req_irq;
1767
1768         i40evf_configure(adapter);
1769
1770         err = i40evf_up_complete(adapter);
1771         if (err)
1772                 goto err_req_irq;
1773
1774         i40evf_irq_enable(adapter, true);
1775
1776         return 0;
1777
1778 err_req_irq:
1779         i40evf_down(adapter);
1780         i40evf_free_traffic_irqs(adapter);
1781 err_setup_rx:
1782         i40evf_free_all_rx_resources(adapter);
1783 err_setup_tx:
1784         i40evf_free_all_tx_resources(adapter);
1785
1786         return err;
1787 }
1788
1789 /**
1790  * i40evf_close - Disables a network interface
1791  * @netdev: network interface device structure
1792  *
1793  * Returns 0, this is not allowed to fail
1794  *
1795  * The close entry point is called when an interface is de-activated
1796  * by the OS.  The hardware is still under the drivers control, but
1797  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
1798  * are freed, along with all transmit and receive resources.
1799  **/
1800 static int i40evf_close(struct net_device *netdev)
1801 {
1802         struct i40evf_adapter *adapter = netdev_priv(netdev);
1803
1804         if (adapter->state <= __I40EVF_DOWN)
1805                 return 0;
1806
1807         /* signal that we are down to the interrupt handler */
1808         adapter->state = __I40EVF_DOWN;
1809
1810         set_bit(__I40E_DOWN, &adapter->vsi.state);
1811
1812         i40evf_down(adapter);
1813         i40evf_free_traffic_irqs(adapter);
1814
1815         i40evf_free_all_tx_resources(adapter);
1816         i40evf_free_all_rx_resources(adapter);
1817
1818         return 0;
1819 }
1820
1821 /**
1822  * i40evf_get_stats - Get System Network Statistics
1823  * @netdev: network interface device structure
1824  *
1825  * Returns the address of the device statistics structure.
1826  * The statistics are actually updated from the timer callback.
1827  **/
1828 static struct net_device_stats *i40evf_get_stats(struct net_device *netdev)
1829 {
1830         struct i40evf_adapter *adapter = netdev_priv(netdev);
1831
1832         /* only return the current stats */
1833         return &adapter->net_stats;
1834 }
1835
1836 /**
1837  * i40evf_reinit_locked - Software reinit
1838  * @adapter: board private structure
1839  *
1840  * Reinititalizes the ring structures in response to a software configuration
1841  * change. Roughly the same as close followed by open, but skips releasing
1842  * and reallocating the interrupts.
1843  **/
1844 void i40evf_reinit_locked(struct i40evf_adapter *adapter)
1845 {
1846         struct net_device *netdev = adapter->netdev;
1847         int err;
1848
1849         WARN_ON(in_interrupt());
1850
1851         adapter->state = __I40EVF_RESETTING;
1852
1853         i40evf_down(adapter);
1854
1855         /* allocate transmit descriptors */
1856         err = i40evf_setup_all_tx_resources(adapter);
1857         if (err)
1858                 goto err_reinit;
1859
1860         /* allocate receive descriptors */
1861         err = i40evf_setup_all_rx_resources(adapter);
1862         if (err)
1863                 goto err_reinit;
1864
1865         i40evf_configure(adapter);
1866
1867         err = i40evf_up_complete(adapter);
1868         if (err)
1869                 goto err_reinit;
1870
1871         i40evf_irq_enable(adapter, true);
1872         return;
1873
1874 err_reinit:
1875         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit.\n");
1876         i40evf_close(netdev);
1877 }
1878
1879 /**
1880  * i40evf_change_mtu - Change the Maximum Transfer Unit
1881  * @netdev: network interface device structure
1882  * @new_mtu: new value for maximum frame size
1883  *
1884  * Returns 0 on success, negative on failure
1885  **/
1886 static int i40evf_change_mtu(struct net_device *netdev, int new_mtu)
1887 {
1888         struct i40evf_adapter *adapter = netdev_priv(netdev);
1889         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
1890
1891         if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER))
1892                 return -EINVAL;
1893
1894         /* must set new MTU before calling down or up */
1895         netdev->mtu = new_mtu;
1896         i40evf_reinit_locked(adapter);
1897         return 0;
1898 }
1899
1900 static const struct net_device_ops i40evf_netdev_ops = {
1901         .ndo_open               = i40evf_open,
1902         .ndo_stop               = i40evf_close,
1903         .ndo_start_xmit         = i40evf_xmit_frame,
1904         .ndo_get_stats          = i40evf_get_stats,
1905         .ndo_set_rx_mode        = i40evf_set_rx_mode,
1906         .ndo_validate_addr      = eth_validate_addr,
1907         .ndo_set_mac_address    = i40evf_set_mac,
1908         .ndo_change_mtu         = i40evf_change_mtu,
1909         .ndo_tx_timeout         = i40evf_tx_timeout,
1910         .ndo_vlan_rx_add_vid    = i40evf_vlan_rx_add_vid,
1911         .ndo_vlan_rx_kill_vid   = i40evf_vlan_rx_kill_vid,
1912 };
1913
1914 /**
1915  * i40evf_check_reset_complete - check that VF reset is complete
1916  * @hw: pointer to hw struct
1917  *
1918  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
1919  **/
1920 static int i40evf_check_reset_complete(struct i40e_hw *hw)
1921 {
1922         u32 rstat;
1923         int i;
1924
1925         for (i = 0; i < 100; i++) {
1926                 rstat = rd32(hw, I40E_VFGEN_RSTAT);
1927                 if (rstat == I40E_VFR_VFACTIVE)
1928                         return 0;
1929                 udelay(10);
1930         }
1931         return -EBUSY;
1932 }
1933
1934 /**
1935  * i40evf_init_task - worker thread to perform delayed initialization
1936  * @work: pointer to work_struct containing our data
1937  *
1938  * This task completes the work that was begun in probe. Due to the nature
1939  * of VF-PF communications, we may need to wait tens of milliseconds to get
1940  * reponses back from the PF. Rather than busy-wait in probe and bog down the
1941  * whole system, we'll do it in a task so we can sleep.
1942  * This task only runs during driver init. Once we've established
1943  * communications with the PF driver and set up our netdev, the watchdog
1944  * takes over.
1945  **/
1946 static void i40evf_init_task(struct work_struct *work)
1947 {
1948         struct i40evf_adapter *adapter = container_of(work,
1949                                                       struct i40evf_adapter,
1950                                                       init_task.work);
1951         struct net_device *netdev = adapter->netdev;
1952         struct i40evf_mac_filter *f;
1953         struct i40e_hw *hw = &adapter->hw;
1954         struct pci_dev *pdev = adapter->pdev;
1955         int i, err, bufsz;
1956
1957         switch (adapter->state) {
1958         case __I40EVF_STARTUP:
1959                 /* driver loaded, probe complete */
1960                 adapter->flags &= ~I40EVF_FLAG_PF_COMMS_FAILED;
1961                 adapter->flags &= ~I40EVF_FLAG_RESET_PENDING;
1962                 err = i40e_set_mac_type(hw);
1963                 if (err) {
1964                         dev_err(&pdev->dev, "Failed to set MAC type (%d)\n",
1965                                 err);
1966                 goto err;
1967                 }
1968                 err = i40evf_check_reset_complete(hw);
1969                 if (err) {
1970                         dev_err(&pdev->dev, "Device is still in reset (%d)\n",
1971                                 err);
1972                         goto err;
1973                 }
1974                 hw->aq.num_arq_entries = I40EVF_AQ_LEN;
1975                 hw->aq.num_asq_entries = I40EVF_AQ_LEN;
1976                 hw->aq.arq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
1977                 hw->aq.asq_buf_size = I40EVF_MAX_AQ_BUF_SIZE;
1978
1979                 err = i40evf_init_adminq(hw);
1980                 if (err) {
1981                         dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
1982                                 err);
1983                         goto err;
1984                 }
1985                 err = i40evf_send_api_ver(adapter);
1986                 if (err) {
1987                         dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err);
1988                         i40evf_shutdown_adminq(hw);
1989                         goto err;
1990                 }
1991                 adapter->state = __I40EVF_INIT_VERSION_CHECK;
1992                 goto restart;
1993                 break;
1994         case __I40EVF_INIT_VERSION_CHECK:
1995                 if (!i40evf_asq_done(hw)) {
1996                         dev_err(&pdev->dev, "Admin queue command never completed.\n");
1997                         goto err;
1998                 }
1999
2000                 /* aq msg sent, awaiting reply */
2001                 err = i40evf_verify_api_ver(adapter);
2002                 if (err) {
2003                         dev_err(&pdev->dev, "Unable to verify API version (%d)\n",
2004                                 err);
2005                         goto err;
2006                 }
2007                 err = i40evf_send_vf_config_msg(adapter);
2008                 if (err) {
2009                         dev_err(&pdev->dev, "Unable send config request (%d)\n",
2010                                 err);
2011                         goto err;
2012                 }
2013                 adapter->state = __I40EVF_INIT_GET_RESOURCES;
2014                 goto restart;
2015                 break;
2016         case __I40EVF_INIT_GET_RESOURCES:
2017                 /* aq msg sent, awaiting reply */
2018                 if (!adapter->vf_res) {
2019                         bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
2020                                 (I40E_MAX_VF_VSI *
2021                                  sizeof(struct i40e_virtchnl_vsi_resource));
2022                         adapter->vf_res = kzalloc(bufsz, GFP_KERNEL);
2023                         if (!adapter->vf_res)
2024                                 goto err;
2025                 }
2026                 err = i40evf_get_vf_config(adapter);
2027                 if (err == I40E_ERR_ADMIN_QUEUE_NO_WORK)
2028                         goto restart;
2029                 if (err) {
2030                         dev_err(&pdev->dev, "Unable to get VF config (%d)\n",
2031                                 err);
2032                         goto err_alloc;
2033                 }
2034                 adapter->state = __I40EVF_INIT_SW;
2035                 break;
2036         default:
2037                 goto err_alloc;
2038         }
2039         /* got VF config message back from PF, now we can parse it */
2040         for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2041                 if (adapter->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
2042                         adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2043         }
2044         if (!adapter->vsi_res) {
2045                 dev_err(&pdev->dev, "No LAN VSI found\n");
2046                 goto err_alloc;
2047         }
2048
2049         adapter->flags |= I40EVF_FLAG_RX_CSUM_ENABLED;
2050
2051         netdev->netdev_ops = &i40evf_netdev_ops;
2052         i40evf_set_ethtool_ops(netdev);
2053         netdev->watchdog_timeo = 5 * HZ;
2054         netdev->features |= NETIF_F_HIGHDMA |
2055                             NETIF_F_SG |
2056                             NETIF_F_IP_CSUM |
2057                             NETIF_F_SCTP_CSUM |
2058                             NETIF_F_IPV6_CSUM |
2059                             NETIF_F_TSO |
2060                             NETIF_F_TSO6 |
2061                             NETIF_F_RXCSUM |
2062                             NETIF_F_GRO;
2063
2064         if (adapter->vf_res->vf_offload_flags
2065             & I40E_VIRTCHNL_VF_OFFLOAD_VLAN) {
2066                 netdev->vlan_features = netdev->features;
2067                 netdev->features |= NETIF_F_HW_VLAN_CTAG_TX |
2068                                     NETIF_F_HW_VLAN_CTAG_RX |
2069                                     NETIF_F_HW_VLAN_CTAG_FILTER;
2070         }
2071
2072         /* copy netdev features into list of user selectable features */
2073         netdev->hw_features |= netdev->features;
2074         netdev->hw_features &= ~NETIF_F_RXCSUM;
2075
2076         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2077                 dev_info(&pdev->dev, "Invalid MAC address %pMAC, using random\n",
2078                          adapter->hw.mac.addr);
2079                 random_ether_addr(adapter->hw.mac.addr);
2080         }
2081         memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
2082         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
2083
2084         INIT_LIST_HEAD(&adapter->mac_filter_list);
2085         INIT_LIST_HEAD(&adapter->vlan_filter_list);
2086         f = kzalloc(sizeof(*f), GFP_ATOMIC);
2087         if (NULL == f)
2088                 goto err_sw_init;
2089
2090         memcpy(f->macaddr, adapter->hw.mac.addr, ETH_ALEN);
2091         f->add = true;
2092         adapter->aq_required |= I40EVF_FLAG_AQ_ADD_MAC_FILTER;
2093
2094         list_add(&f->list, &adapter->mac_filter_list);
2095
2096         init_timer(&adapter->watchdog_timer);
2097         adapter->watchdog_timer.function = &i40evf_watchdog_timer;
2098         adapter->watchdog_timer.data = (unsigned long)adapter;
2099         mod_timer(&adapter->watchdog_timer, jiffies + 1);
2100
2101         err = i40evf_init_interrupt_scheme(adapter);
2102         if (err)
2103                 goto err_sw_init;
2104         i40evf_map_rings_to_vectors(adapter);
2105         i40evf_configure_rss(adapter);
2106         err = i40evf_request_misc_irq(adapter);
2107         if (err)
2108                 goto err_sw_init;
2109
2110         netif_carrier_off(netdev);
2111
2112         adapter->vsi.id = adapter->vsi_res->vsi_id;
2113         adapter->vsi.seid = adapter->vsi_res->vsi_id; /* dummy */
2114         adapter->vsi.back = adapter;
2115         adapter->vsi.base_vector = 1;
2116         adapter->vsi.work_limit = I40E_DEFAULT_IRQ_WORK;
2117         adapter->vsi.rx_itr_setting = I40E_ITR_DYNAMIC;
2118         adapter->vsi.tx_itr_setting = I40E_ITR_DYNAMIC;
2119         adapter->vsi.netdev = adapter->netdev;
2120
2121         if (!adapter->netdev_registered) {
2122                 err = register_netdev(netdev);
2123                 if (err)
2124                         goto err_register;
2125         }
2126
2127         adapter->netdev_registered = true;
2128
2129         netif_tx_stop_all_queues(netdev);
2130
2131         dev_info(&pdev->dev, "MAC address: %pMAC\n", adapter->hw.mac.addr);
2132         if (netdev->features & NETIF_F_GRO)
2133                 dev_info(&pdev->dev, "GRO is enabled\n");
2134
2135         dev_info(&pdev->dev, "%s\n", i40evf_driver_string);
2136         adapter->state = __I40EVF_DOWN;
2137         set_bit(__I40E_DOWN, &adapter->vsi.state);
2138         i40evf_misc_irq_enable(adapter);
2139         return;
2140 restart:
2141         schedule_delayed_work(&adapter->init_task,
2142                               msecs_to_jiffies(50));
2143         return;
2144
2145 err_register:
2146         i40evf_free_misc_irq(adapter);
2147 err_sw_init:
2148         i40evf_reset_interrupt_capability(adapter);
2149 err_alloc:
2150         kfree(adapter->vf_res);
2151         adapter->vf_res = NULL;
2152 err:
2153         /* Things went into the weeds, so try again later */
2154         if (++adapter->aq_wait_count > I40EVF_AQ_MAX_ERR) {
2155                 dev_err(&pdev->dev, "Failed to communicate with PF; giving up.\n");
2156                 adapter->flags |= I40EVF_FLAG_PF_COMMS_FAILED;
2157                 return; /* do not reschedule */
2158         }
2159         schedule_delayed_work(&adapter->init_task, HZ * 3);
2160         return;
2161 }
2162
2163 /**
2164  * i40evf_shutdown - Shutdown the device in preparation for a reboot
2165  * @pdev: pci device structure
2166  **/
2167 static void i40evf_shutdown(struct pci_dev *pdev)
2168 {
2169         struct net_device *netdev = pci_get_drvdata(pdev);
2170
2171         netif_device_detach(netdev);
2172
2173         if (netif_running(netdev))
2174                 i40evf_close(netdev);
2175
2176 #ifdef CONFIG_PM
2177         pci_save_state(pdev);
2178
2179 #endif
2180         pci_disable_device(pdev);
2181 }
2182
2183 /**
2184  * i40evf_probe - Device Initialization Routine
2185  * @pdev: PCI device information struct
2186  * @ent: entry in i40evf_pci_tbl
2187  *
2188  * Returns 0 on success, negative on failure
2189  *
2190  * i40evf_probe initializes an adapter identified by a pci_dev structure.
2191  * The OS initialization, configuring of the adapter private structure,
2192  * and a hardware reset occur.
2193  **/
2194 static int i40evf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2195 {
2196         struct net_device *netdev;
2197         struct i40evf_adapter *adapter = NULL;
2198         struct i40e_hw *hw = NULL;
2199         int err;
2200
2201         err = pci_enable_device(pdev);
2202         if (err)
2203                 return err;
2204
2205         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2206         if (err) {
2207                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2208                 if (err) {
2209                         dev_err(&pdev->dev,
2210                                 "DMA configuration failed: 0x%x\n", err);
2211                         goto err_dma;
2212                 }
2213         }
2214
2215         err = pci_request_regions(pdev, i40evf_driver_name);
2216         if (err) {
2217                 dev_err(&pdev->dev,
2218                         "pci_request_regions failed 0x%x\n", err);
2219                 goto err_pci_reg;
2220         }
2221
2222         pci_enable_pcie_error_reporting(pdev);
2223
2224         pci_set_master(pdev);
2225
2226         netdev = alloc_etherdev_mq(sizeof(struct i40evf_adapter),
2227                                    MAX_TX_QUEUES);
2228         if (!netdev) {
2229                 err = -ENOMEM;
2230                 goto err_alloc_etherdev;
2231         }
2232
2233         SET_NETDEV_DEV(netdev, &pdev->dev);
2234
2235         pci_set_drvdata(pdev, netdev);
2236         adapter = netdev_priv(netdev);
2237
2238         adapter->netdev = netdev;
2239         adapter->pdev = pdev;
2240
2241         hw = &adapter->hw;
2242         hw->back = adapter;
2243
2244         adapter->msg_enable = (1 << DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
2245         adapter->state = __I40EVF_STARTUP;
2246
2247         /* Call save state here because it relies on the adapter struct. */
2248         pci_save_state(pdev);
2249
2250         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
2251                               pci_resource_len(pdev, 0));
2252         if (!hw->hw_addr) {
2253                 err = -EIO;
2254                 goto err_ioremap;
2255         }
2256         hw->vendor_id = pdev->vendor;
2257         hw->device_id = pdev->device;
2258         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
2259         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2260         hw->subsystem_device_id = pdev->subsystem_device;
2261         hw->bus.device = PCI_SLOT(pdev->devfn);
2262         hw->bus.func = PCI_FUNC(pdev->devfn);
2263
2264         INIT_WORK(&adapter->reset_task, i40evf_reset_task);
2265         INIT_WORK(&adapter->adminq_task, i40evf_adminq_task);
2266         INIT_WORK(&adapter->watchdog_task, i40evf_watchdog_task);
2267         INIT_DELAYED_WORK(&adapter->init_task, i40evf_init_task);
2268         schedule_delayed_work(&adapter->init_task, 10);
2269
2270         return 0;
2271
2272 err_ioremap:
2273         free_netdev(netdev);
2274 err_alloc_etherdev:
2275         pci_release_regions(pdev);
2276 err_pci_reg:
2277 err_dma:
2278         pci_disable_device(pdev);
2279         return err;
2280 }
2281
2282 #ifdef CONFIG_PM
2283 /**
2284  * i40evf_suspend - Power management suspend routine
2285  * @pdev: PCI device information struct
2286  * @state: unused
2287  *
2288  * Called when the system (VM) is entering sleep/suspend.
2289  **/
2290 static int i40evf_suspend(struct pci_dev *pdev, pm_message_t state)
2291 {
2292         struct net_device *netdev = pci_get_drvdata(pdev);
2293         struct i40evf_adapter *adapter = netdev_priv(netdev);
2294         int retval = 0;
2295
2296         netif_device_detach(netdev);
2297
2298         if (netif_running(netdev)) {
2299                 rtnl_lock();
2300                 i40evf_down(adapter);
2301                 rtnl_unlock();
2302         }
2303         i40evf_free_misc_irq(adapter);
2304         i40evf_reset_interrupt_capability(adapter);
2305
2306         retval = pci_save_state(pdev);
2307         if (retval)
2308                 return retval;
2309
2310         pci_disable_device(pdev);
2311
2312         return 0;
2313 }
2314
2315 /**
2316  * i40evf_resume - Power managment resume routine
2317  * @pdev: PCI device information struct
2318  *
2319  * Called when the system (VM) is resumed from sleep/suspend.
2320  **/
2321 static int i40evf_resume(struct pci_dev *pdev)
2322 {
2323         struct i40evf_adapter *adapter = pci_get_drvdata(pdev);
2324         struct net_device *netdev = adapter->netdev;
2325         u32 err;
2326
2327         pci_set_power_state(pdev, PCI_D0);
2328         pci_restore_state(pdev);
2329         /* pci_restore_state clears dev->state_saved so call
2330          * pci_save_state to restore it.
2331          */
2332         pci_save_state(pdev);
2333
2334         err = pci_enable_device_mem(pdev);
2335         if (err) {
2336                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend.\n");
2337                 return err;
2338         }
2339         pci_set_master(pdev);
2340
2341         rtnl_lock();
2342         err = i40evf_set_interrupt_capability(adapter);
2343         if (err) {
2344                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
2345                 return err;
2346         }
2347         err = i40evf_request_misc_irq(adapter);
2348         rtnl_unlock();
2349         if (err) {
2350                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
2351                 return err;
2352         }
2353
2354         schedule_work(&adapter->reset_task);
2355
2356         netif_device_attach(netdev);
2357
2358         return err;
2359 }
2360
2361 #endif /* CONFIG_PM */
2362 /**
2363  * i40evf_remove - Device Removal Routine
2364  * @pdev: PCI device information struct
2365  *
2366  * i40evf_remove is called by the PCI subsystem to alert the driver
2367  * that it should release a PCI device.  The could be caused by a
2368  * Hot-Plug event, or because the driver is going to be removed from
2369  * memory.
2370  **/
2371 static void i40evf_remove(struct pci_dev *pdev)
2372 {
2373         struct net_device *netdev = pci_get_drvdata(pdev);
2374         struct i40evf_adapter *adapter = netdev_priv(netdev);
2375         struct i40e_hw *hw = &adapter->hw;
2376
2377         cancel_delayed_work_sync(&adapter->init_task);
2378         cancel_work_sync(&adapter->reset_task);
2379
2380         if (adapter->netdev_registered) {
2381                 unregister_netdev(netdev);
2382                 adapter->netdev_registered = false;
2383         }
2384         adapter->state = __I40EVF_REMOVE;
2385
2386         if (adapter->msix_entries) {
2387                 i40evf_misc_irq_disable(adapter);
2388                 i40evf_free_misc_irq(adapter);
2389                 i40evf_reset_interrupt_capability(adapter);
2390         }
2391
2392         del_timer_sync(&adapter->watchdog_timer);
2393         flush_scheduled_work();
2394
2395         if (hw->aq.asq.count)
2396                 i40evf_shutdown_adminq(hw);
2397
2398         iounmap(hw->hw_addr);
2399         pci_release_regions(pdev);
2400
2401         i40evf_free_queues(adapter);
2402         kfree(adapter->vf_res);
2403
2404         free_netdev(netdev);
2405
2406         pci_disable_pcie_error_reporting(pdev);
2407
2408         pci_disable_device(pdev);
2409 }
2410
2411 static struct pci_driver i40evf_driver = {
2412         .name     = i40evf_driver_name,
2413         .id_table = i40evf_pci_tbl,
2414         .probe    = i40evf_probe,
2415         .remove   = i40evf_remove,
2416 #ifdef CONFIG_PM
2417         .suspend  = i40evf_suspend,
2418         .resume   = i40evf_resume,
2419 #endif
2420         .shutdown = i40evf_shutdown,
2421 };
2422
2423 /**
2424  * i40e_init_module - Driver Registration Routine
2425  *
2426  * i40e_init_module is the first routine called when the driver is
2427  * loaded. All it does is register with the PCI subsystem.
2428  **/
2429 static int __init i40evf_init_module(void)
2430 {
2431         int ret;
2432         pr_info("i40evf: %s - version %s\n", i40evf_driver_string,
2433                i40evf_driver_version);
2434
2435         pr_info("%s\n", i40evf_copyright);
2436
2437         ret = pci_register_driver(&i40evf_driver);
2438         return ret;
2439 }
2440
2441 module_init(i40evf_init_module);
2442
2443 /**
2444  * i40e_exit_module - Driver Exit Cleanup Routine
2445  *
2446  * i40e_exit_module is called just before the driver is removed
2447  * from memory.
2448  **/
2449 static void __exit i40evf_exit_module(void)
2450 {
2451         pci_unregister_driver(&i40evf_driver);
2452 }
2453
2454 module_exit(i40evf_exit_module);
2455
2456 /* i40evf_main.c */