drivers/net/sfc/nic.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2006-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/bitops.h>
  12 #include <linux/delay.h>
  13 #include <linux/pci.h>
  14 #include <linux/module.h>
  15 #include <linux/seq_file.h>
  16 #include "net_driver.h"
  17 #include "bitfield.h"
  18 #include "efx.h"
  19 #include "nic.h"
  20 #include "regs.h"
  21 #include "io.h"
  22 #include "workarounds.h"
  23
  24 /**************************************************************************
  25  *
  26  * Configurable values
  27  *
  28  **************************************************************************
  29  */
  30
  31 /* This is set to 16 for a good reason.  In summary, if larger than
  32  * 16, the descriptor cache holds more than a default socket
  33  * buffer's worth of packets (for UDP we can only have at most one
  34  * socket buffer's worth outstanding).  This combined with the fact
  35  * that we only get 1 TX event per descriptor cache means the NIC
  36  * goes idle.
  37  */
  38 #define TX_DC_ENTRIES 16
  39 #define TX_DC_ENTRIES_ORDER 1
  40
  41 #define RX_DC_ENTRIES 64
  42 #define RX_DC_ENTRIES_ORDER 3
  43
  44 /* If EFX_MAX_INT_ERRORS internal errors occur within
  45  * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
  46  * disable it.
  47  */
  48 #define EFX_INT_ERROR_EXPIRE 3600
  49 #define EFX_MAX_INT_ERRORS 5
  50
  51 /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
  52  */
  53 #define EFX_FLUSH_INTERVAL 10
  54 #define EFX_FLUSH_POLL_COUNT 100
  55
  56 /* Size and alignment of special buffers (4KB) */
  57 #define EFX_BUF_SIZE 4096
  58
  59 /* Depth of RX flush request fifo */
  60 #define EFX_RX_FLUSH_COUNT 4
  61
  62 /* Generated event code for efx_generate_test_event() */
  63 #define EFX_CHANNEL_MAGIC_TEST(_channel)        \
  64         (0x00010100 + (_channel)->channel)
  65
  66 /* Generated event code for efx_generate_fill_event() */
  67 #define EFX_CHANNEL_MAGIC_FILL(_channel)        \
  68         (0x00010200 + (_channel)->channel)
  69
  70 /**************************************************************************
  71  *
  72  * Solarstorm hardware access
  73  *
  74  **************************************************************************/
  75
  76 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
  77                                      unsigned int index)
  78 {
  79         efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
  80                         value, index);
  81 }
  82
  83 /* Read the current event from the event queue */
  84 static inline efx_qword_t *efx_event(struct efx_channel *channel,
  85                                      unsigned int index)
  86 {
  87         return ((efx_qword_t *) (channel->eventq.addr)) + index;
  88 }
  89
  90 /* See if an event is present
  91  *
  92  * We check both the high and low dword of the event for all ones.  We
  93  * wrote all ones when we cleared the event, and no valid event can
  94  * have all ones in either its high or low dwords.  This approach is
  95  * robust against reordering.
  96  *
  97  * Note that using a single 64-bit comparison is incorrect; even
  98  * though the CPU read will be atomic, the DMA write may not be.
  99  */
 100 static inline int efx_event_present(efx_qword_t *event)
 101 {
 102         return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 103                   EFX_DWORD_IS_ALL_ONES(event->dword[1]));
 104 }
 105
 106 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
 107                                      const efx_oword_t *mask)
 108 {
 109         return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
 110                 ((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
 111 }
 112
 113 int efx_nic_test_registers(struct efx_nic *efx,
 114                            const struct efx_nic_register_test *regs,
 115                            size_t n_regs)
 116 {
 117         unsigned address = 0, i, j;
 118         efx_oword_t mask, imask, original, reg, buf;
 119
 120         /* Falcon should be in loopback to isolate the XMAC from the PHY */
 121         WARN_ON(!LOOPBACK_INTERNAL(efx));
 122
 123         for (i = 0; i < n_regs; ++i) {
 124                 address = regs[i].address;
 125                 mask = imask = regs[i].mask;
 126                 EFX_INVERT_OWORD(imask);
 127
 128                 efx_reado(efx, &original, address);
 129
 130                 /* bit sweep on and off */
 131                 for (j = 0; j < 128; j++) {
 132                         if (!EFX_EXTRACT_OWORD32(mask, j, j))
 133                                 continue;
 134
 135                         /* Test this testable bit can be set in isolation */
 136                         EFX_AND_OWORD(reg, original, mask);
 137                         EFX_SET_OWORD32(reg, j, j, 1);
 138
 139                         efx_writeo(efx, &reg, address);
 140                         efx_reado(efx, &buf, address);
 141
 142                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 143                                 goto fail;
 144
 145                         /* Test this testable bit can be cleared in isolation */
 146                         EFX_OR_OWORD(reg, original, mask);
 147                         EFX_SET_OWORD32(reg, j, j, 0);
 148
 149                         efx_writeo(efx, &reg, address);
 150                         efx_reado(efx, &buf, address);
 151
 152                         if (efx_masked_compare_oword(&reg, &buf, &mask))
 153                                 goto fail;
 154                 }
 155
 156                 efx_writeo(efx, &original, address);
 157         }
 158
 159         return 0;
 160
 161 fail:
 162         netif_err(efx, hw, efx->net_dev,
 163                   "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
 164                   " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
 165                   EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
 166         return -EIO;
 167 }
 168
 169 /**************************************************************************
 170  *
 171  * Special buffer handling
 172  * Special buffers are used for event queues and the TX and RX
 173  * descriptor rings.
 174  *
 175  *************************************************************************/
 176
 177 /*
 178  * Initialise a special buffer
 179  *
 180  * This will define a buffer (previously allocated via
 181  * efx_alloc_special_buffer()) in the buffer table, allowing
 182  * it to be used for event queues, descriptor rings etc.
 183  */
 184 static void
 185 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 186 {
 187         efx_qword_t buf_desc;
 188         int index;
 189         dma_addr_t dma_addr;
 190         int i;
 191
 192         EFX_BUG_ON_PARANOID(!buffer->addr);
 193
 194         /* Write buffer descriptors to NIC */
 195         for (i = 0; i < buffer->entries; i++) {
 196                 index = buffer->index + i;
 197                 dma_addr = buffer->dma_addr + (i * 4096);
 198                 netif_dbg(efx, probe, efx->net_dev,
 199                           "mapping special buffer %d at %llx\n",
 200                           index, (unsigned long long)dma_addr);
 201                 EFX_POPULATE_QWORD_3(buf_desc,
 202                                      FRF_AZ_BUF_ADR_REGION, 0,
 203                                      FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
 204                                      FRF_AZ_BUF_OWNER_ID_FBUF, 0);
 205                 efx_write_buf_tbl(efx, &buf_desc, index);
 206         }
 207 }
 208
 209 /* Unmaps a buffer and clears the buffer table entries */
 210 static void
 211 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 212 {
 213         efx_oword_t buf_tbl_upd;
 214         unsigned int start = buffer->index;
 215         unsigned int end = (buffer->index + buffer->entries - 1);
 216
 217         if (!buffer->entries)
 218                 return;
 219
 220         netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
 221                   buffer->index, buffer->index + buffer->entries - 1);
 222
 223         EFX_POPULATE_OWORD_4(buf_tbl_upd,
 224                              FRF_AZ_BUF_UPD_CMD, 0,
 225                              FRF_AZ_BUF_CLR_CMD, 1,
 226                              FRF_AZ_BUF_CLR_END_ID, end,
 227                              FRF_AZ_BUF_CLR_START_ID, start);
 228         efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
 229 }
 230
 231 /*
 232  * Allocate a new special buffer
 233  *
 234  * This allocates memory for a new buffer, clears it and allocates a
 235  * new buffer ID range.  It does not write into the buffer table.
 236  *
 237  * This call will allocate 4KB buffers, since 8KB buffers can't be
 238  * used for event queues and descriptor rings.
 239  */
 240 static int efx_alloc_special_buffer(struct efx_nic *efx,
 241                                     struct efx_special_buffer *buffer,
 242                                     unsigned int len)
 243 {
 244         len = ALIGN(len, EFX_BUF_SIZE);
 245
 246         buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
 247                                           &buffer->dma_addr, GFP_KERNEL);
 248         if (!buffer->addr)
 249                 return -ENOMEM;
 250         buffer->len = len;
 251         buffer->entries = len / EFX_BUF_SIZE;
 252         BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
 253
 254         /* All zeros is a potentially valid event so memset to 0xff */
 255         memset(buffer->addr, 0xff, len);
 256
 257         /* Select new buffer ID */
 258         buffer->index = efx->next_buffer_table;
 259         efx->next_buffer_table += buffer->entries;
 260
 261         netif_dbg(efx, probe, efx->net_dev,
 262                   "allocating special buffers %d-%d at %llx+%x "
 263                   "(virt %p phys %llx)\n", buffer->index,
 264                   buffer->index + buffer->entries - 1,
 265                   (u64)buffer->dma_addr, len,
 266                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 267
 268         return 0;
 269 }
 270
 271 static void
 272 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 273 {
 274         if (!buffer->addr)
 275                 return;
 276
 277         netif_dbg(efx, hw, efx->net_dev,
 278                   "deallocating special buffers %d-%d at %llx+%x "
 279                   "(virt %p phys %llx)\n", buffer->index,
 280                   buffer->index + buffer->entries - 1,
 281                   (u64)buffer->dma_addr, buffer->len,
 282                   buffer->addr, (u64)virt_to_phys(buffer->addr));
 283
 284         dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr,
 285                           buffer->dma_addr);
 286         buffer->addr = NULL;
 287         buffer->entries = 0;
 288 }
 289
 290 /**************************************************************************
 291  *
 292  * Generic buffer handling
 293  * These buffers are used for interrupt status and MAC stats
 294  *
 295  **************************************************************************/
 296
 297 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 298                          unsigned int len)
 299 {
 300         buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
 301                                             &buffer->dma_addr);
 302         if (!buffer->addr)
 303                 return -ENOMEM;
 304         buffer->len = len;
 305         memset(buffer->addr, 0, len);
 306         return 0;
 307 }
 308
 309 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
 310 {
 311         if (buffer->addr) {
 312                 pci_free_consistent(efx->pci_dev, buffer->len,
 313                                     buffer->addr, buffer->dma_addr);
 314                 buffer->addr = NULL;
 315         }
 316 }
 317
 318 /**************************************************************************
 319  *
 320  * TX path
 321  *
 322  **************************************************************************/
 323
 324 /* Returns a pointer to the specified transmit descriptor in the TX
 325  * descriptor queue belonging to the specified channel.
 326  */
 327 static inline efx_qword_t *
 328 efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 329 {
 330         return ((efx_qword_t *) (tx_queue->txd.addr)) + index;
 331 }
 332
 333 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
 334 static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
 335 {
 336         unsigned write_ptr;
 337         efx_dword_t reg;
 338
 339         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 340         EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
 341         efx_writed_page(tx_queue->efx, &reg,
 342                         FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
 343 }
 344
 345 /* Write pointer and first descriptor for TX descriptor ring */
 346 static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue,
 347                                     const efx_qword_t *txd)
 348 {
 349         unsigned write_ptr;
 350         efx_oword_t reg;
 351
 352         BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
 353         BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
 354
 355         write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 356         EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
 357                              FRF_AZ_TX_DESC_WPTR, write_ptr);
 358         reg.qword[0] = *txd;
 359         efx_writeo_page(tx_queue->efx, &reg,
 360                         FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
 361 }
 362
 363 static inline bool
 364 efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count)
 365 {
 366         unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count);
 367
 368         if (empty_read_count == 0)
 369                 return false;
 370
 371         tx_queue->empty_read_count = 0;
 372         return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
 373 }
 374
 375 /* For each entry inserted into the software descriptor ring, create a
 376  * descriptor in the hardware TX descriptor ring (in host memory), and
 377  * write a doorbell.
 378  */
 379 void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
 380 {
 381
 382         struct efx_tx_buffer *buffer;
 383         efx_qword_t *txd;
 384         unsigned write_ptr;
 385         unsigned old_write_count = tx_queue->write_count;
 386
 387         BUG_ON(tx_queue->write_count == tx_queue->insert_count);
 388
 389         do {
 390                 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 391                 buffer = &tx_queue->buffer[write_ptr];
 392                 txd = efx_tx_desc(tx_queue, write_ptr);
 393                 ++tx_queue->write_count;
 394
 395                 /* Create TX descriptor ring entry */
 396                 EFX_POPULATE_QWORD_4(*txd,
 397                                      FSF_AZ_TX_KER_CONT, buffer->continuation,
 398                                      FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
 399                                      FSF_AZ_TX_KER_BUF_REGION, 0,
 400                                      FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
 401         } while (tx_queue->write_count != tx_queue->insert_count);
 402
 403         wmb(); /* Ensure descriptors are written before they are fetched */
 404
 405         if (efx_may_push_tx_desc(tx_queue, old_write_count)) {
 406                 txd = efx_tx_desc(tx_queue,
 407                                   old_write_count & tx_queue->ptr_mask);
 408                 efx_push_tx_desc(tx_queue, txd);
 409                 ++tx_queue->pushes;
 410         } else {
 411                 efx_notify_tx_desc(tx_queue);
 412         }
 413 }
 414
 415 /* Allocate hardware resources for a TX queue */
 416 int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
 417 {
 418         struct efx_nic *efx = tx_queue->efx;
 419         unsigned entries;
 420
 421         entries = tx_queue->ptr_mask + 1;
 422         return efx_alloc_special_buffer(efx, &tx_queue->txd,
 423                                         entries * sizeof(efx_qword_t));
 424 }
 425
 426 void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 427 {
 428         struct efx_nic *efx = tx_queue->efx;
 429         efx_oword_t reg;
 430
 431         tx_queue->flushed = FLUSH_NONE;
 432
 433         /* Pin TX descriptor ring */
 434         efx_init_special_buffer(efx, &tx_queue->txd);
 435
 436         /* Push TX descriptor ring to card */
 437         EFX_POPULATE_OWORD_10(reg,
 438                               FRF_AZ_TX_DESCQ_EN, 1,
 439                               FRF_AZ_TX_ISCSI_DDIG_EN, 0,
 440                               FRF_AZ_TX_ISCSI_HDIG_EN, 0,
 441                               FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
 442                               FRF_AZ_TX_DESCQ_EVQ_ID,
 443                               tx_queue->channel->channel,
 444                               FRF_AZ_TX_DESCQ_OWNER_ID, 0,
 445                               FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
 446                               FRF_AZ_TX_DESCQ_SIZE,
 447                               __ffs(tx_queue->txd.entries),
 448                               FRF_AZ_TX_DESCQ_TYPE, 0,
 449                               FRF_BZ_TX_NON_IP_DROP_DIS, 1);
 450
 451         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 452                 int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
 453                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
 454                 EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS,
 455                                     !csum);
 456         }
 457
 458         efx_writeo_table(efx, &reg, efx->type->txd_ptr_tbl_base,
 459                          tx_queue->queue);
 460
 461         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
 462                 /* Only 128 bits in this register */
 463                 BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128);
 464
 465                 efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 466                 if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
 467                         clear_bit_le(tx_queue->queue, (void *)&reg);
 468                 else
 469                         set_bit_le(tx_queue->queue, (void *)&reg);
 470                 efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 471         }
 472
 473         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 474                 EFX_POPULATE_OWORD_1(reg,
 475                                      FRF_BZ_TX_PACE,
 476                                      (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
 477                                      FFE_BZ_TX_PACE_OFF :
 478                                      FFE_BZ_TX_PACE_RESERVED);
 479                 efx_writeo_table(efx, &reg, FR_BZ_TX_PACE_TBL,
 480                                  tx_queue->queue);
 481         }
 482 }
 483
 484 static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
 485 {
 486         struct efx_nic *efx = tx_queue->efx;
 487         efx_oword_t tx_flush_descq;
 488
 489         tx_queue->flushed = FLUSH_PENDING;
 490
 491         /* Post a flush command */
 492         EFX_POPULATE_OWORD_2(tx_flush_descq,
 493                              FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 494                              FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
 495         efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
 496 }
 497
 498 void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
 499 {
 500         struct efx_nic *efx = tx_queue->efx;
 501         efx_oword_t tx_desc_ptr;
 502
 503         /* The queue should have been flushed */
 504         WARN_ON(tx_queue->flushed != FLUSH_DONE);
 505
 506         /* Remove TX descriptor ring from card */
 507         EFX_ZERO_OWORD(tx_desc_ptr);
 508         efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 509                          tx_queue->queue);
 510
 511         /* Unpin TX descriptor ring */
 512         efx_fini_special_buffer(efx, &tx_queue->txd);
 513 }
 514
 515 /* Free buffers backing TX queue */
 516 void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
 517 {
 518         efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
 519 }
 520
 521 /**************************************************************************
 522  *
 523  * RX path
 524  *
 525  **************************************************************************/
 526
 527 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
 528 static inline efx_qword_t *
 529 efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
 530 {
 531         return ((efx_qword_t *) (rx_queue->rxd.addr)) + index;
 532 }
 533
 534 /* This creates an entry in the RX descriptor queue */
 535 static inline void
 536 efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
 537 {
 538         struct efx_rx_buffer *rx_buf;
 539         efx_qword_t *rxd;
 540
 541         rxd = efx_rx_desc(rx_queue, index);
 542         rx_buf = efx_rx_buffer(rx_queue, index);
 543         EFX_POPULATE_QWORD_3(*rxd,
 544                              FSF_AZ_RX_KER_BUF_SIZE,
 545                              rx_buf->len -
 546                              rx_queue->efx->type->rx_buffer_padding,
 547                              FSF_AZ_RX_KER_BUF_REGION, 0,
 548                              FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
 549 }
 550
 551 /* This writes to the RX_DESC_WPTR register for the specified receive
 552  * descriptor ring.
 553  */
 554 void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
 555 {
 556         struct efx_nic *efx = rx_queue->efx;
 557         efx_dword_t reg;
 558         unsigned write_ptr;
 559
 560         while (rx_queue->notified_count != rx_queue->added_count) {
 561                 efx_build_rx_desc(
 562                         rx_queue,
 563                         rx_queue->notified_count & rx_queue->ptr_mask);
 564                 ++rx_queue->notified_count;
 565         }
 566
 567         wmb();
 568         write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
 569         EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
 570         efx_writed_page(efx, &reg, FR_AZ_RX_DESC_UPD_DWORD_P0,
 571                         efx_rx_queue_index(rx_queue));
 572 }
 573
 574 int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
 575 {
 576         struct efx_nic *efx = rx_queue->efx;
 577         unsigned entries;
 578
 579         entries = rx_queue->ptr_mask + 1;
 580         return efx_alloc_special_buffer(efx, &rx_queue->rxd,
 581                                         entries * sizeof(efx_qword_t));
 582 }
 583
 584 void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
 585 {
 586         efx_oword_t rx_desc_ptr;
 587         struct efx_nic *efx = rx_queue->efx;
 588         bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
 589         bool iscsi_digest_en = is_b0;
 590
 591         netif_dbg(efx, hw, efx->net_dev,
 592                   "RX queue %d ring in special buffers %d-%d\n",
 593                   efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
 594                   rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 595
 596         rx_queue->flushed = FLUSH_NONE;
 597
 598         /* Pin RX descriptor ring */
 599         efx_init_special_buffer(efx, &rx_queue->rxd);
 600
 601         /* Push RX descriptor ring to card */
 602         EFX_POPULATE_OWORD_10(rx_desc_ptr,
 603                               FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
 604                               FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
 605                               FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
 606                               FRF_AZ_RX_DESCQ_EVQ_ID,
 607                               efx_rx_queue_channel(rx_queue)->channel,
 608                               FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 609                               FRF_AZ_RX_DESCQ_LABEL,
 610                               efx_rx_queue_index(rx_queue),
 611                               FRF_AZ_RX_DESCQ_SIZE,
 612                               __ffs(rx_queue->rxd.entries),
 613                               FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
 614                               /* For >=B0 this is scatter so disable */
 615                               FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
 616                               FRF_AZ_RX_DESCQ_EN, 1);
 617         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 618                          efx_rx_queue_index(rx_queue));
 619 }
 620
 621 static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
 622 {
 623         struct efx_nic *efx = rx_queue->efx;
 624         efx_oword_t rx_flush_descq;
 625
 626         rx_queue->flushed = FLUSH_PENDING;
 627
 628         /* Post a flush command */
 629         EFX_POPULATE_OWORD_2(rx_flush_descq,
 630                              FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 631                              FRF_AZ_RX_FLUSH_DESCQ,
 632                              efx_rx_queue_index(rx_queue));
 633         efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
 634 }
 635
 636 void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
 637 {
 638         efx_oword_t rx_desc_ptr;
 639         struct efx_nic *efx = rx_queue->efx;
 640
 641         /* The queue should already have been flushed */
 642         WARN_ON(rx_queue->flushed != FLUSH_DONE);
 643
 644         /* Remove RX descriptor ring from card */
 645         EFX_ZERO_OWORD(rx_desc_ptr);
 646         efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 647                          efx_rx_queue_index(rx_queue));
 648
 649         /* Unpin RX descriptor ring */
 650         efx_fini_special_buffer(efx, &rx_queue->rxd);
 651 }
 652
 653 /* Free buffers backing RX queue */
 654 void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
 655 {
 656         efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
 657 }
 658
 659 /**************************************************************************
 660  *
 661  * Event queue processing
 662  * Event queues are processed by per-channel tasklets.
 663  *
 664  **************************************************************************/
 665
 666 /* Update a channel's event queue's read pointer (RPTR) register
 667  *
 668  * This writes the EVQ_RPTR_REG register for the specified channel's
 669  * event queue.
 670  */
 671 void efx_nic_eventq_read_ack(struct efx_channel *channel)
 672 {
 673         efx_dword_t reg;
 674         struct efx_nic *efx = channel->efx;
 675
 676         EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR, channel->eventq_read_ptr);
 677         efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
 678                          channel->channel);
 679 }
 680
 681 /* Use HW to insert a SW defined event */
 682 static void efx_generate_event(struct efx_channel *channel, efx_qword_t *event)
 683 {
 684         efx_oword_t drv_ev_reg;
 685
 686         BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
 687                      FRF_AZ_DRV_EV_DATA_WIDTH != 64);
 688         drv_ev_reg.u32[0] = event->u32[0];
 689         drv_ev_reg.u32[1] = event->u32[1];
 690         drv_ev_reg.u32[2] = 0;
 691         drv_ev_reg.u32[3] = 0;
 692         EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel);
 693         efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV);
 694 }
 695
 696 /* Handle a transmit completion event
 697  *
 698  * The NIC batches TX completion events; the message we receive is of
 699  * the form "complete all TX events up to this index".
 700  */
 701 static int
 702 efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 703 {
 704         unsigned int tx_ev_desc_ptr;
 705         unsigned int tx_ev_q_label;
 706         struct efx_tx_queue *tx_queue;
 707         struct efx_nic *efx = channel->efx;
 708         int tx_packets = 0;
 709
 710         if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 711                 /* Transmit completion */
 712                 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
 713                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 714                 tx_queue = efx_channel_get_tx_queue(
 715                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 716                 tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
 717                               tx_queue->ptr_mask);
 718                 channel->irq_mod_score += tx_packets;
 719                 efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 720         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 721                 /* Rewrite the FIFO write pointer */
 722                 tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 723                 tx_queue = efx_channel_get_tx_queue(
 724                         channel, tx_ev_q_label % EFX_TXQ_TYPES);
 725
 726                 if (efx_dev_registered(efx))
 727                         netif_tx_lock(efx->net_dev);
 728                 efx_notify_tx_desc(tx_queue);
 729                 if (efx_dev_registered(efx))
 730                         netif_tx_unlock(efx->net_dev);
 731         } else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
 732                    EFX_WORKAROUND_10727(efx)) {
 733                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 734         } else {
 735                 netif_err(efx, tx_err, efx->net_dev,
 736                           "channel %d unexpected TX event "
 737                           EFX_QWORD_FMT"\n", channel->channel,
 738                           EFX_QWORD_VAL(*event));
 739         }
 740
 741         return tx_packets;
 742 }
 743
 744 /* Detect errors included in the rx_evt_pkt_ok bit. */
 745 static void efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
 746                                  const efx_qword_t *event,
 747                                  bool *rx_ev_pkt_ok,
 748                                  bool *discard)
 749 {
 750         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 751         struct efx_nic *efx = rx_queue->efx;
 752         bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
 753         bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
 754         bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
 755         bool rx_ev_other_err, rx_ev_pause_frm;
 756         bool rx_ev_hdr_type, rx_ev_mcast_pkt;
 757         unsigned rx_ev_pkt_type;
 758
 759         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 760         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 761         rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
 762         rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
 763         rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
 764                                                  FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
 765         rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
 766                                                   FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
 767         rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
 768                                                    FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
 769         rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
 770         rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
 771         rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
 772                           0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
 773         rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
 774
 775         /* Every error apart from tobe_disc and pause_frm */
 776         rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
 777                            rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
 778                            rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
 779
 780         /* Count errors that are not in MAC stats.  Ignore expected
 781          * checksum errors during self-test. */
 782         if (rx_ev_frm_trunc)
 783                 ++channel->n_rx_frm_trunc;
 784         else if (rx_ev_tobe_disc)
 785                 ++channel->n_rx_tobe_disc;
 786         else if (!efx->loopback_selftest) {
 787                 if (rx_ev_ip_hdr_chksum_err)
 788                         ++channel->n_rx_ip_hdr_chksum_err;
 789                 else if (rx_ev_tcp_udp_chksum_err)
 790                         ++channel->n_rx_tcp_udp_chksum_err;
 791         }
 792
 793         /* The frame must be discarded if any of these are true. */
 794         *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
 795                     rx_ev_tobe_disc | rx_ev_pause_frm);
 796
 797         /* TOBE_DISC is expected on unicast mismatches; don't print out an
 798          * error message.  FRM_TRUNC indicates RXDP dropped the packet due
 799          * to a FIFO overflow.
 800          */
 801 #ifdef EFX_ENABLE_DEBUG
 802         if (rx_ev_other_err && net_ratelimit()) {
 803                 netif_dbg(efx, rx_err, efx->net_dev,
 804                           " RX queue %d unexpected RX event "
 805                           EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
 806                           efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
 807                           rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
 808                           rx_ev_ip_hdr_chksum_err ?
 809                           " [IP_HDR_CHKSUM_ERR]" : "",
 810                           rx_ev_tcp_udp_chksum_err ?
 811                           " [TCP_UDP_CHKSUM_ERR]" : "",
 812                           rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
 813                           rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
 814                           rx_ev_drib_nib ? " [DRIB_NIB]" : "",
 815                           rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
 816                           rx_ev_pause_frm ? " [PAUSE]" : "");
 817         }
 818 #endif
 819 }
 820
 821 /* Handle receive events that are not in-order. */
 822 static void
 823 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 824 {
 825         struct efx_nic *efx = rx_queue->efx;
 826         unsigned expected, dropped;
 827
 828         expected = rx_queue->removed_count & rx_queue->ptr_mask;
 829         dropped = (index - expected) & rx_queue->ptr_mask;
 830         netif_info(efx, rx_err, efx->net_dev,
 831                    "dropped %d events (index=%d expected=%d)\n",
 832                    dropped, index, expected);
 833
 834         efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
 835                            RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 836 }
 837
 838 /* Handle a packet received event
 839  *
 840  * The NIC gives a "discard" flag if it's a unicast packet with the
 841  * wrong destination address
 842  * Also "is multicast" and "matches multicast filter" flags can be used to
 843  * discard non-matching multicast packets.
 844  */
 845 static void
 846 efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
 847 {
 848         unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
 849         unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
 850         unsigned expected_ptr;
 851         bool rx_ev_pkt_ok, discard = false, checksummed;
 852         struct efx_rx_queue *rx_queue;
 853         struct efx_nic *efx = channel->efx;
 854
 855         /* Basic packet information */
 856         rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
 857         rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
 858         rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 859         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
 860         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
 861         WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
 862                 channel->channel);
 863
 864         rx_queue = efx_channel_get_rx_queue(channel);
 865
 866         rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
 867         expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
 868         if (unlikely(rx_ev_desc_ptr != expected_ptr))
 869                 efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
 870
 871         if (likely(rx_ev_pkt_ok)) {
 872                 /* If packet is marked as OK and packet type is TCP/IP or
 873                  * UDP/IP, then we can rely on the hardware checksum.
 874                  */
 875                 checksummed =
 876                         likely(efx->rx_checksum_enabled) &&
 877                         (rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
 878                          rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP);
 879         } else {
 880                 efx_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, &discard);
 881                 checksummed = false;
 882         }
 883
 884         /* Detect multicast packets that didn't match the filter */
 885         rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 886         if (rx_ev_mcast_pkt) {
 887                 unsigned int rx_ev_mcast_hash_match =
 888                         EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
 889
 890                 if (unlikely(!rx_ev_mcast_hash_match)) {
 891                         ++channel->n_rx_mcast_mismatch;
 892                         discard = true;
 893                 }
 894         }
 895
 896         channel->irq_mod_score += 2;
 897
 898         /* Handle received packet */
 899         efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
 900                       checksummed, discard);
 901 }
 902
 903 static void
 904 efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event)
 905 {
 906         struct efx_nic *efx = channel->efx;
 907         unsigned code;
 908
 909         code = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
 910         if (code == EFX_CHANNEL_MAGIC_TEST(channel))
 911                 ++channel->magic_count;
 912         else if (code == EFX_CHANNEL_MAGIC_FILL(channel))
 913                 /* The queue must be empty, so we won't receive any rx
 914                  * events, so efx_process_channel() won't refill the
 915                  * queue. Refill it here */
 916                 efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel));
 917         else
 918                 netif_dbg(efx, hw, efx->net_dev, "channel %d received "
 919                           "generated event "EFX_QWORD_FMT"\n",
 920                           channel->channel, EFX_QWORD_VAL(*event));
 921 }
 922
 923 static void
 924 efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
 925 {
 926         struct efx_nic *efx = channel->efx;
 927         unsigned int ev_sub_code;
 928         unsigned int ev_sub_data;
 929
 930         ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
 931         ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
 932
 933         switch (ev_sub_code) {
 934         case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
 935                 netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
 936                            channel->channel, ev_sub_data);
 937                 break;
 938         case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
 939                 netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
 940                            channel->channel, ev_sub_data);
 941                 break;
 942         case FSE_AZ_EVQ_INIT_DONE_EV:
 943                 netif_dbg(efx, hw, efx->net_dev,
 944                           "channel %d EVQ %d initialised\n",
 945                           channel->channel, ev_sub_data);
 946                 break;
 947         case FSE_AZ_SRM_UPD_DONE_EV:
 948                 netif_vdbg(efx, hw, efx->net_dev,
 949                            "channel %d SRAM update done\n", channel->channel);
 950                 break;
 951         case FSE_AZ_WAKE_UP_EV:
 952                 netif_vdbg(efx, hw, efx->net_dev,
 953                            "channel %d RXQ %d wakeup event\n",
 954                            channel->channel, ev_sub_data);
 955                 break;
 956         case FSE_AZ_TIMER_EV:
 957                 netif_vdbg(efx, hw, efx->net_dev,
 958                            "channel %d RX queue %d timer expired\n",
 959                            channel->channel, ev_sub_data);
 960                 break;
 961         case FSE_AA_RX_RECOVER_EV:
 962                 netif_err(efx, rx_err, efx->net_dev,
 963                           "channel %d seen DRIVER RX_RESET event. "
 964                         "Resetting.\n", channel->channel);
 965                 atomic_inc(&efx->rx_reset);
 966                 efx_schedule_reset(efx,
 967                                    EFX_WORKAROUND_6555(efx) ?
 968                                    RESET_TYPE_RX_RECOVERY :
 969                                    RESET_TYPE_DISABLE);
 970                 break;
 971         case FSE_BZ_RX_DSC_ERROR_EV:
 972                 netif_err(efx, rx_err, efx->net_dev,
 973                           "RX DMA Q %d reports descriptor fetch error."
 974                           " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 975                 efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
 976                 break;
 977         case FSE_BZ_TX_DSC_ERROR_EV:
 978                 netif_err(efx, tx_err, efx->net_dev,
 979                           "TX DMA Q %d reports descriptor fetch error."
 980                           " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 981                 efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 982                 break;
 983         default:
 984                 netif_vdbg(efx, hw, efx->net_dev,
 985                            "channel %d unknown driver event code %d "
 986                            "data %04x\n", channel->channel, ev_sub_code,
 987                            ev_sub_data);
 988                 break;
 989         }
 990 }
 991
 992 int efx_nic_process_eventq(struct efx_channel *channel, int budget)
 993 {
 994         struct efx_nic *efx = channel->efx;
 995         unsigned int read_ptr;
 996         efx_qword_t event, *p_event;
 997         int ev_code;
 998         int tx_packets = 0;
 999         int spent = 0;
1000
1001         read_ptr = channel->eventq_read_ptr;
1002
1003         for (;;) {
1004                 p_event = efx_event(channel, read_ptr);
1005                 event = *p_event;
1006
1007                 if (!efx_event_present(&event))
1008                         /* End of events */
1009                         break;
1010
1011                 netif_vdbg(channel->efx, intr, channel->efx->net_dev,
1012                            "channel %d event is "EFX_QWORD_FMT"\n",
1013                            channel->channel, EFX_QWORD_VAL(event));
1014
1015                 /* Clear this event by marking it all ones */
1016                 EFX_SET_QWORD(*p_event);
1017
1018                 /* Increment read pointer */
1019                 read_ptr = (read_ptr + 1) & channel->eventq_mask;
1020
1021                 ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
1022
1023                 switch (ev_code) {
1024                 case FSE_AZ_EV_CODE_RX_EV:
1025                         efx_handle_rx_event(channel, &event);
1026                         if (++spent == budget)
1027                                 goto out;
1028                         break;
1029                 case FSE_AZ_EV_CODE_TX_EV:
1030                         tx_packets += efx_handle_tx_event(channel, &event);
1031                         if (tx_packets > efx->txq_entries) {
1032                                 spent = budget;
1033                                 goto out;
1034                         }
1035                         break;
1036                 case FSE_AZ_EV_CODE_DRV_GEN_EV:
1037                         efx_handle_generated_event(channel, &event);
1038                         break;
1039                 case FSE_AZ_EV_CODE_DRIVER_EV:
1040                         efx_handle_driver_event(channel, &event);
1041                         break;
1042                 case FSE_CZ_EV_CODE_MCDI_EV:
1043                         efx_mcdi_process_event(channel, &event);
1044                         break;
1045                 case FSE_AZ_EV_CODE_GLOBAL_EV:
1046                         if (efx->type->handle_global_event &&
1047                             efx->type->handle_global_event(channel, &event))
1048                                 break;
1049                         /* else fall through */
1050                 default:
1051                         netif_err(channel->efx, hw, channel->efx->net_dev,
1052                                   "channel %d unknown event type %d (data "
1053                                   EFX_QWORD_FMT ")\n", channel->channel,
1054                                   ev_code, EFX_QWORD_VAL(event));
1055                 }
1056         }
1057
1058 out:
1059         channel->eventq_read_ptr = read_ptr;
1060         return spent;
1061 }
1062
1063
1064 /* Allocate buffer table entries for event queue */
1065 int efx_nic_probe_eventq(struct efx_channel *channel)
1066 {
1067         struct efx_nic *efx = channel->efx;
1068         unsigned entries;
1069
1070         entries = channel->eventq_mask + 1;
1071         return efx_alloc_special_buffer(efx, &channel->eventq,
1072                                         entries * sizeof(efx_qword_t));
1073 }
1074
1075 void efx_nic_init_eventq(struct efx_channel *channel)
1076 {
1077         efx_oword_t reg;
1078         struct efx_nic *efx = channel->efx;
1079
1080         netif_dbg(efx, hw, efx->net_dev,
1081                   "channel %d event queue in special buffers %d-%d\n",
1082                   channel->channel, channel->eventq.index,
1083                   channel->eventq.index + channel->eventq.entries - 1);
1084
1085         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
1086                 EFX_POPULATE_OWORD_3(reg,
1087                                      FRF_CZ_TIMER_Q_EN, 1,
1088                                      FRF_CZ_HOST_NOTIFY_MODE, 0,
1089                                      FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
1090                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1091         }
1092
1093         /* Pin event queue buffer */
1094         efx_init_special_buffer(efx, &channel->eventq);
1095
1096         /* Fill event queue with all ones (i.e. empty events) */
1097         memset(channel->eventq.addr, 0xff, channel->eventq.len);
1098
1099         /* Push event queue to card */
1100         EFX_POPULATE_OWORD_3(reg,
1101                              FRF_AZ_EVQ_EN, 1,
1102                              FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
1103                              FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
1104         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1105                          channel->channel);
1106
1107         efx->type->push_irq_moderation(channel);
1108 }
1109
1110 void efx_nic_fini_eventq(struct efx_channel *channel)
1111 {
1112         efx_oword_t reg;
1113         struct efx_nic *efx = channel->efx;
1114
1115         /* Remove event queue from card */
1116         EFX_ZERO_OWORD(reg);
1117         efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
1118                          channel->channel);
1119         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1120                 efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
1121
1122         /* Unpin event queue */
1123         efx_fini_special_buffer(efx, &channel->eventq);
1124 }
1125
1126 /* Free buffers backing event queue */
1127 void efx_nic_remove_eventq(struct efx_channel *channel)
1128 {
1129         efx_free_special_buffer(channel->efx, &channel->eventq);
1130 }
1131
1132
1133 void efx_nic_generate_test_event(struct efx_channel *channel)
1134 {
1135         unsigned int magic = EFX_CHANNEL_MAGIC_TEST(channel);
1136         efx_qword_t test_event;
1137
1138         EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
1139                              FSE_AZ_EV_CODE_DRV_GEN_EV,
1140                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
1141         efx_generate_event(channel, &test_event);
1142 }
1143
1144 void efx_nic_generate_fill_event(struct efx_channel *channel)
1145 {
1146         unsigned int magic = EFX_CHANNEL_MAGIC_FILL(channel);
1147         efx_qword_t test_event;
1148
1149         EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
1150                              FSE_AZ_EV_CODE_DRV_GEN_EV,
1151                              FSF_AZ_DRV_GEN_EV_MAGIC, magic);
1152         efx_generate_event(channel, &test_event);
1153 }
1154
1155 /**************************************************************************
1156  *
1157  * Flush handling
1158  *
1159  **************************************************************************/
1160
1161
1162 static void efx_poll_flush_events(struct efx_nic *efx)
1163 {
1164         struct efx_channel *channel = efx_get_channel(efx, 0);
1165         struct efx_tx_queue *tx_queue;
1166         struct efx_rx_queue *rx_queue;
1167         unsigned int read_ptr = channel->eventq_read_ptr;
1168         unsigned int end_ptr = (read_ptr - 1) & channel->eventq_mask;
1169
1170         do {
1171                 efx_qword_t *event = efx_event(channel, read_ptr);
1172                 int ev_code, ev_sub_code, ev_queue;
1173                 bool ev_failed;
1174
1175                 if (!efx_event_present(event))
1176                         break;
1177
1178                 ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE);
1179                 ev_sub_code = EFX_QWORD_FIELD(*event,
1180                                               FSF_AZ_DRIVER_EV_SUBCODE);
1181                 if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1182                     ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) {
1183                         ev_queue = EFX_QWORD_FIELD(*event,
1184                                                    FSF_AZ_DRIVER_EV_SUBDATA);
1185                         if (ev_queue < EFX_TXQ_TYPES * efx->n_tx_channels) {
1186                                 tx_queue = efx_get_tx_queue(
1187                                         efx, ev_queue / EFX_TXQ_TYPES,
1188                                         ev_queue % EFX_TXQ_TYPES);
1189                                 tx_queue->flushed = FLUSH_DONE;
1190                         }
1191                 } else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
1192                            ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) {
1193                         ev_queue = EFX_QWORD_FIELD(
1194                                 *event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
1195                         ev_failed = EFX_QWORD_FIELD(
1196                                 *event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
1197                         if (ev_queue < efx->n_rx_channels) {
1198                                 rx_queue = efx_get_rx_queue(efx, ev_queue);
1199                                 rx_queue->flushed =
1200                                         ev_failed ? FLUSH_FAILED : FLUSH_DONE;
1201                         }
1202                 }
1203
1204                 /* We're about to destroy the queue anyway, so
1205                  * it's ok to throw away every non-flush event */
1206                 EFX_SET_QWORD(*event);
1207
1208                 read_ptr = (read_ptr + 1) & channel->eventq_mask;
1209         } while (read_ptr != end_ptr);
1210
1211         channel->eventq_read_ptr = read_ptr;
1212 }
1213
1214 /* Handle tx and rx flushes at the same time, since they run in
1215  * parallel in the hardware and there's no reason for us to
1216  * serialise them */
1217 int efx_nic_flush_queues(struct efx_nic *efx)
1218 {
1219         struct efx_channel *channel;
1220         struct efx_rx_queue *rx_queue;
1221         struct efx_tx_queue *tx_queue;
1222         int i, tx_pending, rx_pending;
1223
1224         /* If necessary prepare the hardware for flushing */
1225         efx->type->prepare_flush(efx);
1226
1227         /* Flush all tx queues in parallel */
1228         efx_for_each_channel(channel, efx) {
1229                 efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
1230                         if (tx_queue->initialised)
1231                                 efx_flush_tx_queue(tx_queue);
1232                 }
1233         }
1234
1235         /* The hardware supports four concurrent rx flushes, each of which may
1236          * need to be retried if there is an outstanding descriptor fetch */
1237         for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) {
1238                 rx_pending = tx_pending = 0;
1239                 efx_for_each_channel(channel, efx) {
1240                         efx_for_each_channel_rx_queue(rx_queue, channel) {
1241                                 if (rx_queue->flushed == FLUSH_PENDING)
1242                                         ++rx_pending;
1243                         }
1244                 }
1245                 efx_for_each_channel(channel, efx) {
1246                         efx_for_each_channel_rx_queue(rx_queue, channel) {
1247                                 if (rx_pending == EFX_RX_FLUSH_COUNT)
1248                                         break;
1249                                 if (rx_queue->flushed == FLUSH_FAILED ||
1250                                     rx_queue->flushed == FLUSH_NONE) {
1251                                         efx_flush_rx_queue(rx_queue);
1252                                         ++rx_pending;
1253                                 }
1254                         }
1255                         efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
1256                                 if (tx_queue->initialised &&
1257                                     tx_queue->flushed != FLUSH_DONE)
1258                                         ++tx_pending;
1259                         }
1260                 }
1261
1262                 if (rx_pending == 0 && tx_pending == 0)
1263                         return 0;
1264
1265                 msleep(EFX_FLUSH_INTERVAL);
1266                 efx_poll_flush_events(efx);
1267         }
1268
1269         /* Mark the queues as all flushed. We're going to return failure
1270          * leading to a reset, or fake up success anyway */
1271         efx_for_each_channel(channel, efx) {
1272                 efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
1273                         if (tx_queue->initialised &&
1274                             tx_queue->flushed != FLUSH_DONE)
1275                                 netif_err(efx, hw, efx->net_dev,
1276                                           "tx queue %d flush command timed out\n",
1277                                           tx_queue->queue);
1278                         tx_queue->flushed = FLUSH_DONE;
1279                 }
1280                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1281                         if (rx_queue->flushed != FLUSH_DONE)
1282                                 netif_err(efx, hw, efx->net_dev,
1283                                           "rx queue %d flush command timed out\n",
1284                                           efx_rx_queue_index(rx_queue));
1285                         rx_queue->flushed = FLUSH_DONE;
1286                 }
1287         }
1288
1289         return -ETIMEDOUT;
1290 }
1291
1292 /**************************************************************************
1293  *
1294  * Hardware interrupts
1295  * The hardware interrupt handler does very little work; all the event
1296  * queue processing is carried out by per-channel tasklets.
1297  *
1298  **************************************************************************/
1299
1300 /* Enable/disable/generate interrupts */
1301 static inline void efx_nic_interrupts(struct efx_nic *efx,
1302                                       bool enabled, bool force)
1303 {
1304         efx_oword_t int_en_reg_ker;
1305
1306         EFX_POPULATE_OWORD_3(int_en_reg_ker,
1307                              FRF_AZ_KER_INT_LEVE_SEL, efx->fatal_irq_level,
1308                              FRF_AZ_KER_INT_KER, force,
1309                              FRF_AZ_DRV_INT_EN_KER, enabled);
1310         efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
1311 }
1312
1313 void efx_nic_enable_interrupts(struct efx_nic *efx)
1314 {
1315         struct efx_channel *channel;
1316
1317         EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
1318         wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
1319
1320         /* Enable interrupts */
1321         efx_nic_interrupts(efx, true, false);
1322
1323         /* Force processing of all the channels to get the EVQ RPTRs up to
1324            date */
1325         efx_for_each_channel(channel, efx)
1326                 efx_schedule_channel(channel);
1327 }
1328
1329 void efx_nic_disable_interrupts(struct efx_nic *efx)
1330 {
1331         /* Disable interrupts */
1332         efx_nic_interrupts(efx, false, false);
1333 }
1334
1335 /* Generate a test interrupt
1336  * Interrupt must already have been enabled, otherwise nasty things
1337  * may happen.
1338  */
1339 void efx_nic_generate_interrupt(struct efx_nic *efx)
1340 {
1341         efx_nic_interrupts(efx, true, true);
1342 }
1343
1344 /* Process a fatal interrupt
1345  * Disable bus mastering ASAP and schedule a reset
1346  */
1347 irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
1348 {
1349         struct falcon_nic_data *nic_data = efx->nic_data;
1350         efx_oword_t *int_ker = efx->irq_status.addr;
1351         efx_oword_t fatal_intr;
1352         int error, mem_perr;
1353
1354         efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
1355         error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
1356
1357         netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
1358                   EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
1359                   EFX_OWORD_VAL(fatal_intr),
1360                   error ? "disabling bus mastering" : "no recognised error");
1361
1362         /* If this is a memory parity error dump which blocks are offending */
1363         mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
1364                     EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
1365         if (mem_perr) {
1366                 efx_oword_t reg;
1367                 efx_reado(efx, &reg, FR_AZ_MEM_STAT);
1368                 netif_err(efx, hw, efx->net_dev,
1369                           "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
1370                           EFX_OWORD_VAL(reg));
1371         }
1372
1373         /* Disable both devices */
1374         pci_clear_master(efx->pci_dev);
1375         if (efx_nic_is_dual_func(efx))
1376                 pci_clear_master(nic_data->pci_dev2);
1377         efx_nic_disable_interrupts(efx);
1378
1379         /* Count errors and reset or disable the NIC accordingly */
1380         if (efx->int_error_count == 0 ||
1381             time_after(jiffies, efx->int_error_expire)) {
1382                 efx->int_error_count = 0;
1383                 efx->int_error_expire =
1384                         jiffies + EFX_INT_ERROR_EXPIRE * HZ;
1385         }
1386         if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
1387                 netif_err(efx, hw, efx->net_dev,
1388                           "SYSTEM ERROR - reset scheduled\n");
1389                 efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
1390         } else {
1391                 netif_err(efx, hw, efx->net_dev,
1392                           "SYSTEM ERROR - max number of errors seen."
1393                           "NIC will be disabled\n");
1394                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
1395         }
1396
1397         return IRQ_HANDLED;
1398 }
1399
1400 /* Handle a legacy interrupt
1401  * Acknowledges the interrupt and schedule event queue processing.
1402  */
1403 static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
1404 {
1405         struct efx_nic *efx = dev_id;
1406         efx_oword_t *int_ker = efx->irq_status.addr;
1407         irqreturn_t result = IRQ_NONE;
1408         struct efx_channel *channel;
1409         efx_dword_t reg;
1410         u32 queues;
1411         int syserr;
1412
1413         /* Could this be ours?  If interrupts are disabled then the
1414          * channel state may not be valid.
1415          */
1416         if (!efx->legacy_irq_enabled)
1417                 return result;
1418
1419         /* Read the ISR which also ACKs the interrupts */
1420         efx_readd(efx, &reg, FR_BZ_INT_ISR0);
1421         queues = EFX_EXTRACT_DWORD(reg, 0, 31);
1422
1423         /* Check to see if we have a serious error condition */
1424         if (queues & (1U << efx->fatal_irq_level)) {
1425                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1426                 if (unlikely(syserr))
1427                         return efx_nic_fatal_interrupt(efx);
1428         }
1429
1430         if (queues != 0) {
1431                 if (EFX_WORKAROUND_15783(efx))
1432                         efx->irq_zero_count = 0;
1433
1434                 /* Schedule processing of any interrupting queues */
1435                 efx_for_each_channel(channel, efx) {
1436                         if (queues & 1)
1437                                 efx_schedule_channel(channel);
1438                         queues >>= 1;
1439                 }
1440                 result = IRQ_HANDLED;
1441
1442         } else if (EFX_WORKAROUND_15783(efx)) {
1443                 efx_qword_t *event;
1444
1445                 /* We can't return IRQ_HANDLED more than once on seeing ISR=0
1446                  * because this might be a shared interrupt. */
1447                 if (efx->irq_zero_count++ == 0)
1448                         result = IRQ_HANDLED;
1449
1450                 /* Ensure we schedule or rearm all event queues */
1451                 efx_for_each_channel(channel, efx) {
1452                         event = efx_event(channel, channel->eventq_read_ptr);
1453                         if (efx_event_present(event))
1454                                 efx_schedule_channel(channel);
1455                         else
1456                                 efx_nic_eventq_read_ack(channel);
1457                 }
1458         }
1459
1460         if (result == IRQ_HANDLED) {
1461                 efx->last_irq_cpu = raw_smp_processor_id();
1462                 netif_vdbg(efx, intr, efx->net_dev,
1463                            "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
1464                            irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
1465         }
1466
1467         return result;
1468 }
1469
1470 /* Handle an MSI interrupt
1471  *
1472  * Handle an MSI hardware interrupt.  This routine schedules event
1473  * queue processing.  No interrupt acknowledgement cycle is necessary.
1474  * Also, we never need to check that the interrupt is for us, since
1475  * MSI interrupts cannot be shared.
1476  */
1477 static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
1478 {
1479         struct efx_channel *channel = *(struct efx_channel **)dev_id;
1480         struct efx_nic *efx = channel->efx;
1481         efx_oword_t *int_ker = efx->irq_status.addr;
1482         int syserr;
1483
1484         efx->last_irq_cpu = raw_smp_processor_id();
1485         netif_vdbg(efx, intr, efx->net_dev,
1486                    "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
1487                    irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
1488
1489         /* Check to see if we have a serious error condition */
1490         if (channel->channel == efx->fatal_irq_level) {
1491                 syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
1492                 if (unlikely(syserr))
1493                         return efx_nic_fatal_interrupt(efx);
1494         }
1495
1496         /* Schedule processing of the channel */
1497         efx_schedule_channel(channel);
1498
1499         return IRQ_HANDLED;
1500 }
1501
1502
1503 /* Setup RSS indirection table.
1504  * This maps from the hash value of the packet to RXQ
1505  */
1506 void efx_nic_push_rx_indir_table(struct efx_nic *efx)
1507 {
1508         size_t i = 0;
1509         efx_dword_t dword;
1510
1511         if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
1512                 return;
1513
1514         BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
1515                      FR_BZ_RX_INDIRECTION_TBL_ROWS);
1516
1517         for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
1518                 EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
1519                                      efx->rx_indir_table[i]);
1520                 efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i);
1521         }
1522 }
1523
1524 /* Hook interrupt handler(s)
1525  * Try MSI and then legacy interrupts.
1526  */
1527 int efx_nic_init_interrupt(struct efx_nic *efx)
1528 {
1529         struct efx_channel *channel;
1530         int rc;
1531
1532         if (!EFX_INT_MODE_USE_MSI(efx)) {
1533                 irq_handler_t handler;
1534                 if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1535                         handler = efx_legacy_interrupt;
1536                 else
1537                         handler = falcon_legacy_interrupt_a1;
1538
1539                 rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
1540                                  efx->name, efx);
1541                 if (rc) {
1542                         netif_err(efx, drv, efx->net_dev,
1543                                   "failed to hook legacy IRQ %d\n",
1544                                   efx->pci_dev->irq);
1545                         goto fail1;
1546                 }
1547                 return 0;
1548         }
1549
1550         /* Hook MSI or MSI-X interrupt */
1551         efx_for_each_channel(channel, efx) {
1552                 rc = request_irq(channel->irq, efx_msi_interrupt,
1553                                  IRQF_PROBE_SHARED, /* Not shared */
1554                                  efx->channel_name[channel->channel],
1555                                  &efx->channel[channel->channel]);
1556                 if (rc) {
1557                         netif_err(efx, drv, efx->net_dev,
1558                                   "failed to hook IRQ %d\n", channel->irq);
1559                         goto fail2;
1560                 }
1561         }
1562
1563         return 0;
1564
1565  fail2:
1566         efx_for_each_channel(channel, efx)
1567                 free_irq(channel->irq, &efx->channel[channel->channel]);
1568  fail1:
1569         return rc;
1570 }
1571
1572 void efx_nic_fini_interrupt(struct efx_nic *efx)
1573 {
1574         struct efx_channel *channel;
1575         efx_oword_t reg;
1576
1577         /* Disable MSI/MSI-X interrupts */
1578         efx_for_each_channel(channel, efx) {
1579                 if (channel->irq)
1580                         free_irq(channel->irq, &efx->channel[channel->channel]);
1581         }
1582
1583         /* ACK legacy interrupt */
1584         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1585                 efx_reado(efx, &reg, FR_BZ_INT_ISR0);
1586         else
1587                 falcon_irq_ack_a1(efx);
1588
1589         /* Disable legacy interrupt */
1590         if (efx->legacy_irq)
1591                 free_irq(efx->legacy_irq, efx);
1592 }
1593
1594 u32 efx_nic_fpga_ver(struct efx_nic *efx)
1595 {
1596         efx_oword_t altera_build;
1597         efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
1598         return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
1599 }
1600
1601 void efx_nic_init_common(struct efx_nic *efx)
1602 {
1603         efx_oword_t temp;
1604
1605         /* Set positions of descriptor caches in SRAM. */
1606         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR,
1607                              efx->type->tx_dc_base / 8);
1608         efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
1609         EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR,
1610                              efx->type->rx_dc_base / 8);
1611         efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
1612
1613         /* Set TX descriptor cache size. */
1614         BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
1615         EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
1616         efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
1617
1618         /* Set RX descriptor cache size.  Set low watermark to size-8, as
1619          * this allows most efficient prefetching.
1620          */
1621         BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
1622         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
1623         efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
1624         EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
1625         efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
1626
1627         /* Program INT_KER address */
1628         EFX_POPULATE_OWORD_2(temp,
1629                              FRF_AZ_NORM_INT_VEC_DIS_KER,
1630                              EFX_INT_MODE_USE_MSI(efx),
1631                              FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
1632         efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
1633
1634         if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
1635                 /* Use an interrupt level unused by event queues */
1636                 efx->fatal_irq_level = 0x1f;
1637         else
1638                 /* Use a valid MSI-X vector */
1639                 efx->fatal_irq_level = 0;
1640
1641         /* Enable all the genuinely fatal interrupts.  (They are still
1642          * masked by the overall interrupt mask, controlled by
1643          * falcon_interrupts()).
1644          *
1645          * Note: All other fatal interrupts are enabled
1646          */
1647         EFX_POPULATE_OWORD_3(temp,
1648                              FRF_AZ_ILL_ADR_INT_KER_EN, 1,
1649                              FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
1650                              FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
1651         if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
1652                 EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
1653         EFX_INVERT_OWORD(temp);
1654         efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
1655
1656         efx_nic_push_rx_indir_table(efx);
1657
1658         /* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
1659          * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
1660          */
1661         efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
1662         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
1663         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
1664         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
1665         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
1666         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
1667         /* Enable SW_EV to inherit in char driver - assume harmless here */
1668         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
1669         /* Prefetch threshold 2 => fetch when descriptor cache half empty */
1670         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
1671         /* Disable hardware watchdog which can misfire */
1672         EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
1673         /* Squash TX of packets of 16 bytes or less */
1674         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
1675                 EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
1676         efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
1677
1678         if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
1679                 EFX_POPULATE_OWORD_4(temp,
1680                                      /* Default values */
1681                                      FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
1682                                      FRF_BZ_TX_PACE_SB_AF, 0xb,
1683                                      FRF_BZ_TX_PACE_FB_BASE, 0,
1684                                      /* Allow large pace values in the
1685                                       * fast bin. */
1686                                      FRF_BZ_TX_PACE_BIN_TH,
1687                                      FFE_BZ_TX_PACE_RESERVED);
1688                 efx_writeo(efx, &temp, FR_BZ_TX_PACE);
1689         }
1690 }
1691
1692 /* Register dump */
1693
1694 #define REGISTER_REVISION_A     1
1695 #define REGISTER_REVISION_B     2
1696 #define REGISTER_REVISION_C     3
1697 #define REGISTER_REVISION_Z     3       /* latest revision */
1698
1699 struct efx_nic_reg {
1700         u32 offset:24;
1701         u32 min_revision:2, max_revision:2;
1702 };
1703
1704 #define REGISTER(name, min_rev, max_rev) {                              \
1705         FR_ ## min_rev ## max_rev ## _ ## name,                         \
1706         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev    \
1707 }
1708 #define REGISTER_AA(name) REGISTER(name, A, A)
1709 #define REGISTER_AB(name) REGISTER(name, A, B)
1710 #define REGISTER_AZ(name) REGISTER(name, A, Z)
1711 #define REGISTER_BB(name) REGISTER(name, B, B)
1712 #define REGISTER_BZ(name) REGISTER(name, B, Z)
1713 #define REGISTER_CZ(name) REGISTER(name, C, Z)
1714
1715 static const struct efx_nic_reg efx_nic_regs[] = {
1716         REGISTER_AZ(ADR_REGION),
1717         REGISTER_AZ(INT_EN_KER),
1718         REGISTER_BZ(INT_EN_CHAR),
1719         REGISTER_AZ(INT_ADR_KER),
1720         REGISTER_BZ(INT_ADR_CHAR),
1721         /* INT_ACK_KER is WO */
1722         /* INT_ISR0 is RC */
1723         REGISTER_AZ(HW_INIT),
1724         REGISTER_CZ(USR_EV_CFG),
1725         REGISTER_AB(EE_SPI_HCMD),
1726         REGISTER_AB(EE_SPI_HADR),
1727         REGISTER_AB(EE_SPI_HDATA),
1728         REGISTER_AB(EE_BASE_PAGE),
1729         REGISTER_AB(EE_VPD_CFG0),
1730         /* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
1731         /* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
1732         /* PCIE_CORE_INDIRECT is indirect */
1733         REGISTER_AB(NIC_STAT),
1734         REGISTER_AB(GPIO_CTL),
1735         REGISTER_AB(GLB_CTL),
1736         /* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
1737         REGISTER_BZ(DP_CTRL),
1738         REGISTER_AZ(MEM_STAT),
1739         REGISTER_AZ(CS_DEBUG),
1740         REGISTER_AZ(ALTERA_BUILD),
1741         REGISTER_AZ(CSR_SPARE),
1742         REGISTER_AB(PCIE_SD_CTL0123),
1743         REGISTER_AB(PCIE_SD_CTL45),
1744         REGISTER_AB(PCIE_PCS_CTL_STAT),
1745         /* DEBUG_DATA_OUT is not used */
1746         /* DRV_EV is WO */
1747         REGISTER_AZ(EVQ_CTL),
1748         REGISTER_AZ(EVQ_CNT1),
1749         REGISTER_AZ(EVQ_CNT2),
1750         REGISTER_AZ(BUF_TBL_CFG),
1751         REGISTER_AZ(SRM_RX_DC_CFG),
1752         REGISTER_AZ(SRM_TX_DC_CFG),
1753         REGISTER_AZ(SRM_CFG),
1754         /* BUF_TBL_UPD is WO */
1755         REGISTER_AZ(SRM_UPD_EVQ),
1756         REGISTER_AZ(SRAM_PARITY),
1757         REGISTER_AZ(RX_CFG),
1758         REGISTER_BZ(RX_FILTER_CTL),
1759         /* RX_FLUSH_DESCQ is WO */
1760         REGISTER_AZ(RX_DC_CFG),
1761         REGISTER_AZ(RX_DC_PF_WM),
1762         REGISTER_BZ(RX_RSS_TKEY),
1763         /* RX_NODESC_DROP is RC */
1764         REGISTER_AA(RX_SELF_RST),
1765         /* RX_DEBUG, RX_PUSH_DROP are not used */
1766         REGISTER_CZ(RX_RSS_IPV6_REG1),
1767         REGISTER_CZ(RX_RSS_IPV6_REG2),
1768         REGISTER_CZ(RX_RSS_IPV6_REG3),
1769         /* TX_FLUSH_DESCQ is WO */
1770         REGISTER_AZ(TX_DC_CFG),
1771         REGISTER_AA(TX_CHKSM_CFG),
1772         REGISTER_AZ(TX_CFG),
1773         /* TX_PUSH_DROP is not used */
1774         REGISTER_AZ(TX_RESERVED),
1775         REGISTER_BZ(TX_PACE),
1776         /* TX_PACE_DROP_QID is RC */
1777         REGISTER_BB(TX_VLAN),
1778         REGISTER_BZ(TX_IPFIL_PORTEN),
1779         REGISTER_AB(MD_TXD),
1780         REGISTER_AB(MD_RXD),
1781         REGISTER_AB(MD_CS),
1782         REGISTER_AB(MD_PHY_ADR),
1783         REGISTER_AB(MD_ID),
1784         /* MD_STAT is RC */
1785         REGISTER_AB(MAC_STAT_DMA),
1786         REGISTER_AB(MAC_CTRL),
1787         REGISTER_BB(GEN_MODE),
1788         REGISTER_AB(MAC_MC_HASH_REG0),
1789         REGISTER_AB(MAC_MC_HASH_REG1),
1790         REGISTER_AB(GM_CFG1),
1791         REGISTER_AB(GM_CFG2),
1792         /* GM_IPG and GM_HD are not used */
1793         REGISTER_AB(GM_MAX_FLEN),
1794         /* GM_TEST is not used */
1795         REGISTER_AB(GM_ADR1),
1796         REGISTER_AB(GM_ADR2),
1797         REGISTER_AB(GMF_CFG0),
1798         REGISTER_AB(GMF_CFG1),
1799         REGISTER_AB(GMF_CFG2),
1800         REGISTER_AB(GMF_CFG3),
1801         REGISTER_AB(GMF_CFG4),
1802         REGISTER_AB(GMF_CFG5),
1803         REGISTER_BB(TX_SRC_MAC_CTL),
1804         REGISTER_AB(XM_ADR_LO),
1805         REGISTER_AB(XM_ADR_HI),
1806         REGISTER_AB(XM_GLB_CFG),
1807         REGISTER_AB(XM_TX_CFG),
1808         REGISTER_AB(XM_RX_CFG),
1809         REGISTER_AB(XM_MGT_INT_MASK),
1810         REGISTER_AB(XM_FC),
1811         REGISTER_AB(XM_PAUSE_TIME),
1812         REGISTER_AB(XM_TX_PARAM),
1813         REGISTER_AB(XM_RX_PARAM),
1814         /* XM_MGT_INT_MSK (note no 'A') is RC */
1815         REGISTER_AB(XX_PWR_RST),
1816         REGISTER_AB(XX_SD_CTL),
1817         REGISTER_AB(XX_TXDRV_CTL),
1818         /* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
1819         /* XX_CORE_STAT is partly RC */
1820 };
1821
1822 struct efx_nic_reg_table {
1823         u32 offset:24;
1824         u32 min_revision:2, max_revision:2;
1825         u32 step:6, rows:21;
1826 };
1827
1828 #define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \
1829         offset,                                                         \
1830         REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev,   \
1831         step, rows                                                      \
1832 }
1833 #define REGISTER_TABLE(name, min_rev, max_rev)                          \
1834         REGISTER_TABLE_DIMENSIONS(                                      \
1835                 name, FR_ ## min_rev ## max_rev ## _ ## name,           \
1836                 min_rev, max_rev,                                       \
1837                 FR_ ## min_rev ## max_rev ## _ ## name ## _STEP,        \
1838                 FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
1839 #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A)
1840 #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z)
1841 #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B)
1842 #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z)
1843 #define REGISTER_TABLE_BB_CZ(name)                                      \
1844         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B,           \
1845                                   FR_BZ_ ## name ## _STEP,              \
1846                                   FR_BB_ ## name ## _ROWS),             \
1847         REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z,           \
1848                                   FR_BZ_ ## name ## _STEP,              \
1849                                   FR_CZ_ ## name ## _ROWS)
1850 #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z)
1851
1852 static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
1853         /* DRIVER is not used */
1854         /* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
1855         REGISTER_TABLE_BB(TX_IPFIL_TBL),
1856         REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
1857         REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
1858         REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
1859         REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
1860         REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
1861         REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
1862         REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
1863         /* We can't reasonably read all of the buffer table (up to 8MB!).
1864          * However this driver will only use a few entries.  Reading
1865          * 1K entries allows for some expansion of queue count and
1866          * size before we need to change the version. */
1867         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
1868                                   A, A, 8, 1024),
1869         REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
1870                                   B, Z, 8, 1024),
1871         REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
1872         REGISTER_TABLE_BB_CZ(TIMER_TBL),
1873         REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
1874         REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
1875         /* TX_FILTER_TBL0 is huge and not used by this driver */
1876         REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
1877         REGISTER_TABLE_CZ(MC_TREG_SMEM),
1878         /* MSIX_PBA_TABLE is not mapped */
1879         /* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
1880         REGISTER_TABLE_BZ(RX_FILTER_TBL0),
1881 };
1882
1883 size_t efx_nic_get_regs_len(struct efx_nic *efx)
1884 {
1885         const struct efx_nic_reg *reg;
1886         const struct efx_nic_reg_table *table;
1887         size_t len = 0;
1888
1889         for (reg = efx_nic_regs;
1890              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
1891              reg++)
1892                 if (efx->type->revision >= reg->min_revision &&
1893                     efx->type->revision <= reg->max_revision)
1894                         len += sizeof(efx_oword_t);
1895
1896         for (table = efx_nic_reg_tables;
1897              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
1898              table++)
1899                 if (efx->type->revision >= table->min_revision &&
1900                     efx->type->revision <= table->max_revision)
1901                         len += table->rows * min_t(size_t, table->step, 16);
1902
1903         return len;
1904 }
1905
1906 void efx_nic_get_regs(struct efx_nic *efx, void *buf)
1907 {
1908         const struct efx_nic_reg *reg;
1909         const struct efx_nic_reg_table *table;
1910
1911         for (reg = efx_nic_regs;
1912              reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
1913              reg++) {
1914                 if (efx->type->revision >= reg->min_revision &&
1915                     efx->type->revision <= reg->max_revision) {
1916                         efx_reado(efx, (efx_oword_t *)buf, reg->offset);
1917                         buf += sizeof(efx_oword_t);
1918                 }
1919         }
1920
1921         for (table = efx_nic_reg_tables;
1922              table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
1923              table++) {
1924                 size_t size, i;
1925
1926                 if (!(efx->type->revision >= table->min_revision &&
1927                       efx->type->revision <= table->max_revision))
1928                         continue;
1929
1930                 size = min_t(size_t, table->step, 16);
1931
1932                 for (i = 0; i < table->rows; i++) {
1933                         switch (table->step) {
1934                         case 4: /* 32-bit register or SRAM */
1935                                 efx_readd_table(efx, buf, table->offset, i);
1936                                 break;
1937                         case 8: /* 64-bit SRAM */
1938                                 efx_sram_readq(efx,
1939                                                efx->membase + table->offset,
1940                                                buf, i);
1941                                 break;
1942                         case 16: /* 128-bit register */
1943                                 efx_reado_table(efx, buf, table->offset, i);
1944                                 break;
1945                         case 32: /* 128-bit register, interleaved */
1946                                 efx_reado_table(efx, buf, table->offset, 2 * i);
1947                                 break;
1948                         default:
1949                                 WARN_ON(1);
1950                                 return;
1951                         }
1952                         buf += size;
1953                 }
1954         }
1955 }