drivers/net/ethernet/sfc/rx.c

   1 /****************************************************************************
   2  * Driver for Solarflare Solarstorm network controllers and boards
   3  * Copyright 2005-2006 Fen Systems Ltd.
   4  * Copyright 2005-2011 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include <linux/socket.h>
  12 #include <linux/in.h>
  13 #include <linux/slab.h>
  14 #include <linux/ip.h>
  15 #include <linux/tcp.h>
  16 #include <linux/udp.h>
  17 #include <linux/prefetch.h>
  18 #include <linux/moduleparam.h>
  19 #include <net/ip.h>
  20 #include <net/checksum.h>
  21 #include "net_driver.h"
  22 #include "efx.h"
  23 #include "nic.h"
  24 #include "selftest.h"
  25 #include "workarounds.h"
  26
  27 /* Number of RX descriptors pushed at once. */
  28 #define EFX_RX_BATCH  8
  29
  30 /* Maximum length for an RX descriptor sharing a page */
  31 #define EFX_RX_HALF_PAGE ((PAGE_SIZE >> 1) - sizeof(struct efx_rx_page_state) \
  32                           - EFX_PAGE_IP_ALIGN)
  33
  34 /* Size of buffer allocated for skb header area. */
  35 #define EFX_SKB_HEADERS  64u
  36
  37 /* This is the percentage fill level below which new RX descriptors
  38  * will be added to the RX descriptor ring.
  39  */
  40 static unsigned int rx_refill_threshold;
  41
  42 /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */
  43 #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \
  44                                       EFX_RX_USR_BUF_SIZE)
  45
  46 /*
  47  * RX maximum head room required.
  48  *
  49  * This must be at least 1 to prevent overflow, plus one packet-worth
  50  * to allow pipelined receives.
  51  */
  52 #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS)
  53
  54 static inline u8 *efx_rx_buf_va(struct efx_rx_buffer *buf)
  55 {
  56         return page_address(buf->page) + buf->page_offset;
  57 }
  58
  59 static inline u32 efx_rx_buf_hash(const u8 *eh)
  60 {
  61         /* The ethernet header is always directly after any hash. */
  62 #if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) || NET_IP_ALIGN % 4 == 0
  63         return __le32_to_cpup((const __le32 *)(eh - 4));
  64 #else
  65         const u8 *data = eh - 4;
  66         return (u32)data[0]       |
  67                (u32)data[1] << 8  |
  68                (u32)data[2] << 16 |
  69                (u32)data[3] << 24;
  70 #endif
  71 }
  72
  73 static inline struct efx_rx_buffer *
  74 efx_rx_buf_next(struct efx_rx_queue *rx_queue, struct efx_rx_buffer *rx_buf)
  75 {
  76         if (unlikely(rx_buf == efx_rx_buffer(rx_queue, rx_queue->ptr_mask)))
  77                 return efx_rx_buffer(rx_queue, 0);
  78         else
  79                 return rx_buf + 1;
  80 }
  81
  82 /**
  83  * efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers
  84  *
  85  * @rx_queue:           Efx RX queue
  86  *
  87  * This allocates memory for EFX_RX_BATCH receive buffers, maps them for DMA,
  88  * and populates struct efx_rx_buffers for each one. Return a negative error
  89  * code or 0 on success. If a single page can be split between two buffers,
  90  * then the page will either be inserted fully, or not at at all.
  91  */
  92 static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue)
  93 {
  94         struct efx_nic *efx = rx_queue->efx;
  95         struct efx_rx_buffer *rx_buf;
  96         struct page *page;
  97         unsigned int page_offset;
  98         struct efx_rx_page_state *state;
  99         dma_addr_t dma_addr;
 100         unsigned index, count;
 101
 102         /* We can split a page between two buffers */
 103         BUILD_BUG_ON(EFX_RX_BATCH & 1);
 104
 105         for (count = 0; count < EFX_RX_BATCH; ++count) {
 106                 page = alloc_pages(__GFP_COLD | __GFP_COMP | GFP_ATOMIC,
 107                                    efx->rx_buffer_order);
 108                 if (unlikely(page == NULL))
 109                         return -ENOMEM;
 110                 dma_addr = dma_map_page(&efx->pci_dev->dev, page, 0,
 111                                         PAGE_SIZE << efx->rx_buffer_order,
 112                                         DMA_FROM_DEVICE);
 113                 if (unlikely(dma_mapping_error(&efx->pci_dev->dev, dma_addr))) {
 114                         __free_pages(page, efx->rx_buffer_order);
 115                         return -EIO;
 116                 }
 117                 state = page_address(page);
 118                 state->refcnt = 0;
 119                 state->dma_addr = dma_addr;
 120
 121                 dma_addr += sizeof(struct efx_rx_page_state);
 122                 page_offset = sizeof(struct efx_rx_page_state);
 123
 124         split:
 125                 index = rx_queue->added_count & rx_queue->ptr_mask;
 126                 rx_buf = efx_rx_buffer(rx_queue, index);
 127                 rx_buf->dma_addr = dma_addr + EFX_PAGE_IP_ALIGN;
 128                 rx_buf->page = page;
 129                 rx_buf->page_offset = page_offset + EFX_PAGE_IP_ALIGN;
 130                 rx_buf->len = efx->rx_dma_len;
 131                 rx_buf->flags = 0;
 132                 ++rx_queue->added_count;
 133                 ++state->refcnt;
 134
 135                 if ((~count & 1) && (efx->rx_dma_len <= EFX_RX_HALF_PAGE)) {
 136                         /* Use the second half of the page */
 137                         get_page(page);
 138                         dma_addr += (PAGE_SIZE >> 1);
 139                         page_offset += (PAGE_SIZE >> 1);
 140                         ++count;
 141                         goto split;
 142                 }
 143         }
 144
 145         return 0;
 146 }
 147
 148 static void efx_unmap_rx_buffer(struct efx_nic *efx,
 149                                 struct efx_rx_buffer *rx_buf,
 150                                 unsigned int used_len)
 151 {
 152         if (rx_buf->page) {
 153                 struct efx_rx_page_state *state;
 154
 155                 state = page_address(rx_buf->page);
 156                 if (--state->refcnt == 0) {
 157                         dma_unmap_page(&efx->pci_dev->dev,
 158                                        state->dma_addr,
 159                                        PAGE_SIZE << efx->rx_buffer_order,
 160                                        DMA_FROM_DEVICE);
 161                 } else if (used_len) {
 162                         dma_sync_single_for_cpu(&efx->pci_dev->dev,
 163                                                 rx_buf->dma_addr, used_len,
 164                                                 DMA_FROM_DEVICE);
 165                 }
 166         }
 167 }
 168
 169 static void efx_free_rx_buffer(struct efx_nic *efx,
 170                                struct efx_rx_buffer *rx_buf)
 171 {
 172         if (rx_buf->page) {
 173                 __free_pages(rx_buf->page, efx->rx_buffer_order);
 174                 rx_buf->page = NULL;
 175         }
 176 }
 177
 178 static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue,
 179                                struct efx_rx_buffer *rx_buf)
 180 {
 181         efx_unmap_rx_buffer(rx_queue->efx, rx_buf, 0);
 182         efx_free_rx_buffer(rx_queue->efx, rx_buf);
 183 }
 184
 185 /* Attempt to resurrect the other receive buffer that used to share this page,
 186  * which had previously been passed up to the kernel and freed. */
 187 static void efx_resurrect_rx_buffer(struct efx_rx_queue *rx_queue,
 188                                     struct efx_rx_buffer *rx_buf)
 189 {
 190         struct efx_rx_page_state *state = page_address(rx_buf->page);
 191         struct efx_rx_buffer *new_buf;
 192         unsigned fill_level, index;
 193
 194         /* +1 because efx_rx_packet() incremented removed_count. +1 because
 195          * we'd like to insert an additional descriptor whilst leaving
 196          * EFX_RXD_HEAD_ROOM for the non-recycle path */
 197         fill_level = (rx_queue->added_count - rx_queue->removed_count + 2);
 198         if (unlikely(fill_level > rx_queue->max_fill)) {
 199                 /* We could place "state" on a list, and drain the list in
 200                  * efx_fast_push_rx_descriptors(). For now, this will do. */
 201                 return;
 202         }
 203
 204         ++state->refcnt;
 205         get_page(rx_buf->page);
 206
 207         index = rx_queue->added_count & rx_queue->ptr_mask;
 208         new_buf = efx_rx_buffer(rx_queue, index);
 209         new_buf->dma_addr = rx_buf->dma_addr ^ (PAGE_SIZE >> 1);
 210         new_buf->page = rx_buf->page;
 211         new_buf->len = rx_buf->len;
 212         ++rx_queue->added_count;
 213 }
 214
 215 /* Recycle buffers directly back into the rx_queue. There is always
 216  * room to add these buffer, because we've just popped them.
 217  */
 218 static void efx_recycle_rx_buffers(struct efx_channel *channel,
 219                                    struct efx_rx_buffer *rx_buf,
 220                                    unsigned int n_frags)
 221 {
 222         struct efx_nic *efx = channel->efx;
 223         struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
 224         struct efx_rx_buffer *new_buf;
 225         unsigned index;
 226
 227         do {
 228                 rx_buf->flags = 0;
 229
 230                 if (efx->rx_dma_len <= EFX_RX_HALF_PAGE &&
 231                     page_count(rx_buf->page) == 1)
 232                         efx_resurrect_rx_buffer(rx_queue, rx_buf);
 233
 234                 index = rx_queue->added_count & rx_queue->ptr_mask;
 235                 new_buf = efx_rx_buffer(rx_queue, index);
 236
 237                 memcpy(new_buf, rx_buf, sizeof(*new_buf));
 238                 rx_buf->page = NULL;
 239                 ++rx_queue->added_count;
 240
 241                 rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 242         } while (--n_frags);
 243 }
 244
 245 /**
 246  * efx_fast_push_rx_descriptors - push new RX descriptors quickly
 247  * @rx_queue:           RX descriptor queue
 248  *
 249  * This will aim to fill the RX descriptor queue up to
 250  * @rx_queue->@max_fill. If there is insufficient atomic
 251  * memory to do so, a slow fill will be scheduled.
 252  *
 253  * The caller must provide serialisation (none is used here). In practise,
 254  * this means this function must run from the NAPI handler, or be called
 255  * when NAPI is disabled.
 256  */
 257 void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue)
 258 {
 259         unsigned fill_level;
 260         int space, rc = 0;
 261
 262         /* Calculate current fill level, and exit if we don't need to fill */
 263         fill_level = (rx_queue->added_count - rx_queue->removed_count);
 264         EFX_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries);
 265         if (fill_level >= rx_queue->fast_fill_trigger)
 266                 goto out;
 267
 268         /* Record minimum fill level */
 269         if (unlikely(fill_level < rx_queue->min_fill)) {
 270                 if (fill_level)
 271                         rx_queue->min_fill = fill_level;
 272         }
 273
 274         space = rx_queue->max_fill - fill_level;
 275         EFX_BUG_ON_PARANOID(space < EFX_RX_BATCH);
 276
 277         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 278                    "RX queue %d fast-filling descriptor ring from"
 279                    " level %d to level %d\n",
 280                    efx_rx_queue_index(rx_queue), fill_level,
 281                    rx_queue->max_fill);
 282
 283
 284         do {
 285                 rc = efx_init_rx_buffers(rx_queue);
 286                 if (unlikely(rc)) {
 287                         /* Ensure that we don't leave the rx queue empty */
 288                         if (rx_queue->added_count == rx_queue->removed_count)
 289                                 efx_schedule_slow_fill(rx_queue);
 290                         goto out;
 291                 }
 292         } while ((space -= EFX_RX_BATCH) >= EFX_RX_BATCH);
 293
 294         netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev,
 295                    "RX queue %d fast-filled descriptor ring "
 296                    "to level %d\n", efx_rx_queue_index(rx_queue),
 297                    rx_queue->added_count - rx_queue->removed_count);
 298
 299  out:
 300         if (rx_queue->notified_count != rx_queue->added_count)
 301                 efx_nic_notify_rx_desc(rx_queue);
 302 }
 303
 304 void efx_rx_slow_fill(unsigned long context)
 305 {
 306         struct efx_rx_queue *rx_queue = (struct efx_rx_queue *)context;
 307
 308         /* Post an event to cause NAPI to run and refill the queue */
 309         efx_nic_generate_fill_event(rx_queue);
 310         ++rx_queue->slow_fill_count;
 311 }
 312
 313 static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue,
 314                                      struct efx_rx_buffer *rx_buf,
 315                                      int len)
 316 {
 317         struct efx_nic *efx = rx_queue->efx;
 318         unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding;
 319
 320         if (likely(len <= max_len))
 321                 return;
 322
 323         /* The packet must be discarded, but this is only a fatal error
 324          * if the caller indicated it was
 325          */
 326         rx_buf->flags |= EFX_RX_PKT_DISCARD;
 327
 328         if ((len > rx_buf->len) && EFX_WORKAROUND_8071(efx)) {
 329                 if (net_ratelimit())
 330                         netif_err(efx, rx_err, efx->net_dev,
 331                                   " RX queue %d seriously overlength "
 332                                   "RX event (0x%x > 0x%x+0x%x). Leaking\n",
 333                                   efx_rx_queue_index(rx_queue), len, max_len,
 334                                   efx->type->rx_buffer_padding);
 335                 efx_schedule_reset(efx, RESET_TYPE_RX_RECOVERY);
 336         } else {
 337                 if (net_ratelimit())
 338                         netif_err(efx, rx_err, efx->net_dev,
 339                                   " RX queue %d overlength RX event "
 340                                   "(0x%x > 0x%x)\n",
 341                                   efx_rx_queue_index(rx_queue), len, max_len);
 342         }
 343
 344         efx_rx_queue_channel(rx_queue)->n_rx_overlength++;
 345 }
 346
 347 /* Pass a received packet up through GRO.  GRO can handle pages
 348  * regardless of checksum state and skbs with a good checksum.
 349  */
 350 static void
 351 efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf,
 352                   unsigned int n_frags, u8 *eh)
 353 {
 354         struct napi_struct *napi = &channel->napi_str;
 355         gro_result_t gro_result;
 356         struct efx_nic *efx = channel->efx;
 357         struct sk_buff *skb;
 358
 359         skb = napi_get_frags(napi);
 360         if (unlikely(!skb)) {
 361                 while (n_frags--) {
 362                         put_page(rx_buf->page);
 363                         rx_buf->page = NULL;
 364                         rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 365                 }
 366                 return;
 367         }
 368
 369         if (efx->net_dev->features & NETIF_F_RXHASH)
 370                 skb->rxhash = efx_rx_buf_hash(eh);
 371         skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ?
 372                           CHECKSUM_UNNECESSARY : CHECKSUM_NONE);
 373
 374         for (;;) {
 375                 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 376                                    rx_buf->page, rx_buf->page_offset,
 377                                    rx_buf->len);
 378                 rx_buf->page = NULL;
 379                 skb->len += rx_buf->len;
 380                 if (skb_shinfo(skb)->nr_frags == n_frags)
 381                         break;
 382
 383                 rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 384         }
 385
 386         skb->data_len = skb->len;
 387         skb->truesize += n_frags * efx->rx_buffer_truesize;
 388
 389         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 390
 391         gro_result = napi_gro_frags(napi);
 392         if (gro_result != GRO_DROP)
 393                 channel->irq_mod_score += 2;
 394 }
 395
 396 /* Allocate and construct an SKB around page fragments */
 397 static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel,
 398                                      struct efx_rx_buffer *rx_buf,
 399                                      unsigned int n_frags,
 400                                      u8 *eh, int hdr_len)
 401 {
 402         struct efx_nic *efx = channel->efx;
 403         struct sk_buff *skb;
 404
 405         /* Allocate an SKB to store the headers */
 406         skb = netdev_alloc_skb(efx->net_dev, hdr_len + EFX_PAGE_SKB_ALIGN);
 407         if (unlikely(skb == NULL))
 408                 return NULL;
 409
 410         EFX_BUG_ON_PARANOID(rx_buf->len < hdr_len);
 411
 412         skb_reserve(skb, EFX_PAGE_SKB_ALIGN);
 413         memcpy(__skb_put(skb, hdr_len), eh, hdr_len);
 414
 415         /* Append the remaining page(s) onto the frag list */
 416         if (rx_buf->len > hdr_len) {
 417                 rx_buf->page_offset += hdr_len;
 418                 rx_buf->len -= hdr_len;
 419
 420                 for (;;) {
 421                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 422                                            rx_buf->page, rx_buf->page_offset,
 423                                            rx_buf->len);
 424                         rx_buf->page = NULL;
 425                         skb->len += rx_buf->len;
 426                         skb->data_len += rx_buf->len;
 427                         if (skb_shinfo(skb)->nr_frags == n_frags)
 428                                 break;
 429
 430                         rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf);
 431                 }
 432         } else {
 433                 __free_pages(rx_buf->page, efx->rx_buffer_order);
 434                 rx_buf->page = NULL;
 435                 n_frags = 0;
 436         }
 437
 438         skb->truesize += n_frags * efx->rx_buffer_truesize;
 439
 440         /* Move past the ethernet header */
 441         skb->protocol = eth_type_trans(skb, efx->net_dev);
 442
 443         return skb;
 444 }
 445
 446 void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index,
 447                    unsigned int n_frags, unsigned int len, u16 flags)
 448 {
 449         struct efx_nic *efx = rx_queue->efx;
 450         struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 451         struct efx_rx_buffer *rx_buf;
 452
 453         rx_buf = efx_rx_buffer(rx_queue, index);
 454         rx_buf->flags |= flags;
 455
 456         /* Validate the number of fragments and completed length */
 457         if (n_frags == 1) {
 458                 efx_rx_packet__check_len(rx_queue, rx_buf, len);
 459         } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) ||
 460                    unlikely(len <= (n_frags - 1) * EFX_RX_USR_BUF_SIZE) ||
 461                    unlikely(len > n_frags * EFX_RX_USR_BUF_SIZE) ||
 462                    unlikely(!efx->rx_scatter)) {
 463                 /* If this isn't an explicit discard request, either
 464                  * the hardware or the driver is broken.
 465                  */
 466                 WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD));
 467                 rx_buf->flags |= EFX_RX_PKT_DISCARD;
 468         }
 469
 470         netif_vdbg(efx, rx_status, efx->net_dev,
 471                    "RX queue %d received ids %x-%x len %d %s%s\n",
 472                    efx_rx_queue_index(rx_queue), index,
 473                    (index + n_frags - 1) & rx_queue->ptr_mask, len,
 474                    (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "",
 475                    (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : "");
 476
 477         /* Discard packet, if instructed to do so.  Process the
 478          * previous receive first.
 479          */
 480         if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) {
 481                 efx_rx_flush_packet(channel);
 482                 efx_recycle_rx_buffers(channel, rx_buf, n_frags);
 483                 return;
 484         }
 485
 486         if (n_frags == 1)
 487                 rx_buf->len = len;
 488
 489         /* Release and/or sync DMA mapping - assumes all RX buffers
 490          * consumed in-order per RX queue
 491          */
 492         efx_unmap_rx_buffer(efx, rx_buf, rx_buf->len);
 493
 494         /* Prefetch nice and early so data will (hopefully) be in cache by
 495          * the time we look at it.
 496          */
 497         prefetch(efx_rx_buf_va(rx_buf));
 498
 499         rx_buf->page_offset += efx->type->rx_buffer_hash_size;
 500         rx_buf->len -= efx->type->rx_buffer_hash_size;
 501
 502         if (n_frags > 1) {
 503                 /* Release/sync DMA mapping for additional fragments.
 504                  * Fix length for last fragment.
 505                  */
 506                 unsigned int tail_frags = n_frags - 1;
 507
 508                 for (;;) {
 509                         rx_buf = efx_rx_buf_next(rx_queue, rx_buf);
 510                         if (--tail_frags == 0)
 511                                 break;
 512                         efx_unmap_rx_buffer(efx, rx_buf, EFX_RX_USR_BUF_SIZE);
 513                 }
 514                 rx_buf->len = len - (n_frags - 1) * EFX_RX_USR_BUF_SIZE;
 515                 efx_unmap_rx_buffer(efx, rx_buf, rx_buf->len);
 516         }
 517
 518         /* Pipeline receives so that we give time for packet headers to be
 519          * prefetched into cache.
 520          */
 521         efx_rx_flush_packet(channel);
 522         channel->rx_pkt_n_frags = n_frags;
 523         channel->rx_pkt_index = index;
 524 }
 525
 526 static void efx_rx_deliver(struct efx_channel *channel, u8 *eh,
 527                            struct efx_rx_buffer *rx_buf,
 528                            unsigned int n_frags)
 529 {
 530         struct sk_buff *skb;
 531         u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS);
 532
 533         skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len);
 534         if (unlikely(skb == NULL)) {
 535                 efx_free_rx_buffer(channel->efx, rx_buf);
 536                 return;
 537         }
 538         skb_record_rx_queue(skb, channel->rx_queue.core_index);
 539
 540         /* Set the SKB flags */
 541         skb_checksum_none_assert(skb);
 542
 543         if (channel->type->receive_skb)
 544                 if (channel->type->receive_skb(channel, skb))
 545                         return;
 546
 547         /* Pass the packet up */
 548         netif_receive_skb(skb);
 549 }
 550
 551 /* Handle a received packet.  Second half: Touches packet payload. */
 552 void __efx_rx_packet(struct efx_channel *channel)
 553 {
 554         struct efx_nic *efx = channel->efx;
 555         struct efx_rx_buffer *rx_buf =
 556                 efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index);
 557         u8 *eh = efx_rx_buf_va(rx_buf);
 558
 559         /* If we're in loopback test, then pass the packet directly to the
 560          * loopback layer, and free the rx_buf here
 561          */
 562         if (unlikely(efx->loopback_selftest)) {
 563                 efx_loopback_rx_packet(efx, eh, rx_buf->len);
 564                 efx_free_rx_buffer(efx, rx_buf);
 565                 goto out;
 566         }
 567
 568         if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM)))
 569                 rx_buf->flags &= ~EFX_RX_PKT_CSUMMED;
 570
 571         if (!channel->type->receive_skb)
 572                 efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh);
 573         else
 574                 efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags);
 575 out:
 576         channel->rx_pkt_n_frags = 0;
 577 }
 578
 579 int efx_probe_rx_queue(struct efx_rx_queue *rx_queue)
 580 {
 581         struct efx_nic *efx = rx_queue->efx;
 582         unsigned int entries;
 583         int rc;
 584
 585         /* Create the smallest power-of-two aligned ring */
 586         entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE);
 587         EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
 588         rx_queue->ptr_mask = entries - 1;
 589
 590         netif_dbg(efx, probe, efx->net_dev,
 591                   "creating RX queue %d size %#x mask %#x\n",
 592                   efx_rx_queue_index(rx_queue), efx->rxq_entries,
 593                   rx_queue->ptr_mask);
 594
 595         /* Allocate RX buffers */
 596         rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer),
 597                                    GFP_KERNEL);
 598         if (!rx_queue->buffer)
 599                 return -ENOMEM;
 600
 601         rc = efx_nic_probe_rx(rx_queue);
 602         if (rc) {
 603                 kfree(rx_queue->buffer);
 604                 rx_queue->buffer = NULL;
 605         }
 606         return rc;
 607 }
 608
 609 void efx_init_rx_queue(struct efx_rx_queue *rx_queue)
 610 {
 611         struct efx_nic *efx = rx_queue->efx;
 612         unsigned int max_fill, trigger, max_trigger;
 613
 614         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 615                   "initialising RX queue %d\n", efx_rx_queue_index(rx_queue));
 616
 617         /* Initialise ptr fields */
 618         rx_queue->added_count = 0;
 619         rx_queue->notified_count = 0;
 620         rx_queue->removed_count = 0;
 621         rx_queue->min_fill = -1U;
 622
 623         /* Initialise limit fields */
 624         max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM;
 625         max_trigger = max_fill - EFX_RX_BATCH;
 626         if (rx_refill_threshold != 0) {
 627                 trigger = max_fill * min(rx_refill_threshold, 100U) / 100U;
 628                 if (trigger > max_trigger)
 629                         trigger = max_trigger;
 630         } else {
 631                 trigger = max_trigger;
 632         }
 633
 634         rx_queue->max_fill = max_fill;
 635         rx_queue->fast_fill_trigger = trigger;
 636
 637         /* Set up RX descriptor ring */
 638         rx_queue->enabled = true;
 639         efx_nic_init_rx(rx_queue);
 640 }
 641
 642 void efx_fini_rx_queue(struct efx_rx_queue *rx_queue)
 643 {
 644         int i;
 645         struct efx_rx_buffer *rx_buf;
 646
 647         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 648                   "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue));
 649
 650         /* A flush failure might have left rx_queue->enabled */
 651         rx_queue->enabled = false;
 652
 653         del_timer_sync(&rx_queue->slow_fill);
 654         efx_nic_fini_rx(rx_queue);
 655
 656         /* Release RX buffers NB start at index 0 not current HW ptr */
 657         if (rx_queue->buffer) {
 658                 for (i = 0; i <= rx_queue->ptr_mask; i++) {
 659                         rx_buf = efx_rx_buffer(rx_queue, i);
 660                         efx_fini_rx_buffer(rx_queue, rx_buf);
 661                 }
 662         }
 663 }
 664
 665 void efx_remove_rx_queue(struct efx_rx_queue *rx_queue)
 666 {
 667         netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev,
 668                   "destroying RX queue %d\n", efx_rx_queue_index(rx_queue));
 669
 670         efx_nic_remove_rx(rx_queue);
 671
 672         kfree(rx_queue->buffer);
 673         rx_queue->buffer = NULL;
 674 }
 675
 676
 677 module_param(rx_refill_threshold, uint, 0444);
 678 MODULE_PARM_DESC(rx_refill_threshold,
 679                  "RX descriptor ring refill threshold (%)");
 680