drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2011 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/firmware.h>
  27 #include <linux/prefetch.h>
  28 #include "bnx2x_cmn.h"
  29 #include "bnx2x_init.h"
  30 #include "bnx2x_sp.h"
  31
  32
  33
  34 /**
  35  * bnx2x_bz_fp - zero content of the fastpath structure.
  36  *
  37  * @bp:         driver handle
  38  * @index:      fastpath index to be zeroed
  39  *
  40  * Makes sure the contents of the bp->fp[index].napi is kept
  41  * intact.
  42  */
  43 static inline void bnx2x_bz_fp(struct bnx2x *bp, int index)
  44 {
  45         struct bnx2x_fastpath *fp = &bp->fp[index];
  46         struct napi_struct orig_napi = fp->napi;
  47         /* bzero bnx2x_fastpath contents */
  48         memset(fp, 0, sizeof(*fp));
  49
  50         /* Restore the NAPI object as it has been already initialized */
  51         fp->napi = orig_napi;
  52
  53         fp->bp = bp;
  54         fp->index = index;
  55         if (IS_ETH_FP(fp))
  56                 fp->max_cos = bp->max_cos;
  57         else
  58                 /* Special queues support only one CoS */
  59                 fp->max_cos = 1;
  60
  61         /*
  62          * set the tpa flag for each queue. The tpa flag determines the queue
  63          * minimal size so it must be set prior to queue memory allocation
  64          */
  65         fp->disable_tpa = ((bp->flags & TPA_ENABLE_FLAG) == 0);
  66
  67 #ifdef BCM_CNIC
  68         /* We don't want TPA on an FCoE L2 ring */
  69         if (IS_FCOE_FP(fp))
  70                 fp->disable_tpa = 1;
  71 #endif
  72 }
  73
  74 /**
  75  * bnx2x_move_fp - move content of the fastpath structure.
  76  *
  77  * @bp:         driver handle
  78  * @from:       source FP index
  79  * @to:         destination FP index
  80  *
  81  * Makes sure the contents of the bp->fp[to].napi is kept
  82  * intact.
  83  */
  84 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  85 {
  86         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  87         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  88         struct napi_struct orig_napi = to_fp->napi;
  89         /* Move bnx2x_fastpath contents */
  90         memcpy(to_fp, from_fp, sizeof(*to_fp));
  91         to_fp->index = to;
  92
  93         /* Restore the NAPI object as it has been already initialized */
  94         to_fp->napi = orig_napi;
  95 }
  96
  97 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  98
  99 /* free skb in the packet ring at pos idx
 100  * return idx of last bd freed
 101  */
 102 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
 103                              u16 idx)
 104 {
 105         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
 106         struct eth_tx_start_bd *tx_start_bd;
 107         struct eth_tx_bd *tx_data_bd;
 108         struct sk_buff *skb = tx_buf->skb;
 109         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
 110         int nbd;
 111
 112         /* prefetch skb end pointer to speedup dev_kfree_skb() */
 113         prefetch(&skb->end);
 114
 115         DP(BNX2X_MSG_FP, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
 116            txdata->txq_index, idx, tx_buf, skb);
 117
 118         /* unmap first bd */
 119         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
 120         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
 121         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
 122                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
 123
 124
 125         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
 126 #ifdef BNX2X_STOP_ON_ERROR
 127         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 128                 BNX2X_ERR("BAD nbd!\n");
 129                 bnx2x_panic();
 130         }
 131 #endif
 132         new_cons = nbd + tx_buf->first_bd;
 133
 134         /* Get the next bd */
 135         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 136
 137         /* Skip a parse bd... */
 138         --nbd;
 139         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 140
 141         /* ...and the TSO split header bd since they have no mapping */
 142         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 143                 --nbd;
 144                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 145         }
 146
 147         /* now free frags */
 148         while (nbd > 0) {
 149
 150                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
 151                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 152                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 153                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 154                 if (--nbd)
 155                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 156         }
 157
 158         /* release skb */
 159         WARN_ON(!skb);
 160         dev_kfree_skb_any(skb);
 161         tx_buf->first_bd = 0;
 162         tx_buf->skb = NULL;
 163
 164         return new_cons;
 165 }
 166
 167 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 168 {
 169         struct netdev_queue *txq;
 170         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 171
 172 #ifdef BNX2X_STOP_ON_ERROR
 173         if (unlikely(bp->panic))
 174                 return -1;
 175 #endif
 176
 177         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 178         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 179         sw_cons = txdata->tx_pkt_cons;
 180
 181         while (sw_cons != hw_cons) {
 182                 u16 pkt_cons;
 183
 184                 pkt_cons = TX_BD(sw_cons);
 185
 186                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 187                                       " pkt_cons %u\n",
 188                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 189
 190                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons);
 191                 sw_cons++;
 192         }
 193
 194         txdata->tx_pkt_cons = sw_cons;
 195         txdata->tx_bd_cons = bd_cons;
 196
 197         /* Need to make the tx_bd_cons update visible to start_xmit()
 198          * before checking for netif_tx_queue_stopped().  Without the
 199          * memory barrier, there is a small possibility that
 200          * start_xmit() will miss it and cause the queue to be stopped
 201          * forever.
 202          * On the other hand we need an rmb() here to ensure the proper
 203          * ordering of bit testing in the following
 204          * netif_tx_queue_stopped(txq) call.
 205          */
 206         smp_mb();
 207
 208         if (unlikely(netif_tx_queue_stopped(txq))) {
 209                 /* Taking tx_lock() is needed to prevent reenabling the queue
 210                  * while it's empty. This could have happen if rx_action() gets
 211                  * suspended in bnx2x_tx_int() after the condition before
 212                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 213                  *
 214                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 215                  * sends some packets consuming the whole queue again->
 216                  * stops the queue
 217                  */
 218
 219                 __netif_tx_lock(txq, smp_processor_id());
 220
 221                 if ((netif_tx_queue_stopped(txq)) &&
 222                     (bp->state == BNX2X_STATE_OPEN) &&
 223                     (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
 224                         netif_tx_wake_queue(txq);
 225
 226                 __netif_tx_unlock(txq);
 227         }
 228         return 0;
 229 }
 230
 231 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 232                                              u16 idx)
 233 {
 234         u16 last_max = fp->last_max_sge;
 235
 236         if (SUB_S16(idx, last_max) > 0)
 237                 fp->last_max_sge = idx;
 238 }
 239
 240 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 241                                   struct eth_fast_path_rx_cqe *fp_cqe)
 242 {
 243         struct bnx2x *bp = fp->bp;
 244         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 245                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 246                       SGE_PAGE_SHIFT;
 247         u16 last_max, last_elem, first_elem;
 248         u16 delta = 0;
 249         u16 i;
 250
 251         if (!sge_len)
 252                 return;
 253
 254         /* First mark all used pages */
 255         for (i = 0; i < sge_len; i++)
 256                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 257                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 258
 259         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 260            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 261
 262         /* Here we assume that the last SGE index is the biggest */
 263         prefetch((void *)(fp->sge_mask));
 264         bnx2x_update_last_max_sge(fp,
 265                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 266
 267         last_max = RX_SGE(fp->last_max_sge);
 268         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 269         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 270
 271         /* If ring is not full */
 272         if (last_elem + 1 != first_elem)
 273                 last_elem++;
 274
 275         /* Now update the prod */
 276         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 277                 if (likely(fp->sge_mask[i]))
 278                         break;
 279
 280                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 281                 delta += BIT_VEC64_ELEM_SZ;
 282         }
 283
 284         if (delta > 0) {
 285                 fp->rx_sge_prod += delta;
 286                 /* clear page-end entries */
 287                 bnx2x_clear_sge_mask_next_elems(fp);
 288         }
 289
 290         DP(NETIF_MSG_RX_STATUS,
 291            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 292            fp->last_max_sge, fp->rx_sge_prod);
 293 }
 294
 295 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 296                             struct sk_buff *skb, u16 cons, u16 prod,
 297                             struct eth_fast_path_rx_cqe *cqe)
 298 {
 299         struct bnx2x *bp = fp->bp;
 300         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 301         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 302         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 303         dma_addr_t mapping;
 304         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 305         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 306
 307         /* print error if current state != stop */
 308         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 309                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 310
 311         /* Try to map an empty skb from the aggregation info  */
 312         mapping = dma_map_single(&bp->pdev->dev,
 313                                  first_buf->skb->data,
 314                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 315         /*
 316          *  ...if it fails - move the skb from the consumer to the producer
 317          *  and set the current aggregation state as ERROR to drop it
 318          *  when TPA_STOP arrives.
 319          */
 320
 321         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 322                 /* Move the BD from the consumer to the producer */
 323                 bnx2x_reuse_rx_skb(fp, cons, prod);
 324                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 325                 return;
 326         }
 327
 328         /* move empty skb from pool to prod */
 329         prod_rx_buf->skb = first_buf->skb;
 330         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 331         /* point prod_bd to new skb */
 332         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 333         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 334
 335         /* move partial skb from cons to pool (don't unmap yet) */
 336         *first_buf = *cons_rx_buf;
 337
 338         /* mark bin state as START */
 339         tpa_info->parsing_flags =
 340                 le16_to_cpu(cqe->pars_flags.flags);
 341         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 342         tpa_info->tpa_state = BNX2X_TPA_START;
 343         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 344         tpa_info->placement_offset = cqe->placement_offset;
 345
 346 #ifdef BNX2X_STOP_ON_ERROR
 347         fp->tpa_queue_used |= (1 << queue);
 348 #ifdef _ASM_GENERIC_INT_L64_H
 349         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 350 #else
 351         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 352 #endif
 353            fp->tpa_queue_used);
 354 #endif
 355 }
 356
 357 /* Timestamp option length allowed for TPA aggregation:
 358  *
 359  *              nop nop kind length echo val
 360  */
 361 #define TPA_TSTAMP_OPT_LEN      12
 362 /**
 363  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 364  *
 365  * @bp:                 driver handle
 366  * @parsing_flags:      parsing flags from the START CQE
 367  * @len_on_bd:          total length of the first packet for the
 368  *                      aggregation.
 369  *
 370  * Approximate value of the MSS for this aggregation calculated using
 371  * the first packet of it.
 372  */
 373 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 374                                     u16 len_on_bd)
 375 {
 376         /*
 377          * TPA arrgregation won't have either IP options or TCP options
 378          * other than timestamp or IPv6 extension headers.
 379          */
 380         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 381
 382         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 383             PRS_FLAG_OVERETH_IPV6)
 384                 hdrs_len += sizeof(struct ipv6hdr);
 385         else /* IPv4 */
 386                 hdrs_len += sizeof(struct iphdr);
 387
 388
 389         /* Check if there was a TCP timestamp, if there is it's will
 390          * always be 12 bytes length: nop nop kind length echo val.
 391          *
 392          * Otherwise FW would close the aggregation.
 393          */
 394         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 395                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 396
 397         return len_on_bd - hdrs_len;
 398 }
 399
 400 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 401                                u16 queue, struct sk_buff *skb,
 402                                struct eth_end_agg_rx_cqe *cqe,
 403                                u16 cqe_idx)
 404 {
 405         struct sw_rx_page *rx_pg, old_rx_pg;
 406         u32 i, frag_len, frag_size, pages;
 407         int err;
 408         int j;
 409         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 410         u16 len_on_bd = tpa_info->len_on_bd;
 411
 412         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 413         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 414
 415         /* This is needed in order to enable forwarding support */
 416         if (frag_size)
 417                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 418                                         tpa_info->parsing_flags, len_on_bd);
 419
 420 #ifdef BNX2X_STOP_ON_ERROR
 421         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 422                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 423                           pages, cqe_idx);
 424                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 425                 bnx2x_panic();
 426                 return -EINVAL;
 427         }
 428 #endif
 429
 430         /* Run through the SGL and compose the fragmented skb */
 431         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 432                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 433
 434                 /* FW gives the indices of the SGE as if the ring is an array
 435                    (meaning that "next" element will consume 2 indices) */
 436                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 437                 rx_pg = &fp->rx_page_ring[sge_idx];
 438                 old_rx_pg = *rx_pg;
 439
 440                 /* If we fail to allocate a substitute page, we simply stop
 441                    where we are and drop the whole packet */
 442                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 443                 if (unlikely(err)) {
 444                         fp->eth_q_stats.rx_skb_alloc_failed++;
 445                         return err;
 446                 }
 447
 448                 /* Unmap the page as we r going to pass it to the stack */
 449                 dma_unmap_page(&bp->pdev->dev,
 450                                dma_unmap_addr(&old_rx_pg, mapping),
 451                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 452
 453                 /* Add one frag and update the appropriate fields in the skb */
 454                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 455
 456                 skb->data_len += frag_len;
 457                 skb->truesize += frag_len;
 458                 skb->len += frag_len;
 459
 460                 frag_size -= frag_len;
 461         }
 462
 463         return 0;
 464 }
 465
 466 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 467                            u16 queue, struct eth_end_agg_rx_cqe *cqe,
 468                            u16 cqe_idx)
 469 {
 470         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 471         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 472         u8 pad = tpa_info->placement_offset;
 473         u16 len = tpa_info->len_on_bd;
 474         struct sk_buff *skb = rx_buf->skb;
 475         /* alloc new skb */
 476         struct sk_buff *new_skb;
 477         u8 old_tpa_state = tpa_info->tpa_state;
 478
 479         tpa_info->tpa_state = BNX2X_TPA_STOP;
 480
 481         /* If we there was an error during the handling of the TPA_START -
 482          * drop this aggregation.
 483          */
 484         if (old_tpa_state == BNX2X_TPA_ERROR)
 485                 goto drop;
 486
 487         /* Try to allocate the new skb */
 488         new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 489
 490         /* Unmap skb in the pool anyway, as we are going to change
 491            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 492            fails. */
 493         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 494                          fp->rx_buf_size, DMA_FROM_DEVICE);
 495
 496         if (likely(new_skb)) {
 497                 prefetch(skb);
 498                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 499
 500 #ifdef BNX2X_STOP_ON_ERROR
 501                 if (pad + len > fp->rx_buf_size) {
 502                         BNX2X_ERR("skb_put is about to fail...  "
 503                                   "pad %d  len %d  rx_buf_size %d\n",
 504                                   pad, len, fp->rx_buf_size);
 505                         bnx2x_panic();
 506                         return;
 507                 }
 508 #endif
 509
 510                 skb_reserve(skb, pad);
 511                 skb_put(skb, len);
 512
 513                 skb->protocol = eth_type_trans(skb, bp->dev);
 514                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 515
 516                 if (!bnx2x_fill_frag_skb(bp, fp, queue, skb, cqe, cqe_idx)) {
 517                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 518                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 519                         napi_gro_receive(&fp->napi, skb);
 520                 } else {
 521                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 522                            " - dropping packet!\n");
 523                         dev_kfree_skb_any(skb);
 524                 }
 525
 526
 527                 /* put new skb in bin */
 528                 rx_buf->skb = new_skb;
 529
 530                 return;
 531         }
 532
 533 drop:
 534         /* drop the packet and keep the buffer in the bin */
 535         DP(NETIF_MSG_RX_STATUS,
 536            "Failed to allocate or map a new skb - dropping packet!\n");
 537         fp->eth_q_stats.rx_skb_alloc_failed++;
 538 }
 539
 540 /* Set Toeplitz hash value in the skb using the value from the
 541  * CQE (calculated by HW).
 542  */
 543 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 544                                         struct sk_buff *skb)
 545 {
 546         /* Set Toeplitz hash from CQE */
 547         if ((bp->dev->features & NETIF_F_RXHASH) &&
 548             (cqe->fast_path_cqe.status_flags &
 549              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 550                 skb->rxhash =
 551                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 552 }
 553
 554 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 555 {
 556         struct bnx2x *bp = fp->bp;
 557         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 558         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 559         int rx_pkt = 0;
 560
 561 #ifdef BNX2X_STOP_ON_ERROR
 562         if (unlikely(bp->panic))
 563                 return 0;
 564 #endif
 565
 566         /* CQ "next element" is of the size of the regular element,
 567            that's why it's ok here */
 568         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 569         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 570                 hw_comp_cons++;
 571
 572         bd_cons = fp->rx_bd_cons;
 573         bd_prod = fp->rx_bd_prod;
 574         bd_prod_fw = bd_prod;
 575         sw_comp_cons = fp->rx_comp_cons;
 576         sw_comp_prod = fp->rx_comp_prod;
 577
 578         /* Memory barrier necessary as speculative reads of the rx
 579          * buffer can be ahead of the index in the status block
 580          */
 581         rmb();
 582
 583         DP(NETIF_MSG_RX_STATUS,
 584            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 585            fp->index, hw_comp_cons, sw_comp_cons);
 586
 587         while (sw_comp_cons != hw_comp_cons) {
 588                 struct sw_rx_bd *rx_buf = NULL;
 589                 struct sk_buff *skb;
 590                 union eth_rx_cqe *cqe;
 591                 struct eth_fast_path_rx_cqe *cqe_fp;
 592                 u8 cqe_fp_flags;
 593                 enum eth_rx_cqe_type cqe_fp_type;
 594                 u16 len, pad;
 595
 596 #ifdef BNX2X_STOP_ON_ERROR
 597                 if (unlikely(bp->panic))
 598                         return 0;
 599 #endif
 600
 601                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 602                 bd_prod = RX_BD(bd_prod);
 603                 bd_cons = RX_BD(bd_cons);
 604
 605                 /* Prefetch the page containing the BD descriptor
 606                    at producer's index. It will be needed when new skb is
 607                    allocated */
 608                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 609                                              (&fp->rx_desc_ring[bd_prod])) -
 610                                   PAGE_SIZE + 1));
 611
 612                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 613                 cqe_fp = &cqe->fast_path_cqe;
 614                 cqe_fp_flags = cqe_fp->type_error_flags;
 615                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 616
 617                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 618                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 619                    cqe_fp_flags, cqe_fp->status_flags,
 620                    le32_to_cpu(cqe_fp->rss_hash_result),
 621                    le16_to_cpu(cqe_fp->vlan_tag), le16_to_cpu(cqe_fp->pkt_len));
 622
 623                 /* is this a slowpath msg? */
 624                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 625                         bnx2x_sp_event(fp, cqe);
 626                         goto next_cqe;
 627
 628                 /* this is an rx packet */
 629                 } else {
 630                         rx_buf = &fp->rx_buf_ring[bd_cons];
 631                         skb = rx_buf->skb;
 632                         prefetch(skb);
 633
 634                         if (!CQE_TYPE_FAST(cqe_fp_type)) {
 635 #ifdef BNX2X_STOP_ON_ERROR
 636                                 /* sanity check */
 637                                 if (fp->disable_tpa &&
 638                                     (CQE_TYPE_START(cqe_fp_type) ||
 639                                      CQE_TYPE_STOP(cqe_fp_type)))
 640                                         BNX2X_ERR("START/STOP packet while "
 641                                                   "disable_tpa type %x\n",
 642                                                   CQE_TYPE(cqe_fp_type));
 643 #endif
 644
 645                                 if (CQE_TYPE_START(cqe_fp_type)) {
 646                                         u16 queue = cqe_fp->queue_index;
 647                                         DP(NETIF_MSG_RX_STATUS,
 648                                            "calling tpa_start on queue %d\n",
 649                                            queue);
 650
 651                                         bnx2x_tpa_start(fp, queue, skb,
 652                                                         bd_cons, bd_prod,
 653                                                         cqe_fp);
 654
 655                                         /* Set Toeplitz hash for LRO skb */
 656                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 657
 658                                         goto next_rx;
 659
 660                                 } else {
 661                                         u16 queue =
 662                                                 cqe->end_agg_cqe.queue_index;
 663                                         DP(NETIF_MSG_RX_STATUS,
 664                                            "calling tpa_stop on queue %d\n",
 665                                            queue);
 666
 667                                         bnx2x_tpa_stop(bp, fp, queue,
 668                                                        &cqe->end_agg_cqe,
 669                                                        comp_ring_cons);
 670 #ifdef BNX2X_STOP_ON_ERROR
 671                                         if (bp->panic)
 672                                                 return 0;
 673 #endif
 674
 675                                         bnx2x_update_sge_prod(fp, cqe_fp);
 676                                         goto next_cqe;
 677                                 }
 678                         }
 679                         /* non TPA */
 680                         len = le16_to_cpu(cqe_fp->pkt_len);
 681                         pad = cqe_fp->placement_offset;
 682                         dma_sync_single_for_cpu(&bp->pdev->dev,
 683                                         dma_unmap_addr(rx_buf, mapping),
 684                                                        pad + RX_COPY_THRESH,
 685                                                        DMA_FROM_DEVICE);
 686                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 687
 688                         /* is this an error packet? */
 689                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 690                                 DP(NETIF_MSG_RX_ERR,
 691                                    "ERROR  flags %x  rx packet %u\n",
 692                                    cqe_fp_flags, sw_comp_cons);
 693                                 fp->eth_q_stats.rx_err_discard_pkt++;
 694                                 goto reuse_rx;
 695                         }
 696
 697                         /* Since we don't have a jumbo ring
 698                          * copy small packets if mtu > 1500
 699                          */
 700                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 701                             (len <= RX_COPY_THRESH)) {
 702                                 struct sk_buff *new_skb;
 703
 704                                 new_skb = netdev_alloc_skb(bp->dev, len + pad);
 705                                 if (new_skb == NULL) {
 706                                         DP(NETIF_MSG_RX_ERR,
 707                                            "ERROR  packet dropped "
 708                                            "because of alloc failure\n");
 709                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 710                                         goto reuse_rx;
 711                                 }
 712
 713                                 /* aligned copy */
 714                                 skb_copy_from_linear_data_offset(skb, pad,
 715                                                     new_skb->data + pad, len);
 716                                 skb_reserve(new_skb, pad);
 717                                 skb_put(new_skb, len);
 718
 719                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 720
 721                                 skb = new_skb;
 722
 723                         } else
 724                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 725                                 dma_unmap_single(&bp->pdev->dev,
 726                                         dma_unmap_addr(rx_buf, mapping),
 727                                                  fp->rx_buf_size,
 728                                                  DMA_FROM_DEVICE);
 729                                 skb_reserve(skb, pad);
 730                                 skb_put(skb, len);
 731
 732                         } else {
 733                                 DP(NETIF_MSG_RX_ERR,
 734                                    "ERROR  packet dropped because "
 735                                    "of alloc failure\n");
 736                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 737 reuse_rx:
 738                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 739                                 goto next_rx;
 740                         }
 741
 742                         skb->protocol = eth_type_trans(skb, bp->dev);
 743
 744                         /* Set Toeplitz hash for a none-LRO skb */
 745                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 746
 747                         skb_checksum_none_assert(skb);
 748
 749                         if (bp->dev->features & NETIF_F_RXCSUM) {
 750
 751                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 752                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 753                                 else
 754                                         fp->eth_q_stats.hw_csum_err++;
 755                         }
 756                 }
 757
 758                 skb_record_rx_queue(skb, fp->index);
 759
 760                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 761                     PARSING_FLAGS_VLAN)
 762                         __vlan_hwaccel_put_tag(skb,
 763                                                le16_to_cpu(cqe_fp->vlan_tag));
 764                 napi_gro_receive(&fp->napi, skb);
 765
 766
 767 next_rx:
 768                 rx_buf->skb = NULL;
 769
 770                 bd_cons = NEXT_RX_IDX(bd_cons);
 771                 bd_prod = NEXT_RX_IDX(bd_prod);
 772                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 773                 rx_pkt++;
 774 next_cqe:
 775                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 776                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 777
 778                 if (rx_pkt == budget)
 779                         break;
 780         } /* while */
 781
 782         fp->rx_bd_cons = bd_cons;
 783         fp->rx_bd_prod = bd_prod_fw;
 784         fp->rx_comp_cons = sw_comp_cons;
 785         fp->rx_comp_prod = sw_comp_prod;
 786
 787         /* Update producers */
 788         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 789                              fp->rx_sge_prod);
 790
 791         fp->rx_pkt += rx_pkt;
 792         fp->rx_calls++;
 793
 794         return rx_pkt;
 795 }
 796
 797 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 798 {
 799         struct bnx2x_fastpath *fp = fp_cookie;
 800         struct bnx2x *bp = fp->bp;
 801         u8 cos;
 802
 803         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 804                          "[fp %d fw_sd %d igusb %d]\n",
 805            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 806         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 807
 808 #ifdef BNX2X_STOP_ON_ERROR
 809         if (unlikely(bp->panic))
 810                 return IRQ_HANDLED;
 811 #endif
 812
 813         /* Handle Rx and Tx according to MSI-X vector */
 814         prefetch(fp->rx_cons_sb);
 815
 816         for_each_cos_in_tx_queue(fp, cos)
 817                 prefetch(fp->txdata[cos].tx_cons_sb);
 818
 819         prefetch(&fp->sb_running_index[SM_RX_ID]);
 820         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 821
 822         return IRQ_HANDLED;
 823 }
 824
 825 /* HW Lock for shared dual port PHYs */
 826 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 827 {
 828         mutex_lock(&bp->port.phy_mutex);
 829
 830         if (bp->port.need_hw_lock)
 831                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 832 }
 833
 834 void bnx2x_release_phy_lock(struct bnx2x *bp)
 835 {
 836         if (bp->port.need_hw_lock)
 837                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 838
 839         mutex_unlock(&bp->port.phy_mutex);
 840 }
 841
 842 /* calculates MF speed according to current linespeed and MF configuration */
 843 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 844 {
 845         u16 line_speed = bp->link_vars.line_speed;
 846         if (IS_MF(bp)) {
 847                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 848                                                    bp->mf_config[BP_VN(bp)]);
 849
 850                 /* Calculate the current MAX line speed limit for the MF
 851                  * devices
 852                  */
 853                 if (IS_MF_SI(bp))
 854                         line_speed = (line_speed * maxCfg) / 100;
 855                 else { /* SD mode */
 856                         u16 vn_max_rate = maxCfg * 100;
 857
 858                         if (vn_max_rate < line_speed)
 859                                 line_speed = vn_max_rate;
 860                 }
 861         }
 862
 863         return line_speed;
 864 }
 865
 866 /**
 867  * bnx2x_fill_report_data - fill link report data to report
 868  *
 869  * @bp:         driver handle
 870  * @data:       link state to update
 871  *
 872  * It uses a none-atomic bit operations because is called under the mutex.
 873  */
 874 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
 875                                           struct bnx2x_link_report_data *data)
 876 {
 877         u16 line_speed = bnx2x_get_mf_speed(bp);
 878
 879         memset(data, 0, sizeof(*data));
 880
 881         /* Fill the report data: efective line speed */
 882         data->line_speed = line_speed;
 883
 884         /* Link is down */
 885         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
 886                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 887                           &data->link_report_flags);
 888
 889         /* Full DUPLEX */
 890         if (bp->link_vars.duplex == DUPLEX_FULL)
 891                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
 892
 893         /* Rx Flow Control is ON */
 894         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
 895                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
 896
 897         /* Tx Flow Control is ON */
 898         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
 899                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
 900 }
 901
 902 /**
 903  * bnx2x_link_report - report link status to OS.
 904  *
 905  * @bp:         driver handle
 906  *
 907  * Calls the __bnx2x_link_report() under the same locking scheme
 908  * as a link/PHY state managing code to ensure a consistent link
 909  * reporting.
 910  */
 911
 912 void bnx2x_link_report(struct bnx2x *bp)
 913 {
 914         bnx2x_acquire_phy_lock(bp);
 915         __bnx2x_link_report(bp);
 916         bnx2x_release_phy_lock(bp);
 917 }
 918
 919 /**
 920  * __bnx2x_link_report - report link status to OS.
 921  *
 922  * @bp:         driver handle
 923  *
 924  * None atomic inmlementation.
 925  * Should be called under the phy_lock.
 926  */
 927 void __bnx2x_link_report(struct bnx2x *bp)
 928 {
 929         struct bnx2x_link_report_data cur_data;
 930
 931         /* reread mf_cfg */
 932         if (!CHIP_IS_E1(bp))
 933                 bnx2x_read_mf_cfg(bp);
 934
 935         /* Read the current link report info */
 936         bnx2x_fill_report_data(bp, &cur_data);
 937
 938         /* Don't report link down or exactly the same link status twice */
 939         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
 940             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 941                       &bp->last_reported_link.link_report_flags) &&
 942              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 943                       &cur_data.link_report_flags)))
 944                 return;
 945
 946         bp->link_cnt++;
 947
 948         /* We are going to report a new link parameters now -
 949          * remember the current data for the next time.
 950          */
 951         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
 952
 953         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 954                      &cur_data.link_report_flags)) {
 955                 netif_carrier_off(bp->dev);
 956                 netdev_err(bp->dev, "NIC Link is Down\n");
 957                 return;
 958         } else {
 959                 const char *duplex;
 960                 const char *flow;
 961
 962                 netif_carrier_on(bp->dev);
 963
 964                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
 965                                        &cur_data.link_report_flags))
 966                         duplex = "full";
 967                 else
 968                         duplex = "half";
 969
 970                 /* Handle the FC at the end so that only these flags would be
 971                  * possibly set. This way we may easily check if there is no FC
 972                  * enabled.
 973                  */
 974                 if (cur_data.link_report_flags) {
 975                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
 976                                      &cur_data.link_report_flags)) {
 977                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
 978                                      &cur_data.link_report_flags))
 979                                         flow = "ON - receive & transmit";
 980                                 else
 981                                         flow = "ON - receive";
 982                         } else {
 983                                 flow = "ON - transmit";
 984                         }
 985                 } else {
 986                         flow = "none";
 987                 }
 988                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
 989                             cur_data.line_speed, duplex, flow);
 990         }
 991 }
 992
 993 void bnx2x_init_rx_rings(struct bnx2x *bp)
 994 {
 995         int func = BP_FUNC(bp);
 996         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 997                                               ETH_MAX_AGGREGATION_QUEUES_E1H_E2;
 998         u16 ring_prod;
 999         int i, j;
1000
1001         /* Allocate TPA resources */
1002         for_each_rx_queue(bp, j) {
1003                 struct bnx2x_fastpath *fp = &bp->fp[j];
1004
1005                 DP(NETIF_MSG_IFUP,
1006                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1007
1008                 if (!fp->disable_tpa) {
1009                         /* Fill the per-aggregtion pool */
1010                         for (i = 0; i < max_agg_queues; i++) {
1011                                 struct bnx2x_agg_info *tpa_info =
1012                                         &fp->tpa_info[i];
1013                                 struct sw_rx_bd *first_buf =
1014                                         &tpa_info->first_buf;
1015
1016                                 first_buf->skb = netdev_alloc_skb(bp->dev,
1017                                                        fp->rx_buf_size);
1018                                 if (!first_buf->skb) {
1019                                         BNX2X_ERR("Failed to allocate TPA "
1020                                                   "skb pool for queue[%d] - "
1021                                                   "disabling TPA on this "
1022                                                   "queue!\n", j);
1023                                         bnx2x_free_tpa_pool(bp, fp, i);
1024                                         fp->disable_tpa = 1;
1025                                         break;
1026                                 }
1027                                 dma_unmap_addr_set(first_buf, mapping, 0);
1028                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1029                         }
1030
1031                         /* "next page" elements initialization */
1032                         bnx2x_set_next_page_sgl(fp);
1033
1034                         /* set SGEs bit mask */
1035                         bnx2x_init_sge_ring_bit_mask(fp);
1036
1037                         /* Allocate SGEs and initialize the ring elements */
1038                         for (i = 0, ring_prod = 0;
1039                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1040
1041                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1042                                         BNX2X_ERR("was only able to allocate "
1043                                                   "%d rx sges\n", i);
1044                                         BNX2X_ERR("disabling TPA for "
1045                                                   "queue[%d]\n", j);
1046                                         /* Cleanup already allocated elements */
1047                                         bnx2x_free_rx_sge_range(bp, fp,
1048                                                                 ring_prod);
1049                                         bnx2x_free_tpa_pool(bp, fp,
1050                                                             max_agg_queues);
1051                                         fp->disable_tpa = 1;
1052                                         ring_prod = 0;
1053                                         break;
1054                                 }
1055                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1056                         }
1057
1058                         fp->rx_sge_prod = ring_prod;
1059                 }
1060         }
1061
1062         for_each_rx_queue(bp, j) {
1063                 struct bnx2x_fastpath *fp = &bp->fp[j];
1064
1065                 fp->rx_bd_cons = 0;
1066
1067                 /* Activate BD ring */
1068                 /* Warning!
1069                  * this will generate an interrupt (to the TSTORM)
1070                  * must only be done after chip is initialized
1071                  */
1072                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1073                                      fp->rx_sge_prod);
1074
1075                 if (j != 0)
1076                         continue;
1077
1078                 if (CHIP_IS_E1(bp)) {
1079                         REG_WR(bp, BAR_USTRORM_INTMEM +
1080                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1081                                U64_LO(fp->rx_comp_mapping));
1082                         REG_WR(bp, BAR_USTRORM_INTMEM +
1083                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1084                                U64_HI(fp->rx_comp_mapping));
1085                 }
1086         }
1087 }
1088
1089 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1090 {
1091         int i;
1092         u8 cos;
1093
1094         for_each_tx_queue(bp, i) {
1095                 struct bnx2x_fastpath *fp = &bp->fp[i];
1096                 for_each_cos_in_tx_queue(fp, cos) {
1097                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1098
1099                         u16 bd_cons = txdata->tx_bd_cons;
1100                         u16 sw_prod = txdata->tx_pkt_prod;
1101                         u16 sw_cons = txdata->tx_pkt_cons;
1102
1103                         while (sw_cons != sw_prod) {
1104                                 bd_cons = bnx2x_free_tx_pkt(bp, txdata,
1105                                                             TX_BD(sw_cons));
1106                                 sw_cons++;
1107                         }
1108                 }
1109         }
1110 }
1111
1112 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1113 {
1114         struct bnx2x *bp = fp->bp;
1115         int i;
1116
1117         /* ring wasn't allocated */
1118         if (fp->rx_buf_ring == NULL)
1119                 return;
1120
1121         for (i = 0; i < NUM_RX_BD; i++) {
1122                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1123                 struct sk_buff *skb = rx_buf->skb;
1124
1125                 if (skb == NULL)
1126                         continue;
1127                 dma_unmap_single(&bp->pdev->dev,
1128                                  dma_unmap_addr(rx_buf, mapping),
1129                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1130
1131                 rx_buf->skb = NULL;
1132                 dev_kfree_skb(skb);
1133         }
1134 }
1135
1136 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1137 {
1138         int j;
1139
1140         for_each_rx_queue(bp, j) {
1141                 struct bnx2x_fastpath *fp = &bp->fp[j];
1142
1143                 bnx2x_free_rx_bds(fp);
1144
1145                 if (!fp->disable_tpa)
1146                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
1147                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
1148                                             ETH_MAX_AGGREGATION_QUEUES_E1H_E2);
1149         }
1150 }
1151
1152 void bnx2x_free_skbs(struct bnx2x *bp)
1153 {
1154         bnx2x_free_tx_skbs(bp);
1155         bnx2x_free_rx_skbs(bp);
1156 }
1157
1158 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1159 {
1160         /* load old values */
1161         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1162
1163         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1164                 /* leave all but MAX value */
1165                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1166
1167                 /* set new MAX value */
1168                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1169                                 & FUNC_MF_CFG_MAX_BW_MASK;
1170
1171                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1172         }
1173 }
1174
1175 /**
1176  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1177  *
1178  * @bp:         driver handle
1179  * @nvecs:      number of vectors to be released
1180  */
1181 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1182 {
1183         int i, offset = 0;
1184
1185         if (nvecs == offset)
1186                 return;
1187         free_irq(bp->msix_table[offset].vector, bp->dev);
1188         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1189            bp->msix_table[offset].vector);
1190         offset++;
1191 #ifdef BCM_CNIC
1192         if (nvecs == offset)
1193                 return;
1194         offset++;
1195 #endif
1196
1197         for_each_eth_queue(bp, i) {
1198                 if (nvecs == offset)
1199                         return;
1200                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d "
1201                    "irq\n", i, bp->msix_table[offset].vector);
1202
1203                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1204         }
1205 }
1206
1207 void bnx2x_free_irq(struct bnx2x *bp)
1208 {
1209         if (bp->flags & USING_MSIX_FLAG)
1210                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1211                                      CNIC_PRESENT + 1);
1212         else if (bp->flags & USING_MSI_FLAG)
1213                 free_irq(bp->pdev->irq, bp->dev);
1214         else
1215                 free_irq(bp->pdev->irq, bp->dev);
1216 }
1217
1218 int bnx2x_enable_msix(struct bnx2x *bp)
1219 {
1220         int msix_vec = 0, i, rc, req_cnt;
1221
1222         bp->msix_table[msix_vec].entry = msix_vec;
1223         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1224            bp->msix_table[0].entry);
1225         msix_vec++;
1226
1227 #ifdef BCM_CNIC
1228         bp->msix_table[msix_vec].entry = msix_vec;
1229         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1230            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1231         msix_vec++;
1232 #endif
1233         /* We need separate vectors for ETH queues only (not FCoE) */
1234         for_each_eth_queue(bp, i) {
1235                 bp->msix_table[msix_vec].entry = msix_vec;
1236                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1237                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1238                 msix_vec++;
1239         }
1240
1241         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1242
1243         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1244
1245         /*
1246          * reconfigure number of tx/rx queues according to available
1247          * MSI-X vectors
1248          */
1249         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1250                 /* how less vectors we will have? */
1251                 int diff = req_cnt - rc;
1252
1253                 DP(NETIF_MSG_IFUP,
1254                    "Trying to use less MSI-X vectors: %d\n", rc);
1255
1256                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1257
1258                 if (rc) {
1259                         DP(NETIF_MSG_IFUP,
1260                            "MSI-X is not attainable  rc %d\n", rc);
1261                         return rc;
1262                 }
1263                 /*
1264                  * decrease number of queues by number of unallocated entries
1265                  */
1266                 bp->num_queues -= diff;
1267
1268                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1269                                   bp->num_queues);
1270         } else if (rc) {
1271                 /* fall to INTx if not enough memory */
1272                 if (rc == -ENOMEM)
1273                         bp->flags |= DISABLE_MSI_FLAG;
1274                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1275                 return rc;
1276         }
1277
1278         bp->flags |= USING_MSIX_FLAG;
1279
1280         return 0;
1281 }
1282
1283 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1284 {
1285         int i, rc, offset = 0;
1286
1287         rc = request_irq(bp->msix_table[offset++].vector,
1288                          bnx2x_msix_sp_int, 0,
1289                          bp->dev->name, bp->dev);
1290         if (rc) {
1291                 BNX2X_ERR("request sp irq failed\n");
1292                 return -EBUSY;
1293         }
1294
1295 #ifdef BCM_CNIC
1296         offset++;
1297 #endif
1298         for_each_eth_queue(bp, i) {
1299                 struct bnx2x_fastpath *fp = &bp->fp[i];
1300                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1301                          bp->dev->name, i);
1302
1303                 rc = request_irq(bp->msix_table[offset].vector,
1304                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1305                 if (rc) {
1306                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1307                               bp->msix_table[offset].vector, rc);
1308                         bnx2x_free_msix_irqs(bp, offset);
1309                         return -EBUSY;
1310                 }
1311
1312                 offset++;
1313         }
1314
1315         i = BNX2X_NUM_ETH_QUEUES(bp);
1316         offset = 1 + CNIC_PRESENT;
1317         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1318                " ... fp[%d] %d\n",
1319                bp->msix_table[0].vector,
1320                0, bp->msix_table[offset].vector,
1321                i - 1, bp->msix_table[offset + i - 1].vector);
1322
1323         return 0;
1324 }
1325
1326 int bnx2x_enable_msi(struct bnx2x *bp)
1327 {
1328         int rc;
1329
1330         rc = pci_enable_msi(bp->pdev);
1331         if (rc) {
1332                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1333                 return -1;
1334         }
1335         bp->flags |= USING_MSI_FLAG;
1336
1337         return 0;
1338 }
1339
1340 static int bnx2x_req_irq(struct bnx2x *bp)
1341 {
1342         unsigned long flags;
1343         int rc;
1344
1345         if (bp->flags & USING_MSI_FLAG)
1346                 flags = 0;
1347         else
1348                 flags = IRQF_SHARED;
1349
1350         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1351                          bp->dev->name, bp->dev);
1352         return rc;
1353 }
1354
1355 static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1356 {
1357         int rc = 0;
1358         if (bp->flags & USING_MSIX_FLAG) {
1359                 rc = bnx2x_req_msix_irqs(bp);
1360                 if (rc)
1361                         return rc;
1362         } else {
1363                 bnx2x_ack_int(bp);
1364                 rc = bnx2x_req_irq(bp);
1365                 if (rc) {
1366                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1367                         return rc;
1368                 }
1369                 if (bp->flags & USING_MSI_FLAG) {
1370                         bp->dev->irq = bp->pdev->irq;
1371                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
1372                                bp->pdev->irq);
1373                 }
1374         }
1375
1376         return 0;
1377 }
1378
1379 static inline void bnx2x_napi_enable(struct bnx2x *bp)
1380 {
1381         int i;
1382
1383         for_each_rx_queue(bp, i)
1384                 napi_enable(&bnx2x_fp(bp, i, napi));
1385 }
1386
1387 static inline void bnx2x_napi_disable(struct bnx2x *bp)
1388 {
1389         int i;
1390
1391         for_each_rx_queue(bp, i)
1392                 napi_disable(&bnx2x_fp(bp, i, napi));
1393 }
1394
1395 void bnx2x_netif_start(struct bnx2x *bp)
1396 {
1397         if (netif_running(bp->dev)) {
1398                 bnx2x_napi_enable(bp);
1399                 bnx2x_int_enable(bp);
1400                 if (bp->state == BNX2X_STATE_OPEN)
1401                         netif_tx_wake_all_queues(bp->dev);
1402         }
1403 }
1404
1405 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1406 {
1407         bnx2x_int_disable_sync(bp, disable_hw);
1408         bnx2x_napi_disable(bp);
1409 }
1410
1411 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1412 {
1413         struct bnx2x *bp = netdev_priv(dev);
1414
1415 #ifdef BCM_CNIC
1416         if (!NO_FCOE(bp)) {
1417                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1418                 u16 ether_type = ntohs(hdr->h_proto);
1419
1420                 /* Skip VLAN tag if present */
1421                 if (ether_type == ETH_P_8021Q) {
1422                         struct vlan_ethhdr *vhdr =
1423                                 (struct vlan_ethhdr *)skb->data;
1424
1425                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1426                 }
1427
1428                 /* If ethertype is FCoE or FIP - use FCoE ring */
1429                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1430                         return bnx2x_fcoe_tx(bp, txq_index);
1431         }
1432 #endif
1433         /* select a non-FCoE queue */
1434         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1435 }
1436
1437 void bnx2x_set_num_queues(struct bnx2x *bp)
1438 {
1439         switch (bp->multi_mode) {
1440         case ETH_RSS_MODE_DISABLED:
1441                 bp->num_queues = 1;
1442                 break;
1443         case ETH_RSS_MODE_REGULAR:
1444                 bp->num_queues = bnx2x_calc_num_queues(bp);
1445                 break;
1446
1447         default:
1448                 bp->num_queues = 1;
1449                 break;
1450         }
1451
1452         /* Add special queues */
1453         bp->num_queues += NON_ETH_CONTEXT_USE;
1454 }
1455
1456 /**
1457  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1458  *
1459  * @bp:         Driver handle
1460  *
1461  * We currently support for at most 16 Tx queues for each CoS thus we will
1462  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1463  * bp->max_cos.
1464  *
1465  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1466  * index after all ETH L2 indices.
1467  *
1468  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1469  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1470  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1471  *
1472  * The proper configuration of skb->queue_mapping is handled by
1473  * bnx2x_select_queue() and __skb_tx_hash().
1474  *
1475  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1476  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1477  */
1478 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1479 {
1480         int rc, tx, rx;
1481
1482         tx = MAX_TXQS_PER_COS * bp->max_cos;
1483         rx = BNX2X_NUM_ETH_QUEUES(bp);
1484
1485 /* account for fcoe queue */
1486 #ifdef BCM_CNIC
1487         if (!NO_FCOE(bp)) {
1488                 rx += FCOE_PRESENT;
1489                 tx += FCOE_PRESENT;
1490         }
1491 #endif
1492
1493         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1494         if (rc) {
1495                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1496                 return rc;
1497         }
1498         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1499         if (rc) {
1500                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1501                 return rc;
1502         }
1503
1504         DP(NETIF_MSG_DRV, "Setting real num queues to (tx, rx) (%d, %d)\n",
1505                           tx, rx);
1506
1507         return rc;
1508 }
1509
1510 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1511 {
1512         int i;
1513
1514         for_each_queue(bp, i) {
1515                 struct bnx2x_fastpath *fp = &bp->fp[i];
1516
1517                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1518                 if (IS_FCOE_IDX(i))
1519                         /*
1520                          * Although there are no IP frames expected to arrive to
1521                          * this ring we still want to add an
1522                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1523                          * overrun attack.
1524                          */
1525                         fp->rx_buf_size =
1526                                 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1527                                 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1528                 else
1529                         fp->rx_buf_size =
1530                                 bp->dev->mtu + ETH_OVREHEAD +
1531                                 BNX2X_FW_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1532         }
1533 }
1534
1535 static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1536 {
1537         int i;
1538         u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1539         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1540
1541         /*
1542          * Prepare the inital contents fo the indirection table if RSS is
1543          * enabled
1544          */
1545         if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1546                 for (i = 0; i < sizeof(ind_table); i++)
1547                         ind_table[i] =
1548                                 bp->fp->cl_id + (i % num_eth_queues);
1549         }
1550
1551         /*
1552          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1553          * per-port, so if explicit configuration is needed , do it only
1554          * for a PMF.
1555          *
1556          * For 57712 and newer on the other hand it's a per-function
1557          * configuration.
1558          */
1559         return bnx2x_config_rss_pf(bp, ind_table,
1560                                    bp->port.pmf || !CHIP_IS_E1x(bp));
1561 }
1562
1563 int bnx2x_config_rss_pf(struct bnx2x *bp, u8 *ind_table, bool config_hash)
1564 {
1565         struct bnx2x_config_rss_params params = {0};
1566         int i;
1567
1568         /* Although RSS is meaningless when there is a single HW queue we
1569          * still need it enabled in order to have HW Rx hash generated.
1570          *
1571          * if (!is_eth_multi(bp))
1572          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1573          */
1574
1575         params.rss_obj = &bp->rss_conf_obj;
1576
1577         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1578
1579         /* RSS mode */
1580         switch (bp->multi_mode) {
1581         case ETH_RSS_MODE_DISABLED:
1582                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
1583                 break;
1584         case ETH_RSS_MODE_REGULAR:
1585                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1586                 break;
1587         case ETH_RSS_MODE_VLAN_PRI:
1588                 __set_bit(BNX2X_RSS_MODE_VLAN_PRI, &params.rss_flags);
1589                 break;
1590         case ETH_RSS_MODE_E1HOV_PRI:
1591                 __set_bit(BNX2X_RSS_MODE_E1HOV_PRI, &params.rss_flags);
1592                 break;
1593         case ETH_RSS_MODE_IP_DSCP:
1594                 __set_bit(BNX2X_RSS_MODE_IP_DSCP, &params.rss_flags);
1595                 break;
1596         default:
1597                 BNX2X_ERR("Unknown multi_mode: %d\n", bp->multi_mode);
1598                 return -EINVAL;
1599         }
1600
1601         /* If RSS is enabled */
1602         if (bp->multi_mode != ETH_RSS_MODE_DISABLED) {
1603                 /* RSS configuration */
1604                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1605                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1606                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1607                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1608
1609                 /* Hash bits */
1610                 params.rss_result_mask = MULTI_MASK;
1611
1612                 memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1613
1614                 if (config_hash) {
1615                         /* RSS keys */
1616                         for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1617                                 params.rss_key[i] = random32();
1618
1619                         __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1620                 }
1621         }
1622
1623         return bnx2x_config_rss(bp, &params);
1624 }
1625
1626 static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1627 {
1628         struct bnx2x_func_state_params func_params = {0};
1629
1630         /* Prepare parameters for function state transitions */
1631         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1632
1633         func_params.f_obj = &bp->func_obj;
1634         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1635
1636         func_params.params.hw_init.load_phase = load_code;
1637
1638         return bnx2x_func_state_change(bp, &func_params);
1639 }
1640
1641 /*
1642  * Cleans the object that have internal lists without sending
1643  * ramrods. Should be run when interrutps are disabled.
1644  */
1645 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1646 {
1647         int rc;
1648         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1649         struct bnx2x_mcast_ramrod_params rparam = {0};
1650         struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1651
1652         /***************** Cleanup MACs' object first *************************/
1653
1654         /* Wait for completion of requested */
1655         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1656         /* Perform a dry cleanup */
1657         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1658
1659         /* Clean ETH primary MAC */
1660         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1661         rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1662                                  &ramrod_flags);
1663         if (rc != 0)
1664                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1665
1666         /* Cleanup UC list */
1667         vlan_mac_flags = 0;
1668         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1669         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1670                                  &ramrod_flags);
1671         if (rc != 0)
1672                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1673
1674         /***************** Now clean mcast object *****************************/
1675         rparam.mcast_obj = &bp->mcast_obj;
1676         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1677
1678         /* Add a DEL command... */
1679         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1680         if (rc < 0)
1681                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast "
1682                           "object: %d\n", rc);
1683
1684         /* ...and wait until all pending commands are cleared */
1685         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1686         while (rc != 0) {
1687                 if (rc < 0) {
1688                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1689                                   rc);
1690                         return;
1691                 }
1692
1693                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1694         }
1695 }
1696
1697 #ifndef BNX2X_STOP_ON_ERROR
1698 #define LOAD_ERROR_EXIT(bp, label) \
1699         do { \
1700                 (bp)->state = BNX2X_STATE_ERROR; \
1701                 goto label; \
1702         } while (0)
1703 #else
1704 #define LOAD_ERROR_EXIT(bp, label) \
1705         do { \
1706                 (bp)->state = BNX2X_STATE_ERROR; \
1707                 (bp)->panic = 1; \
1708                 return -EBUSY; \
1709         } while (0)
1710 #endif
1711
1712 /* must be called with rtnl_lock */
1713 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1714 {
1715         int port = BP_PORT(bp);
1716         u32 load_code;
1717         int i, rc;
1718
1719 #ifdef BNX2X_STOP_ON_ERROR
1720         if (unlikely(bp->panic))
1721                 return -EPERM;
1722 #endif
1723
1724         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1725
1726         /* Set the initial link reported state to link down */
1727         bnx2x_acquire_phy_lock(bp);
1728         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1729         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1730                 &bp->last_reported_link.link_report_flags);
1731         bnx2x_release_phy_lock(bp);
1732
1733         /* must be called before memory allocation and HW init */
1734         bnx2x_ilt_set_info(bp);
1735
1736         /*
1737          * Zero fastpath structures preserving invariants like napi, which are
1738          * allocated only once, fp index, max_cos, bp pointer.
1739          * Also set fp->disable_tpa.
1740          */
1741         for_each_queue(bp, i)
1742                 bnx2x_bz_fp(bp, i);
1743
1744
1745         /* Set the receive queues buffer size */
1746         bnx2x_set_rx_buf_size(bp);
1747
1748         if (bnx2x_alloc_mem(bp))
1749                 return -ENOMEM;
1750
1751         /* As long as bnx2x_alloc_mem() may possibly update
1752          * bp->num_queues, bnx2x_set_real_num_queues() should always
1753          * come after it.
1754          */
1755         rc = bnx2x_set_real_num_queues(bp);
1756         if (rc) {
1757                 BNX2X_ERR("Unable to set real_num_queues\n");
1758                 LOAD_ERROR_EXIT(bp, load_error0);
1759         }
1760
1761         /* configure multi cos mappings in kernel.
1762          * this configuration may be overriden by a multi class queue discipline
1763          * or by a dcbx negotiation result.
1764          */
1765         bnx2x_setup_tc(bp->dev, bp->max_cos);
1766
1767         bnx2x_napi_enable(bp);
1768
1769         /* Send LOAD_REQUEST command to MCP
1770          * Returns the type of LOAD command:
1771          * if it is the first port to be initialized
1772          * common blocks should be initialized, otherwise - not
1773          */
1774         if (!BP_NOMCP(bp)) {
1775                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1776                 if (!load_code) {
1777                         BNX2X_ERR("MCP response failure, aborting\n");
1778                         rc = -EBUSY;
1779                         LOAD_ERROR_EXIT(bp, load_error1);
1780                 }
1781                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1782                         rc = -EBUSY; /* other port in diagnostic mode */
1783                         LOAD_ERROR_EXIT(bp, load_error1);
1784                 }
1785
1786         } else {
1787                 int path = BP_PATH(bp);
1788
1789                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1790                    path, load_count[path][0], load_count[path][1],
1791                    load_count[path][2]);
1792                 load_count[path][0]++;
1793                 load_count[path][1 + port]++;
1794                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1795                    path, load_count[path][0], load_count[path][1],
1796                    load_count[path][2]);
1797                 if (load_count[path][0] == 1)
1798                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1799                 else if (load_count[path][1 + port] == 1)
1800                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1801                 else
1802                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1803         }
1804
1805         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1806             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1807             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1808                 bp->port.pmf = 1;
1809                 /*
1810                  * We need the barrier to ensure the ordering between the
1811                  * writing to bp->port.pmf here and reading it from the
1812                  * bnx2x_periodic_task().
1813                  */
1814                 smp_mb();
1815                 queue_delayed_work(bnx2x_wq, &bp->period_task, 0);
1816         } else
1817                 bp->port.pmf = 0;
1818
1819         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1820
1821         /* Init Function state controlling object */
1822         bnx2x__init_func_obj(bp);
1823
1824         /* Initialize HW */
1825         rc = bnx2x_init_hw(bp, load_code);
1826         if (rc) {
1827                 BNX2X_ERR("HW init failed, aborting\n");
1828                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1829                 LOAD_ERROR_EXIT(bp, load_error2);
1830         }
1831
1832         /* Connect to IRQs */
1833         rc = bnx2x_setup_irqs(bp);
1834         if (rc) {
1835                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1836                 LOAD_ERROR_EXIT(bp, load_error2);
1837         }
1838
1839         /* Setup NIC internals and enable interrupts */
1840         bnx2x_nic_init(bp, load_code);
1841
1842         /* Init per-function objects */
1843         bnx2x_init_bp_objs(bp);
1844
1845         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1846             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1847             (bp->common.shmem2_base)) {
1848                 if (SHMEM2_HAS(bp, dcc_support))
1849                         SHMEM2_WR(bp, dcc_support,
1850                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1851                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1852         }
1853
1854         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1855         rc = bnx2x_func_start(bp);
1856         if (rc) {
1857                 BNX2X_ERR("Function start failed!\n");
1858                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1859                 LOAD_ERROR_EXIT(bp, load_error3);
1860         }
1861
1862         /* Send LOAD_DONE command to MCP */
1863         if (!BP_NOMCP(bp)) {
1864                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1865                 if (!load_code) {
1866                         BNX2X_ERR("MCP response failure, aborting\n");
1867                         rc = -EBUSY;
1868                         LOAD_ERROR_EXIT(bp, load_error3);
1869                 }
1870         }
1871
1872         rc = bnx2x_setup_leading(bp);
1873         if (rc) {
1874                 BNX2X_ERR("Setup leading failed!\n");
1875                 LOAD_ERROR_EXIT(bp, load_error3);
1876         }
1877
1878 #ifdef BCM_CNIC
1879         /* Enable Timer scan */
1880         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1881 #endif
1882
1883         for_each_nondefault_queue(bp, i) {
1884                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1885                 if (rc)
1886                         LOAD_ERROR_EXIT(bp, load_error4);
1887         }
1888
1889         rc = bnx2x_init_rss_pf(bp);
1890         if (rc)
1891                 LOAD_ERROR_EXIT(bp, load_error4);
1892
1893         /* Now when Clients are configured we are ready to work */
1894         bp->state = BNX2X_STATE_OPEN;
1895
1896         /* Configure a ucast MAC */
1897         rc = bnx2x_set_eth_mac(bp, true);
1898         if (rc)
1899                 LOAD_ERROR_EXIT(bp, load_error4);
1900
1901         if (bp->pending_max) {
1902                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1903                 bp->pending_max = 0;
1904         }
1905
1906         if (bp->port.pmf)
1907                 bnx2x_initial_phy_init(bp, load_mode);
1908
1909         /* Start fast path */
1910
1911         /* Initialize Rx filter. */
1912         netif_addr_lock_bh(bp->dev);
1913         bnx2x_set_rx_mode(bp->dev);
1914         netif_addr_unlock_bh(bp->dev);
1915
1916         /* Start the Tx */
1917         switch (load_mode) {
1918         case LOAD_NORMAL:
1919                 /* Tx queue should be only reenabled */
1920                 netif_tx_wake_all_queues(bp->dev);
1921                 break;
1922
1923         case LOAD_OPEN:
1924                 netif_tx_start_all_queues(bp->dev);
1925                 smp_mb__after_clear_bit();
1926                 break;
1927
1928         case LOAD_DIAG:
1929                 bp->state = BNX2X_STATE_DIAG;
1930                 break;
1931
1932         default:
1933                 break;
1934         }
1935
1936         if (!bp->port.pmf)
1937                 bnx2x__link_status_update(bp);
1938
1939         /* start the timer */
1940         mod_timer(&bp->timer, jiffies + bp->current_interval);
1941
1942 #ifdef BCM_CNIC
1943         bnx2x_setup_cnic_irq_info(bp);
1944         if (bp->state == BNX2X_STATE_OPEN)
1945                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1946 #endif
1947         bnx2x_inc_load_cnt(bp);
1948
1949         /* Wait for all pending SP commands to complete */
1950         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
1951                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
1952                 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
1953                 return -EBUSY;
1954         }
1955
1956         bnx2x_dcbx_init(bp);
1957         return 0;
1958
1959 #ifndef BNX2X_STOP_ON_ERROR
1960 load_error4:
1961 #ifdef BCM_CNIC
1962         /* Disable Timer scan */
1963         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
1964 #endif
1965 load_error3:
1966         bnx2x_int_disable_sync(bp, 1);
1967
1968         /* Clean queueable objects */
1969         bnx2x_squeeze_objects(bp);
1970
1971         /* Free SKBs, SGEs, TPA pool and driver internals */
1972         bnx2x_free_skbs(bp);
1973         for_each_rx_queue(bp, i)
1974                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1975
1976         /* Release IRQs */
1977         bnx2x_free_irq(bp);
1978 load_error2:
1979         if (!BP_NOMCP(bp)) {
1980                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1981                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1982         }
1983
1984         bp->port.pmf = 0;
1985 load_error1:
1986         bnx2x_napi_disable(bp);
1987 load_error0:
1988         bnx2x_free_mem(bp);
1989
1990         return rc;
1991 #endif /* ! BNX2X_STOP_ON_ERROR */
1992 }
1993
1994 /* must be called with rtnl_lock */
1995 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1996 {
1997         int i;
1998         bool global = false;
1999
2000         if ((bp->state == BNX2X_STATE_CLOSED) ||
2001             (bp->state == BNX2X_STATE_ERROR)) {
2002                 /* We can get here if the driver has been unloaded
2003                  * during parity error recovery and is either waiting for a
2004                  * leader to complete or for other functions to unload and
2005                  * then ifdown has been issued. In this case we want to
2006                  * unload and let other functions to complete a recovery
2007                  * process.
2008                  */
2009                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2010                 bp->is_leader = 0;
2011                 bnx2x_release_leader_lock(bp);
2012                 smp_mb();
2013
2014                 DP(NETIF_MSG_HW, "Releasing a leadership...\n");
2015
2016                 return -EINVAL;
2017         }
2018
2019         /*
2020          * It's important to set the bp->state to the value different from
2021          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2022          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2023          */
2024         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2025         smp_mb();
2026
2027         /* Stop Tx */
2028         bnx2x_tx_disable(bp);
2029
2030 #ifdef BCM_CNIC
2031         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2032 #endif
2033
2034         bp->rx_mode = BNX2X_RX_MODE_NONE;
2035
2036         del_timer_sync(&bp->timer);
2037
2038         /* Set ALWAYS_ALIVE bit in shmem */
2039         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2040
2041         bnx2x_drv_pulse(bp);
2042
2043         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2044
2045         /* Cleanup the chip if needed */
2046         if (unload_mode != UNLOAD_RECOVERY)
2047                 bnx2x_chip_cleanup(bp, unload_mode);
2048         else {
2049                 /* Send the UNLOAD_REQUEST to the MCP */
2050                 bnx2x_send_unload_req(bp, unload_mode);
2051
2052                 /*
2053                  * Prevent transactions to host from the functions on the
2054                  * engine that doesn't reset global blocks in case of global
2055                  * attention once gloabl blocks are reset and gates are opened
2056                  * (the engine which leader will perform the recovery
2057                  * last).
2058                  */
2059                 if (!CHIP_IS_E1x(bp))
2060                         bnx2x_pf_disable(bp);
2061
2062                 /* Disable HW interrupts, NAPI */
2063                 bnx2x_netif_stop(bp, 1);
2064
2065                 /* Release IRQs */
2066                 bnx2x_free_irq(bp);
2067
2068                 /* Report UNLOAD_DONE to MCP */
2069                 bnx2x_send_unload_done(bp);
2070         }
2071
2072         /*
2073          * At this stage no more interrupts will arrive so we may safly clean
2074          * the queueable objects here in case they failed to get cleaned so far.
2075          */
2076         bnx2x_squeeze_objects(bp);
2077
2078         /* There should be no more pending SP commands at this stage */
2079         bp->sp_state = 0;
2080
2081         bp->port.pmf = 0;
2082
2083         /* Free SKBs, SGEs, TPA pool and driver internals */
2084         bnx2x_free_skbs(bp);
2085         for_each_rx_queue(bp, i)
2086                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2087
2088         bnx2x_free_mem(bp);
2089
2090         bp->state = BNX2X_STATE_CLOSED;
2091
2092         /* Check if there are pending parity attentions. If there are - set
2093          * RECOVERY_IN_PROGRESS.
2094          */
2095         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2096                 bnx2x_set_reset_in_progress(bp);
2097
2098                 /* Set RESET_IS_GLOBAL if needed */
2099                 if (global)
2100                         bnx2x_set_reset_global(bp);
2101         }
2102
2103
2104         /* The last driver must disable a "close the gate" if there is no
2105          * parity attention or "process kill" pending.
2106          */
2107         if (!bnx2x_dec_load_cnt(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2108                 bnx2x_disable_close_the_gate(bp);
2109
2110         return 0;
2111 }
2112
2113 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2114 {
2115         u16 pmcsr;
2116
2117         /* If there is no power capability, silently succeed */
2118         if (!bp->pm_cap) {
2119                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
2120                 return 0;
2121         }
2122
2123         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2124
2125         switch (state) {
2126         case PCI_D0:
2127                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2128                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2129                                        PCI_PM_CTRL_PME_STATUS));
2130
2131                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2132                         /* delay required during transition out of D3hot */
2133                         msleep(20);
2134                 break;
2135
2136         case PCI_D3hot:
2137                 /* If there are other clients above don't
2138                    shut down the power */
2139                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2140                         return 0;
2141                 /* Don't shut down the power for emulation and FPGA */
2142                 if (CHIP_REV_IS_SLOW(bp))
2143                         return 0;
2144
2145                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2146                 pmcsr |= 3;
2147
2148                 if (bp->wol)
2149                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2150
2151                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2152                                       pmcsr);
2153
2154                 /* No more memory access after this point until
2155                 * device is brought back to D0.
2156                 */
2157                 break;
2158
2159         default:
2160                 return -EINVAL;
2161         }
2162         return 0;
2163 }
2164
2165 /*
2166  * net_device service functions
2167  */
2168 int bnx2x_poll(struct napi_struct *napi, int budget)
2169 {
2170         int work_done = 0;
2171         u8 cos;
2172         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2173                                                  napi);
2174         struct bnx2x *bp = fp->bp;
2175
2176         while (1) {
2177 #ifdef BNX2X_STOP_ON_ERROR
2178                 if (unlikely(bp->panic)) {
2179                         napi_complete(napi);
2180                         return 0;
2181                 }
2182 #endif
2183
2184                 for_each_cos_in_tx_queue(fp, cos)
2185                         if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2186                                 bnx2x_tx_int(bp, &fp->txdata[cos]);
2187
2188
2189                 if (bnx2x_has_rx_work(fp)) {
2190                         work_done += bnx2x_rx_int(fp, budget - work_done);
2191
2192                         /* must not complete if we consumed full budget */
2193                         if (work_done >= budget)
2194                                 break;
2195                 }
2196
2197                 /* Fall out from the NAPI loop if needed */
2198                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2199 #ifdef BCM_CNIC
2200                         /* No need to update SB for FCoE L2 ring as long as
2201                          * it's connected to the default SB and the SB
2202                          * has been updated when NAPI was scheduled.
2203                          */
2204                         if (IS_FCOE_FP(fp)) {
2205                                 napi_complete(napi);
2206                                 break;
2207                         }
2208 #endif
2209
2210                         bnx2x_update_fpsb_idx(fp);
2211                         /* bnx2x_has_rx_work() reads the status block,
2212                          * thus we need to ensure that status block indices
2213                          * have been actually read (bnx2x_update_fpsb_idx)
2214                          * prior to this check (bnx2x_has_rx_work) so that
2215                          * we won't write the "newer" value of the status block
2216                          * to IGU (if there was a DMA right after
2217                          * bnx2x_has_rx_work and if there is no rmb, the memory
2218                          * reading (bnx2x_update_fpsb_idx) may be postponed
2219                          * to right before bnx2x_ack_sb). In this case there
2220                          * will never be another interrupt until there is
2221                          * another update of the status block, while there
2222                          * is still unhandled work.
2223                          */
2224                         rmb();
2225
2226                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2227                                 napi_complete(napi);
2228                                 /* Re-enable interrupts */
2229                                 DP(NETIF_MSG_HW,
2230                                    "Update index to %d\n", fp->fp_hc_idx);
2231                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2232                                              le16_to_cpu(fp->fp_hc_idx),
2233                                              IGU_INT_ENABLE, 1);
2234                                 break;
2235                         }
2236                 }
2237         }
2238
2239         return work_done;
2240 }
2241
2242 /* we split the first BD into headers and data BDs
2243  * to ease the pain of our fellow microcode engineers
2244  * we use one mapping for both BDs
2245  * So far this has only been observed to happen
2246  * in Other Operating Systems(TM)
2247  */
2248 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2249                                    struct bnx2x_fp_txdata *txdata,
2250                                    struct sw_tx_bd *tx_buf,
2251                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2252                                    u16 bd_prod, int nbd)
2253 {
2254         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2255         struct eth_tx_bd *d_tx_bd;
2256         dma_addr_t mapping;
2257         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2258
2259         /* first fix first BD */
2260         h_tx_bd->nbd = cpu_to_le16(nbd);
2261         h_tx_bd->nbytes = cpu_to_le16(hlen);
2262
2263         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
2264            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
2265            h_tx_bd->addr_lo, h_tx_bd->nbd);
2266
2267         /* now get a new data BD
2268          * (after the pbd) and fill it */
2269         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2270         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2271
2272         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2273                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2274
2275         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2276         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2277         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2278
2279         /* this marks the BD as one that has no individual mapping */
2280         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2281
2282         DP(NETIF_MSG_TX_QUEUED,
2283            "TSO split data size is %d (%x:%x)\n",
2284            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2285
2286         /* update tx_bd */
2287         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2288
2289         return bd_prod;
2290 }
2291
2292 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2293 {
2294         if (fix > 0)
2295                 csum = (u16) ~csum_fold(csum_sub(csum,
2296                                 csum_partial(t_header - fix, fix, 0)));
2297
2298         else if (fix < 0)
2299                 csum = (u16) ~csum_fold(csum_add(csum,
2300                                 csum_partial(t_header, -fix, 0)));
2301
2302         return swab16(csum);
2303 }
2304
2305 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2306 {
2307         u32 rc;
2308
2309         if (skb->ip_summed != CHECKSUM_PARTIAL)
2310                 rc = XMIT_PLAIN;
2311
2312         else {
2313                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2314                         rc = XMIT_CSUM_V6;
2315                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2316                                 rc |= XMIT_CSUM_TCP;
2317
2318                 } else {
2319                         rc = XMIT_CSUM_V4;
2320                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2321                                 rc |= XMIT_CSUM_TCP;
2322                 }
2323         }
2324
2325         if (skb_is_gso_v6(skb))
2326                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2327         else if (skb_is_gso(skb))
2328                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2329
2330         return rc;
2331 }
2332
2333 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2334 /* check if packet requires linearization (packet is too fragmented)
2335    no need to check fragmentation if page size > 8K (there will be no
2336    violation to FW restrictions) */
2337 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2338                              u32 xmit_type)
2339 {
2340         int to_copy = 0;
2341         int hlen = 0;
2342         int first_bd_sz = 0;
2343
2344         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2345         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2346
2347                 if (xmit_type & XMIT_GSO) {
2348                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2349                         /* Check if LSO packet needs to be copied:
2350                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2351                         int wnd_size = MAX_FETCH_BD - 3;
2352                         /* Number of windows to check */
2353                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2354                         int wnd_idx = 0;
2355                         int frag_idx = 0;
2356                         u32 wnd_sum = 0;
2357
2358                         /* Headers length */
2359                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2360                                 tcp_hdrlen(skb);
2361
2362                         /* Amount of data (w/o headers) on linear part of SKB*/
2363                         first_bd_sz = skb_headlen(skb) - hlen;
2364
2365                         wnd_sum  = first_bd_sz;
2366
2367                         /* Calculate the first sum - it's special */
2368                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2369                                 wnd_sum +=
2370                                         skb_shinfo(skb)->frags[frag_idx].size;
2371
2372                         /* If there was data on linear skb data - check it */
2373                         if (first_bd_sz > 0) {
2374                                 if (unlikely(wnd_sum < lso_mss)) {
2375                                         to_copy = 1;
2376                                         goto exit_lbl;
2377                                 }
2378
2379                                 wnd_sum -= first_bd_sz;
2380                         }
2381
2382                         /* Others are easier: run through the frag list and
2383                            check all windows */
2384                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2385                                 wnd_sum +=
2386                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
2387
2388                                 if (unlikely(wnd_sum < lso_mss)) {
2389                                         to_copy = 1;
2390                                         break;
2391                                 }
2392                                 wnd_sum -=
2393                                         skb_shinfo(skb)->frags[wnd_idx].size;
2394                         }
2395                 } else {
2396                         /* in non-LSO too fragmented packet should always
2397                            be linearized */
2398                         to_copy = 1;
2399                 }
2400         }
2401
2402 exit_lbl:
2403         if (unlikely(to_copy))
2404                 DP(NETIF_MSG_TX_QUEUED,
2405                    "Linearization IS REQUIRED for %s packet. "
2406                    "num_frags %d  hlen %d  first_bd_sz %d\n",
2407                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2408                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2409
2410         return to_copy;
2411 }
2412 #endif
2413
2414 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2415                                         u32 xmit_type)
2416 {
2417         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2418                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2419                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2420         if ((xmit_type & XMIT_GSO_V6) &&
2421             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2422                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2423 }
2424
2425 /**
2426  * bnx2x_set_pbd_gso - update PBD in GSO case.
2427  *
2428  * @skb:        packet skb
2429  * @pbd:        parse BD
2430  * @xmit_type:  xmit flags
2431  */
2432 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2433                                      struct eth_tx_parse_bd_e1x *pbd,
2434                                      u32 xmit_type)
2435 {
2436         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2437         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2438         pbd->tcp_flags = pbd_tcp_flags(skb);
2439
2440         if (xmit_type & XMIT_GSO_V4) {
2441                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2442                 pbd->tcp_pseudo_csum =
2443                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2444                                                   ip_hdr(skb)->daddr,
2445                                                   0, IPPROTO_TCP, 0));
2446
2447         } else
2448                 pbd->tcp_pseudo_csum =
2449                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2450                                                 &ipv6_hdr(skb)->daddr,
2451                                                 0, IPPROTO_TCP, 0));
2452
2453         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2454 }
2455
2456 /**
2457  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2458  *
2459  * @bp:                 driver handle
2460  * @skb:                packet skb
2461  * @parsing_data:       data to be updated
2462  * @xmit_type:          xmit flags
2463  *
2464  * 57712 related
2465  */
2466 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2467         u32 *parsing_data, u32 xmit_type)
2468 {
2469         *parsing_data |=
2470                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2471                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2472                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2473
2474         if (xmit_type & XMIT_CSUM_TCP) {
2475                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2476                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2477                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2478
2479                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2480         } else
2481                 /* We support checksum offload for TCP and UDP only.
2482                  * No need to pass the UDP header length - it's a constant.
2483                  */
2484                 return skb_transport_header(skb) +
2485                                 sizeof(struct udphdr) - skb->data;
2486 }
2487
2488 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2489         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2490 {
2491         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2492
2493         if (xmit_type & XMIT_CSUM_V4)
2494                 tx_start_bd->bd_flags.as_bitfield |=
2495                                         ETH_TX_BD_FLAGS_IP_CSUM;
2496         else
2497                 tx_start_bd->bd_flags.as_bitfield |=
2498                                         ETH_TX_BD_FLAGS_IPV6;
2499
2500         if (!(xmit_type & XMIT_CSUM_TCP))
2501                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2502 }
2503
2504 /**
2505  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2506  *
2507  * @bp:         driver handle
2508  * @skb:        packet skb
2509  * @pbd:        parse BD to be updated
2510  * @xmit_type:  xmit flags
2511  */
2512 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2513         struct eth_tx_parse_bd_e1x *pbd,
2514         u32 xmit_type)
2515 {
2516         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2517
2518         /* for now NS flag is not used in Linux */
2519         pbd->global_data =
2520                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2521                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2522
2523         pbd->ip_hlen_w = (skb_transport_header(skb) -
2524                         skb_network_header(skb)) >> 1;
2525
2526         hlen += pbd->ip_hlen_w;
2527
2528         /* We support checksum offload for TCP and UDP only */
2529         if (xmit_type & XMIT_CSUM_TCP)
2530                 hlen += tcp_hdrlen(skb) / 2;
2531         else
2532                 hlen += sizeof(struct udphdr) / 2;
2533
2534         pbd->total_hlen_w = cpu_to_le16(hlen);
2535         hlen = hlen*2;
2536
2537         if (xmit_type & XMIT_CSUM_TCP) {
2538                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2539
2540         } else {
2541                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2542
2543                 DP(NETIF_MSG_TX_QUEUED,
2544                    "hlen %d  fix %d  csum before fix %x\n",
2545                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2546
2547                 /* HW bug: fixup the CSUM */
2548                 pbd->tcp_pseudo_csum =
2549                         bnx2x_csum_fix(skb_transport_header(skb),
2550                                        SKB_CS(skb), fix);
2551
2552                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2553                    pbd->tcp_pseudo_csum);
2554         }
2555
2556         return hlen;
2557 }
2558
2559 /* called with netif_tx_lock
2560  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2561  * netif_wake_queue()
2562  */
2563 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2564 {
2565         struct bnx2x *bp = netdev_priv(dev);
2566
2567         struct bnx2x_fastpath *fp;
2568         struct netdev_queue *txq;
2569         struct bnx2x_fp_txdata *txdata;
2570         struct sw_tx_bd *tx_buf;
2571         struct eth_tx_start_bd *tx_start_bd, *first_bd;
2572         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2573         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2574         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2575         u32 pbd_e2_parsing_data = 0;
2576         u16 pkt_prod, bd_prod;
2577         int nbd, txq_index, fp_index, txdata_index;
2578         dma_addr_t mapping;
2579         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2580         int i;
2581         u8 hlen = 0;
2582         __le16 pkt_size = 0;
2583         struct ethhdr *eth;
2584         u8 mac_type = UNICAST_ADDRESS;
2585
2586 #ifdef BNX2X_STOP_ON_ERROR
2587         if (unlikely(bp->panic))
2588                 return NETDEV_TX_BUSY;
2589 #endif
2590
2591         txq_index = skb_get_queue_mapping(skb);
2592         txq = netdev_get_tx_queue(dev, txq_index);
2593
2594         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2595
2596         /* decode the fastpath index and the cos index from the txq */
2597         fp_index = TXQ_TO_FP(txq_index);
2598         txdata_index = TXQ_TO_COS(txq_index);
2599
2600 #ifdef BCM_CNIC
2601         /*
2602          * Override the above for the FCoE queue:
2603          *   - FCoE fp entry is right after the ETH entries.
2604          *   - FCoE L2 queue uses bp->txdata[0] only.
2605          */
2606         if (unlikely(!NO_FCOE(bp) && (txq_index ==
2607                                       bnx2x_fcoe_tx(bp, txq_index)))) {
2608                 fp_index = FCOE_IDX;
2609                 txdata_index = 0;
2610         }
2611 #endif
2612
2613         /* enable this debug print to view the transmission queue being used
2614         DP(BNX2X_MSG_FP, "indices: txq %d, fp %d, txdata %d\n",
2615            txq_index, fp_index, txdata_index); */
2616
2617         /* locate the fastpath and the txdata */
2618         fp = &bp->fp[fp_index];
2619         txdata = &fp->txdata[txdata_index];
2620
2621         /* enable this debug print to view the tranmission details
2622         DP(BNX2X_MSG_FP,"transmitting packet cid %d fp index %d txdata_index %d"
2623                         " tx_data ptr %p fp pointer %p\n",
2624            txdata->cid, fp_index, txdata_index, txdata, fp); */
2625
2626         if (unlikely(bnx2x_tx_avail(bp, txdata) <
2627                      (skb_shinfo(skb)->nr_frags + 3))) {
2628                 fp->eth_q_stats.driver_xoff++;
2629                 netif_tx_stop_queue(txq);
2630                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2631                 return NETDEV_TX_BUSY;
2632         }
2633
2634         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
2635                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
2636            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2637            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2638
2639         eth = (struct ethhdr *)skb->data;
2640
2641         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2642         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2643                 if (is_broadcast_ether_addr(eth->h_dest))
2644                         mac_type = BROADCAST_ADDRESS;
2645                 else
2646                         mac_type = MULTICAST_ADDRESS;
2647         }
2648
2649 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2650         /* First, check if we need to linearize the skb (due to FW
2651            restrictions). No need to check fragmentation if page size > 8K
2652            (there will be no violation to FW restrictions) */
2653         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2654                 /* Statistics of linearization */
2655                 bp->lin_cnt++;
2656                 if (skb_linearize(skb) != 0) {
2657                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2658                            "silently dropping this SKB\n");
2659                         dev_kfree_skb_any(skb);
2660                         return NETDEV_TX_OK;
2661                 }
2662         }
2663 #endif
2664         /* Map skb linear data for DMA */
2665         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2666                                  skb_headlen(skb), DMA_TO_DEVICE);
2667         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2668                 DP(NETIF_MSG_TX_QUEUED, "SKB mapping failed - "
2669                    "silently dropping this SKB\n");
2670                 dev_kfree_skb_any(skb);
2671                 return NETDEV_TX_OK;
2672         }
2673         /*
2674         Please read carefully. First we use one BD which we mark as start,
2675         then we have a parsing info BD (used for TSO or xsum),
2676         and only then we have the rest of the TSO BDs.
2677         (don't forget to mark the last one as last,
2678         and to unmap only AFTER you write to the BD ...)
2679         And above all, all pdb sizes are in words - NOT DWORDS!
2680         */
2681
2682         /* get current pkt produced now - advance it just before sending packet
2683          * since mapping of pages may fail and cause packet to be dropped
2684          */
2685         pkt_prod = txdata->tx_pkt_prod;
2686         bd_prod = TX_BD(txdata->tx_bd_prod);
2687
2688         /* get a tx_buf and first BD
2689          * tx_start_bd may be changed during SPLIT,
2690          * but first_bd will always stay first
2691          */
2692         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2693         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2694         first_bd = tx_start_bd;
2695
2696         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2697         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2698                  mac_type);
2699
2700         /* header nbd */
2701         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2702
2703         /* remember the first BD of the packet */
2704         tx_buf->first_bd = txdata->tx_bd_prod;
2705         tx_buf->skb = skb;
2706         tx_buf->flags = 0;
2707
2708         DP(NETIF_MSG_TX_QUEUED,
2709            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2710            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2711
2712         if (vlan_tx_tag_present(skb)) {
2713                 tx_start_bd->vlan_or_ethertype =
2714                     cpu_to_le16(vlan_tx_tag_get(skb));
2715                 tx_start_bd->bd_flags.as_bitfield |=
2716                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2717         } else
2718                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2719
2720         /* turn on parsing and get a BD */
2721         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2722
2723         if (xmit_type & XMIT_CSUM)
2724                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2725
2726         if (!CHIP_IS_E1x(bp)) {
2727                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
2728                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2729                 /* Set PBD in checksum offload case */
2730                 if (xmit_type & XMIT_CSUM)
2731                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2732                                                      &pbd_e2_parsing_data,
2733                                                      xmit_type);
2734                 if (IS_MF_SI(bp)) {
2735                         /*
2736                          * fill in the MAC addresses in the PBD - for local
2737                          * switching
2738                          */
2739                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2740                                               &pbd_e2->src_mac_addr_mid,
2741                                               &pbd_e2->src_mac_addr_lo,
2742                                               eth->h_source);
2743                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2744                                               &pbd_e2->dst_mac_addr_mid,
2745                                               &pbd_e2->dst_mac_addr_lo,
2746                                               eth->h_dest);
2747                 }
2748         } else {
2749                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
2750                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2751                 /* Set PBD in checksum offload case */
2752                 if (xmit_type & XMIT_CSUM)
2753                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2754
2755         }
2756
2757         /* Setup the data pointer of the first BD of the packet */
2758         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2759         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2760         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2761         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2762         pkt_size = tx_start_bd->nbytes;
2763
2764         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2765            "  nbytes %d  flags %x  vlan %x\n",
2766            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2767            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2768            tx_start_bd->bd_flags.as_bitfield,
2769            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2770
2771         if (xmit_type & XMIT_GSO) {
2772
2773                 DP(NETIF_MSG_TX_QUEUED,
2774                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2775                    skb->len, hlen, skb_headlen(skb),
2776                    skb_shinfo(skb)->gso_size);
2777
2778                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2779
2780                 if (unlikely(skb_headlen(skb) > hlen))
2781                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2782                                                  &tx_start_bd, hlen,
2783                                                  bd_prod, ++nbd);
2784                 if (!CHIP_IS_E1x(bp))
2785                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2786                                              xmit_type);
2787                 else
2788                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2789         }
2790
2791         /* Set the PBD's parsing_data field if not zero
2792          * (for the chips newer than 57711).
2793          */
2794         if (pbd_e2_parsing_data)
2795                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2796
2797         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2798
2799         /* Handle fragmented skb */
2800         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2801                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2802
2803                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2804                                        frag->page_offset, frag->size,
2805                                        DMA_TO_DEVICE);
2806                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2807
2808                         DP(NETIF_MSG_TX_QUEUED, "Unable to map page - "
2809                                                 "dropping packet...\n");
2810
2811                         /* we need unmap all buffers already mapped
2812                          * for this SKB;
2813                          * first_bd->nbd need to be properly updated
2814                          * before call to bnx2x_free_tx_pkt
2815                          */
2816                         first_bd->nbd = cpu_to_le16(nbd);
2817                         bnx2x_free_tx_pkt(bp, txdata,
2818                                           TX_BD(txdata->tx_pkt_prod));
2819                         return NETDEV_TX_OK;
2820                 }
2821
2822                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2823                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2824                 if (total_pkt_bd == NULL)
2825                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2826
2827                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2828                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2829                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2830                 le16_add_cpu(&pkt_size, frag->size);
2831                 nbd++;
2832
2833                 DP(NETIF_MSG_TX_QUEUED,
2834                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2835                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2836                    le16_to_cpu(tx_data_bd->nbytes));
2837         }
2838
2839         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2840
2841         /* update with actual num BDs */
2842         first_bd->nbd = cpu_to_le16(nbd);
2843
2844         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2845
2846         /* now send a tx doorbell, counting the next BD
2847          * if the packet contains or ends with it
2848          */
2849         if (TX_BD_POFF(bd_prod) < nbd)
2850                 nbd++;
2851
2852         /* total_pkt_bytes should be set on the first data BD if
2853          * it's not an LSO packet and there is more than one
2854          * data BD. In this case pkt_size is limited by an MTU value.
2855          * However we prefer to set it for an LSO packet (while we don't
2856          * have to) in order to save some CPU cycles in a none-LSO
2857          * case, when we much more care about them.
2858          */
2859         if (total_pkt_bd != NULL)
2860                 total_pkt_bd->total_pkt_bytes = pkt_size;
2861
2862         if (pbd_e1x)
2863                 DP(NETIF_MSG_TX_QUEUED,
2864                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2865                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2866                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2867                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2868                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2869                     le16_to_cpu(pbd_e1x->total_hlen_w));
2870         if (pbd_e2)
2871                 DP(NETIF_MSG_TX_QUEUED,
2872                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2873                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2874                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2875                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2876                    pbd_e2->parsing_data);
2877         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2878
2879         txdata->tx_pkt_prod++;
2880         /*
2881          * Make sure that the BD data is updated before updating the producer
2882          * since FW might read the BD right after the producer is updated.
2883          * This is only applicable for weak-ordered memory model archs such
2884          * as IA-64. The following barrier is also mandatory since FW will
2885          * assumes packets must have BDs.
2886          */
2887         wmb();
2888
2889         txdata->tx_db.data.prod += nbd;
2890         barrier();
2891
2892         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2893
2894         mmiowb();
2895
2896         txdata->tx_bd_prod += nbd;
2897
2898         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2899                 netif_tx_stop_queue(txq);
2900
2901                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2902                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2903                  * fp->bd_tx_cons */
2904                 smp_mb();
2905
2906                 fp->eth_q_stats.driver_xoff++;
2907                 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
2908                         netif_tx_wake_queue(txq);
2909         }
2910         txdata->tx_pkt++;
2911
2912         return NETDEV_TX_OK;
2913 }
2914
2915 /**
2916  * bnx2x_setup_tc - routine to configure net_device for multi tc
2917  *
2918  * @netdev: net device to configure
2919  * @tc: number of traffic classes to enable
2920  *
2921  * callback connected to the ndo_setup_tc function pointer
2922  */
2923 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
2924 {
2925         int cos, prio, count, offset;
2926         struct bnx2x *bp = netdev_priv(dev);
2927
2928         /* setup tc must be called under rtnl lock */
2929         ASSERT_RTNL();
2930
2931         /* no traffic classes requested. aborting */
2932         if (!num_tc) {
2933                 netdev_reset_tc(dev);
2934                 return 0;
2935         }
2936
2937         /* requested to support too many traffic classes */
2938         if (num_tc > bp->max_cos) {
2939                 DP(NETIF_MSG_TX_ERR, "support for too many traffic classes"
2940                                      " requested: %d. max supported is %d\n",
2941                                      num_tc, bp->max_cos);
2942                 return -EINVAL;
2943         }
2944
2945         /* declare amount of supported traffic classes */
2946         if (netdev_set_num_tc(dev, num_tc)) {
2947                 DP(NETIF_MSG_TX_ERR, "failed to declare %d traffic classes\n",
2948                                      num_tc);
2949                 return -EINVAL;
2950         }
2951
2952         /* configure priority to traffic class mapping */
2953         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
2954                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
2955                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n",
2956                    prio, bp->prio_to_cos[prio]);
2957         }
2958
2959
2960         /* Use this configuration to diffrentiate tc0 from other COSes
2961            This can be used for ets or pfc, and save the effort of setting
2962            up a multio class queue disc or negotiating DCBX with a switch
2963         netdev_set_prio_tc_map(dev, 0, 0);
2964         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
2965         for (prio = 1; prio < 16; prio++) {
2966                 netdev_set_prio_tc_map(dev, prio, 1);
2967                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
2968         } */
2969
2970         /* configure traffic class to transmission queue mapping */
2971         for (cos = 0; cos < bp->max_cos; cos++) {
2972                 count = BNX2X_NUM_ETH_QUEUES(bp);
2973                 offset = cos * MAX_TXQS_PER_COS;
2974                 netdev_set_tc_queue(dev, cos, count, offset);
2975                 DP(BNX2X_MSG_SP, "mapping tc %d to offset %d count %d\n",
2976                    cos, offset, count);
2977         }
2978
2979         return 0;
2980 }
2981
2982 /* called with rtnl_lock */
2983 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2984 {
2985         struct sockaddr *addr = p;
2986         struct bnx2x *bp = netdev_priv(dev);
2987         int rc = 0;
2988
2989         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2990                 return -EINVAL;
2991
2992         if (netif_running(dev))  {
2993                 rc = bnx2x_set_eth_mac(bp, false);
2994                 if (rc)
2995                         return rc;
2996         }
2997
2998         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2999
3000         if (netif_running(dev))
3001                 rc = bnx2x_set_eth_mac(bp, true);
3002
3003         return rc;
3004 }
3005
3006 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3007 {
3008         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3009         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3010         u8 cos;
3011
3012         /* Common */
3013 #ifdef BCM_CNIC
3014         if (IS_FCOE_IDX(fp_index)) {
3015                 memset(sb, 0, sizeof(union host_hc_status_block));
3016                 fp->status_blk_mapping = 0;
3017
3018         } else {
3019 #endif
3020                 /* status blocks */
3021                 if (!CHIP_IS_E1x(bp))
3022                         BNX2X_PCI_FREE(sb->e2_sb,
3023                                        bnx2x_fp(bp, fp_index,
3024                                                 status_blk_mapping),
3025                                        sizeof(struct host_hc_status_block_e2));
3026                 else
3027                         BNX2X_PCI_FREE(sb->e1x_sb,
3028                                        bnx2x_fp(bp, fp_index,
3029                                                 status_blk_mapping),
3030                                        sizeof(struct host_hc_status_block_e1x));
3031 #ifdef BCM_CNIC
3032         }
3033 #endif
3034         /* Rx */
3035         if (!skip_rx_queue(bp, fp_index)) {
3036                 bnx2x_free_rx_bds(fp);
3037
3038                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3039                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3040                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3041                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3042                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3043
3044                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3045                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3046                                sizeof(struct eth_fast_path_rx_cqe) *
3047                                NUM_RCQ_BD);
3048
3049                 /* SGE ring */
3050                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3051                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3052                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3053                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3054         }
3055
3056         /* Tx */
3057         if (!skip_tx_queue(bp, fp_index)) {
3058                 /* fastpath tx rings: tx_buf tx_desc */
3059                 for_each_cos_in_tx_queue(fp, cos) {
3060                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3061
3062                         DP(BNX2X_MSG_SP,
3063                            "freeing tx memory of fp %d cos %d cid %d\n",
3064                            fp_index, cos, txdata->cid);
3065
3066                         BNX2X_FREE(txdata->tx_buf_ring);
3067                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3068                                 txdata->tx_desc_mapping,
3069                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3070                 }
3071         }
3072         /* end of fastpath */
3073 }
3074
3075 void bnx2x_free_fp_mem(struct bnx2x *bp)
3076 {
3077         int i;
3078         for_each_queue(bp, i)
3079                 bnx2x_free_fp_mem_at(bp, i);
3080 }
3081
3082 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3083 {
3084         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3085         if (!CHIP_IS_E1x(bp)) {
3086                 bnx2x_fp(bp, index, sb_index_values) =
3087                         (__le16 *)status_blk.e2_sb->sb.index_values;
3088                 bnx2x_fp(bp, index, sb_running_index) =
3089                         (__le16 *)status_blk.e2_sb->sb.running_index;
3090         } else {
3091                 bnx2x_fp(bp, index, sb_index_values) =
3092                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3093                 bnx2x_fp(bp, index, sb_running_index) =
3094                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3095         }
3096 }
3097
3098 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3099 {
3100         union host_hc_status_block *sb;
3101         struct bnx2x_fastpath *fp = &bp->fp[index];
3102         int ring_size = 0;
3103         u8 cos;
3104
3105         /* if rx_ring_size specified - use it */
3106         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
3107                            MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3108
3109         /* allocate at least number of buffers required by FW */
3110         rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3111                                                     MIN_RX_SIZE_TPA,
3112                                   rx_ring_size);
3113
3114         /* Common */
3115         sb = &bnx2x_fp(bp, index, status_blk);
3116 #ifdef BCM_CNIC
3117         if (!IS_FCOE_IDX(index)) {
3118 #endif
3119                 /* status blocks */
3120                 if (!CHIP_IS_E1x(bp))
3121                         BNX2X_PCI_ALLOC(sb->e2_sb,
3122                                 &bnx2x_fp(bp, index, status_blk_mapping),
3123                                 sizeof(struct host_hc_status_block_e2));
3124                 else
3125                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3126                                 &bnx2x_fp(bp, index, status_blk_mapping),
3127                             sizeof(struct host_hc_status_block_e1x));
3128 #ifdef BCM_CNIC
3129         }
3130 #endif
3131
3132         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3133          * set shortcuts for it.
3134          */
3135         if (!IS_FCOE_IDX(index))
3136                 set_sb_shortcuts(bp, index);
3137
3138         /* Tx */
3139         if (!skip_tx_queue(bp, index)) {
3140                 /* fastpath tx rings: tx_buf tx_desc */
3141                 for_each_cos_in_tx_queue(fp, cos) {
3142                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3143
3144                         DP(BNX2X_MSG_SP, "allocating tx memory of "
3145                                          "fp %d cos %d\n",
3146                            index, cos);
3147
3148                         BNX2X_ALLOC(txdata->tx_buf_ring,
3149                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3150                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3151                                 &txdata->tx_desc_mapping,
3152                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3153                 }
3154         }
3155
3156         /* Rx */
3157         if (!skip_rx_queue(bp, index)) {
3158                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3159                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3160                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3161                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3162                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3163                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3164
3165                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3166                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3167                                 sizeof(struct eth_fast_path_rx_cqe) *
3168                                 NUM_RCQ_BD);
3169
3170                 /* SGE ring */
3171                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3172                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3173                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3174                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3175                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3176                 /* RX BD ring */
3177                 bnx2x_set_next_page_rx_bd(fp);
3178
3179                 /* CQ ring */
3180                 bnx2x_set_next_page_rx_cq(fp);
3181
3182                 /* BDs */
3183                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3184                 if (ring_size < rx_ring_size)
3185                         goto alloc_mem_err;
3186         }
3187
3188         return 0;
3189
3190 /* handles low memory cases */
3191 alloc_mem_err:
3192         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3193                                                 index, ring_size);
3194         /* FW will drop all packets if queue is not big enough,
3195          * In these cases we disable the queue
3196          * Min size is different for OOO, TPA and non-TPA queues
3197          */
3198         if (ring_size < (fp->disable_tpa ?
3199                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3200                         /* release memory allocated for this queue */
3201                         bnx2x_free_fp_mem_at(bp, index);
3202                         return -ENOMEM;
3203         }
3204         return 0;
3205 }
3206
3207 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3208 {
3209         int i;
3210
3211         /**
3212          * 1. Allocate FP for leading - fatal if error
3213          * 2. {CNIC} Allocate FCoE FP - fatal if error
3214          * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3215          * 4. Allocate RSS - fix number of queues if error
3216          */
3217
3218         /* leading */
3219         if (bnx2x_alloc_fp_mem_at(bp, 0))
3220                 return -ENOMEM;
3221
3222 #ifdef BCM_CNIC
3223         if (!NO_FCOE(bp))
3224                 /* FCoE */
3225                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3226                         /* we will fail load process instead of mark
3227                          * NO_FCOE_FLAG
3228                          */
3229                         return -ENOMEM;
3230 #endif
3231
3232         /* RSS */
3233         for_each_nondefault_eth_queue(bp, i)
3234                 if (bnx2x_alloc_fp_mem_at(bp, i))
3235                         break;
3236
3237         /* handle memory failures */
3238         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3239                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3240
3241                 WARN_ON(delta < 0);
3242 #ifdef BCM_CNIC
3243                 /**
3244                  * move non eth FPs next to last eth FP
3245                  * must be done in that order
3246                  * FCOE_IDX < FWD_IDX < OOO_IDX
3247                  */
3248
3249                 /* move FCoE fp even NO_FCOE_FLAG is on */
3250                 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3251 #endif
3252                 bp->num_queues -= delta;
3253                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3254                           bp->num_queues + delta, bp->num_queues);
3255         }
3256
3257         return 0;
3258 }
3259
3260 void bnx2x_free_mem_bp(struct bnx2x *bp)
3261 {
3262         kfree(bp->fp);
3263         kfree(bp->msix_table);
3264         kfree(bp->ilt);
3265 }
3266
3267 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3268 {
3269         struct bnx2x_fastpath *fp;
3270         struct msix_entry *tbl;
3271         struct bnx2x_ilt *ilt;
3272         int msix_table_size = 0;
3273
3274         /*
3275          * The biggest MSI-X table we might need is as a maximum number of fast
3276          * path IGU SBs plus default SB (for PF).
3277          */
3278         msix_table_size = bp->igu_sb_cnt + 1;
3279
3280         /* fp array: RSS plus CNIC related L2 queues */
3281         fp = kzalloc((BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE) *
3282                      sizeof(*fp), GFP_KERNEL);
3283         if (!fp)
3284                 goto alloc_err;
3285         bp->fp = fp;
3286
3287         /* msix table */
3288         tbl = kzalloc(msix_table_size * sizeof(*tbl), GFP_KERNEL);
3289         if (!tbl)
3290                 goto alloc_err;
3291         bp->msix_table = tbl;
3292
3293         /* ilt */
3294         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3295         if (!ilt)
3296                 goto alloc_err;
3297         bp->ilt = ilt;
3298
3299         return 0;
3300 alloc_err:
3301         bnx2x_free_mem_bp(bp);
3302         return -ENOMEM;
3303
3304 }
3305
3306 int bnx2x_reload_if_running(struct net_device *dev)
3307 {
3308         struct bnx2x *bp = netdev_priv(dev);
3309
3310         if (unlikely(!netif_running(dev)))
3311                 return 0;
3312
3313         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3314         return bnx2x_nic_load(bp, LOAD_NORMAL);
3315 }
3316
3317 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3318 {
3319         u32 sel_phy_idx = 0;
3320         if (bp->link_params.num_phys <= 1)
3321                 return INT_PHY;
3322
3323         if (bp->link_vars.link_up) {
3324                 sel_phy_idx = EXT_PHY1;
3325                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3326                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3327                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3328                         sel_phy_idx = EXT_PHY2;
3329         } else {
3330
3331                 switch (bnx2x_phy_selection(&bp->link_params)) {
3332                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3333                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3334                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3335                        sel_phy_idx = EXT_PHY1;
3336                        break;
3337                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3338                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3339                        sel_phy_idx = EXT_PHY2;
3340                        break;
3341                 }
3342         }
3343
3344         return sel_phy_idx;
3345
3346 }
3347 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3348 {
3349         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3350         /*
3351          * The selected actived PHY is always after swapping (in case PHY
3352          * swapping is enabled). So when swapping is enabled, we need to reverse
3353          * the configuration
3354          */
3355
3356         if (bp->link_params.multi_phy_config &
3357             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3358                 if (sel_phy_idx == EXT_PHY1)
3359                         sel_phy_idx = EXT_PHY2;
3360                 else if (sel_phy_idx == EXT_PHY2)
3361                         sel_phy_idx = EXT_PHY1;
3362         }
3363         return LINK_CONFIG_IDX(sel_phy_idx);
3364 }
3365
3366 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3367 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3368 {
3369         struct bnx2x *bp = netdev_priv(dev);
3370         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3371
3372         switch (type) {
3373         case NETDEV_FCOE_WWNN:
3374                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3375                                 cp->fcoe_wwn_node_name_lo);
3376                 break;
3377         case NETDEV_FCOE_WWPN:
3378                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3379                                 cp->fcoe_wwn_port_name_lo);
3380                 break;
3381         default:
3382                 return -EINVAL;
3383         }
3384
3385         return 0;
3386 }
3387 #endif
3388
3389 /* called with rtnl_lock */
3390 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3391 {
3392         struct bnx2x *bp = netdev_priv(dev);
3393
3394         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3395                 pr_err("Handling parity error recovery. Try again later\n");
3396                 return -EAGAIN;
3397         }
3398
3399         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3400             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
3401                 return -EINVAL;
3402
3403         /* This does not race with packet allocation
3404          * because the actual alloc size is
3405          * only updated as part of load
3406          */
3407         dev->mtu = new_mtu;
3408
3409         return bnx2x_reload_if_running(dev);
3410 }
3411
3412 u32 bnx2x_fix_features(struct net_device *dev, u32 features)
3413 {
3414         struct bnx2x *bp = netdev_priv(dev);
3415
3416         /* TPA requires Rx CSUM offloading */
3417         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
3418                 features &= ~NETIF_F_LRO;
3419
3420         return features;
3421 }
3422
3423 int bnx2x_set_features(struct net_device *dev, u32 features)
3424 {
3425         struct bnx2x *bp = netdev_priv(dev);
3426         u32 flags = bp->flags;
3427         bool bnx2x_reload = false;
3428
3429         if (features & NETIF_F_LRO)
3430                 flags |= TPA_ENABLE_FLAG;
3431         else
3432                 flags &= ~TPA_ENABLE_FLAG;
3433
3434         if (features & NETIF_F_LOOPBACK) {
3435                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3436                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
3437                         bnx2x_reload = true;
3438                 }
3439         } else {
3440                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3441                         bp->link_params.loopback_mode = LOOPBACK_NONE;
3442                         bnx2x_reload = true;
3443                 }
3444         }
3445
3446         if (flags ^ bp->flags) {
3447                 bp->flags = flags;
3448                 bnx2x_reload = true;
3449         }
3450
3451         if (bnx2x_reload) {
3452                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3453                         return bnx2x_reload_if_running(dev);
3454                 /* else: bnx2x_nic_load() will be called at end of recovery */
3455         }
3456
3457         return 0;
3458 }
3459
3460 void bnx2x_tx_timeout(struct net_device *dev)
3461 {
3462         struct bnx2x *bp = netdev_priv(dev);
3463
3464 #ifdef BNX2X_STOP_ON_ERROR
3465         if (!bp->panic)
3466                 bnx2x_panic();
3467 #endif
3468
3469         smp_mb__before_clear_bit();
3470         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3471         smp_mb__after_clear_bit();
3472
3473         /* This allows the netif to be shutdown gracefully before resetting */
3474         schedule_delayed_work(&bp->sp_rtnl_task, 0);
3475 }
3476
3477 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3478 {
3479         struct net_device *dev = pci_get_drvdata(pdev);
3480         struct bnx2x *bp;
3481
3482         if (!dev) {
3483                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3484                 return -ENODEV;
3485         }
3486         bp = netdev_priv(dev);
3487
3488         rtnl_lock();
3489
3490         pci_save_state(pdev);
3491
3492         if (!netif_running(dev)) {
3493                 rtnl_unlock();
3494                 return 0;
3495         }
3496
3497         netif_device_detach(dev);
3498
3499         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3500
3501         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3502
3503         rtnl_unlock();
3504
3505         return 0;
3506 }
3507
3508 int bnx2x_resume(struct pci_dev *pdev)
3509 {
3510         struct net_device *dev = pci_get_drvdata(pdev);
3511         struct bnx2x *bp;
3512         int rc;
3513
3514         if (!dev) {
3515                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3516                 return -ENODEV;
3517         }
3518         bp = netdev_priv(dev);
3519
3520         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3521                 pr_err("Handling parity error recovery. Try again later\n");
3522                 return -EAGAIN;
3523         }
3524
3525         rtnl_lock();
3526
3527         pci_restore_state(pdev);
3528
3529         if (!netif_running(dev)) {
3530                 rtnl_unlock();
3531                 return 0;
3532         }
3533
3534         bnx2x_set_power_state(bp, PCI_D0);
3535         netif_device_attach(dev);
3536
3537         /* Since the chip was reset, clear the FW sequence number */
3538         bp->fw_seq = 0;
3539         rc = bnx2x_nic_load(bp, LOAD_OPEN);
3540
3541         rtnl_unlock();
3542
3543         return rc;
3544 }
3545
3546
3547 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3548                               u32 cid)
3549 {
3550         /* ustorm cxt validation */
3551         cxt->ustorm_ag_context.cdu_usage =
3552                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3553                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3554         /* xcontext validation */
3555         cxt->xstorm_ag_context.cdu_reserved =
3556                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3557                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3558 }
3559
3560 static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3561                                              u8 fw_sb_id, u8 sb_index,
3562                                              u8 ticks)
3563 {
3564
3565         u32 addr = BAR_CSTRORM_INTMEM +
3566                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3567         REG_WR8(bp, addr, ticks);
3568         DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d ticks %d\n",
3569                           port, fw_sb_id, sb_index, ticks);
3570 }
3571
3572 static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3573                                              u16 fw_sb_id, u8 sb_index,
3574                                              u8 disable)
3575 {
3576         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3577         u32 addr = BAR_CSTRORM_INTMEM +
3578                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3579         u16 flags = REG_RD16(bp, addr);
3580         /* clear and set */
3581         flags &= ~HC_INDEX_DATA_HC_ENABLED;
3582         flags |= enable_flag;
3583         REG_WR16(bp, addr, flags);
3584         DP(NETIF_MSG_HW, "port %x fw_sb_id %d sb_index %d disable %d\n",
3585                           port, fw_sb_id, sb_index, disable);
3586 }
3587
3588 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3589                                     u8 sb_index, u8 disable, u16 usec)
3590 {
3591         int port = BP_PORT(bp);
3592         u8 ticks = usec / BNX2X_BTR;
3593
3594         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3595
3596         disable = disable ? 1 : (usec ? 0 : 1);
3597         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3598 }