drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2012 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/etherdevice.h>
  21 #include <linux/if_vlan.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/ip.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip6_checksum.h>
  26 #include <linux/prefetch.h>
  27 #include "bnx2x_cmn.h"
  28 #include "bnx2x_init.h"
  29 #include "bnx2x_sp.h"
  30
  31
  32
  33 /**
  34  * bnx2x_move_fp - move content of the fastpath structure.
  35  *
  36  * @bp:         driver handle
  37  * @from:       source FP index
  38  * @to:         destination FP index
  39  *
  40  * Makes sure the contents of the bp->fp[to].napi is kept
  41  * intact. This is done by first copying the napi struct from
  42  * the target to the source, and then mem copying the entire
  43  * source onto the target
  44  */
  45 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  46 {
  47         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  48         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  49
  50         /* Copy the NAPI object as it has been already initialized */
  51         from_fp->napi = to_fp->napi;
  52
  53         /* Move bnx2x_fastpath contents */
  54         memcpy(to_fp, from_fp, sizeof(*to_fp));
  55         to_fp->index = to;
  56 }
  57
  58 int load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
  59
  60 /* free skb in the packet ring at pos idx
  61  * return idx of last bd freed
  62  */
  63 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
  64                              u16 idx, unsigned int *pkts_compl,
  65                              unsigned int *bytes_compl)
  66 {
  67         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
  68         struct eth_tx_start_bd *tx_start_bd;
  69         struct eth_tx_bd *tx_data_bd;
  70         struct sk_buff *skb = tx_buf->skb;
  71         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  72         int nbd;
  73
  74         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  75         prefetch(&skb->end);
  76
  77         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
  78            txdata->txq_index, idx, tx_buf, skb);
  79
  80         /* unmap first bd */
  81         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
  82         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  83                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  84
  85
  86         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  87 #ifdef BNX2X_STOP_ON_ERROR
  88         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  89                 BNX2X_ERR("BAD nbd!\n");
  90                 bnx2x_panic();
  91         }
  92 #endif
  93         new_cons = nbd + tx_buf->first_bd;
  94
  95         /* Get the next bd */
  96         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  97
  98         /* Skip a parse bd... */
  99         --nbd;
 100         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 101
 102         /* ...and the TSO split header bd since they have no mapping */
 103         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 104                 --nbd;
 105                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 106         }
 107
 108         /* now free frags */
 109         while (nbd > 0) {
 110
 111                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
 112                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 113                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 114                 if (--nbd)
 115                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 116         }
 117
 118         /* release skb */
 119         WARN_ON(!skb);
 120         if (likely(skb)) {
 121                 (*pkts_compl)++;
 122                 (*bytes_compl) += skb->len;
 123         }
 124
 125         dev_kfree_skb_any(skb);
 126         tx_buf->first_bd = 0;
 127         tx_buf->skb = NULL;
 128
 129         return new_cons;
 130 }
 131
 132 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
 133 {
 134         struct netdev_queue *txq;
 135         u16 hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
 136         unsigned int pkts_compl = 0, bytes_compl = 0;
 137
 138 #ifdef BNX2X_STOP_ON_ERROR
 139         if (unlikely(bp->panic))
 140                 return -1;
 141 #endif
 142
 143         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
 144         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
 145         sw_cons = txdata->tx_pkt_cons;
 146
 147         while (sw_cons != hw_cons) {
 148                 u16 pkt_cons;
 149
 150                 pkt_cons = TX_BD(sw_cons);
 151
 152                 DP(NETIF_MSG_TX_DONE,
 153                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
 154                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
 155
 156                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
 157                     &pkts_compl, &bytes_compl);
 158
 159                 sw_cons++;
 160         }
 161
 162         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
 163
 164         txdata->tx_pkt_cons = sw_cons;
 165         txdata->tx_bd_cons = bd_cons;
 166
 167         /* Need to make the tx_bd_cons update visible to start_xmit()
 168          * before checking for netif_tx_queue_stopped().  Without the
 169          * memory barrier, there is a small possibility that
 170          * start_xmit() will miss it and cause the queue to be stopped
 171          * forever.
 172          * On the other hand we need an rmb() here to ensure the proper
 173          * ordering of bit testing in the following
 174          * netif_tx_queue_stopped(txq) call.
 175          */
 176         smp_mb();
 177
 178         if (unlikely(netif_tx_queue_stopped(txq))) {
 179                 /* Taking tx_lock() is needed to prevent reenabling the queue
 180                  * while it's empty. This could have happen if rx_action() gets
 181                  * suspended in bnx2x_tx_int() after the condition before
 182                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 183                  *
 184                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 185                  * sends some packets consuming the whole queue again->
 186                  * stops the queue
 187                  */
 188
 189                 __netif_tx_lock(txq, smp_processor_id());
 190
 191                 if ((netif_tx_queue_stopped(txq)) &&
 192                     (bp->state == BNX2X_STATE_OPEN) &&
 193                     (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3))
 194                         netif_tx_wake_queue(txq);
 195
 196                 __netif_tx_unlock(txq);
 197         }
 198         return 0;
 199 }
 200
 201 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 202                                              u16 idx)
 203 {
 204         u16 last_max = fp->last_max_sge;
 205
 206         if (SUB_S16(idx, last_max) > 0)
 207                 fp->last_max_sge = idx;
 208 }
 209
 210 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 211                                          u16 sge_len,
 212                                          struct eth_end_agg_rx_cqe *cqe)
 213 {
 214         struct bnx2x *bp = fp->bp;
 215         u16 last_max, last_elem, first_elem;
 216         u16 delta = 0;
 217         u16 i;
 218
 219         if (!sge_len)
 220                 return;
 221
 222         /* First mark all used pages */
 223         for (i = 0; i < sge_len; i++)
 224                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
 225                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
 226
 227         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 228            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 229
 230         /* Here we assume that the last SGE index is the biggest */
 231         prefetch((void *)(fp->sge_mask));
 232         bnx2x_update_last_max_sge(fp,
 233                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 234
 235         last_max = RX_SGE(fp->last_max_sge);
 236         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
 237         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
 238
 239         /* If ring is not full */
 240         if (last_elem + 1 != first_elem)
 241                 last_elem++;
 242
 243         /* Now update the prod */
 244         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 245                 if (likely(fp->sge_mask[i]))
 246                         break;
 247
 248                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
 249                 delta += BIT_VEC64_ELEM_SZ;
 250         }
 251
 252         if (delta > 0) {
 253                 fp->rx_sge_prod += delta;
 254                 /* clear page-end entries */
 255                 bnx2x_clear_sge_mask_next_elems(fp);
 256         }
 257
 258         DP(NETIF_MSG_RX_STATUS,
 259            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 260            fp->last_max_sge, fp->rx_sge_prod);
 261 }
 262
 263 /* Set Toeplitz hash value in the skb using the value from the
 264  * CQE (calculated by HW).
 265  */
 266 static u32 bnx2x_get_rxhash(const struct bnx2x *bp,
 267                             const struct eth_fast_path_rx_cqe *cqe)
 268 {
 269         /* Set Toeplitz hash from CQE */
 270         if ((bp->dev->features & NETIF_F_RXHASH) &&
 271             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 272                 return le32_to_cpu(cqe->rss_hash_result);
 273         return 0;
 274 }
 275
 276 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 277                             u16 cons, u16 prod,
 278                             struct eth_fast_path_rx_cqe *cqe)
 279 {
 280         struct bnx2x *bp = fp->bp;
 281         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 282         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 283         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 284         dma_addr_t mapping;
 285         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
 286         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
 287
 288         /* print error if current state != stop */
 289         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
 290                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 291
 292         /* Try to map an empty data buffer from the aggregation info  */
 293         mapping = dma_map_single(&bp->pdev->dev,
 294                                  first_buf->data + NET_SKB_PAD,
 295                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 296         /*
 297          *  ...if it fails - move the skb from the consumer to the producer
 298          *  and set the current aggregation state as ERROR to drop it
 299          *  when TPA_STOP arrives.
 300          */
 301
 302         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
 303                 /* Move the BD from the consumer to the producer */
 304                 bnx2x_reuse_rx_data(fp, cons, prod);
 305                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
 306                 return;
 307         }
 308
 309         /* move empty data from pool to prod */
 310         prod_rx_buf->data = first_buf->data;
 311         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 312         /* point prod_bd to new data */
 313         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 314         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 315
 316         /* move partial skb from cons to pool (don't unmap yet) */
 317         *first_buf = *cons_rx_buf;
 318
 319         /* mark bin state as START */
 320         tpa_info->parsing_flags =
 321                 le16_to_cpu(cqe->pars_flags.flags);
 322         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
 323         tpa_info->tpa_state = BNX2X_TPA_START;
 324         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
 325         tpa_info->placement_offset = cqe->placement_offset;
 326         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe);
 327         if (fp->mode == TPA_MODE_GRO) {
 328                 u16 gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
 329                 tpa_info->full_page =
 330                         SGE_PAGE_SIZE * PAGES_PER_SGE / gro_size * gro_size;
 331                 tpa_info->gro_size = gro_size;
 332         }
 333
 334 #ifdef BNX2X_STOP_ON_ERROR
 335         fp->tpa_queue_used |= (1 << queue);
 336 #ifdef _ASM_GENERIC_INT_L64_H
 337         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 338 #else
 339         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 340 #endif
 341            fp->tpa_queue_used);
 342 #endif
 343 }
 344
 345 /* Timestamp option length allowed for TPA aggregation:
 346  *
 347  *              nop nop kind length echo val
 348  */
 349 #define TPA_TSTAMP_OPT_LEN      12
 350 /**
 351  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 352  *
 353  * @bp:                 driver handle
 354  * @parsing_flags:      parsing flags from the START CQE
 355  * @len_on_bd:          total length of the first packet for the
 356  *                      aggregation.
 357  *
 358  * Approximate value of the MSS for this aggregation calculated using
 359  * the first packet of it.
 360  */
 361 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 362                                     u16 len_on_bd)
 363 {
 364         /*
 365          * TPA arrgregation won't have either IP options or TCP options
 366          * other than timestamp or IPv6 extension headers.
 367          */
 368         u16 hdrs_len = ETH_HLEN + sizeof(struct tcphdr);
 369
 370         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 371             PRS_FLAG_OVERETH_IPV6)
 372                 hdrs_len += sizeof(struct ipv6hdr);
 373         else /* IPv4 */
 374                 hdrs_len += sizeof(struct iphdr);
 375
 376
 377         /* Check if there was a TCP timestamp, if there is it's will
 378          * always be 12 bytes length: nop nop kind length echo val.
 379          *
 380          * Otherwise FW would close the aggregation.
 381          */
 382         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 383                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 384
 385         return len_on_bd - hdrs_len;
 386 }
 387
 388 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 389                                struct bnx2x_agg_info *tpa_info,
 390                                u16 pages,
 391                                struct sk_buff *skb,
 392                                struct eth_end_agg_rx_cqe *cqe,
 393                                u16 cqe_idx)
 394 {
 395         struct sw_rx_page *rx_pg, old_rx_pg;
 396         u32 i, frag_len, frag_size;
 397         int err, j, frag_id = 0;
 398         u16 len_on_bd = tpa_info->len_on_bd;
 399         u16 full_page = 0, gro_size = 0;
 400
 401         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
 402
 403         if (fp->mode == TPA_MODE_GRO) {
 404                 gro_size = tpa_info->gro_size;
 405                 full_page = tpa_info->full_page;
 406         }
 407
 408         /* This is needed in order to enable forwarding support */
 409         if (frag_size) {
 410                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp,
 411                                         tpa_info->parsing_flags, len_on_bd);
 412
 413                 /* set for GRO */
 414                 if (fp->mode == TPA_MODE_GRO)
 415                         skb_shinfo(skb)->gso_type =
 416                             (GET_FLAG(tpa_info->parsing_flags,
 417                                       PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
 418                                                 PRS_FLAG_OVERETH_IPV6) ?
 419                                 SKB_GSO_TCPV6 : SKB_GSO_TCPV4;
 420         }
 421
 422
 423 #ifdef BNX2X_STOP_ON_ERROR
 424         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 425                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 426                           pages, cqe_idx);
 427                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
 428                 bnx2x_panic();
 429                 return -EINVAL;
 430         }
 431 #endif
 432
 433         /* Run through the SGL and compose the fragmented skb */
 434         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 435                 u16 sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
 436
 437                 /* FW gives the indices of the SGE as if the ring is an array
 438                    (meaning that "next" element will consume 2 indices) */
 439                 if (fp->mode == TPA_MODE_GRO)
 440                         frag_len = min_t(u32, frag_size, (u32)full_page);
 441                 else /* LRO */
 442                         frag_len = min_t(u32, frag_size,
 443                                          (u32)(SGE_PAGE_SIZE * PAGES_PER_SGE));
 444
 445                 rx_pg = &fp->rx_page_ring[sge_idx];
 446                 old_rx_pg = *rx_pg;
 447
 448                 /* If we fail to allocate a substitute page, we simply stop
 449                    where we are and drop the whole packet */
 450                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 451                 if (unlikely(err)) {
 452                         fp->eth_q_stats.rx_skb_alloc_failed++;
 453                         return err;
 454                 }
 455
 456                 /* Unmap the page as we r going to pass it to the stack */
 457                 dma_unmap_page(&bp->pdev->dev,
 458                                dma_unmap_addr(&old_rx_pg, mapping),
 459                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 460                 /* Add one frag and update the appropriate fields in the skb */
 461                 if (fp->mode == TPA_MODE_LRO)
 462                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 463                 else { /* GRO */
 464                         int rem;
 465                         int offset = 0;
 466                         for (rem = frag_len; rem > 0; rem -= gro_size) {
 467                                 int len = rem > gro_size ? gro_size : rem;
 468                                 skb_fill_page_desc(skb, frag_id++,
 469                                                    old_rx_pg.page, offset, len);
 470                                 if (offset)
 471                                         get_page(old_rx_pg.page);
 472                                 offset += len;
 473                         }
 474                 }
 475
 476                 skb->data_len += frag_len;
 477                 skb->truesize += SGE_PAGE_SIZE * PAGES_PER_SGE;
 478                 skb->len += frag_len;
 479
 480                 frag_size -= frag_len;
 481         }
 482
 483         return 0;
 484 }
 485
 486 static inline void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 487                                   struct bnx2x_agg_info *tpa_info,
 488                                   u16 pages,
 489                                   struct eth_end_agg_rx_cqe *cqe,
 490                                   u16 cqe_idx)
 491 {
 492         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
 493         u8 pad = tpa_info->placement_offset;
 494         u16 len = tpa_info->len_on_bd;
 495         struct sk_buff *skb = NULL;
 496         u8 *new_data, *data = rx_buf->data;
 497         u8 old_tpa_state = tpa_info->tpa_state;
 498
 499         tpa_info->tpa_state = BNX2X_TPA_STOP;
 500
 501         /* If we there was an error during the handling of the TPA_START -
 502          * drop this aggregation.
 503          */
 504         if (old_tpa_state == BNX2X_TPA_ERROR)
 505                 goto drop;
 506
 507         /* Try to allocate the new data */
 508         new_data = kmalloc(fp->rx_buf_size + NET_SKB_PAD, GFP_ATOMIC);
 509
 510         /* Unmap skb in the pool anyway, as we are going to change
 511            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 512            fails. */
 513         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 514                          fp->rx_buf_size, DMA_FROM_DEVICE);
 515         if (likely(new_data))
 516                 skb = build_skb(data, 0);
 517
 518         if (likely(skb)) {
 519 #ifdef BNX2X_STOP_ON_ERROR
 520                 if (pad + len > fp->rx_buf_size) {
 521                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 522                                   pad, len, fp->rx_buf_size);
 523                         bnx2x_panic();
 524                         return;
 525                 }
 526 #endif
 527
 528                 skb_reserve(skb, pad + NET_SKB_PAD);
 529                 skb_put(skb, len);
 530                 skb->rxhash = tpa_info->rxhash;
 531
 532                 skb->protocol = eth_type_trans(skb, bp->dev);
 533                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 534
 535                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
 536                                          skb, cqe, cqe_idx)) {
 537                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
 538                                 __vlan_hwaccel_put_tag(skb, tpa_info->vlan_tag);
 539                         napi_gro_receive(&fp->napi, skb);
 540                 } else {
 541                         DP(NETIF_MSG_RX_STATUS,
 542                            "Failed to allocate new pages - dropping packet!\n");
 543                         dev_kfree_skb_any(skb);
 544                 }
 545
 546
 547                 /* put new data in bin */
 548                 rx_buf->data = new_data;
 549
 550                 return;
 551         }
 552         kfree(new_data);
 553 drop:
 554         /* drop the packet and keep the buffer in the bin */
 555         DP(NETIF_MSG_RX_STATUS,
 556            "Failed to allocate or map a new skb - dropping packet!\n");
 557         fp->eth_q_stats.rx_skb_alloc_failed++;
 558 }
 559
 560
 561 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 562 {
 563         struct bnx2x *bp = fp->bp;
 564         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 565         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 566         int rx_pkt = 0;
 567
 568 #ifdef BNX2X_STOP_ON_ERROR
 569         if (unlikely(bp->panic))
 570                 return 0;
 571 #endif
 572
 573         /* CQ "next element" is of the size of the regular element,
 574            that's why it's ok here */
 575         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 576         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 577                 hw_comp_cons++;
 578
 579         bd_cons = fp->rx_bd_cons;
 580         bd_prod = fp->rx_bd_prod;
 581         bd_prod_fw = bd_prod;
 582         sw_comp_cons = fp->rx_comp_cons;
 583         sw_comp_prod = fp->rx_comp_prod;
 584
 585         /* Memory barrier necessary as speculative reads of the rx
 586          * buffer can be ahead of the index in the status block
 587          */
 588         rmb();
 589
 590         DP(NETIF_MSG_RX_STATUS,
 591            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 592            fp->index, hw_comp_cons, sw_comp_cons);
 593
 594         while (sw_comp_cons != hw_comp_cons) {
 595                 struct sw_rx_bd *rx_buf = NULL;
 596                 struct sk_buff *skb;
 597                 union eth_rx_cqe *cqe;
 598                 struct eth_fast_path_rx_cqe *cqe_fp;
 599                 u8 cqe_fp_flags;
 600                 enum eth_rx_cqe_type cqe_fp_type;
 601                 u16 len, pad, queue;
 602                 u8 *data;
 603
 604 #ifdef BNX2X_STOP_ON_ERROR
 605                 if (unlikely(bp->panic))
 606                         return 0;
 607 #endif
 608
 609                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 610                 bd_prod = RX_BD(bd_prod);
 611                 bd_cons = RX_BD(bd_cons);
 612
 613                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 614                 cqe_fp = &cqe->fast_path_cqe;
 615                 cqe_fp_flags = cqe_fp->type_error_flags;
 616                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
 617
 618                 DP(NETIF_MSG_RX_STATUS,
 619                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
 620                    CQE_TYPE(cqe_fp_flags),
 621                    cqe_fp_flags, cqe_fp->status_flags,
 622                    le32_to_cpu(cqe_fp->rss_hash_result),
 623                    le16_to_cpu(cqe_fp->vlan_tag),
 624                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
 625
 626                 /* is this a slowpath msg? */
 627                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
 628                         bnx2x_sp_event(fp, cqe);
 629                         goto next_cqe;
 630                 }
 631
 632                 rx_buf = &fp->rx_buf_ring[bd_cons];
 633                 data = rx_buf->data;
 634
 635                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
 636                         struct bnx2x_agg_info *tpa_info;
 637                         u16 frag_size, pages;
 638 #ifdef BNX2X_STOP_ON_ERROR
 639                         /* sanity check */
 640                         if (fp->disable_tpa &&
 641                             (CQE_TYPE_START(cqe_fp_type) ||
 642                              CQE_TYPE_STOP(cqe_fp_type)))
 643                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
 644                                           CQE_TYPE(cqe_fp_type));
 645 #endif
 646
 647                         if (CQE_TYPE_START(cqe_fp_type)) {
 648                                 u16 queue = cqe_fp->queue_index;
 649                                 DP(NETIF_MSG_RX_STATUS,
 650                                    "calling tpa_start on queue %d\n",
 651                                    queue);
 652
 653                                 bnx2x_tpa_start(fp, queue,
 654                                                 bd_cons, bd_prod,
 655                                                 cqe_fp);
 656
 657                                 goto next_rx;
 658
 659                         }
 660                         queue = cqe->end_agg_cqe.queue_index;
 661                         tpa_info = &fp->tpa_info[queue];
 662                         DP(NETIF_MSG_RX_STATUS,
 663                            "calling tpa_stop on queue %d\n",
 664                            queue);
 665
 666                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
 667                                     tpa_info->len_on_bd;
 668
 669                         if (fp->mode == TPA_MODE_GRO)
 670                                 pages = (frag_size + tpa_info->full_page - 1) /
 671                                          tpa_info->full_page;
 672                         else
 673                                 pages = SGE_PAGE_ALIGN(frag_size) >>
 674                                         SGE_PAGE_SHIFT;
 675
 676                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
 677                                        &cqe->end_agg_cqe, comp_ring_cons);
 678 #ifdef BNX2X_STOP_ON_ERROR
 679                         if (bp->panic)
 680                                 return 0;
 681 #endif
 682
 683                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
 684                         goto next_cqe;
 685                 }
 686                 /* non TPA */
 687                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
 688                 pad = cqe_fp->placement_offset;
 689                 dma_sync_single_for_cpu(&bp->pdev->dev,
 690                                         dma_unmap_addr(rx_buf, mapping),
 691                                         pad + RX_COPY_THRESH,
 692                                         DMA_FROM_DEVICE);
 693                 pad += NET_SKB_PAD;
 694                 prefetch(data + pad); /* speedup eth_type_trans() */
 695                 /* is this an error packet? */
 696                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 697                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 698                            "ERROR  flags %x  rx packet %u\n",
 699                            cqe_fp_flags, sw_comp_cons);
 700                         fp->eth_q_stats.rx_err_discard_pkt++;
 701                         goto reuse_rx;
 702                 }
 703
 704                 /* Since we don't have a jumbo ring
 705                  * copy small packets if mtu > 1500
 706                  */
 707                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 708                     (len <= RX_COPY_THRESH)) {
 709                         skb = netdev_alloc_skb_ip_align(bp->dev, len);
 710                         if (skb == NULL) {
 711                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 712                                    "ERROR  packet dropped because of alloc failure\n");
 713                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 714                                 goto reuse_rx;
 715                         }
 716                         memcpy(skb->data, data + pad, len);
 717                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 718                 } else {
 719                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod) == 0)) {
 720                                 dma_unmap_single(&bp->pdev->dev,
 721                                                  dma_unmap_addr(rx_buf, mapping),
 722                                                  fp->rx_buf_size,
 723                                                  DMA_FROM_DEVICE);
 724                                 skb = build_skb(data, 0);
 725                                 if (unlikely(!skb)) {
 726                                         kfree(data);
 727                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 728                                         goto next_rx;
 729                                 }
 730                                 skb_reserve(skb, pad);
 731                         } else {
 732                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
 733                                    "ERROR  packet dropped because of alloc failure\n");
 734                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 735 reuse_rx:
 736                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
 737                                 goto next_rx;
 738                         }
 739                 }
 740
 741                 skb_put(skb, len);
 742                 skb->protocol = eth_type_trans(skb, bp->dev);
 743
 744                 /* Set Toeplitz hash for a none-LRO skb */
 745                 skb->rxhash = bnx2x_get_rxhash(bp, cqe_fp);
 746
 747                 skb_checksum_none_assert(skb);
 748
 749                 if (bp->dev->features & NETIF_F_RXCSUM) {
 750
 751                         if (likely(BNX2X_RX_CSUM_OK(cqe)))
 752                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 753                         else
 754                                 fp->eth_q_stats.hw_csum_err++;
 755                 }
 756
 757                 skb_record_rx_queue(skb, fp->rx_queue);
 758
 759                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
 760                     PARSING_FLAGS_VLAN)
 761                         __vlan_hwaccel_put_tag(skb,
 762                                                le16_to_cpu(cqe_fp->vlan_tag));
 763                 napi_gro_receive(&fp->napi, skb);
 764
 765
 766 next_rx:
 767                 rx_buf->data = NULL;
 768
 769                 bd_cons = NEXT_RX_IDX(bd_cons);
 770                 bd_prod = NEXT_RX_IDX(bd_prod);
 771                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 772                 rx_pkt++;
 773 next_cqe:
 774                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 775                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 776
 777                 if (rx_pkt == budget)
 778                         break;
 779         } /* while */
 780
 781         fp->rx_bd_cons = bd_cons;
 782         fp->rx_bd_prod = bd_prod_fw;
 783         fp->rx_comp_cons = sw_comp_cons;
 784         fp->rx_comp_prod = sw_comp_prod;
 785
 786         /* Update producers */
 787         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 788                              fp->rx_sge_prod);
 789
 790         fp->rx_pkt += rx_pkt;
 791         fp->rx_calls++;
 792
 793         return rx_pkt;
 794 }
 795
 796 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 797 {
 798         struct bnx2x_fastpath *fp = fp_cookie;
 799         struct bnx2x *bp = fp->bp;
 800         u8 cos;
 801
 802         DP(NETIF_MSG_INTR,
 803            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
 804            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 805         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 806
 807 #ifdef BNX2X_STOP_ON_ERROR
 808         if (unlikely(bp->panic))
 809                 return IRQ_HANDLED;
 810 #endif
 811
 812         /* Handle Rx and Tx according to MSI-X vector */
 813         prefetch(fp->rx_cons_sb);
 814
 815         for_each_cos_in_tx_queue(fp, cos)
 816                 prefetch(fp->txdata[cos].tx_cons_sb);
 817
 818         prefetch(&fp->sb_running_index[SM_RX_ID]);
 819         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 820
 821         return IRQ_HANDLED;
 822 }
 823
 824 /* HW Lock for shared dual port PHYs */
 825 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 826 {
 827         mutex_lock(&bp->port.phy_mutex);
 828
 829         if (bp->port.need_hw_lock)
 830                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 831 }
 832
 833 void bnx2x_release_phy_lock(struct bnx2x *bp)
 834 {
 835         if (bp->port.need_hw_lock)
 836                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 837
 838         mutex_unlock(&bp->port.phy_mutex);
 839 }
 840
 841 /* calculates MF speed according to current linespeed and MF configuration */
 842 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 843 {
 844         u16 line_speed = bp->link_vars.line_speed;
 845         if (IS_MF(bp)) {
 846                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 847                                                    bp->mf_config[BP_VN(bp)]);
 848
 849                 /* Calculate the current MAX line speed limit for the MF
 850                  * devices
 851                  */
 852                 if (IS_MF_SI(bp))
 853                         line_speed = (line_speed * maxCfg) / 100;
 854                 else { /* SD mode */
 855                         u16 vn_max_rate = maxCfg * 100;
 856
 857                         if (vn_max_rate < line_speed)
 858                                 line_speed = vn_max_rate;
 859                 }
 860         }
 861
 862         return line_speed;
 863 }
 864
 865 /**
 866  * bnx2x_fill_report_data - fill link report data to report
 867  *
 868  * @bp:         driver handle
 869  * @data:       link state to update
 870  *
 871  * It uses a none-atomic bit operations because is called under the mutex.
 872  */
 873 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
 874                                           struct bnx2x_link_report_data *data)
 875 {
 876         u16 line_speed = bnx2x_get_mf_speed(bp);
 877
 878         memset(data, 0, sizeof(*data));
 879
 880         /* Fill the report data: efective line speed */
 881         data->line_speed = line_speed;
 882
 883         /* Link is down */
 884         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
 885                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 886                           &data->link_report_flags);
 887
 888         /* Full DUPLEX */
 889         if (bp->link_vars.duplex == DUPLEX_FULL)
 890                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
 891
 892         /* Rx Flow Control is ON */
 893         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
 894                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
 895
 896         /* Tx Flow Control is ON */
 897         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
 898                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
 899 }
 900
 901 /**
 902  * bnx2x_link_report - report link status to OS.
 903  *
 904  * @bp:         driver handle
 905  *
 906  * Calls the __bnx2x_link_report() under the same locking scheme
 907  * as a link/PHY state managing code to ensure a consistent link
 908  * reporting.
 909  */
 910
 911 void bnx2x_link_report(struct bnx2x *bp)
 912 {
 913         bnx2x_acquire_phy_lock(bp);
 914         __bnx2x_link_report(bp);
 915         bnx2x_release_phy_lock(bp);
 916 }
 917
 918 /**
 919  * __bnx2x_link_report - report link status to OS.
 920  *
 921  * @bp:         driver handle
 922  *
 923  * None atomic inmlementation.
 924  * Should be called under the phy_lock.
 925  */
 926 void __bnx2x_link_report(struct bnx2x *bp)
 927 {
 928         struct bnx2x_link_report_data cur_data;
 929
 930         /* reread mf_cfg */
 931         if (!CHIP_IS_E1(bp))
 932                 bnx2x_read_mf_cfg(bp);
 933
 934         /* Read the current link report info */
 935         bnx2x_fill_report_data(bp, &cur_data);
 936
 937         /* Don't report link down or exactly the same link status twice */
 938         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
 939             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 940                       &bp->last_reported_link.link_report_flags) &&
 941              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 942                       &cur_data.link_report_flags)))
 943                 return;
 944
 945         bp->link_cnt++;
 946
 947         /* We are going to report a new link parameters now -
 948          * remember the current data for the next time.
 949          */
 950         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
 951
 952         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 953                      &cur_data.link_report_flags)) {
 954                 netif_carrier_off(bp->dev);
 955                 netdev_err(bp->dev, "NIC Link is Down\n");
 956                 return;
 957         } else {
 958                 const char *duplex;
 959                 const char *flow;
 960
 961                 netif_carrier_on(bp->dev);
 962
 963                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
 964                                        &cur_data.link_report_flags))
 965                         duplex = "full";
 966                 else
 967                         duplex = "half";
 968
 969                 /* Handle the FC at the end so that only these flags would be
 970                  * possibly set. This way we may easily check if there is no FC
 971                  * enabled.
 972                  */
 973                 if (cur_data.link_report_flags) {
 974                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
 975                                      &cur_data.link_report_flags)) {
 976                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
 977                                      &cur_data.link_report_flags))
 978                                         flow = "ON - receive & transmit";
 979                                 else
 980                                         flow = "ON - receive";
 981                         } else {
 982                                 flow = "ON - transmit";
 983                         }
 984                 } else {
 985                         flow = "none";
 986                 }
 987                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
 988                             cur_data.line_speed, duplex, flow);
 989         }
 990 }
 991
 992 void bnx2x_init_rx_rings(struct bnx2x *bp)
 993 {
 994         int func = BP_FUNC(bp);
 995         u16 ring_prod;
 996         int i, j;
 997
 998         /* Allocate TPA resources */
 999         for_each_rx_queue(bp, j) {
1000                 struct bnx2x_fastpath *fp = &bp->fp[j];
1001
1002                 DP(NETIF_MSG_IFUP,
1003                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1004
1005                 if (!fp->disable_tpa) {
1006                         /* Fill the per-aggregtion pool */
1007                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1008                                 struct bnx2x_agg_info *tpa_info =
1009                                         &fp->tpa_info[i];
1010                                 struct sw_rx_bd *first_buf =
1011                                         &tpa_info->first_buf;
1012
1013                                 first_buf->data = kmalloc(fp->rx_buf_size + NET_SKB_PAD,
1014                                                           GFP_ATOMIC);
1015                                 if (!first_buf->data) {
1016                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1017                                                   j);
1018                                         bnx2x_free_tpa_pool(bp, fp, i);
1019                                         fp->disable_tpa = 1;
1020                                         break;
1021                                 }
1022                                 dma_unmap_addr_set(first_buf, mapping, 0);
1023                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1024                         }
1025
1026                         /* "next page" elements initialization */
1027                         bnx2x_set_next_page_sgl(fp);
1028
1029                         /* set SGEs bit mask */
1030                         bnx2x_init_sge_ring_bit_mask(fp);
1031
1032                         /* Allocate SGEs and initialize the ring elements */
1033                         for (i = 0, ring_prod = 0;
1034                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1035
1036                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
1037                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1038                                                   i);
1039                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1040                                                   j);
1041                                         /* Cleanup already allocated elements */
1042                                         bnx2x_free_rx_sge_range(bp, fp,
1043                                                                 ring_prod);
1044                                         bnx2x_free_tpa_pool(bp, fp,
1045                                                             MAX_AGG_QS(bp));
1046                                         fp->disable_tpa = 1;
1047                                         ring_prod = 0;
1048                                         break;
1049                                 }
1050                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1051                         }
1052
1053                         fp->rx_sge_prod = ring_prod;
1054                 }
1055         }
1056
1057         for_each_rx_queue(bp, j) {
1058                 struct bnx2x_fastpath *fp = &bp->fp[j];
1059
1060                 fp->rx_bd_cons = 0;
1061
1062                 /* Activate BD ring */
1063                 /* Warning!
1064                  * this will generate an interrupt (to the TSTORM)
1065                  * must only be done after chip is initialized
1066                  */
1067                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1068                                      fp->rx_sge_prod);
1069
1070                 if (j != 0)
1071                         continue;
1072
1073                 if (CHIP_IS_E1(bp)) {
1074                         REG_WR(bp, BAR_USTRORM_INTMEM +
1075                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1076                                U64_LO(fp->rx_comp_mapping));
1077                         REG_WR(bp, BAR_USTRORM_INTMEM +
1078                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1079                                U64_HI(fp->rx_comp_mapping));
1080                 }
1081         }
1082 }
1083
1084 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1085 {
1086         int i;
1087         u8 cos;
1088
1089         for_each_tx_queue(bp, i) {
1090                 struct bnx2x_fastpath *fp = &bp->fp[i];
1091                 for_each_cos_in_tx_queue(fp, cos) {
1092                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
1093                         unsigned pkts_compl = 0, bytes_compl = 0;
1094
1095                         u16 sw_prod = txdata->tx_pkt_prod;
1096                         u16 sw_cons = txdata->tx_pkt_cons;
1097
1098                         while (sw_cons != sw_prod) {
1099                                 bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1100                                     &pkts_compl, &bytes_compl);
1101                                 sw_cons++;
1102                         }
1103                         netdev_tx_reset_queue(
1104                             netdev_get_tx_queue(bp->dev, txdata->txq_index));
1105                 }
1106         }
1107 }
1108
1109 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1110 {
1111         struct bnx2x *bp = fp->bp;
1112         int i;
1113
1114         /* ring wasn't allocated */
1115         if (fp->rx_buf_ring == NULL)
1116                 return;
1117
1118         for (i = 0; i < NUM_RX_BD; i++) {
1119                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1120                 u8 *data = rx_buf->data;
1121
1122                 if (data == NULL)
1123                         continue;
1124                 dma_unmap_single(&bp->pdev->dev,
1125                                  dma_unmap_addr(rx_buf, mapping),
1126                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1127
1128                 rx_buf->data = NULL;
1129                 kfree(data);
1130         }
1131 }
1132
1133 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1134 {
1135         int j;
1136
1137         for_each_rx_queue(bp, j) {
1138                 struct bnx2x_fastpath *fp = &bp->fp[j];
1139
1140                 bnx2x_free_rx_bds(fp);
1141
1142                 if (!fp->disable_tpa)
1143                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1144         }
1145 }
1146
1147 void bnx2x_free_skbs(struct bnx2x *bp)
1148 {
1149         bnx2x_free_tx_skbs(bp);
1150         bnx2x_free_rx_skbs(bp);
1151 }
1152
1153 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1154 {
1155         /* load old values */
1156         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1157
1158         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1159                 /* leave all but MAX value */
1160                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1161
1162                 /* set new MAX value */
1163                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1164                                 & FUNC_MF_CFG_MAX_BW_MASK;
1165
1166                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1167         }
1168 }
1169
1170 /**
1171  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1172  *
1173  * @bp:         driver handle
1174  * @nvecs:      number of vectors to be released
1175  */
1176 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1177 {
1178         int i, offset = 0;
1179
1180         if (nvecs == offset)
1181                 return;
1182         free_irq(bp->msix_table[offset].vector, bp->dev);
1183         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1184            bp->msix_table[offset].vector);
1185         offset++;
1186 #ifdef BCM_CNIC
1187         if (nvecs == offset)
1188                 return;
1189         offset++;
1190 #endif
1191
1192         for_each_eth_queue(bp, i) {
1193                 if (nvecs == offset)
1194                         return;
1195                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1196                    i, bp->msix_table[offset].vector);
1197
1198                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1199         }
1200 }
1201
1202 void bnx2x_free_irq(struct bnx2x *bp)
1203 {
1204         if (bp->flags & USING_MSIX_FLAG &&
1205             !(bp->flags & USING_SINGLE_MSIX_FLAG))
1206                 bnx2x_free_msix_irqs(bp, BNX2X_NUM_ETH_QUEUES(bp) +
1207                                      CNIC_PRESENT + 1);
1208         else
1209                 free_irq(bp->dev->irq, bp->dev);
1210 }
1211
1212 int __devinit bnx2x_enable_msix(struct bnx2x *bp)
1213 {
1214         int msix_vec = 0, i, rc, req_cnt;
1215
1216         bp->msix_table[msix_vec].entry = msix_vec;
1217         BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1218            bp->msix_table[0].entry);
1219         msix_vec++;
1220
1221 #ifdef BCM_CNIC
1222         bp->msix_table[msix_vec].entry = msix_vec;
1223         BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1224            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1225         msix_vec++;
1226 #endif
1227         /* We need separate vectors for ETH queues only (not FCoE) */
1228         for_each_eth_queue(bp, i) {
1229                 bp->msix_table[msix_vec].entry = msix_vec;
1230                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1231                                msix_vec, msix_vec, i);
1232                 msix_vec++;
1233         }
1234
1235         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_PRESENT + 1;
1236
1237         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1238
1239         /*
1240          * reconfigure number of tx/rx queues according to available
1241          * MSI-X vectors
1242          */
1243         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1244                 /* how less vectors we will have? */
1245                 int diff = req_cnt - rc;
1246
1247                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1248
1249                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1250
1251                 if (rc) {
1252                         BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1253                         goto no_msix;
1254                 }
1255                 /*
1256                  * decrease number of queues by number of unallocated entries
1257                  */
1258                 bp->num_queues -= diff;
1259
1260                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1261                                bp->num_queues);
1262         } else if (rc > 0) {
1263                 /* Get by with single vector */
1264                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], 1);
1265                 if (rc) {
1266                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1267                                        rc);
1268                         goto no_msix;
1269                 }
1270
1271                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1272                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1273
1274         } else if (rc < 0) {
1275                 BNX2X_DEV_INFO("MSI-X is not attainable  rc %d\n", rc);
1276                 goto no_msix;
1277         }
1278
1279         bp->flags |= USING_MSIX_FLAG;
1280
1281         return 0;
1282
1283 no_msix:
1284         /* fall to INTx if not enough memory */
1285         if (rc == -ENOMEM)
1286                 bp->flags |= DISABLE_MSI_FLAG;
1287
1288         return rc;
1289 }
1290
1291 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1292 {
1293         int i, rc, offset = 0;
1294
1295         rc = request_irq(bp->msix_table[offset++].vector,
1296                          bnx2x_msix_sp_int, 0,
1297                          bp->dev->name, bp->dev);
1298         if (rc) {
1299                 BNX2X_ERR("request sp irq failed\n");
1300                 return -EBUSY;
1301         }
1302
1303 #ifdef BCM_CNIC
1304         offset++;
1305 #endif
1306         for_each_eth_queue(bp, i) {
1307                 struct bnx2x_fastpath *fp = &bp->fp[i];
1308                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1309                          bp->dev->name, i);
1310
1311                 rc = request_irq(bp->msix_table[offset].vector,
1312                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1313                 if (rc) {
1314                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1315                               bp->msix_table[offset].vector, rc);
1316                         bnx2x_free_msix_irqs(bp, offset);
1317                         return -EBUSY;
1318                 }
1319
1320                 offset++;
1321         }
1322
1323         i = BNX2X_NUM_ETH_QUEUES(bp);
1324         offset = 1 + CNIC_PRESENT;
1325         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1326                bp->msix_table[0].vector,
1327                0, bp->msix_table[offset].vector,
1328                i - 1, bp->msix_table[offset + i - 1].vector);
1329
1330         return 0;
1331 }
1332
1333 int bnx2x_enable_msi(struct bnx2x *bp)
1334 {
1335         int rc;
1336
1337         rc = pci_enable_msi(bp->pdev);
1338         if (rc) {
1339                 BNX2X_DEV_INFO("MSI is not attainable\n");
1340                 return -1;
1341         }
1342         bp->flags |= USING_MSI_FLAG;
1343
1344         return 0;
1345 }
1346
1347 static int bnx2x_req_irq(struct bnx2x *bp)
1348 {
1349         unsigned long flags;
1350         unsigned int irq;
1351
1352         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1353                 flags = 0;
1354         else
1355                 flags = IRQF_SHARED;
1356
1357         if (bp->flags & USING_MSIX_FLAG)
1358                 irq = bp->msix_table[0].vector;
1359         else
1360                 irq = bp->pdev->irq;
1361
1362         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1363 }
1364
1365 static inline int bnx2x_setup_irqs(struct bnx2x *bp)
1366 {
1367         int rc = 0;
1368         if (bp->flags & USING_MSIX_FLAG &&
1369             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1370                 rc = bnx2x_req_msix_irqs(bp);
1371                 if (rc)
1372                         return rc;
1373         } else {
1374                 bnx2x_ack_int(bp);
1375                 rc = bnx2x_req_irq(bp);
1376                 if (rc) {
1377                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1378                         return rc;
1379                 }
1380                 if (bp->flags & USING_MSI_FLAG) {
1381                         bp->dev->irq = bp->pdev->irq;
1382                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1383                                     bp->dev->irq);
1384                 }
1385                 if (bp->flags & USING_MSIX_FLAG) {
1386                         bp->dev->irq = bp->msix_table[0].vector;
1387                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1388                                     bp->dev->irq);
1389                 }
1390         }
1391
1392         return 0;
1393 }
1394
1395 static inline void bnx2x_napi_enable(struct bnx2x *bp)
1396 {
1397         int i;
1398
1399         for_each_rx_queue(bp, i)
1400                 napi_enable(&bnx2x_fp(bp, i, napi));
1401 }
1402
1403 static inline void bnx2x_napi_disable(struct bnx2x *bp)
1404 {
1405         int i;
1406
1407         for_each_rx_queue(bp, i)
1408                 napi_disable(&bnx2x_fp(bp, i, napi));
1409 }
1410
1411 void bnx2x_netif_start(struct bnx2x *bp)
1412 {
1413         if (netif_running(bp->dev)) {
1414                 bnx2x_napi_enable(bp);
1415                 bnx2x_int_enable(bp);
1416                 if (bp->state == BNX2X_STATE_OPEN)
1417                         netif_tx_wake_all_queues(bp->dev);
1418         }
1419 }
1420
1421 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1422 {
1423         bnx2x_int_disable_sync(bp, disable_hw);
1424         bnx2x_napi_disable(bp);
1425 }
1426
1427 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1428 {
1429         struct bnx2x *bp = netdev_priv(dev);
1430
1431 #ifdef BCM_CNIC
1432         if (!NO_FCOE(bp)) {
1433                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1434                 u16 ether_type = ntohs(hdr->h_proto);
1435
1436                 /* Skip VLAN tag if present */
1437                 if (ether_type == ETH_P_8021Q) {
1438                         struct vlan_ethhdr *vhdr =
1439                                 (struct vlan_ethhdr *)skb->data;
1440
1441                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1442                 }
1443
1444                 /* If ethertype is FCoE or FIP - use FCoE ring */
1445                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1446                         return bnx2x_fcoe_tx(bp, txq_index);
1447         }
1448 #endif
1449         /* select a non-FCoE queue */
1450         return __skb_tx_hash(dev, skb, BNX2X_NUM_ETH_QUEUES(bp));
1451 }
1452
1453
1454 void bnx2x_set_num_queues(struct bnx2x *bp)
1455 {
1456         /* RSS queues */
1457         bp->num_queues = bnx2x_calc_num_queues(bp);
1458
1459 #ifdef BCM_CNIC
1460         /* override in STORAGE SD modes */
1461         if (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))
1462                 bp->num_queues = 1;
1463 #endif
1464         /* Add special queues */
1465         bp->num_queues += NON_ETH_CONTEXT_USE;
1466 }
1467
1468 /**
1469  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1470  *
1471  * @bp:         Driver handle
1472  *
1473  * We currently support for at most 16 Tx queues for each CoS thus we will
1474  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1475  * bp->max_cos.
1476  *
1477  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1478  * index after all ETH L2 indices.
1479  *
1480  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1481  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1482  * 16..31,...) with indicies that are not coupled with any real Tx queue.
1483  *
1484  * The proper configuration of skb->queue_mapping is handled by
1485  * bnx2x_select_queue() and __skb_tx_hash().
1486  *
1487  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1488  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1489  */
1490 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1491 {
1492         int rc, tx, rx;
1493
1494         tx = MAX_TXQS_PER_COS * bp->max_cos;
1495         rx = BNX2X_NUM_ETH_QUEUES(bp);
1496
1497 /* account for fcoe queue */
1498 #ifdef BCM_CNIC
1499         if (!NO_FCOE(bp)) {
1500                 rx += FCOE_PRESENT;
1501                 tx += FCOE_PRESENT;
1502         }
1503 #endif
1504
1505         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1506         if (rc) {
1507                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1508                 return rc;
1509         }
1510         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1511         if (rc) {
1512                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1513                 return rc;
1514         }
1515
1516         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1517                           tx, rx);
1518
1519         return rc;
1520 }
1521
1522 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1523 {
1524         int i;
1525
1526         for_each_queue(bp, i) {
1527                 struct bnx2x_fastpath *fp = &bp->fp[i];
1528                 u32 mtu;
1529
1530                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1531                 if (IS_FCOE_IDX(i))
1532                         /*
1533                          * Although there are no IP frames expected to arrive to
1534                          * this ring we still want to add an
1535                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1536                          * overrun attack.
1537                          */
1538                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
1539                 else
1540                         mtu = bp->dev->mtu;
1541                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
1542                                   IP_HEADER_ALIGNMENT_PADDING +
1543                                   ETH_OVREHEAD +
1544                                   mtu +
1545                                   BNX2X_FW_RX_ALIGN_END;
1546                 /* Note : rx_buf_size doesnt take into account NET_SKB_PAD */
1547         }
1548 }
1549
1550 static inline int bnx2x_init_rss_pf(struct bnx2x *bp)
1551 {
1552         int i;
1553         u8 ind_table[T_ETH_INDIRECTION_TABLE_SIZE] = {0};
1554         u8 num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
1555
1556         /* Prepare the initial contents fo the indirection table if RSS is
1557          * enabled
1558          */
1559         for (i = 0; i < sizeof(ind_table); i++)
1560                 ind_table[i] =
1561                         bp->fp->cl_id +
1562                         ethtool_rxfh_indir_default(i, num_eth_queues);
1563
1564         /*
1565          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
1566          * per-port, so if explicit configuration is needed , do it only
1567          * for a PMF.
1568          *
1569          * For 57712 and newer on the other hand it's a per-function
1570          * configuration.
1571          */
1572         return bnx2x_config_rss_eth(bp, ind_table,
1573                                     bp->port.pmf || !CHIP_IS_E1x(bp));
1574 }
1575
1576 int bnx2x_config_rss_pf(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
1577                         u8 *ind_table, bool config_hash)
1578 {
1579         struct bnx2x_config_rss_params params = {NULL};
1580         int i;
1581
1582         /* Although RSS is meaningless when there is a single HW queue we
1583          * still need it enabled in order to have HW Rx hash generated.
1584          *
1585          * if (!is_eth_multi(bp))
1586          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
1587          */
1588
1589         params.rss_obj = rss_obj;
1590
1591         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
1592
1593         __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
1594
1595         /* RSS configuration */
1596         __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
1597         __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
1598         __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
1599         __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
1600
1601         /* Hash bits */
1602         params.rss_result_mask = MULTI_MASK;
1603
1604         memcpy(params.ind_table, ind_table, sizeof(params.ind_table));
1605
1606         if (config_hash) {
1607                 /* RSS keys */
1608                 for (i = 0; i < sizeof(params.rss_key) / 4; i++)
1609                         params.rss_key[i] = random32();
1610
1611                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
1612         }
1613
1614         return bnx2x_config_rss(bp, &params);
1615 }
1616
1617 static inline int bnx2x_init_hw(struct bnx2x *bp, u32 load_code)
1618 {
1619         struct bnx2x_func_state_params func_params = {NULL};
1620
1621         /* Prepare parameters for function state transitions */
1622         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
1623
1624         func_params.f_obj = &bp->func_obj;
1625         func_params.cmd = BNX2X_F_CMD_HW_INIT;
1626
1627         func_params.params.hw_init.load_phase = load_code;
1628
1629         return bnx2x_func_state_change(bp, &func_params);
1630 }
1631
1632 /*
1633  * Cleans the object that have internal lists without sending
1634  * ramrods. Should be run when interrutps are disabled.
1635  */
1636 static void bnx2x_squeeze_objects(struct bnx2x *bp)
1637 {
1638         int rc;
1639         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
1640         struct bnx2x_mcast_ramrod_params rparam = {NULL};
1641         struct bnx2x_vlan_mac_obj *mac_obj = &bp->fp->mac_obj;
1642
1643         /***************** Cleanup MACs' object first *************************/
1644
1645         /* Wait for completion of requested */
1646         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
1647         /* Perform a dry cleanup */
1648         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
1649
1650         /* Clean ETH primary MAC */
1651         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
1652         rc = mac_obj->delete_all(bp, &bp->fp->mac_obj, &vlan_mac_flags,
1653                                  &ramrod_flags);
1654         if (rc != 0)
1655                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
1656
1657         /* Cleanup UC list */
1658         vlan_mac_flags = 0;
1659         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
1660         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
1661                                  &ramrod_flags);
1662         if (rc != 0)
1663                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
1664
1665         /***************** Now clean mcast object *****************************/
1666         rparam.mcast_obj = &bp->mcast_obj;
1667         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
1668
1669         /* Add a DEL command... */
1670         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
1671         if (rc < 0)
1672                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
1673                           rc);
1674
1675         /* ...and wait until all pending commands are cleared */
1676         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1677         while (rc != 0) {
1678                 if (rc < 0) {
1679                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
1680                                   rc);
1681                         return;
1682                 }
1683
1684                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
1685         }
1686 }
1687
1688 #ifndef BNX2X_STOP_ON_ERROR
1689 #define LOAD_ERROR_EXIT(bp, label) \
1690         do { \
1691                 (bp)->state = BNX2X_STATE_ERROR; \
1692                 goto label; \
1693         } while (0)
1694 #else
1695 #define LOAD_ERROR_EXIT(bp, label) \
1696         do { \
1697                 (bp)->state = BNX2X_STATE_ERROR; \
1698                 (bp)->panic = 1; \
1699                 return -EBUSY; \
1700         } while (0)
1701 #endif
1702
1703 bool bnx2x_test_firmware_version(struct bnx2x *bp, bool is_err)
1704 {
1705         /* build FW version dword */
1706         u32 my_fw = (BCM_5710_FW_MAJOR_VERSION) +
1707                     (BCM_5710_FW_MINOR_VERSION << 8) +
1708                     (BCM_5710_FW_REVISION_VERSION << 16) +
1709                     (BCM_5710_FW_ENGINEERING_VERSION << 24);
1710
1711         /* read loaded FW from chip */
1712         u32 loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
1713
1714         DP(NETIF_MSG_IFUP, "loaded fw %x, my fw %x\n", loaded_fw, my_fw);
1715
1716         if (loaded_fw != my_fw) {
1717                 if (is_err)
1718                         BNX2X_ERR("bnx2x with FW %x was already loaded, which mismatches my %x FW. aborting\n",
1719                                   loaded_fw, my_fw);
1720                 return false;
1721         }
1722
1723         return true;
1724 }
1725
1726 /* must be called with rtnl_lock */
1727 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1728 {
1729         int port = BP_PORT(bp);
1730         u32 load_code;
1731         int i, rc;
1732
1733 #ifdef BNX2X_STOP_ON_ERROR
1734         if (unlikely(bp->panic)) {
1735                 BNX2X_ERR("Can't load NIC when there is panic\n");
1736                 return -EPERM;
1737         }
1738 #endif
1739
1740         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1741
1742         /* Set the initial link reported state to link down */
1743         bnx2x_acquire_phy_lock(bp);
1744         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1745         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1746                 &bp->last_reported_link.link_report_flags);
1747         bnx2x_release_phy_lock(bp);
1748
1749         /* must be called before memory allocation and HW init */
1750         bnx2x_ilt_set_info(bp);
1751
1752         /*
1753          * Zero fastpath structures preserving invariants like napi, which are
1754          * allocated only once, fp index, max_cos, bp pointer.
1755          * Also set fp->disable_tpa.
1756          */
1757         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
1758         for_each_queue(bp, i)
1759                 bnx2x_bz_fp(bp, i);
1760
1761
1762         /* Set the receive queues buffer size */
1763         bnx2x_set_rx_buf_size(bp);
1764
1765         if (bnx2x_alloc_mem(bp))
1766                 return -ENOMEM;
1767
1768         /* As long as bnx2x_alloc_mem() may possibly update
1769          * bp->num_queues, bnx2x_set_real_num_queues() should always
1770          * come after it.
1771          */
1772         rc = bnx2x_set_real_num_queues(bp);
1773         if (rc) {
1774                 BNX2X_ERR("Unable to set real_num_queues\n");
1775                 LOAD_ERROR_EXIT(bp, load_error0);
1776         }
1777
1778         /* configure multi cos mappings in kernel.
1779          * this configuration may be overriden by a multi class queue discipline
1780          * or by a dcbx negotiation result.
1781          */
1782         bnx2x_setup_tc(bp->dev, bp->max_cos);
1783
1784         bnx2x_napi_enable(bp);
1785
1786         /* set pf load just before approaching the MCP */
1787         bnx2x_set_pf_load(bp);
1788
1789         /* Send LOAD_REQUEST command to MCP
1790          * Returns the type of LOAD command:
1791          * if it is the first port to be initialized
1792          * common blocks should be initialized, otherwise - not
1793          */
1794         if (!BP_NOMCP(bp)) {
1795                 /* init fw_seq */
1796                 bp->fw_seq =
1797                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
1798                          DRV_MSG_SEQ_NUMBER_MASK);
1799                 BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
1800
1801                 /* Get current FW pulse sequence */
1802                 bp->fw_drv_pulse_wr_seq =
1803                         (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
1804                          DRV_PULSE_SEQ_MASK);
1805                 BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
1806
1807                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1808                 if (!load_code) {
1809                         BNX2X_ERR("MCP response failure, aborting\n");
1810                         rc = -EBUSY;
1811                         LOAD_ERROR_EXIT(bp, load_error1);
1812                 }
1813                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1814                         BNX2X_ERR("Driver load refused\n");
1815                         rc = -EBUSY; /* other port in diagnostic mode */
1816                         LOAD_ERROR_EXIT(bp, load_error1);
1817                 }
1818                 if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
1819                     load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
1820                         /* abort nic load if version mismatch */
1821                         if (!bnx2x_test_firmware_version(bp, true)) {
1822                                 rc = -EBUSY;
1823                                 LOAD_ERROR_EXIT(bp, load_error2);
1824                         }
1825                 }
1826
1827         } else {
1828                 int path = BP_PATH(bp);
1829
1830                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1831                    path, load_count[path][0], load_count[path][1],
1832                    load_count[path][2]);
1833                 load_count[path][0]++;
1834                 load_count[path][1 + port]++;
1835                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1836                    path, load_count[path][0], load_count[path][1],
1837                    load_count[path][2]);
1838                 if (load_count[path][0] == 1)
1839                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1840                 else if (load_count[path][1 + port] == 1)
1841                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1842                 else
1843                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1844         }
1845
1846         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1847             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1848             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
1849                 bp->port.pmf = 1;
1850                 /*
1851                  * We need the barrier to ensure the ordering between the
1852                  * writing to bp->port.pmf here and reading it from the
1853                  * bnx2x_periodic_task().
1854                  */
1855                 smp_mb();
1856         } else
1857                 bp->port.pmf = 0;
1858
1859         DP(NETIF_MSG_IFUP, "pmf %d\n", bp->port.pmf);
1860
1861         /* Init Function state controlling object */
1862         bnx2x__init_func_obj(bp);
1863
1864         /* Initialize HW */
1865         rc = bnx2x_init_hw(bp, load_code);
1866         if (rc) {
1867                 BNX2X_ERR("HW init failed, aborting\n");
1868                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1869                 LOAD_ERROR_EXIT(bp, load_error2);
1870         }
1871
1872         /* Connect to IRQs */
1873         rc = bnx2x_setup_irqs(bp);
1874         if (rc) {
1875                 BNX2X_ERR("IRQs setup failed\n");
1876                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1877                 LOAD_ERROR_EXIT(bp, load_error2);
1878         }
1879
1880         /* Setup NIC internals and enable interrupts */
1881         bnx2x_nic_init(bp, load_code);
1882
1883         /* Init per-function objects */
1884         bnx2x_init_bp_objs(bp);
1885
1886         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1887             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1888             (bp->common.shmem2_base)) {
1889                 if (SHMEM2_HAS(bp, dcc_support))
1890                         SHMEM2_WR(bp, dcc_support,
1891                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1892                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1893                 if (SHMEM2_HAS(bp, afex_driver_support))
1894                         SHMEM2_WR(bp, afex_driver_support,
1895                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
1896         }
1897
1898         /* Set AFEX default VLAN tag to an invalid value */
1899         bp->afex_def_vlan_tag = -1;
1900
1901         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1902         rc = bnx2x_func_start(bp);
1903         if (rc) {
1904                 BNX2X_ERR("Function start failed!\n");
1905                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1906                 LOAD_ERROR_EXIT(bp, load_error3);
1907         }
1908
1909         /* Send LOAD_DONE command to MCP */
1910         if (!BP_NOMCP(bp)) {
1911                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1912                 if (!load_code) {
1913                         BNX2X_ERR("MCP response failure, aborting\n");
1914                         rc = -EBUSY;
1915                         LOAD_ERROR_EXIT(bp, load_error3);
1916                 }
1917         }
1918
1919         rc = bnx2x_setup_leading(bp);
1920         if (rc) {
1921                 BNX2X_ERR("Setup leading failed!\n");
1922                 LOAD_ERROR_EXIT(bp, load_error3);
1923         }
1924
1925 #ifdef BCM_CNIC
1926         /* Enable Timer scan */
1927         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
1928 #endif
1929
1930         for_each_nondefault_queue(bp, i) {
1931                 rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
1932                 if (rc) {
1933                         BNX2X_ERR("Queue setup failed\n");
1934                         LOAD_ERROR_EXIT(bp, load_error4);
1935                 }
1936         }
1937
1938         rc = bnx2x_init_rss_pf(bp);
1939         if (rc) {
1940                 BNX2X_ERR("PF RSS init failed\n");
1941                 LOAD_ERROR_EXIT(bp, load_error4);
1942         }
1943
1944         /* Now when Clients are configured we are ready to work */
1945         bp->state = BNX2X_STATE_OPEN;
1946
1947         /* Configure a ucast MAC */
1948         rc = bnx2x_set_eth_mac(bp, true);
1949         if (rc) {
1950                 BNX2X_ERR("Setting Ethernet MAC failed\n");
1951                 LOAD_ERROR_EXIT(bp, load_error4);
1952         }
1953
1954         if (bp->pending_max) {
1955                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1956                 bp->pending_max = 0;
1957         }
1958
1959         if (bp->port.pmf)
1960                 bnx2x_initial_phy_init(bp, load_mode);
1961
1962         /* Start fast path */
1963
1964         /* Initialize Rx filter. */
1965         netif_addr_lock_bh(bp->dev);
1966         bnx2x_set_rx_mode(bp->dev);
1967         netif_addr_unlock_bh(bp->dev);
1968
1969         /* Start the Tx */
1970         switch (load_mode) {
1971         case LOAD_NORMAL:
1972                 /* Tx queue should be only reenabled */
1973                 netif_tx_wake_all_queues(bp->dev);
1974                 break;
1975
1976         case LOAD_OPEN:
1977                 netif_tx_start_all_queues(bp->dev);
1978                 smp_mb__after_clear_bit();
1979                 break;
1980
1981         case LOAD_DIAG:
1982                 bp->state = BNX2X_STATE_DIAG;
1983                 break;
1984
1985         default:
1986                 break;
1987         }
1988
1989         if (bp->port.pmf)
1990                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_DCB_CONFIGURED, 0);
1991         else
1992                 bnx2x__link_status_update(bp);
1993
1994         /* start the timer */
1995         mod_timer(&bp->timer, jiffies + bp->current_interval);
1996
1997 #ifdef BCM_CNIC
1998         /* re-read iscsi info */
1999         bnx2x_get_iscsi_info(bp);
2000         bnx2x_setup_cnic_irq_info(bp);
2001         if (bp->state == BNX2X_STATE_OPEN)
2002                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2003 #endif
2004
2005         /* mark driver is loaded in shmem2 */
2006         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2007                 u32 val;
2008                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2009                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2010                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2011                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2012         }
2013
2014         /* Wait for all pending SP commands to complete */
2015         if (!bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2016                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2017                 bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2018                 return -EBUSY;
2019         }
2020
2021         bnx2x_dcbx_init(bp);
2022         return 0;
2023
2024 #ifndef BNX2X_STOP_ON_ERROR
2025 load_error4:
2026 #ifdef BCM_CNIC
2027         /* Disable Timer scan */
2028         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2029 #endif
2030 load_error3:
2031         bnx2x_int_disable_sync(bp, 1);
2032
2033         /* Clean queueable objects */
2034         bnx2x_squeeze_objects(bp);
2035
2036         /* Free SKBs, SGEs, TPA pool and driver internals */
2037         bnx2x_free_skbs(bp);
2038         for_each_rx_queue(bp, i)
2039                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2040
2041         /* Release IRQs */
2042         bnx2x_free_irq(bp);
2043 load_error2:
2044         if (!BP_NOMCP(bp)) {
2045                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2046                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2047         }
2048
2049         bp->port.pmf = 0;
2050 load_error1:
2051         bnx2x_napi_disable(bp);
2052         /* clear pf_load status, as it was already set */
2053         bnx2x_clear_pf_load(bp);
2054 load_error0:
2055         bnx2x_free_mem(bp);
2056
2057         return rc;
2058 #endif /* ! BNX2X_STOP_ON_ERROR */
2059 }
2060
2061 /* must be called with rtnl_lock */
2062 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
2063 {
2064         int i;
2065         bool global = false;
2066
2067         /* mark driver is unloaded in shmem2 */
2068         if (SHMEM2_HAS(bp, drv_capabilities_flag)) {
2069                 u32 val;
2070                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2071                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2072                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2073         }
2074
2075         if ((bp->state == BNX2X_STATE_CLOSED) ||
2076             (bp->state == BNX2X_STATE_ERROR)) {
2077                 /* We can get here if the driver has been unloaded
2078                  * during parity error recovery and is either waiting for a
2079                  * leader to complete or for other functions to unload and
2080                  * then ifdown has been issued. In this case we want to
2081                  * unload and let other functions to complete a recovery
2082                  * process.
2083                  */
2084                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2085                 bp->is_leader = 0;
2086                 bnx2x_release_leader_lock(bp);
2087                 smp_mb();
2088
2089                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2090                 BNX2X_ERR("Can't unload in closed or error state\n");
2091                 return -EINVAL;
2092         }
2093
2094         /*
2095          * It's important to set the bp->state to the value different from
2096          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2097          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2098          */
2099         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2100         smp_mb();
2101
2102         /* Stop Tx */
2103         bnx2x_tx_disable(bp);
2104
2105 #ifdef BCM_CNIC
2106         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2107 #endif
2108
2109         bp->rx_mode = BNX2X_RX_MODE_NONE;
2110
2111         del_timer_sync(&bp->timer);
2112
2113         /* Set ALWAYS_ALIVE bit in shmem */
2114         bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2115
2116         bnx2x_drv_pulse(bp);
2117
2118         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2119         bnx2x_save_statistics(bp);
2120
2121         /* Cleanup the chip if needed */
2122         if (unload_mode != UNLOAD_RECOVERY)
2123                 bnx2x_chip_cleanup(bp, unload_mode);
2124         else {
2125                 /* Send the UNLOAD_REQUEST to the MCP */
2126                 bnx2x_send_unload_req(bp, unload_mode);
2127
2128                 /*
2129                  * Prevent transactions to host from the functions on the
2130                  * engine that doesn't reset global blocks in case of global
2131                  * attention once gloabl blocks are reset and gates are opened
2132                  * (the engine which leader will perform the recovery
2133                  * last).
2134                  */
2135                 if (!CHIP_IS_E1x(bp))
2136                         bnx2x_pf_disable(bp);
2137
2138                 /* Disable HW interrupts, NAPI */
2139                 bnx2x_netif_stop(bp, 1);
2140
2141                 /* Release IRQs */
2142                 bnx2x_free_irq(bp);
2143
2144                 /* Report UNLOAD_DONE to MCP */
2145                 bnx2x_send_unload_done(bp);
2146         }
2147
2148         /*
2149          * At this stage no more interrupts will arrive so we may safly clean
2150          * the queueable objects here in case they failed to get cleaned so far.
2151          */
2152         bnx2x_squeeze_objects(bp);
2153
2154         /* There should be no more pending SP commands at this stage */
2155         bp->sp_state = 0;
2156
2157         bp->port.pmf = 0;
2158
2159         /* Free SKBs, SGEs, TPA pool and driver internals */
2160         bnx2x_free_skbs(bp);
2161         for_each_rx_queue(bp, i)
2162                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2163
2164         bnx2x_free_mem(bp);
2165
2166         bp->state = BNX2X_STATE_CLOSED;
2167
2168         /* Check if there are pending parity attentions. If there are - set
2169          * RECOVERY_IN_PROGRESS.
2170          */
2171         if (bnx2x_chk_parity_attn(bp, &global, false)) {
2172                 bnx2x_set_reset_in_progress(bp);
2173
2174                 /* Set RESET_IS_GLOBAL if needed */
2175                 if (global)
2176                         bnx2x_set_reset_global(bp);
2177         }
2178
2179
2180         /* The last driver must disable a "close the gate" if there is no
2181          * parity attention or "process kill" pending.
2182          */
2183         if (!bnx2x_clear_pf_load(bp) && bnx2x_reset_is_done(bp, BP_PATH(bp)))
2184                 bnx2x_disable_close_the_gate(bp);
2185
2186         return 0;
2187 }
2188
2189 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
2190 {
2191         u16 pmcsr;
2192
2193         /* If there is no power capability, silently succeed */
2194         if (!bp->pm_cap) {
2195                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
2196                 return 0;
2197         }
2198
2199         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
2200
2201         switch (state) {
2202         case PCI_D0:
2203                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2204                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
2205                                        PCI_PM_CTRL_PME_STATUS));
2206
2207                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
2208                         /* delay required during transition out of D3hot */
2209                         msleep(20);
2210                 break;
2211
2212         case PCI_D3hot:
2213                 /* If there are other clients above don't
2214                    shut down the power */
2215                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
2216                         return 0;
2217                 /* Don't shut down the power for emulation and FPGA */
2218                 if (CHIP_REV_IS_SLOW(bp))
2219                         return 0;
2220
2221                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2222                 pmcsr |= 3;
2223
2224                 if (bp->wol)
2225                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2226
2227                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
2228                                       pmcsr);
2229
2230                 /* No more memory access after this point until
2231                 * device is brought back to D0.
2232                 */
2233                 break;
2234
2235         default:
2236                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
2237                 return -EINVAL;
2238         }
2239         return 0;
2240 }
2241
2242 /*
2243  * net_device service functions
2244  */
2245 int bnx2x_poll(struct napi_struct *napi, int budget)
2246 {
2247         int work_done = 0;
2248         u8 cos;
2249         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
2250                                                  napi);
2251         struct bnx2x *bp = fp->bp;
2252
2253         while (1) {
2254 #ifdef BNX2X_STOP_ON_ERROR
2255                 if (unlikely(bp->panic)) {
2256                         napi_complete(napi);
2257                         return 0;
2258                 }
2259 #endif
2260
2261                 for_each_cos_in_tx_queue(fp, cos)
2262                         if (bnx2x_tx_queue_has_work(&fp->txdata[cos]))
2263                                 bnx2x_tx_int(bp, &fp->txdata[cos]);
2264
2265
2266                 if (bnx2x_has_rx_work(fp)) {
2267                         work_done += bnx2x_rx_int(fp, budget - work_done);
2268
2269                         /* must not complete if we consumed full budget */
2270                         if (work_done >= budget)
2271                                 break;
2272                 }
2273
2274                 /* Fall out from the NAPI loop if needed */
2275                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2276 #ifdef BCM_CNIC
2277                         /* No need to update SB for FCoE L2 ring as long as
2278                          * it's connected to the default SB and the SB
2279                          * has been updated when NAPI was scheduled.
2280                          */
2281                         if (IS_FCOE_FP(fp)) {
2282                                 napi_complete(napi);
2283                                 break;
2284                         }
2285 #endif
2286
2287                         bnx2x_update_fpsb_idx(fp);
2288                         /* bnx2x_has_rx_work() reads the status block,
2289                          * thus we need to ensure that status block indices
2290                          * have been actually read (bnx2x_update_fpsb_idx)
2291                          * prior to this check (bnx2x_has_rx_work) so that
2292                          * we won't write the "newer" value of the status block
2293                          * to IGU (if there was a DMA right after
2294                          * bnx2x_has_rx_work and if there is no rmb, the memory
2295                          * reading (bnx2x_update_fpsb_idx) may be postponed
2296                          * to right before bnx2x_ack_sb). In this case there
2297                          * will never be another interrupt until there is
2298                          * another update of the status block, while there
2299                          * is still unhandled work.
2300                          */
2301                         rmb();
2302
2303                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
2304                                 napi_complete(napi);
2305                                 /* Re-enable interrupts */
2306                                 DP(NETIF_MSG_RX_STATUS,
2307                                    "Update index to %d\n", fp->fp_hc_idx);
2308                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
2309                                              le16_to_cpu(fp->fp_hc_idx),
2310                                              IGU_INT_ENABLE, 1);
2311                                 break;
2312                         }
2313                 }
2314         }
2315
2316         return work_done;
2317 }
2318
2319 /* we split the first BD into headers and data BDs
2320  * to ease the pain of our fellow microcode engineers
2321  * we use one mapping for both BDs
2322  * So far this has only been observed to happen
2323  * in Other Operating Systems(TM)
2324  */
2325 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
2326                                    struct bnx2x_fp_txdata *txdata,
2327                                    struct sw_tx_bd *tx_buf,
2328                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
2329                                    u16 bd_prod, int nbd)
2330 {
2331         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
2332         struct eth_tx_bd *d_tx_bd;
2333         dma_addr_t mapping;
2334         int old_len = le16_to_cpu(h_tx_bd->nbytes);
2335
2336         /* first fix first BD */
2337         h_tx_bd->nbd = cpu_to_le16(nbd);
2338         h_tx_bd->nbytes = cpu_to_le16(hlen);
2339
2340         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x) nbd %d\n",
2341            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo, h_tx_bd->nbd);
2342
2343         /* now get a new data BD
2344          * (after the pbd) and fill it */
2345         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2346         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2347
2348         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
2349                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
2350
2351         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2352         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2353         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
2354
2355         /* this marks the BD as one that has no individual mapping */
2356         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
2357
2358         DP(NETIF_MSG_TX_QUEUED,
2359            "TSO split data size is %d (%x:%x)\n",
2360            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
2361
2362         /* update tx_bd */
2363         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
2364
2365         return bd_prod;
2366 }
2367
2368 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
2369 {
2370         if (fix > 0)
2371                 csum = (u16) ~csum_fold(csum_sub(csum,
2372                                 csum_partial(t_header - fix, fix, 0)));
2373
2374         else if (fix < 0)
2375                 csum = (u16) ~csum_fold(csum_add(csum,
2376                                 csum_partial(t_header, -fix, 0)));
2377
2378         return swab16(csum);
2379 }
2380
2381 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
2382 {
2383         u32 rc;
2384
2385         if (skb->ip_summed != CHECKSUM_PARTIAL)
2386                 rc = XMIT_PLAIN;
2387
2388         else {
2389                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
2390                         rc = XMIT_CSUM_V6;
2391                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
2392                                 rc |= XMIT_CSUM_TCP;
2393
2394                 } else {
2395                         rc = XMIT_CSUM_V4;
2396                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
2397                                 rc |= XMIT_CSUM_TCP;
2398                 }
2399         }
2400
2401         if (skb_is_gso_v6(skb))
2402                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
2403         else if (skb_is_gso(skb))
2404                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
2405
2406         return rc;
2407 }
2408
2409 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2410 /* check if packet requires linearization (packet is too fragmented)
2411    no need to check fragmentation if page size > 8K (there will be no
2412    violation to FW restrictions) */
2413 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2414                              u32 xmit_type)
2415 {
2416         int to_copy = 0;
2417         int hlen = 0;
2418         int first_bd_sz = 0;
2419
2420         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2421         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2422
2423                 if (xmit_type & XMIT_GSO) {
2424                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2425                         /* Check if LSO packet needs to be copied:
2426                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2427                         int wnd_size = MAX_FETCH_BD - 3;
2428                         /* Number of windows to check */
2429                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2430                         int wnd_idx = 0;
2431                         int frag_idx = 0;
2432                         u32 wnd_sum = 0;
2433
2434                         /* Headers length */
2435                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2436                                 tcp_hdrlen(skb);
2437
2438                         /* Amount of data (w/o headers) on linear part of SKB*/
2439                         first_bd_sz = skb_headlen(skb) - hlen;
2440
2441                         wnd_sum  = first_bd_sz;
2442
2443                         /* Calculate the first sum - it's special */
2444                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2445                                 wnd_sum +=
2446                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
2447
2448                         /* If there was data on linear skb data - check it */
2449                         if (first_bd_sz > 0) {
2450                                 if (unlikely(wnd_sum < lso_mss)) {
2451                                         to_copy = 1;
2452                                         goto exit_lbl;
2453                                 }
2454
2455                                 wnd_sum -= first_bd_sz;
2456                         }
2457
2458                         /* Others are easier: run through the frag list and
2459                            check all windows */
2460                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2461                                 wnd_sum +=
2462                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
2463
2464                                 if (unlikely(wnd_sum < lso_mss)) {
2465                                         to_copy = 1;
2466                                         break;
2467                                 }
2468                                 wnd_sum -=
2469                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
2470                         }
2471                 } else {
2472                         /* in non-LSO too fragmented packet should always
2473                            be linearized */
2474                         to_copy = 1;
2475                 }
2476         }
2477
2478 exit_lbl:
2479         if (unlikely(to_copy))
2480                 DP(NETIF_MSG_TX_QUEUED,
2481                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
2482                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2483                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2484
2485         return to_copy;
2486 }
2487 #endif
2488
2489 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2490                                         u32 xmit_type)
2491 {
2492         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2493                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2494                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2495         if ((xmit_type & XMIT_GSO_V6) &&
2496             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2497                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2498 }
2499
2500 /**
2501  * bnx2x_set_pbd_gso - update PBD in GSO case.
2502  *
2503  * @skb:        packet skb
2504  * @pbd:        parse BD
2505  * @xmit_type:  xmit flags
2506  */
2507 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2508                                      struct eth_tx_parse_bd_e1x *pbd,
2509                                      u32 xmit_type)
2510 {
2511         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2512         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2513         pbd->tcp_flags = pbd_tcp_flags(skb);
2514
2515         if (xmit_type & XMIT_GSO_V4) {
2516                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2517                 pbd->tcp_pseudo_csum =
2518                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2519                                                   ip_hdr(skb)->daddr,
2520                                                   0, IPPROTO_TCP, 0));
2521
2522         } else
2523                 pbd->tcp_pseudo_csum =
2524                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2525                                                 &ipv6_hdr(skb)->daddr,
2526                                                 0, IPPROTO_TCP, 0));
2527
2528         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2529 }
2530
2531 /**
2532  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2533  *
2534  * @bp:                 driver handle
2535  * @skb:                packet skb
2536  * @parsing_data:       data to be updated
2537  * @xmit_type:          xmit flags
2538  *
2539  * 57712 related
2540  */
2541 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2542         u32 *parsing_data, u32 xmit_type)
2543 {
2544         *parsing_data |=
2545                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2546                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2547                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2548
2549         if (xmit_type & XMIT_CSUM_TCP) {
2550                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2551                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2552                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2553
2554                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2555         } else
2556                 /* We support checksum offload for TCP and UDP only.
2557                  * No need to pass the UDP header length - it's a constant.
2558                  */
2559                 return skb_transport_header(skb) +
2560                                 sizeof(struct udphdr) - skb->data;
2561 }
2562
2563 static inline void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2564         struct eth_tx_start_bd *tx_start_bd, u32 xmit_type)
2565 {
2566         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2567
2568         if (xmit_type & XMIT_CSUM_V4)
2569                 tx_start_bd->bd_flags.as_bitfield |=
2570                                         ETH_TX_BD_FLAGS_IP_CSUM;
2571         else
2572                 tx_start_bd->bd_flags.as_bitfield |=
2573                                         ETH_TX_BD_FLAGS_IPV6;
2574
2575         if (!(xmit_type & XMIT_CSUM_TCP))
2576                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
2577 }
2578
2579 /**
2580  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2581  *
2582  * @bp:         driver handle
2583  * @skb:        packet skb
2584  * @pbd:        parse BD to be updated
2585  * @xmit_type:  xmit flags
2586  */
2587 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2588         struct eth_tx_parse_bd_e1x *pbd,
2589         u32 xmit_type)
2590 {
2591         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2592
2593         /* for now NS flag is not used in Linux */
2594         pbd->global_data =
2595                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2596                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2597
2598         pbd->ip_hlen_w = (skb_transport_header(skb) -
2599                         skb_network_header(skb)) >> 1;
2600
2601         hlen += pbd->ip_hlen_w;
2602
2603         /* We support checksum offload for TCP and UDP only */
2604         if (xmit_type & XMIT_CSUM_TCP)
2605                 hlen += tcp_hdrlen(skb) / 2;
2606         else
2607                 hlen += sizeof(struct udphdr) / 2;
2608
2609         pbd->total_hlen_w = cpu_to_le16(hlen);
2610         hlen = hlen*2;
2611
2612         if (xmit_type & XMIT_CSUM_TCP) {
2613                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2614
2615         } else {
2616                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2617
2618                 DP(NETIF_MSG_TX_QUEUED,
2619                    "hlen %d  fix %d  csum before fix %x\n",
2620                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2621
2622                 /* HW bug: fixup the CSUM */
2623                 pbd->tcp_pseudo_csum =
2624                         bnx2x_csum_fix(skb_transport_header(skb),
2625                                        SKB_CS(skb), fix);
2626
2627                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2628                    pbd->tcp_pseudo_csum);
2629         }
2630
2631         return hlen;
2632 }
2633
2634 /* called with netif_tx_lock
2635  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2636  * netif_wake_queue()
2637  */
2638 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2639 {
2640         struct bnx2x *bp = netdev_priv(dev);
2641
2642         struct bnx2x_fastpath *fp;
2643         struct netdev_queue *txq;
2644         struct bnx2x_fp_txdata *txdata;
2645         struct sw_tx_bd *tx_buf;
2646         struct eth_tx_start_bd *tx_start_bd, *first_bd;
2647         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2648         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2649         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2650         u32 pbd_e2_parsing_data = 0;
2651         u16 pkt_prod, bd_prod;
2652         int nbd, txq_index, fp_index, txdata_index;
2653         dma_addr_t mapping;
2654         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2655         int i;
2656         u8 hlen = 0;
2657         __le16 pkt_size = 0;
2658         struct ethhdr *eth;
2659         u8 mac_type = UNICAST_ADDRESS;
2660
2661 #ifdef BNX2X_STOP_ON_ERROR
2662         if (unlikely(bp->panic))
2663                 return NETDEV_TX_BUSY;
2664 #endif
2665
2666         txq_index = skb_get_queue_mapping(skb);
2667         txq = netdev_get_tx_queue(dev, txq_index);
2668
2669         BUG_ON(txq_index >= MAX_ETH_TXQ_IDX(bp) + FCOE_PRESENT);
2670
2671         /* decode the fastpath index and the cos index from the txq */
2672         fp_index = TXQ_TO_FP(txq_index);
2673         txdata_index = TXQ_TO_COS(txq_index);
2674
2675 #ifdef BCM_CNIC
2676         /*
2677          * Override the above for the FCoE queue:
2678          *   - FCoE fp entry is right after the ETH entries.
2679          *   - FCoE L2 queue uses bp->txdata[0] only.
2680          */
2681         if (unlikely(!NO_FCOE(bp) && (txq_index ==
2682                                       bnx2x_fcoe_tx(bp, txq_index)))) {
2683                 fp_index = FCOE_IDX;
2684                 txdata_index = 0;
2685         }
2686 #endif
2687
2688         /* enable this debug print to view the transmission queue being used
2689         DP(NETIF_MSG_TX_QUEUED, "indices: txq %d, fp %d, txdata %d\n",
2690            txq_index, fp_index, txdata_index); */
2691
2692         /* locate the fastpath and the txdata */
2693         fp = &bp->fp[fp_index];
2694         txdata = &fp->txdata[txdata_index];
2695
2696         /* enable this debug print to view the tranmission details
2697         DP(NETIF_MSG_TX_QUEUED,
2698            "transmitting packet cid %d fp index %d txdata_index %d tx_data ptr %p fp pointer %p\n",
2699            txdata->cid, fp_index, txdata_index, txdata, fp); */
2700
2701         if (unlikely(bnx2x_tx_avail(bp, txdata) <
2702                      (skb_shinfo(skb)->nr_frags + 3))) {
2703                 fp->eth_q_stats.driver_xoff++;
2704                 netif_tx_stop_queue(txq);
2705                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2706                 return NETDEV_TX_BUSY;
2707         }
2708
2709         DP(NETIF_MSG_TX_QUEUED,
2710            "queue[%d]: SKB: summed %x  protocol %x protocol(%x,%x) gso type %x  xmit_type %x\n",
2711            txq_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2712            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2713
2714         eth = (struct ethhdr *)skb->data;
2715
2716         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2717         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2718                 if (is_broadcast_ether_addr(eth->h_dest))
2719                         mac_type = BROADCAST_ADDRESS;
2720                 else
2721                         mac_type = MULTICAST_ADDRESS;
2722         }
2723
2724 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2725         /* First, check if we need to linearize the skb (due to FW
2726            restrictions). No need to check fragmentation if page size > 8K
2727            (there will be no violation to FW restrictions) */
2728         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2729                 /* Statistics of linearization */
2730                 bp->lin_cnt++;
2731                 if (skb_linearize(skb) != 0) {
2732                         DP(NETIF_MSG_TX_QUEUED,
2733                            "SKB linearization failed - silently dropping this SKB\n");
2734                         dev_kfree_skb_any(skb);
2735                         return NETDEV_TX_OK;
2736                 }
2737         }
2738 #endif
2739         /* Map skb linear data for DMA */
2740         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2741                                  skb_headlen(skb), DMA_TO_DEVICE);
2742         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2743                 DP(NETIF_MSG_TX_QUEUED,
2744                    "SKB mapping failed - silently dropping this SKB\n");
2745                 dev_kfree_skb_any(skb);
2746                 return NETDEV_TX_OK;
2747         }
2748         /*
2749         Please read carefully. First we use one BD which we mark as start,
2750         then we have a parsing info BD (used for TSO or xsum),
2751         and only then we have the rest of the TSO BDs.
2752         (don't forget to mark the last one as last,
2753         and to unmap only AFTER you write to the BD ...)
2754         And above all, all pdb sizes are in words - NOT DWORDS!
2755         */
2756
2757         /* get current pkt produced now - advance it just before sending packet
2758          * since mapping of pages may fail and cause packet to be dropped
2759          */
2760         pkt_prod = txdata->tx_pkt_prod;
2761         bd_prod = TX_BD(txdata->tx_bd_prod);
2762
2763         /* get a tx_buf and first BD
2764          * tx_start_bd may be changed during SPLIT,
2765          * but first_bd will always stay first
2766          */
2767         tx_buf = &txdata->tx_buf_ring[TX_BD(pkt_prod)];
2768         tx_start_bd = &txdata->tx_desc_ring[bd_prod].start_bd;
2769         first_bd = tx_start_bd;
2770
2771         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2772         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2773                  mac_type);
2774
2775         /* header nbd */
2776         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2777
2778         /* remember the first BD of the packet */
2779         tx_buf->first_bd = txdata->tx_bd_prod;
2780         tx_buf->skb = skb;
2781         tx_buf->flags = 0;
2782
2783         DP(NETIF_MSG_TX_QUEUED,
2784            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2785            pkt_prod, tx_buf, txdata->tx_pkt_prod, bd_prod, tx_start_bd);
2786
2787         if (vlan_tx_tag_present(skb)) {
2788                 tx_start_bd->vlan_or_ethertype =
2789                     cpu_to_le16(vlan_tx_tag_get(skb));
2790                 tx_start_bd->bd_flags.as_bitfield |=
2791                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2792         } else
2793                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2794
2795         /* turn on parsing and get a BD */
2796         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2797
2798         if (xmit_type & XMIT_CSUM)
2799                 bnx2x_set_sbd_csum(bp, skb, tx_start_bd, xmit_type);
2800
2801         if (!CHIP_IS_E1x(bp)) {
2802                 pbd_e2 = &txdata->tx_desc_ring[bd_prod].parse_bd_e2;
2803                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2804                 /* Set PBD in checksum offload case */
2805                 if (xmit_type & XMIT_CSUM)
2806                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2807                                                      &pbd_e2_parsing_data,
2808                                                      xmit_type);
2809                 if (IS_MF_SI(bp)) {
2810                         /*
2811                          * fill in the MAC addresses in the PBD - for local
2812                          * switching
2813                          */
2814                         bnx2x_set_fw_mac_addr(&pbd_e2->src_mac_addr_hi,
2815                                               &pbd_e2->src_mac_addr_mid,
2816                                               &pbd_e2->src_mac_addr_lo,
2817                                               eth->h_source);
2818                         bnx2x_set_fw_mac_addr(&pbd_e2->dst_mac_addr_hi,
2819                                               &pbd_e2->dst_mac_addr_mid,
2820                                               &pbd_e2->dst_mac_addr_lo,
2821                                               eth->h_dest);
2822                 }
2823         } else {
2824                 pbd_e1x = &txdata->tx_desc_ring[bd_prod].parse_bd_e1x;
2825                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2826                 /* Set PBD in checksum offload case */
2827                 if (xmit_type & XMIT_CSUM)
2828                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2829
2830         }
2831
2832         /* Setup the data pointer of the first BD of the packet */
2833         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2834         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2835         nbd = 2; /* start_bd + pbd + frags (updated when pages are mapped) */
2836         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2837         pkt_size = tx_start_bd->nbytes;
2838
2839         DP(NETIF_MSG_TX_QUEUED,
2840            "first bd @%p  addr (%x:%x)  nbd %d  nbytes %d  flags %x  vlan %x\n",
2841            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2842            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2843            tx_start_bd->bd_flags.as_bitfield,
2844            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2845
2846         if (xmit_type & XMIT_GSO) {
2847
2848                 DP(NETIF_MSG_TX_QUEUED,
2849                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2850                    skb->len, hlen, skb_headlen(skb),
2851                    skb_shinfo(skb)->gso_size);
2852
2853                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2854
2855                 if (unlikely(skb_headlen(skb) > hlen))
2856                         bd_prod = bnx2x_tx_split(bp, txdata, tx_buf,
2857                                                  &tx_start_bd, hlen,
2858                                                  bd_prod, ++nbd);
2859                 if (!CHIP_IS_E1x(bp))
2860                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2861                                              xmit_type);
2862                 else
2863                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2864         }
2865
2866         /* Set the PBD's parsing_data field if not zero
2867          * (for the chips newer than 57711).
2868          */
2869         if (pbd_e2_parsing_data)
2870                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2871
2872         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2873
2874         /* Handle fragmented skb */
2875         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2876                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2877
2878                 mapping = skb_frag_dma_map(&bp->pdev->dev, frag, 0,
2879                                            skb_frag_size(frag), DMA_TO_DEVICE);
2880                 if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
2881                         unsigned int pkts_compl = 0, bytes_compl = 0;
2882
2883                         DP(NETIF_MSG_TX_QUEUED,
2884                            "Unable to map page - dropping packet...\n");
2885
2886                         /* we need unmap all buffers already mapped
2887                          * for this SKB;
2888                          * first_bd->nbd need to be properly updated
2889                          * before call to bnx2x_free_tx_pkt
2890                          */
2891                         first_bd->nbd = cpu_to_le16(nbd);
2892                         bnx2x_free_tx_pkt(bp, txdata,
2893                                           TX_BD(txdata->tx_pkt_prod),
2894                                           &pkts_compl, &bytes_compl);
2895                         return NETDEV_TX_OK;
2896                 }
2897
2898                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2899                 tx_data_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2900                 if (total_pkt_bd == NULL)
2901                         total_pkt_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
2902
2903                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2904                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2905                 tx_data_bd->nbytes = cpu_to_le16(skb_frag_size(frag));
2906                 le16_add_cpu(&pkt_size, skb_frag_size(frag));
2907                 nbd++;
2908
2909                 DP(NETIF_MSG_TX_QUEUED,
2910                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2911                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2912                    le16_to_cpu(tx_data_bd->nbytes));
2913         }
2914
2915         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2916
2917         /* update with actual num BDs */
2918         first_bd->nbd = cpu_to_le16(nbd);
2919
2920         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2921
2922         /* now send a tx doorbell, counting the next BD
2923          * if the packet contains or ends with it
2924          */
2925         if (TX_BD_POFF(bd_prod) < nbd)
2926                 nbd++;
2927
2928         /* total_pkt_bytes should be set on the first data BD if
2929          * it's not an LSO packet and there is more than one
2930          * data BD. In this case pkt_size is limited by an MTU value.
2931          * However we prefer to set it for an LSO packet (while we don't
2932          * have to) in order to save some CPU cycles in a none-LSO
2933          * case, when we much more care about them.
2934          */
2935         if (total_pkt_bd != NULL)
2936                 total_pkt_bd->total_pkt_bytes = pkt_size;
2937
2938         if (pbd_e1x)
2939                 DP(NETIF_MSG_TX_QUEUED,
2940                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2941                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2942                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2943                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2944                     le16_to_cpu(pbd_e1x->total_hlen_w));
2945         if (pbd_e2)
2946                 DP(NETIF_MSG_TX_QUEUED,
2947                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2948                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2949                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2950                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2951                    pbd_e2->parsing_data);
2952         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2953
2954         netdev_tx_sent_queue(txq, skb->len);
2955
2956         skb_tx_timestamp(skb);
2957
2958         txdata->tx_pkt_prod++;
2959         /*
2960          * Make sure that the BD data is updated before updating the producer
2961          * since FW might read the BD right after the producer is updated.
2962          * This is only applicable for weak-ordered memory model archs such
2963          * as IA-64. The following barrier is also mandatory since FW will
2964          * assumes packets must have BDs.
2965          */
2966         wmb();
2967
2968         txdata->tx_db.data.prod += nbd;
2969         barrier();
2970
2971         DOORBELL(bp, txdata->cid, txdata->tx_db.raw);
2972
2973         mmiowb();
2974
2975         txdata->tx_bd_prod += nbd;
2976
2977         if (unlikely(bnx2x_tx_avail(bp, txdata) < MAX_SKB_FRAGS + 3)) {
2978                 netif_tx_stop_queue(txq);
2979
2980                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2981                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2982                  * fp->bd_tx_cons */
2983                 smp_mb();
2984
2985                 fp->eth_q_stats.driver_xoff++;
2986                 if (bnx2x_tx_avail(bp, txdata) >= MAX_SKB_FRAGS + 3)
2987                         netif_tx_wake_queue(txq);
2988         }
2989         txdata->tx_pkt++;
2990
2991         return NETDEV_TX_OK;
2992 }
2993
2994 /**
2995  * bnx2x_setup_tc - routine to configure net_device for multi tc
2996  *
2997  * @netdev: net device to configure
2998  * @tc: number of traffic classes to enable
2999  *
3000  * callback connected to the ndo_setup_tc function pointer
3001  */
3002 int bnx2x_setup_tc(struct net_device *dev, u8 num_tc)
3003 {
3004         int cos, prio, count, offset;
3005         struct bnx2x *bp = netdev_priv(dev);
3006
3007         /* setup tc must be called under rtnl lock */
3008         ASSERT_RTNL();
3009
3010         /* no traffic classes requested. aborting */
3011         if (!num_tc) {
3012                 netdev_reset_tc(dev);
3013                 return 0;
3014         }
3015
3016         /* requested to support too many traffic classes */
3017         if (num_tc > bp->max_cos) {
3018                 BNX2X_ERR("support for too many traffic classes requested: %d. max supported is %d\n",
3019                           num_tc, bp->max_cos);
3020                 return -EINVAL;
3021         }
3022
3023         /* declare amount of supported traffic classes */
3024         if (netdev_set_num_tc(dev, num_tc)) {
3025                 BNX2X_ERR("failed to declare %d traffic classes\n", num_tc);
3026                 return -EINVAL;
3027         }
3028
3029         /* configure priority to traffic class mapping */
3030         for (prio = 0; prio < BNX2X_MAX_PRIORITY; prio++) {
3031                 netdev_set_prio_tc_map(dev, prio, bp->prio_to_cos[prio]);
3032                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3033                    "mapping priority %d to tc %d\n",
3034                    prio, bp->prio_to_cos[prio]);
3035         }
3036
3037
3038         /* Use this configuration to diffrentiate tc0 from other COSes
3039            This can be used for ets or pfc, and save the effort of setting
3040            up a multio class queue disc or negotiating DCBX with a switch
3041         netdev_set_prio_tc_map(dev, 0, 0);
3042         DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", 0, 0);
3043         for (prio = 1; prio < 16; prio++) {
3044                 netdev_set_prio_tc_map(dev, prio, 1);
3045                 DP(BNX2X_MSG_SP, "mapping priority %d to tc %d\n", prio, 1);
3046         } */
3047
3048         /* configure traffic class to transmission queue mapping */
3049         for (cos = 0; cos < bp->max_cos; cos++) {
3050                 count = BNX2X_NUM_ETH_QUEUES(bp);
3051                 offset = cos * MAX_TXQS_PER_COS;
3052                 netdev_set_tc_queue(dev, cos, count, offset);
3053                 DP(BNX2X_MSG_SP | NETIF_MSG_IFUP,
3054                    "mapping tc %d to offset %d count %d\n",
3055                    cos, offset, count);
3056         }
3057
3058         return 0;
3059 }
3060
3061 /* called with rtnl_lock */
3062 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
3063 {
3064         struct sockaddr *addr = p;
3065         struct bnx2x *bp = netdev_priv(dev);
3066         int rc = 0;
3067
3068         if (!bnx2x_is_valid_ether_addr(bp, addr->sa_data)) {
3069                 BNX2X_ERR("Requested MAC address is not valid\n");
3070                 return -EINVAL;
3071         }
3072
3073 #ifdef BCM_CNIC
3074         if ((IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp)) &&
3075             !is_zero_ether_addr(addr->sa_data)) {
3076                 BNX2X_ERR("Can't configure non-zero address on iSCSI or FCoE functions in MF-SD mode\n");
3077                 return -EINVAL;
3078         }
3079 #endif
3080
3081         if (netif_running(dev))  {
3082                 rc = bnx2x_set_eth_mac(bp, false);
3083                 if (rc)
3084                         return rc;
3085         }
3086
3087         dev->addr_assign_type &= ~NET_ADDR_RANDOM;
3088         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
3089
3090         if (netif_running(dev))
3091                 rc = bnx2x_set_eth_mac(bp, true);
3092
3093         return rc;
3094 }
3095
3096 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
3097 {
3098         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
3099         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
3100         u8 cos;
3101
3102         /* Common */
3103 #ifdef BCM_CNIC
3104         if (IS_FCOE_IDX(fp_index)) {
3105                 memset(sb, 0, sizeof(union host_hc_status_block));
3106                 fp->status_blk_mapping = 0;
3107
3108         } else {
3109 #endif
3110                 /* status blocks */
3111                 if (!CHIP_IS_E1x(bp))
3112                         BNX2X_PCI_FREE(sb->e2_sb,
3113                                        bnx2x_fp(bp, fp_index,
3114                                                 status_blk_mapping),
3115                                        sizeof(struct host_hc_status_block_e2));
3116                 else
3117                         BNX2X_PCI_FREE(sb->e1x_sb,
3118                                        bnx2x_fp(bp, fp_index,
3119                                                 status_blk_mapping),
3120                                        sizeof(struct host_hc_status_block_e1x));
3121 #ifdef BCM_CNIC
3122         }
3123 #endif
3124         /* Rx */
3125         if (!skip_rx_queue(bp, fp_index)) {
3126                 bnx2x_free_rx_bds(fp);
3127
3128                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3129                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
3130                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
3131                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
3132                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
3133
3134                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
3135                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
3136                                sizeof(struct eth_fast_path_rx_cqe) *
3137                                NUM_RCQ_BD);
3138
3139                 /* SGE ring */
3140                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
3141                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
3142                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
3143                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3144         }
3145
3146         /* Tx */
3147         if (!skip_tx_queue(bp, fp_index)) {
3148                 /* fastpath tx rings: tx_buf tx_desc */
3149                 for_each_cos_in_tx_queue(fp, cos) {
3150                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3151
3152                         DP(NETIF_MSG_IFDOWN,
3153                            "freeing tx memory of fp %d cos %d cid %d\n",
3154                            fp_index, cos, txdata->cid);
3155
3156                         BNX2X_FREE(txdata->tx_buf_ring);
3157                         BNX2X_PCI_FREE(txdata->tx_desc_ring,
3158                                 txdata->tx_desc_mapping,
3159                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3160                 }
3161         }
3162         /* end of fastpath */
3163 }
3164
3165 void bnx2x_free_fp_mem(struct bnx2x *bp)
3166 {
3167         int i;
3168         for_each_queue(bp, i)
3169                 bnx2x_free_fp_mem_at(bp, i);
3170 }
3171
3172 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
3173 {
3174         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
3175         if (!CHIP_IS_E1x(bp)) {
3176                 bnx2x_fp(bp, index, sb_index_values) =
3177                         (__le16 *)status_blk.e2_sb->sb.index_values;
3178                 bnx2x_fp(bp, index, sb_running_index) =
3179                         (__le16 *)status_blk.e2_sb->sb.running_index;
3180         } else {
3181                 bnx2x_fp(bp, index, sb_index_values) =
3182                         (__le16 *)status_blk.e1x_sb->sb.index_values;
3183                 bnx2x_fp(bp, index, sb_running_index) =
3184                         (__le16 *)status_blk.e1x_sb->sb.running_index;
3185         }
3186 }
3187
3188 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
3189 {
3190         union host_hc_status_block *sb;
3191         struct bnx2x_fastpath *fp = &bp->fp[index];
3192         int ring_size = 0;
3193         u8 cos;
3194         int rx_ring_size = 0;
3195
3196 #ifdef BCM_CNIC
3197         if (!bp->rx_ring_size &&
3198             (IS_MF_STORAGE_SD(bp) || IS_MF_FCOE_AFEX(bp))) {
3199                 rx_ring_size = MIN_RX_SIZE_NONTPA;
3200                 bp->rx_ring_size = rx_ring_size;
3201         } else
3202 #endif
3203         if (!bp->rx_ring_size) {
3204                 u32 cfg = SHMEM_RD(bp,
3205                              dev_info.port_hw_config[BP_PORT(bp)].default_cfg);
3206
3207                 rx_ring_size = MAX_RX_AVAIL/BNX2X_NUM_RX_QUEUES(bp);
3208
3209                 /* Dercease ring size for 1G functions */
3210                 if ((cfg & PORT_HW_CFG_NET_SERDES_IF_MASK) ==
3211                     PORT_HW_CFG_NET_SERDES_IF_SGMII)
3212                         rx_ring_size /= 10;
3213
3214                 /* allocate at least number of buffers required by FW */
3215                 rx_ring_size = max_t(int, bp->disable_tpa ? MIN_RX_SIZE_NONTPA :
3216                                      MIN_RX_SIZE_TPA, rx_ring_size);
3217
3218                 bp->rx_ring_size = rx_ring_size;
3219         } else /* if rx_ring_size specified - use it */
3220                 rx_ring_size = bp->rx_ring_size;
3221
3222         /* Common */
3223         sb = &bnx2x_fp(bp, index, status_blk);
3224 #ifdef BCM_CNIC
3225         if (!IS_FCOE_IDX(index)) {
3226 #endif
3227                 /* status blocks */
3228                 if (!CHIP_IS_E1x(bp))
3229                         BNX2X_PCI_ALLOC(sb->e2_sb,
3230                                 &bnx2x_fp(bp, index, status_blk_mapping),
3231                                 sizeof(struct host_hc_status_block_e2));
3232                 else
3233                         BNX2X_PCI_ALLOC(sb->e1x_sb,
3234                                 &bnx2x_fp(bp, index, status_blk_mapping),
3235                             sizeof(struct host_hc_status_block_e1x));
3236 #ifdef BCM_CNIC
3237         }
3238 #endif
3239
3240         /* FCoE Queue uses Default SB and doesn't ACK the SB, thus no need to
3241          * set shortcuts for it.
3242          */
3243         if (!IS_FCOE_IDX(index))
3244                 set_sb_shortcuts(bp, index);
3245
3246         /* Tx */
3247         if (!skip_tx_queue(bp, index)) {
3248                 /* fastpath tx rings: tx_buf tx_desc */
3249                 for_each_cos_in_tx_queue(fp, cos) {
3250                         struct bnx2x_fp_txdata *txdata = &fp->txdata[cos];
3251
3252                         DP(NETIF_MSG_IFUP,
3253                            "allocating tx memory of fp %d cos %d\n",
3254                            index, cos);
3255
3256                         BNX2X_ALLOC(txdata->tx_buf_ring,
3257                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
3258                         BNX2X_PCI_ALLOC(txdata->tx_desc_ring,
3259                                 &txdata->tx_desc_mapping,
3260                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
3261                 }
3262         }
3263
3264         /* Rx */
3265         if (!skip_rx_queue(bp, index)) {
3266                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
3267                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
3268                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
3269                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
3270                                 &bnx2x_fp(bp, index, rx_desc_mapping),
3271                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
3272
3273                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
3274                                 &bnx2x_fp(bp, index, rx_comp_mapping),
3275                                 sizeof(struct eth_fast_path_rx_cqe) *
3276                                 NUM_RCQ_BD);
3277
3278                 /* SGE ring */
3279                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
3280                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
3281                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
3282                                 &bnx2x_fp(bp, index, rx_sge_mapping),
3283                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
3284                 /* RX BD ring */
3285                 bnx2x_set_next_page_rx_bd(fp);
3286
3287                 /* CQ ring */
3288                 bnx2x_set_next_page_rx_cq(fp);
3289
3290                 /* BDs */
3291                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
3292                 if (ring_size < rx_ring_size)
3293                         goto alloc_mem_err;
3294         }
3295
3296         return 0;
3297
3298 /* handles low memory cases */
3299 alloc_mem_err:
3300         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
3301                                                 index, ring_size);
3302         /* FW will drop all packets if queue is not big enough,
3303          * In these cases we disable the queue
3304          * Min size is different for OOO, TPA and non-TPA queues
3305          */
3306         if (ring_size < (fp->disable_tpa ?
3307                                 MIN_RX_SIZE_NONTPA : MIN_RX_SIZE_TPA)) {
3308                         /* release memory allocated for this queue */
3309                         bnx2x_free_fp_mem_at(bp, index);
3310                         return -ENOMEM;
3311         }
3312         return 0;
3313 }
3314
3315 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
3316 {
3317         int i;
3318
3319         /**
3320          * 1. Allocate FP for leading - fatal if error
3321          * 2. {CNIC} Allocate FCoE FP - fatal if error
3322          * 3. {CNIC} Allocate OOO + FWD - disable OOO if error
3323          * 4. Allocate RSS - fix number of queues if error
3324          */
3325
3326         /* leading */
3327         if (bnx2x_alloc_fp_mem_at(bp, 0))
3328                 return -ENOMEM;
3329
3330 #ifdef BCM_CNIC
3331         if (!NO_FCOE(bp))
3332                 /* FCoE */
3333                 if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
3334                         /* we will fail load process instead of mark
3335                          * NO_FCOE_FLAG
3336                          */
3337                         return -ENOMEM;
3338 #endif
3339
3340         /* RSS */
3341         for_each_nondefault_eth_queue(bp, i)
3342                 if (bnx2x_alloc_fp_mem_at(bp, i))
3343                         break;
3344
3345         /* handle memory failures */
3346         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
3347                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
3348
3349                 WARN_ON(delta < 0);
3350 #ifdef BCM_CNIC
3351                 /**
3352                  * move non eth FPs next to last eth FP
3353                  * must be done in that order
3354                  * FCOE_IDX < FWD_IDX < OOO_IDX
3355                  */
3356
3357                 /* move FCoE fp even NO_FCOE_FLAG is on */
3358                 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
3359 #endif
3360                 bp->num_queues -= delta;
3361                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
3362                           bp->num_queues + delta, bp->num_queues);
3363         }
3364
3365         return 0;
3366 }
3367
3368 void bnx2x_free_mem_bp(struct bnx2x *bp)
3369 {
3370         kfree(bp->fp);
3371         kfree(bp->msix_table);
3372         kfree(bp->ilt);
3373 }
3374
3375 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
3376 {
3377         struct bnx2x_fastpath *fp;
3378         struct msix_entry *tbl;
3379         struct bnx2x_ilt *ilt;
3380         int msix_table_size = 0;
3381
3382         /*
3383          * The biggest MSI-X table we might need is as a maximum number of fast
3384          * path IGU SBs plus default SB (for PF).
3385          */
3386         msix_table_size = bp->igu_sb_cnt + 1;
3387
3388         /* fp array: RSS plus CNIC related L2 queues */
3389         fp = kcalloc(BNX2X_MAX_RSS_COUNT(bp) + NON_ETH_CONTEXT_USE,
3390                      sizeof(*fp), GFP_KERNEL);
3391         if (!fp)
3392                 goto alloc_err;
3393         bp->fp = fp;
3394
3395         /* msix table */
3396         tbl = kcalloc(msix_table_size, sizeof(*tbl), GFP_KERNEL);
3397         if (!tbl)
3398                 goto alloc_err;
3399         bp->msix_table = tbl;
3400
3401         /* ilt */
3402         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
3403         if (!ilt)
3404                 goto alloc_err;
3405         bp->ilt = ilt;
3406
3407         return 0;
3408 alloc_err:
3409         bnx2x_free_mem_bp(bp);
3410         return -ENOMEM;
3411
3412 }
3413
3414 int bnx2x_reload_if_running(struct net_device *dev)
3415 {
3416         struct bnx2x *bp = netdev_priv(dev);
3417
3418         if (unlikely(!netif_running(dev)))
3419                 return 0;
3420
3421         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
3422         return bnx2x_nic_load(bp, LOAD_NORMAL);
3423 }
3424
3425 int bnx2x_get_cur_phy_idx(struct bnx2x *bp)
3426 {
3427         u32 sel_phy_idx = 0;
3428         if (bp->link_params.num_phys <= 1)
3429                 return INT_PHY;
3430
3431         if (bp->link_vars.link_up) {
3432                 sel_phy_idx = EXT_PHY1;
3433                 /* In case link is SERDES, check if the EXT_PHY2 is the one */
3434                 if ((bp->link_vars.link_status & LINK_STATUS_SERDES_LINK) &&
3435                     (bp->link_params.phy[EXT_PHY2].supported & SUPPORTED_FIBRE))
3436                         sel_phy_idx = EXT_PHY2;
3437         } else {
3438
3439                 switch (bnx2x_phy_selection(&bp->link_params)) {
3440                 case PORT_HW_CFG_PHY_SELECTION_HARDWARE_DEFAULT:
3441                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY:
3442                 case PORT_HW_CFG_PHY_SELECTION_FIRST_PHY_PRIORITY:
3443                        sel_phy_idx = EXT_PHY1;
3444                        break;
3445                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY:
3446                 case PORT_HW_CFG_PHY_SELECTION_SECOND_PHY_PRIORITY:
3447                        sel_phy_idx = EXT_PHY2;
3448                        break;
3449                 }
3450         }
3451
3452         return sel_phy_idx;
3453
3454 }
3455 int bnx2x_get_link_cfg_idx(struct bnx2x *bp)
3456 {
3457         u32 sel_phy_idx = bnx2x_get_cur_phy_idx(bp);
3458         /*
3459          * The selected actived PHY is always after swapping (in case PHY
3460          * swapping is enabled). So when swapping is enabled, we need to reverse
3461          * the configuration
3462          */
3463
3464         if (bp->link_params.multi_phy_config &
3465             PORT_HW_CFG_PHY_SWAPPED_ENABLED) {
3466                 if (sel_phy_idx == EXT_PHY1)
3467                         sel_phy_idx = EXT_PHY2;
3468                 else if (sel_phy_idx == EXT_PHY2)
3469                         sel_phy_idx = EXT_PHY1;
3470         }
3471         return LINK_CONFIG_IDX(sel_phy_idx);
3472 }
3473
3474 #if defined(NETDEV_FCOE_WWNN) && defined(BCM_CNIC)
3475 int bnx2x_fcoe_get_wwn(struct net_device *dev, u64 *wwn, int type)
3476 {
3477         struct bnx2x *bp = netdev_priv(dev);
3478         struct cnic_eth_dev *cp = &bp->cnic_eth_dev;
3479
3480         switch (type) {
3481         case NETDEV_FCOE_WWNN:
3482                 *wwn = HILO_U64(cp->fcoe_wwn_node_name_hi,
3483                                 cp->fcoe_wwn_node_name_lo);
3484                 break;
3485         case NETDEV_FCOE_WWPN:
3486                 *wwn = HILO_U64(cp->fcoe_wwn_port_name_hi,
3487                                 cp->fcoe_wwn_port_name_lo);
3488                 break;
3489         default:
3490                 BNX2X_ERR("Wrong WWN type requested - %d\n", type);
3491                 return -EINVAL;
3492         }
3493
3494         return 0;
3495 }
3496 #endif
3497
3498 /* called with rtnl_lock */
3499 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
3500 {
3501         struct bnx2x *bp = netdev_priv(dev);
3502
3503         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3504                 BNX2X_ERR("Can't perform change MTU during parity recovery\n");
3505                 return -EAGAIN;
3506         }
3507
3508         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
3509             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE)) {
3510                 BNX2X_ERR("Can't support requested MTU size\n");
3511                 return -EINVAL;
3512         }
3513
3514         /* This does not race with packet allocation
3515          * because the actual alloc size is
3516          * only updated as part of load
3517          */
3518         dev->mtu = new_mtu;
3519
3520         return bnx2x_reload_if_running(dev);
3521 }
3522
3523 netdev_features_t bnx2x_fix_features(struct net_device *dev,
3524                                      netdev_features_t features)
3525 {
3526         struct bnx2x *bp = netdev_priv(dev);
3527
3528         /* TPA requires Rx CSUM offloading */
3529         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa) {
3530                 features &= ~NETIF_F_LRO;
3531                 features &= ~NETIF_F_GRO;
3532         }
3533
3534         return features;
3535 }
3536
3537 int bnx2x_set_features(struct net_device *dev, netdev_features_t features)
3538 {
3539         struct bnx2x *bp = netdev_priv(dev);
3540         u32 flags = bp->flags;
3541         bool bnx2x_reload = false;
3542
3543         if (features & NETIF_F_LRO)
3544                 flags |= TPA_ENABLE_FLAG;
3545         else
3546                 flags &= ~TPA_ENABLE_FLAG;
3547
3548         if (features & NETIF_F_GRO)
3549                 flags |= GRO_ENABLE_FLAG;
3550         else
3551                 flags &= ~GRO_ENABLE_FLAG;
3552
3553         if (features & NETIF_F_LOOPBACK) {
3554                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
3555                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
3556                         bnx2x_reload = true;
3557                 }
3558         } else {
3559                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
3560                         bp->link_params.loopback_mode = LOOPBACK_NONE;
3561                         bnx2x_reload = true;
3562                 }
3563         }
3564
3565         if (flags ^ bp->flags) {
3566                 bp->flags = flags;
3567                 bnx2x_reload = true;
3568         }
3569
3570         if (bnx2x_reload) {
3571                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
3572                         return bnx2x_reload_if_running(dev);
3573                 /* else: bnx2x_nic_load() will be called at end of recovery */
3574         }
3575
3576         return 0;
3577 }
3578
3579 void bnx2x_tx_timeout(struct net_device *dev)
3580 {
3581         struct bnx2x *bp = netdev_priv(dev);
3582
3583 #ifdef BNX2X_STOP_ON_ERROR
3584         if (!bp->panic)
3585                 bnx2x_panic();
3586 #endif
3587
3588         smp_mb__before_clear_bit();
3589         set_bit(BNX2X_SP_RTNL_TX_TIMEOUT, &bp->sp_rtnl_state);
3590         smp_mb__after_clear_bit();
3591
3592         /* This allows the netif to be shutdown gracefully before resetting */
3593         schedule_delayed_work(&bp->sp_rtnl_task, 0);
3594 }
3595
3596 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
3597 {
3598         struct net_device *dev = pci_get_drvdata(pdev);
3599         struct bnx2x *bp;
3600
3601         if (!dev) {
3602                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3603                 return -ENODEV;
3604         }
3605         bp = netdev_priv(dev);
3606
3607         rtnl_lock();
3608
3609         pci_save_state(pdev);
3610
3611         if (!netif_running(dev)) {
3612                 rtnl_unlock();
3613                 return 0;
3614         }
3615
3616         netif_device_detach(dev);
3617
3618         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
3619
3620         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
3621
3622         rtnl_unlock();
3623
3624         return 0;
3625 }
3626
3627 int bnx2x_resume(struct pci_dev *pdev)
3628 {
3629         struct net_device *dev = pci_get_drvdata(pdev);
3630         struct bnx2x *bp;
3631         int rc;
3632
3633         if (!dev) {
3634                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
3635                 return -ENODEV;
3636         }
3637         bp = netdev_priv(dev);
3638
3639         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
3640                 BNX2X_ERR("Handling parity error recovery. Try again later\n");
3641                 return -EAGAIN;
3642         }
3643
3644         rtnl_lock();
3645
3646         pci_restore_state(pdev);
3647
3648         if (!netif_running(dev)) {
3649                 rtnl_unlock();
3650                 return 0;
3651         }
3652
3653         bnx2x_set_power_state(bp, PCI_D0);
3654         netif_device_attach(dev);
3655
3656         rc = bnx2x_nic_load(bp, LOAD_OPEN);
3657
3658         rtnl_unlock();
3659
3660         return rc;
3661 }
3662
3663
3664 void bnx2x_set_ctx_validation(struct bnx2x *bp, struct eth_context *cxt,
3665                               u32 cid)
3666 {
3667         /* ustorm cxt validation */
3668         cxt->ustorm_ag_context.cdu_usage =
3669                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3670                         CDU_REGION_NUMBER_UCM_AG, ETH_CONNECTION_TYPE);
3671         /* xcontext validation */
3672         cxt->xstorm_ag_context.cdu_reserved =
3673                 CDU_RSRVD_VALUE_TYPE_A(HW_CID(bp, cid),
3674                         CDU_REGION_NUMBER_XCM_AG, ETH_CONNECTION_TYPE);
3675 }
3676
3677 static inline void storm_memset_hc_timeout(struct bnx2x *bp, u8 port,
3678                                              u8 fw_sb_id, u8 sb_index,
3679                                              u8 ticks)
3680 {
3681
3682         u32 addr = BAR_CSTRORM_INTMEM +
3683                    CSTORM_STATUS_BLOCK_DATA_TIMEOUT_OFFSET(fw_sb_id, sb_index);
3684         REG_WR8(bp, addr, ticks);
3685         DP(NETIF_MSG_IFUP,
3686            "port %x fw_sb_id %d sb_index %d ticks %d\n",
3687            port, fw_sb_id, sb_index, ticks);
3688 }
3689
3690 static inline void storm_memset_hc_disable(struct bnx2x *bp, u8 port,
3691                                              u16 fw_sb_id, u8 sb_index,
3692                                              u8 disable)
3693 {
3694         u32 enable_flag = disable ? 0 : (1 << HC_INDEX_DATA_HC_ENABLED_SHIFT);
3695         u32 addr = BAR_CSTRORM_INTMEM +
3696                    CSTORM_STATUS_BLOCK_DATA_FLAGS_OFFSET(fw_sb_id, sb_index);
3697         u16 flags = REG_RD16(bp, addr);
3698         /* clear and set */
3699         flags &= ~HC_INDEX_DATA_HC_ENABLED;
3700         flags |= enable_flag;
3701         REG_WR16(bp, addr, flags);
3702         DP(NETIF_MSG_IFUP,
3703            "port %x fw_sb_id %d sb_index %d disable %d\n",
3704            port, fw_sb_id, sb_index, disable);
3705 }
3706
3707 void bnx2x_update_coalesce_sb_index(struct bnx2x *bp, u8 fw_sb_id,
3708                                     u8 sb_index, u8 disable, u16 usec)
3709 {
3710         int port = BP_PORT(bp);
3711         u8 ticks = usec / BNX2X_BTR;
3712
3713         storm_memset_hc_timeout(bp, port, fw_sb_id, sb_index, ticks);
3714
3715         disable = disable ? 1 : (usec ? 0 : 1);
3716         storm_memset_hc_disable(bp, port, fw_sb_id, sb_index, disable);
3717 }