drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2011 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <linux/if_vlan.h>
  20 #include <linux/ip.h>
  21 #include <net/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #include "bnx2x_init.h"
  27
  28 static int bnx2x_setup_irqs(struct bnx2x *bp);
  29
  30 /**
  31  * bnx2x_bz_fp - zero content of the fastpath structure.
  32  *
  33  * @bp:         driver handle
  34  * @index:      fastpath index to be zeroed
  35  *
  36  * Makes sure the contents of the bp->fp[index].napi is kept
  37  * intact.
  38  */
  39 static inline void bnx2x_bz_fp(struct bnx2x *bp, int index)
  40 {
  41         struct bnx2x_fastpath *fp = &bp->fp[index];
  42         struct napi_struct orig_napi = fp->napi;
  43         /* bzero bnx2x_fastpath contents */
  44         memset(fp, 0, sizeof(*fp));
  45
  46         /* Restore the NAPI object as it has been already initialized */
  47         fp->napi = orig_napi;
  48 }
  49
  50 /**
  51  * bnx2x_move_fp - move content of the fastpath structure.
  52  *
  53  * @bp:         driver handle
  54  * @from:       source FP index
  55  * @to:         destination FP index
  56  *
  57  * Makes sure the contents of the bp->fp[to].napi is kept
  58  * intact.
  59  */
  60 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
  61 {
  62         struct bnx2x_fastpath *from_fp = &bp->fp[from];
  63         struct bnx2x_fastpath *to_fp = &bp->fp[to];
  64         struct napi_struct orig_napi = to_fp->napi;
  65         /* Move bnx2x_fastpath contents */
  66         memcpy(to_fp, from_fp, sizeof(*to_fp));
  67         to_fp->index = to;
  68
  69         /* Restore the NAPI object as it has been already initialized */
  70         to_fp->napi = orig_napi;
  71 }
  72
  73 /* free skb in the packet ring at pos idx
  74  * return idx of last bd freed
  75  */
  76 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  77                              u16 idx)
  78 {
  79         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  80         struct eth_tx_start_bd *tx_start_bd;
  81         struct eth_tx_bd *tx_data_bd;
  82         struct sk_buff *skb = tx_buf->skb;
  83         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  84         int nbd;
  85
  86         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  87         prefetch(&skb->end);
  88
  89         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  90            idx, tx_buf, skb);
  91
  92         /* unmap first bd */
  93         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  94         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  95         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  96                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  97
  98         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  99 #ifdef BNX2X_STOP_ON_ERROR
 100         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
 101                 BNX2X_ERR("BAD nbd!\n");
 102                 bnx2x_panic();
 103         }
 104 #endif
 105         new_cons = nbd + tx_buf->first_bd;
 106
 107         /* Get the next bd */
 108         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 109
 110         /* Skip a parse bd... */
 111         --nbd;
 112         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 113
 114         /* ...and the TSO split header bd since they have no mapping */
 115         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
 116                 --nbd;
 117                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 118         }
 119
 120         /* now free frags */
 121         while (nbd > 0) {
 122
 123                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
 124                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
 125                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
 126                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
 127                 if (--nbd)
 128                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
 129         }
 130
 131         /* release skb */
 132         WARN_ON(!skb);
 133         dev_kfree_skb(skb);
 134         tx_buf->first_bd = 0;
 135         tx_buf->skb = NULL;
 136
 137         return new_cons;
 138 }
 139
 140 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
 141 {
 142         struct bnx2x *bp = fp->bp;
 143         struct netdev_queue *txq;
 144         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 145
 146 #ifdef BNX2X_STOP_ON_ERROR
 147         if (unlikely(bp->panic))
 148                 return -1;
 149 #endif
 150
 151         txq = netdev_get_tx_queue(bp->dev, fp->index);
 152         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 153         sw_cons = fp->tx_pkt_cons;
 154
 155         while (sw_cons != hw_cons) {
 156                 u16 pkt_cons;
 157
 158                 pkt_cons = TX_BD(sw_cons);
 159
 160                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 161                                       " pkt_cons %u\n",
 162                    fp->index, hw_cons, sw_cons, pkt_cons);
 163
 164                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 165                 sw_cons++;
 166         }
 167
 168         fp->tx_pkt_cons = sw_cons;
 169         fp->tx_bd_cons = bd_cons;
 170
 171         /* Need to make the tx_bd_cons update visible to start_xmit()
 172          * before checking for netif_tx_queue_stopped().  Without the
 173          * memory barrier, there is a small possibility that
 174          * start_xmit() will miss it and cause the queue to be stopped
 175          * forever.
 176          */
 177         smp_mb();
 178
 179         if (unlikely(netif_tx_queue_stopped(txq))) {
 180                 /* Taking tx_lock() is needed to prevent reenabling the queue
 181                  * while it's empty. This could have happen if rx_action() gets
 182                  * suspended in bnx2x_tx_int() after the condition before
 183                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 184                  *
 185                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 186                  * sends some packets consuming the whole queue again->
 187                  * stops the queue
 188                  */
 189
 190                 __netif_tx_lock(txq, smp_processor_id());
 191
 192                 if ((netif_tx_queue_stopped(txq)) &&
 193                     (bp->state == BNX2X_STATE_OPEN) &&
 194                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 195                         netif_tx_wake_queue(txq);
 196
 197                 __netif_tx_unlock(txq);
 198         }
 199         return 0;
 200 }
 201
 202 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 203                                              u16 idx)
 204 {
 205         u16 last_max = fp->last_max_sge;
 206
 207         if (SUB_S16(idx, last_max) > 0)
 208                 fp->last_max_sge = idx;
 209 }
 210
 211 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 212                                   struct eth_fast_path_rx_cqe *fp_cqe)
 213 {
 214         struct bnx2x *bp = fp->bp;
 215         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 216                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 217                       SGE_PAGE_SHIFT;
 218         u16 last_max, last_elem, first_elem;
 219         u16 delta = 0;
 220         u16 i;
 221
 222         if (!sge_len)
 223                 return;
 224
 225         /* First mark all used pages */
 226         for (i = 0; i < sge_len; i++)
 227                 SGE_MASK_CLEAR_BIT(fp,
 228                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 229
 230         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 231            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 232
 233         /* Here we assume that the last SGE index is the biggest */
 234         prefetch((void *)(fp->sge_mask));
 235         bnx2x_update_last_max_sge(fp,
 236                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 237
 238         last_max = RX_SGE(fp->last_max_sge);
 239         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 240         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 241
 242         /* If ring is not full */
 243         if (last_elem + 1 != first_elem)
 244                 last_elem++;
 245
 246         /* Now update the prod */
 247         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 248                 if (likely(fp->sge_mask[i]))
 249                         break;
 250
 251                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 252                 delta += RX_SGE_MASK_ELEM_SZ;
 253         }
 254
 255         if (delta > 0) {
 256                 fp->rx_sge_prod += delta;
 257                 /* clear page-end entries */
 258                 bnx2x_clear_sge_mask_next_elems(fp);
 259         }
 260
 261         DP(NETIF_MSG_RX_STATUS,
 262            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 263            fp->last_max_sge, fp->rx_sge_prod);
 264 }
 265
 266 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 267                             struct sk_buff *skb, u16 cons, u16 prod)
 268 {
 269         struct bnx2x *bp = fp->bp;
 270         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 271         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 272         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 273         dma_addr_t mapping;
 274
 275         /* move empty skb from pool to prod and map it */
 276         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 277         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 278                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 279         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 280
 281         /* move partial skb from cons to pool (don't unmap yet) */
 282         fp->tpa_pool[queue] = *cons_rx_buf;
 283
 284         /* mark bin state as start - print error if current state != stop */
 285         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 286                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 287
 288         fp->tpa_state[queue] = BNX2X_TPA_START;
 289
 290         /* point prod_bd to new skb */
 291         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 292         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 293
 294 #ifdef BNX2X_STOP_ON_ERROR
 295         fp->tpa_queue_used |= (1 << queue);
 296 #ifdef _ASM_GENERIC_INT_L64_H
 297         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 298 #else
 299         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 300 #endif
 301            fp->tpa_queue_used);
 302 #endif
 303 }
 304
 305 /* Timestamp option length allowed for TPA aggregation:
 306  *
 307  *              nop nop kind length echo val
 308  */
 309 #define TPA_TSTAMP_OPT_LEN      12
 310 /**
 311  * bnx2x_set_lro_mss - calculate the approximate value of the MSS
 312  *
 313  * @bp:                 driver handle
 314  * @parsing_flags:      parsing flags from the START CQE
 315  * @len_on_bd:          total length of the first packet for the
 316  *                      aggregation.
 317  *
 318  * Approximate value of the MSS for this aggregation calculated using
 319  * the first packet of it.
 320  */
 321 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 322                                     u16 len_on_bd)
 323 {
 324         /* TPA arrgregation won't have an IP options and TCP options
 325          * other than timestamp.
 326          */
 327         u16 hdrs_len = ETH_HLEN + sizeof(struct iphdr) + sizeof(struct tcphdr);
 328
 329
 330         /* Check if there was a TCP timestamp, if there is it's will
 331          * always be 12 bytes length: nop nop kind length echo val.
 332          *
 333          * Otherwise FW would close the aggregation.
 334          */
 335         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 336                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 337
 338         return len_on_bd - hdrs_len;
 339 }
 340
 341 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 342                                struct sk_buff *skb,
 343                                struct eth_fast_path_rx_cqe *fp_cqe,
 344                                u16 cqe_idx, u16 parsing_flags)
 345 {
 346         struct sw_rx_page *rx_pg, old_rx_pg;
 347         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 348         u32 i, frag_len, frag_size, pages;
 349         int err;
 350         int j;
 351
 352         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 353         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 354
 355         /* This is needed in order to enable forwarding support */
 356         if (frag_size)
 357                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp, parsing_flags,
 358                                                               len_on_bd);
 359
 360 #ifdef BNX2X_STOP_ON_ERROR
 361         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 362                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 363                           pages, cqe_idx);
 364                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 365                           fp_cqe->pkt_len, len_on_bd);
 366                 bnx2x_panic();
 367                 return -EINVAL;
 368         }
 369 #endif
 370
 371         /* Run through the SGL and compose the fragmented skb */
 372         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 373                 u16 sge_idx =
 374                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
 375
 376                 /* FW gives the indices of the SGE as if the ring is an array
 377                    (meaning that "next" element will consume 2 indices) */
 378                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 379                 rx_pg = &fp->rx_page_ring[sge_idx];
 380                 old_rx_pg = *rx_pg;
 381
 382                 /* If we fail to allocate a substitute page, we simply stop
 383                    where we are and drop the whole packet */
 384                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 385                 if (unlikely(err)) {
 386                         fp->eth_q_stats.rx_skb_alloc_failed++;
 387                         return err;
 388                 }
 389
 390                 /* Unmap the page as we r going to pass it to the stack */
 391                 dma_unmap_page(&bp->pdev->dev,
 392                                dma_unmap_addr(&old_rx_pg, mapping),
 393                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 394
 395                 /* Add one frag and update the appropriate fields in the skb */
 396                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 397
 398                 skb->data_len += frag_len;
 399                 skb->truesize += frag_len;
 400                 skb->len += frag_len;
 401
 402                 frag_size -= frag_len;
 403         }
 404
 405         return 0;
 406 }
 407
 408 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 409                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 410                            u16 cqe_idx)
 411 {
 412         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 413         struct sk_buff *skb = rx_buf->skb;
 414         /* alloc new skb */
 415         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 416
 417         /* Unmap skb in the pool anyway, as we are going to change
 418            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 419            fails. */
 420         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 421                          fp->rx_buf_size, DMA_FROM_DEVICE);
 422
 423         if (likely(new_skb)) {
 424                 /* fix ip xsum and give it to the stack */
 425                 /* (no need to map the new skb) */
 426                 u16 parsing_flags =
 427                         le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags);
 428
 429                 prefetch(skb);
 430                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 431
 432 #ifdef BNX2X_STOP_ON_ERROR
 433                 if (pad + len > fp->rx_buf_size) {
 434                         BNX2X_ERR("skb_put is about to fail...  "
 435                                   "pad %d  len %d  rx_buf_size %d\n",
 436                                   pad, len, fp->rx_buf_size);
 437                         bnx2x_panic();
 438                         return;
 439                 }
 440 #endif
 441
 442                 skb_reserve(skb, pad);
 443                 skb_put(skb, len);
 444
 445                 skb->protocol = eth_type_trans(skb, bp->dev);
 446                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 447
 448                 {
 449                         struct iphdr *iph;
 450
 451                         iph = (struct iphdr *)skb->data;
 452                         iph->check = 0;
 453                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 454                 }
 455
 456                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 457                                          &cqe->fast_path_cqe, cqe_idx,
 458                                          parsing_flags)) {
 459                         if (parsing_flags & PARSING_FLAGS_VLAN)
 460                                 __vlan_hwaccel_put_tag(skb,
 461                                                  le16_to_cpu(cqe->fast_path_cqe.
 462                                                              vlan_tag));
 463                         napi_gro_receive(&fp->napi, skb);
 464                 } else {
 465                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 466                            " - dropping packet!\n");
 467                         dev_kfree_skb(skb);
 468                 }
 469
 470
 471                 /* put new skb in bin */
 472                 fp->tpa_pool[queue].skb = new_skb;
 473
 474         } else {
 475                 /* else drop the packet and keep the buffer in the bin */
 476                 DP(NETIF_MSG_RX_STATUS,
 477                    "Failed to allocate new skb - dropping packet!\n");
 478                 fp->eth_q_stats.rx_skb_alloc_failed++;
 479         }
 480
 481         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 482 }
 483
 484 /* Set Toeplitz hash value in the skb using the value from the
 485  * CQE (calculated by HW).
 486  */
 487 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 488                                         struct sk_buff *skb)
 489 {
 490         /* Set Toeplitz hash from CQE */
 491         if ((bp->dev->features & NETIF_F_RXHASH) &&
 492             (cqe->fast_path_cqe.status_flags &
 493              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 494                 skb->rxhash =
 495                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 496 }
 497
 498 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 499 {
 500         struct bnx2x *bp = fp->bp;
 501         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 502         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 503         int rx_pkt = 0;
 504
 505 #ifdef BNX2X_STOP_ON_ERROR
 506         if (unlikely(bp->panic))
 507                 return 0;
 508 #endif
 509
 510         /* CQ "next element" is of the size of the regular element,
 511            that's why it's ok here */
 512         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 513         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 514                 hw_comp_cons++;
 515
 516         bd_cons = fp->rx_bd_cons;
 517         bd_prod = fp->rx_bd_prod;
 518         bd_prod_fw = bd_prod;
 519         sw_comp_cons = fp->rx_comp_cons;
 520         sw_comp_prod = fp->rx_comp_prod;
 521
 522         /* Memory barrier necessary as speculative reads of the rx
 523          * buffer can be ahead of the index in the status block
 524          */
 525         rmb();
 526
 527         DP(NETIF_MSG_RX_STATUS,
 528            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 529            fp->index, hw_comp_cons, sw_comp_cons);
 530
 531         while (sw_comp_cons != hw_comp_cons) {
 532                 struct sw_rx_bd *rx_buf = NULL;
 533                 struct sk_buff *skb;
 534                 union eth_rx_cqe *cqe;
 535                 u8 cqe_fp_flags;
 536                 u16 len, pad;
 537
 538                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 539                 bd_prod = RX_BD(bd_prod);
 540                 bd_cons = RX_BD(bd_cons);
 541
 542                 /* Prefetch the page containing the BD descriptor
 543                    at producer's index. It will be needed when new skb is
 544                    allocated */
 545                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 546                                              (&fp->rx_desc_ring[bd_prod])) -
 547                                   PAGE_SIZE + 1));
 548
 549                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 550                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 551
 552                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 553                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 554                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 555                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 556                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 557                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 558
 559                 /* is this a slowpath msg? */
 560                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 561                         bnx2x_sp_event(fp, cqe);
 562                         goto next_cqe;
 563
 564                 /* this is an rx packet */
 565                 } else {
 566                         rx_buf = &fp->rx_buf_ring[bd_cons];
 567                         skb = rx_buf->skb;
 568                         prefetch(skb);
 569                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 570                         pad = cqe->fast_path_cqe.placement_offset;
 571
 572                         /* - If CQE is marked both TPA_START and TPA_END it is
 573                          *   a non-TPA CQE.
 574                          * - FP CQE will always have either TPA_START or/and
 575                          *   TPA_STOP flags set.
 576                          */
 577                         if ((!fp->disable_tpa) &&
 578                             (TPA_TYPE(cqe_fp_flags) !=
 579                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 580                                 u16 queue = cqe->fast_path_cqe.queue_index;
 581
 582                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 583                                         DP(NETIF_MSG_RX_STATUS,
 584                                            "calling tpa_start on queue %d\n",
 585                                            queue);
 586
 587                                         bnx2x_tpa_start(fp, queue, skb,
 588                                                         bd_cons, bd_prod);
 589
 590                                         /* Set Toeplitz hash for an LRO skb */
 591                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 592
 593                                         goto next_rx;
 594                                 } else { /* TPA_STOP */
 595                                         DP(NETIF_MSG_RX_STATUS,
 596                                            "calling tpa_stop on queue %d\n",
 597                                            queue);
 598
 599                                         if (!BNX2X_RX_SUM_FIX(cqe))
 600                                                 BNX2X_ERR("STOP on none TCP "
 601                                                           "data\n");
 602
 603                                         /* This is a size of the linear data
 604                                            on this skb */
 605                                         len = le16_to_cpu(cqe->fast_path_cqe.
 606                                                                 len_on_bd);
 607                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 608                                                     len, cqe, comp_ring_cons);
 609 #ifdef BNX2X_STOP_ON_ERROR
 610                                         if (bp->panic)
 611                                                 return 0;
 612 #endif
 613
 614                                         bnx2x_update_sge_prod(fp,
 615                                                         &cqe->fast_path_cqe);
 616                                         goto next_cqe;
 617                                 }
 618                         }
 619
 620                         dma_sync_single_for_device(&bp->pdev->dev,
 621                                         dma_unmap_addr(rx_buf, mapping),
 622                                                    pad + RX_COPY_THRESH,
 623                                                    DMA_FROM_DEVICE);
 624                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 625
 626                         /* is this an error packet? */
 627                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 628                                 DP(NETIF_MSG_RX_ERR,
 629                                    "ERROR  flags %x  rx packet %u\n",
 630                                    cqe_fp_flags, sw_comp_cons);
 631                                 fp->eth_q_stats.rx_err_discard_pkt++;
 632                                 goto reuse_rx;
 633                         }
 634
 635                         /* Since we don't have a jumbo ring
 636                          * copy small packets if mtu > 1500
 637                          */
 638                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 639                             (len <= RX_COPY_THRESH)) {
 640                                 struct sk_buff *new_skb;
 641
 642                                 new_skb = netdev_alloc_skb(bp->dev,
 643                                                            len + pad);
 644                                 if (new_skb == NULL) {
 645                                         DP(NETIF_MSG_RX_ERR,
 646                                            "ERROR  packet dropped "
 647                                            "because of alloc failure\n");
 648                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 649                                         goto reuse_rx;
 650                                 }
 651
 652                                 /* aligned copy */
 653                                 skb_copy_from_linear_data_offset(skb, pad,
 654                                                     new_skb->data + pad, len);
 655                                 skb_reserve(new_skb, pad);
 656                                 skb_put(new_skb, len);
 657
 658                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 659
 660                                 skb = new_skb;
 661
 662                         } else
 663                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 664                                 dma_unmap_single(&bp->pdev->dev,
 665                                         dma_unmap_addr(rx_buf, mapping),
 666                                                  fp->rx_buf_size,
 667                                                  DMA_FROM_DEVICE);
 668                                 skb_reserve(skb, pad);
 669                                 skb_put(skb, len);
 670
 671                         } else {
 672                                 DP(NETIF_MSG_RX_ERR,
 673                                    "ERROR  packet dropped because "
 674                                    "of alloc failure\n");
 675                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 676 reuse_rx:
 677                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 678                                 goto next_rx;
 679                         }
 680
 681                         skb->protocol = eth_type_trans(skb, bp->dev);
 682
 683                         /* Set Toeplitz hash for a none-LRO skb */
 684                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 685
 686                         skb_checksum_none_assert(skb);
 687
 688                         if (bp->dev->features & NETIF_F_RXCSUM) {
 689                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 690                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 691                                 else
 692                                         fp->eth_q_stats.hw_csum_err++;
 693                         }
 694                 }
 695
 696                 skb_record_rx_queue(skb, fp->index);
 697
 698                 if (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 699                      PARSING_FLAGS_VLAN)
 700                         __vlan_hwaccel_put_tag(skb,
 701                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
 702                 napi_gro_receive(&fp->napi, skb);
 703
 704
 705 next_rx:
 706                 rx_buf->skb = NULL;
 707
 708                 bd_cons = NEXT_RX_IDX(bd_cons);
 709                 bd_prod = NEXT_RX_IDX(bd_prod);
 710                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 711                 rx_pkt++;
 712 next_cqe:
 713                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 714                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 715
 716                 if (rx_pkt == budget)
 717                         break;
 718         } /* while */
 719
 720         fp->rx_bd_cons = bd_cons;
 721         fp->rx_bd_prod = bd_prod_fw;
 722         fp->rx_comp_cons = sw_comp_cons;
 723         fp->rx_comp_prod = sw_comp_prod;
 724
 725         /* Update producers */
 726         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 727                              fp->rx_sge_prod);
 728
 729         fp->rx_pkt += rx_pkt;
 730         fp->rx_calls++;
 731
 732         return rx_pkt;
 733 }
 734
 735 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 736 {
 737         struct bnx2x_fastpath *fp = fp_cookie;
 738         struct bnx2x *bp = fp->bp;
 739
 740         /* Return here if interrupt is disabled */
 741         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 742                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 743                 return IRQ_HANDLED;
 744         }
 745
 746         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 747                          "[fp %d fw_sd %d igusb %d]\n",
 748            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 749         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 750
 751 #ifdef BNX2X_STOP_ON_ERROR
 752         if (unlikely(bp->panic))
 753                 return IRQ_HANDLED;
 754 #endif
 755
 756         /* Handle Rx and Tx according to MSI-X vector */
 757         prefetch(fp->rx_cons_sb);
 758         prefetch(fp->tx_cons_sb);
 759         prefetch(&fp->sb_running_index[SM_RX_ID]);
 760         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 761
 762         return IRQ_HANDLED;
 763 }
 764
 765 /* HW Lock for shared dual port PHYs */
 766 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 767 {
 768         mutex_lock(&bp->port.phy_mutex);
 769
 770         if (bp->port.need_hw_lock)
 771                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 772 }
 773
 774 void bnx2x_release_phy_lock(struct bnx2x *bp)
 775 {
 776         if (bp->port.need_hw_lock)
 777                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 778
 779         mutex_unlock(&bp->port.phy_mutex);
 780 }
 781
 782 /* calculates MF speed according to current linespeed and MF configuration */
 783 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 784 {
 785         u16 line_speed = bp->link_vars.line_speed;
 786         if (IS_MF(bp)) {
 787                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 788                                                    bp->mf_config[BP_VN(bp)]);
 789
 790                 /* Calculate the current MAX line speed limit for the MF
 791                  * devices
 792                  */
 793                 if (IS_MF_SI(bp))
 794                         line_speed = (line_speed * maxCfg) / 100;
 795                 else { /* SD mode */
 796                         u16 vn_max_rate = maxCfg * 100;
 797
 798                         if (vn_max_rate < line_speed)
 799                                 line_speed = vn_max_rate;
 800                 }
 801         }
 802
 803         return line_speed;
 804 }
 805
 806 /**
 807  * bnx2x_fill_report_data - fill link report data to report
 808  *
 809  * @bp:         driver handle
 810  * @data:       link state to update
 811  *
 812  * It uses a none-atomic bit operations because is called under the mutex.
 813  */
 814 static inline void bnx2x_fill_report_data(struct bnx2x *bp,
 815                                           struct bnx2x_link_report_data *data)
 816 {
 817         u16 line_speed = bnx2x_get_mf_speed(bp);
 818
 819         memset(data, 0, sizeof(*data));
 820
 821         /* Fill the report data: efective line speed */
 822         data->line_speed = line_speed;
 823
 824         /* Link is down */
 825         if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
 826                 __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 827                           &data->link_report_flags);
 828
 829         /* Full DUPLEX */
 830         if (bp->link_vars.duplex == DUPLEX_FULL)
 831                 __set_bit(BNX2X_LINK_REPORT_FD, &data->link_report_flags);
 832
 833         /* Rx Flow Control is ON */
 834         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
 835                 __set_bit(BNX2X_LINK_REPORT_RX_FC_ON, &data->link_report_flags);
 836
 837         /* Tx Flow Control is ON */
 838         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
 839                 __set_bit(BNX2X_LINK_REPORT_TX_FC_ON, &data->link_report_flags);
 840 }
 841
 842 /**
 843  * bnx2x_link_report - report link status to OS.
 844  *
 845  * @bp:         driver handle
 846  *
 847  * Calls the __bnx2x_link_report() under the same locking scheme
 848  * as a link/PHY state managing code to ensure a consistent link
 849  * reporting.
 850  */
 851
 852 void bnx2x_link_report(struct bnx2x *bp)
 853 {
 854         bnx2x_acquire_phy_lock(bp);
 855         __bnx2x_link_report(bp);
 856         bnx2x_release_phy_lock(bp);
 857 }
 858
 859 /**
 860  * __bnx2x_link_report - report link status to OS.
 861  *
 862  * @bp:         driver handle
 863  *
 864  * None atomic inmlementation.
 865  * Should be called under the phy_lock.
 866  */
 867 void __bnx2x_link_report(struct bnx2x *bp)
 868 {
 869         struct bnx2x_link_report_data cur_data;
 870
 871         /* reread mf_cfg */
 872         if (!CHIP_IS_E1(bp))
 873                 bnx2x_read_mf_cfg(bp);
 874
 875         /* Read the current link report info */
 876         bnx2x_fill_report_data(bp, &cur_data);
 877
 878         /* Don't report link down or exactly the same link status twice */
 879         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
 880             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 881                       &bp->last_reported_link.link_report_flags) &&
 882              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 883                       &cur_data.link_report_flags)))
 884                 return;
 885
 886         bp->link_cnt++;
 887
 888         /* We are going to report a new link parameters now -
 889          * remember the current data for the next time.
 890          */
 891         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
 892
 893         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
 894                      &cur_data.link_report_flags)) {
 895                 netif_carrier_off(bp->dev);
 896                 netdev_err(bp->dev, "NIC Link is Down\n");
 897                 return;
 898         } else {
 899                 netif_carrier_on(bp->dev);
 900                 netdev_info(bp->dev, "NIC Link is Up, ");
 901                 pr_cont("%d Mbps ", cur_data.line_speed);
 902
 903                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
 904                                        &cur_data.link_report_flags))
 905                         pr_cont("full duplex");
 906                 else
 907                         pr_cont("half duplex");
 908
 909                 /* Handle the FC at the end so that only these flags would be
 910                  * possibly set. This way we may easily check if there is no FC
 911                  * enabled.
 912                  */
 913                 if (cur_data.link_report_flags) {
 914                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
 915                                      &cur_data.link_report_flags)) {
 916                                 pr_cont(", receive ");
 917                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
 918                                      &cur_data.link_report_flags))
 919                                         pr_cont("& transmit ");
 920                         } else {
 921                                 pr_cont(", transmit ");
 922                         }
 923                         pr_cont("flow control ON");
 924                 }
 925                 pr_cont("\n");
 926         }
 927 }
 928
 929 void bnx2x_init_rx_rings(struct bnx2x *bp)
 930 {
 931         int func = BP_FUNC(bp);
 932         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 933                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 934         u16 ring_prod;
 935         int i, j;
 936
 937         /* Allocate TPA resources */
 938         for_each_rx_queue(bp, j) {
 939                 struct bnx2x_fastpath *fp = &bp->fp[j];
 940
 941                 DP(NETIF_MSG_IFUP,
 942                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
 943
 944                 if (!fp->disable_tpa) {
 945                         /* Fill the per-aggregation pool */
 946                         for (i = 0; i < max_agg_queues; i++) {
 947                                 fp->tpa_pool[i].skb =
 948                                    netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 949                                 if (!fp->tpa_pool[i].skb) {
 950                                         BNX2X_ERR("Failed to allocate TPA "
 951                                                   "skb pool for queue[%d] - "
 952                                                   "disabling TPA on this "
 953                                                   "queue!\n", j);
 954                                         bnx2x_free_tpa_pool(bp, fp, i);
 955                                         fp->disable_tpa = 1;
 956                                         break;
 957                                 }
 958                                 dma_unmap_addr_set((struct sw_rx_bd *)
 959                                                         &bp->fp->tpa_pool[i],
 960                                                    mapping, 0);
 961                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 962                         }
 963
 964                         /* "next page" elements initialization */
 965                         bnx2x_set_next_page_sgl(fp);
 966
 967                         /* set SGEs bit mask */
 968                         bnx2x_init_sge_ring_bit_mask(fp);
 969
 970                         /* Allocate SGEs and initialize the ring elements */
 971                         for (i = 0, ring_prod = 0;
 972                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 973
 974                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 975                                         BNX2X_ERR("was only able to allocate "
 976                                                   "%d rx sges\n", i);
 977                                         BNX2X_ERR("disabling TPA for"
 978                                                   " queue[%d]\n", j);
 979                                         /* Cleanup already allocated elements */
 980                                         bnx2x_free_rx_sge_range(bp,
 981                                                                 fp, ring_prod);
 982                                         bnx2x_free_tpa_pool(bp,
 983                                                             fp, max_agg_queues);
 984                                         fp->disable_tpa = 1;
 985                                         ring_prod = 0;
 986                                         break;
 987                                 }
 988                                 ring_prod = NEXT_SGE_IDX(ring_prod);
 989                         }
 990
 991                         fp->rx_sge_prod = ring_prod;
 992                 }
 993         }
 994
 995         for_each_rx_queue(bp, j) {
 996                 struct bnx2x_fastpath *fp = &bp->fp[j];
 997
 998                 fp->rx_bd_cons = 0;
 999
1000                 /* Activate BD ring */
1001                 /* Warning!
1002                  * this will generate an interrupt (to the TSTORM)
1003                  * must only be done after chip is initialized
1004                  */
1005                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1006                                      fp->rx_sge_prod);
1007
1008                 if (j != 0)
1009                         continue;
1010
1011                 if (!CHIP_IS_E2(bp)) {
1012                         REG_WR(bp, BAR_USTRORM_INTMEM +
1013                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1014                                U64_LO(fp->rx_comp_mapping));
1015                         REG_WR(bp, BAR_USTRORM_INTMEM +
1016                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1017                                U64_HI(fp->rx_comp_mapping));
1018                 }
1019         }
1020 }
1021
1022 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1023 {
1024         int i;
1025
1026         for_each_tx_queue(bp, i) {
1027                 struct bnx2x_fastpath *fp = &bp->fp[i];
1028
1029                 u16 bd_cons = fp->tx_bd_cons;
1030                 u16 sw_prod = fp->tx_pkt_prod;
1031                 u16 sw_cons = fp->tx_pkt_cons;
1032
1033                 while (sw_cons != sw_prod) {
1034                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
1035                         sw_cons++;
1036                 }
1037         }
1038 }
1039
1040 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1041 {
1042         struct bnx2x *bp = fp->bp;
1043         int i;
1044
1045         /* ring wasn't allocated */
1046         if (fp->rx_buf_ring == NULL)
1047                 return;
1048
1049         for (i = 0; i < NUM_RX_BD; i++) {
1050                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1051                 struct sk_buff *skb = rx_buf->skb;
1052
1053                 if (skb == NULL)
1054                         continue;
1055
1056                 dma_unmap_single(&bp->pdev->dev,
1057                                  dma_unmap_addr(rx_buf, mapping),
1058                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1059
1060                 rx_buf->skb = NULL;
1061                 dev_kfree_skb(skb);
1062         }
1063 }
1064
1065 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1066 {
1067         int j;
1068
1069         for_each_rx_queue(bp, j) {
1070                 struct bnx2x_fastpath *fp = &bp->fp[j];
1071
1072                 bnx2x_free_rx_bds(fp);
1073
1074                 if (!fp->disable_tpa)
1075                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
1076                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
1077                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
1078         }
1079 }
1080
1081 void bnx2x_free_skbs(struct bnx2x *bp)
1082 {
1083         bnx2x_free_tx_skbs(bp);
1084         bnx2x_free_rx_skbs(bp);
1085 }
1086
1087 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
1088 {
1089         /* load old values */
1090         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1091
1092         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1093                 /* leave all but MAX value */
1094                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1095
1096                 /* set new MAX value */
1097                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1098                                 & FUNC_MF_CFG_MAX_BW_MASK;
1099
1100                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1101         }
1102 }
1103
1104 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
1105 {
1106         int i, offset = 1;
1107
1108         free_irq(bp->msix_table[0].vector, bp->dev);
1109         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1110            bp->msix_table[0].vector);
1111
1112 #ifdef BCM_CNIC
1113         offset++;
1114 #endif
1115         for_each_eth_queue(bp, i) {
1116                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
1117                    "state %x\n", i, bp->msix_table[i + offset].vector,
1118                    bnx2x_fp(bp, i, state));
1119
1120                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
1121         }
1122 }
1123
1124 void bnx2x_free_irq(struct bnx2x *bp)
1125 {
1126         if (bp->flags & USING_MSIX_FLAG)
1127                 bnx2x_free_msix_irqs(bp);
1128         else if (bp->flags & USING_MSI_FLAG)
1129                 free_irq(bp->pdev->irq, bp->dev);
1130         else
1131                 free_irq(bp->pdev->irq, bp->dev);
1132 }
1133
1134 int bnx2x_enable_msix(struct bnx2x *bp)
1135 {
1136         int msix_vec = 0, i, rc, req_cnt;
1137
1138         bp->msix_table[msix_vec].entry = msix_vec;
1139         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1140            bp->msix_table[0].entry);
1141         msix_vec++;
1142
1143 #ifdef BCM_CNIC
1144         bp->msix_table[msix_vec].entry = msix_vec;
1145         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1146            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1147         msix_vec++;
1148 #endif
1149         for_each_eth_queue(bp, i) {
1150                 bp->msix_table[msix_vec].entry = msix_vec;
1151                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1152                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1153                 msix_vec++;
1154         }
1155
1156         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1157
1158         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1159
1160         /*
1161          * reconfigure number of tx/rx queues according to available
1162          * MSI-X vectors
1163          */
1164         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1165                 /* how less vectors we will have? */
1166                 int diff = req_cnt - rc;
1167
1168                 DP(NETIF_MSG_IFUP,
1169                    "Trying to use less MSI-X vectors: %d\n", rc);
1170
1171                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1172
1173                 if (rc) {
1174                         DP(NETIF_MSG_IFUP,
1175                            "MSI-X is not attainable  rc %d\n", rc);
1176                         return rc;
1177                 }
1178                 /*
1179                  * decrease number of queues by number of unallocated entries
1180                  */
1181                 bp->num_queues -= diff;
1182
1183                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1184                                   bp->num_queues);
1185         } else if (rc) {
1186                 /* fall to INTx if not enough memory */
1187                 if (rc == -ENOMEM)
1188                         bp->flags |= DISABLE_MSI_FLAG;
1189                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1190                 return rc;
1191         }
1192
1193         bp->flags |= USING_MSIX_FLAG;
1194
1195         return 0;
1196 }
1197
1198 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1199 {
1200         int i, rc, offset = 1;
1201
1202         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1203                          bp->dev->name, bp->dev);
1204         if (rc) {
1205                 BNX2X_ERR("request sp irq failed\n");
1206                 return -EBUSY;
1207         }
1208
1209 #ifdef BCM_CNIC
1210         offset++;
1211 #endif
1212         for_each_eth_queue(bp, i) {
1213                 struct bnx2x_fastpath *fp = &bp->fp[i];
1214                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1215                          bp->dev->name, i);
1216
1217                 rc = request_irq(bp->msix_table[offset].vector,
1218                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1219                 if (rc) {
1220                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1221                         bnx2x_free_msix_irqs(bp);
1222                         return -EBUSY;
1223                 }
1224
1225                 offset++;
1226                 fp->state = BNX2X_FP_STATE_IRQ;
1227         }
1228
1229         i = BNX2X_NUM_ETH_QUEUES(bp);
1230         offset = 1 + CNIC_CONTEXT_USE;
1231         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1232                " ... fp[%d] %d\n",
1233                bp->msix_table[0].vector,
1234                0, bp->msix_table[offset].vector,
1235                i - 1, bp->msix_table[offset + i - 1].vector);
1236
1237         return 0;
1238 }
1239
1240 int bnx2x_enable_msi(struct bnx2x *bp)
1241 {
1242         int rc;
1243
1244         rc = pci_enable_msi(bp->pdev);
1245         if (rc) {
1246                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1247                 return -1;
1248         }
1249         bp->flags |= USING_MSI_FLAG;
1250
1251         return 0;
1252 }
1253
1254 static int bnx2x_req_irq(struct bnx2x *bp)
1255 {
1256         unsigned long flags;
1257         int rc;
1258
1259         if (bp->flags & USING_MSI_FLAG)
1260                 flags = 0;
1261         else
1262                 flags = IRQF_SHARED;
1263
1264         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1265                          bp->dev->name, bp->dev);
1266         if (!rc)
1267                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1268
1269         return rc;
1270 }
1271
1272 static void bnx2x_napi_enable(struct bnx2x *bp)
1273 {
1274         int i;
1275
1276         for_each_napi_queue(bp, i)
1277                 napi_enable(&bnx2x_fp(bp, i, napi));
1278 }
1279
1280 static void bnx2x_napi_disable(struct bnx2x *bp)
1281 {
1282         int i;
1283
1284         for_each_napi_queue(bp, i)
1285                 napi_disable(&bnx2x_fp(bp, i, napi));
1286 }
1287
1288 void bnx2x_netif_start(struct bnx2x *bp)
1289 {
1290         int intr_sem;
1291
1292         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1293         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1294
1295         if (intr_sem) {
1296                 if (netif_running(bp->dev)) {
1297                         bnx2x_napi_enable(bp);
1298                         bnx2x_int_enable(bp);
1299                         if (bp->state == BNX2X_STATE_OPEN)
1300                                 netif_tx_wake_all_queues(bp->dev);
1301                 }
1302         }
1303 }
1304
1305 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1306 {
1307         bnx2x_int_disable_sync(bp, disable_hw);
1308         bnx2x_napi_disable(bp);
1309         netif_tx_disable(bp->dev);
1310 }
1311
1312 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1313 {
1314 #ifdef BCM_CNIC
1315         struct bnx2x *bp = netdev_priv(dev);
1316         if (NO_FCOE(bp))
1317                 return skb_tx_hash(dev, skb);
1318         else {
1319                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1320                 u16 ether_type = ntohs(hdr->h_proto);
1321
1322                 /* Skip VLAN tag if present */
1323                 if (ether_type == ETH_P_8021Q) {
1324                         struct vlan_ethhdr *vhdr =
1325                                 (struct vlan_ethhdr *)skb->data;
1326
1327                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1328                 }
1329
1330                 /* If ethertype is FCoE or FIP - use FCoE ring */
1331                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1332                         return bnx2x_fcoe(bp, index);
1333         }
1334 #endif
1335         /* Select a none-FCoE queue:  if FCoE is enabled, exclude FCoE L2 ring
1336          */
1337         return __skb_tx_hash(dev, skb,
1338                         dev->real_num_tx_queues - FCOE_CONTEXT_USE);
1339 }
1340
1341 void bnx2x_set_num_queues(struct bnx2x *bp)
1342 {
1343         switch (bp->multi_mode) {
1344         case ETH_RSS_MODE_DISABLED:
1345                 bp->num_queues = 1;
1346                 break;
1347         case ETH_RSS_MODE_REGULAR:
1348                 bp->num_queues = bnx2x_calc_num_queues(bp);
1349                 break;
1350
1351         default:
1352                 bp->num_queues = 1;
1353                 break;
1354         }
1355
1356         /* Add special queues */
1357         bp->num_queues += NONE_ETH_CONTEXT_USE;
1358 }
1359
1360 #ifdef BCM_CNIC
1361 static inline void bnx2x_set_fcoe_eth_macs(struct bnx2x *bp)
1362 {
1363         if (!NO_FCOE(bp)) {
1364                 if (!IS_MF_SD(bp))
1365                         bnx2x_set_fip_eth_mac_addr(bp, 1);
1366                 bnx2x_set_all_enode_macs(bp, 1);
1367                 bp->flags |= FCOE_MACS_SET;
1368         }
1369 }
1370 #endif
1371
1372 static void bnx2x_release_firmware(struct bnx2x *bp)
1373 {
1374         kfree(bp->init_ops_offsets);
1375         kfree(bp->init_ops);
1376         kfree(bp->init_data);
1377         release_firmware(bp->firmware);
1378 }
1379
1380 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1381 {
1382         int rc, num = bp->num_queues;
1383
1384 #ifdef BCM_CNIC
1385         if (NO_FCOE(bp))
1386                 num -= FCOE_CONTEXT_USE;
1387
1388 #endif
1389         netif_set_real_num_tx_queues(bp->dev, num);
1390         rc = netif_set_real_num_rx_queues(bp->dev, num);
1391         return rc;
1392 }
1393
1394 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1395 {
1396         int i;
1397
1398         for_each_queue(bp, i) {
1399                 struct bnx2x_fastpath *fp = &bp->fp[i];
1400
1401                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1402                 if (IS_FCOE_IDX(i))
1403                         /*
1404                          * Although there are no IP frames expected to arrive to
1405                          * this ring we still want to add an
1406                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1407                          * overrun attack.
1408                          */
1409                         fp->rx_buf_size =
1410                                 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1411                                 BNX2X_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1412                 else
1413                         fp->rx_buf_size =
1414                                 bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
1415                                 IP_HEADER_ALIGNMENT_PADDING;
1416         }
1417 }
1418
1419 /* must be called with rtnl_lock */
1420 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1421 {
1422         u32 load_code;
1423         int i, rc;
1424
1425         /* Set init arrays */
1426         rc = bnx2x_init_firmware(bp);
1427         if (rc) {
1428                 BNX2X_ERR("Error loading firmware\n");
1429                 return rc;
1430         }
1431
1432 #ifdef BNX2X_STOP_ON_ERROR
1433         if (unlikely(bp->panic))
1434                 return -EPERM;
1435 #endif
1436
1437         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1438
1439         /* Set the initial link reported state to link down */
1440         bnx2x_acquire_phy_lock(bp);
1441         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
1442         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1443                 &bp->last_reported_link.link_report_flags);
1444         bnx2x_release_phy_lock(bp);
1445
1446         /* must be called before memory allocation and HW init */
1447         bnx2x_ilt_set_info(bp);
1448
1449         /* zero fastpath structures preserving invariants like napi which are
1450          * allocated only once
1451          */
1452         for_each_queue(bp, i)
1453                 bnx2x_bz_fp(bp, i);
1454
1455         /* Set the receive queues buffer size */
1456         bnx2x_set_rx_buf_size(bp);
1457
1458         for_each_queue(bp, i)
1459                 bnx2x_fp(bp, i, disable_tpa) =
1460                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1461
1462 #ifdef BCM_CNIC
1463         /* We don't want TPA on FCoE L2 ring */
1464         bnx2x_fcoe(bp, disable_tpa) = 1;
1465 #endif
1466
1467         if (bnx2x_alloc_mem(bp))
1468                 return -ENOMEM;
1469
1470         /* As long as bnx2x_alloc_mem() may possibly update
1471          * bp->num_queues, bnx2x_set_real_num_queues() should always
1472          * come after it.
1473          */
1474         rc = bnx2x_set_real_num_queues(bp);
1475         if (rc) {
1476                 BNX2X_ERR("Unable to set real_num_queues\n");
1477                 goto load_error0;
1478         }
1479
1480         bnx2x_napi_enable(bp);
1481
1482         /* Send LOAD_REQUEST command to MCP
1483            Returns the type of LOAD command:
1484            if it is the first port to be initialized
1485            common blocks should be initialized, otherwise - not
1486         */
1487         if (!BP_NOMCP(bp)) {
1488                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1489                 if (!load_code) {
1490                         BNX2X_ERR("MCP response failure, aborting\n");
1491                         rc = -EBUSY;
1492                         goto load_error1;
1493                 }
1494                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1495                         rc = -EBUSY; /* other port in diagnostic mode */
1496                         goto load_error1;
1497                 }
1498
1499         } else {
1500                 int path = BP_PATH(bp);
1501                 int port = BP_PORT(bp);
1502
1503                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1504                    path, load_count[path][0], load_count[path][1],
1505                    load_count[path][2]);
1506                 load_count[path][0]++;
1507                 load_count[path][1 + port]++;
1508                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1509                    path, load_count[path][0], load_count[path][1],
1510                    load_count[path][2]);
1511                 if (load_count[path][0] == 1)
1512                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1513                 else if (load_count[path][1 + port] == 1)
1514                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1515                 else
1516                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1517         }
1518
1519         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1520             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1521             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1522                 bp->port.pmf = 1;
1523         else
1524                 bp->port.pmf = 0;
1525         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1526
1527         /* Initialize HW */
1528         rc = bnx2x_init_hw(bp, load_code);
1529         if (rc) {
1530                 BNX2X_ERR("HW init failed, aborting\n");
1531                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1532                 goto load_error2;
1533         }
1534
1535         /* Connect to IRQs */
1536         rc = bnx2x_setup_irqs(bp);
1537         if (rc) {
1538                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1539                 goto load_error2;
1540         }
1541
1542         /* Setup NIC internals and enable interrupts */
1543         bnx2x_nic_init(bp, load_code);
1544
1545         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1546             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1547             (bp->common.shmem2_base))
1548                 SHMEM2_WR(bp, dcc_support,
1549                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1550                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1551
1552         /* Send LOAD_DONE command to MCP */
1553         if (!BP_NOMCP(bp)) {
1554                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1555                 if (!load_code) {
1556                         BNX2X_ERR("MCP response failure, aborting\n");
1557                         rc = -EBUSY;
1558                         goto load_error3;
1559                 }
1560         }
1561
1562         bnx2x_dcbx_init(bp);
1563
1564         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1565
1566         rc = bnx2x_func_start(bp);
1567         if (rc) {
1568                 BNX2X_ERR("Function start failed!\n");
1569 #ifndef BNX2X_STOP_ON_ERROR
1570                 goto load_error3;
1571 #else
1572                 bp->panic = 1;
1573                 return -EBUSY;
1574 #endif
1575         }
1576
1577         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1578         if (rc) {
1579                 BNX2X_ERR("Setup leading failed!\n");
1580 #ifndef BNX2X_STOP_ON_ERROR
1581                 goto load_error3;
1582 #else
1583                 bp->panic = 1;
1584                 return -EBUSY;
1585 #endif
1586         }
1587
1588         if (!CHIP_IS_E1(bp) &&
1589             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1590                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1591                 bp->flags |= MF_FUNC_DIS;
1592         }
1593
1594 #ifdef BCM_CNIC
1595         /* Enable Timer scan */
1596         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1597 #endif
1598
1599         for_each_nondefault_queue(bp, i) {
1600                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1601                 if (rc)
1602 #ifdef BCM_CNIC
1603                         goto load_error4;
1604 #else
1605                         goto load_error3;
1606 #endif
1607         }
1608
1609         /* Now when Clients are configured we are ready to work */
1610         bp->state = BNX2X_STATE_OPEN;
1611
1612 #ifdef BCM_CNIC
1613         bnx2x_set_fcoe_eth_macs(bp);
1614 #endif
1615
1616         bnx2x_set_eth_mac(bp, 1);
1617
1618         /* Clear MC configuration */
1619         if (CHIP_IS_E1(bp))
1620                 bnx2x_invalidate_e1_mc_list(bp);
1621         else
1622                 bnx2x_invalidate_e1h_mc_list(bp);
1623
1624         /* Clear UC lists configuration */
1625         bnx2x_invalidate_uc_list(bp);
1626
1627         if (bp->pending_max) {
1628                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1629                 bp->pending_max = 0;
1630         }
1631
1632         if (bp->port.pmf)
1633                 bnx2x_initial_phy_init(bp, load_mode);
1634
1635         /* Initialize Rx filtering */
1636         bnx2x_set_rx_mode(bp->dev);
1637
1638         /* Start fast path */
1639         switch (load_mode) {
1640         case LOAD_NORMAL:
1641                 /* Tx queue should be only reenabled */
1642                 netif_tx_wake_all_queues(bp->dev);
1643                 /* Initialize the receive filter. */
1644                 break;
1645
1646         case LOAD_OPEN:
1647                 netif_tx_start_all_queues(bp->dev);
1648                 smp_mb__after_clear_bit();
1649                 break;
1650
1651         case LOAD_DIAG:
1652                 bp->state = BNX2X_STATE_DIAG;
1653                 break;
1654
1655         default:
1656                 break;
1657         }
1658
1659         if (!bp->port.pmf)
1660                 bnx2x__link_status_update(bp);
1661
1662         /* start the timer */
1663         mod_timer(&bp->timer, jiffies + bp->current_interval);
1664
1665 #ifdef BCM_CNIC
1666         bnx2x_setup_cnic_irq_info(bp);
1667         if (bp->state == BNX2X_STATE_OPEN)
1668                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1669 #endif
1670         bnx2x_inc_load_cnt(bp);
1671
1672         bnx2x_release_firmware(bp);
1673
1674         return 0;
1675
1676 #ifdef BCM_CNIC
1677 load_error4:
1678         /* Disable Timer scan */
1679         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1680 #endif
1681 load_error3:
1682         bnx2x_int_disable_sync(bp, 1);
1683
1684         /* Free SKBs, SGEs, TPA pool and driver internals */
1685         bnx2x_free_skbs(bp);
1686         for_each_rx_queue(bp, i)
1687                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1688
1689         /* Release IRQs */
1690         bnx2x_free_irq(bp);
1691 load_error2:
1692         if (!BP_NOMCP(bp)) {
1693                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1694                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1695         }
1696
1697         bp->port.pmf = 0;
1698 load_error1:
1699         bnx2x_napi_disable(bp);
1700 load_error0:
1701         bnx2x_free_mem(bp);
1702
1703         bnx2x_release_firmware(bp);
1704
1705         return rc;
1706 }
1707
1708 /* must be called with rtnl_lock */
1709 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1710 {
1711         int i;
1712
1713         if (bp->state == BNX2X_STATE_CLOSED) {
1714                 /* Interface has been removed - nothing to recover */
1715                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1716                 bp->is_leader = 0;
1717                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1718                 smp_wmb();
1719
1720                 return -EINVAL;
1721         }
1722
1723 #ifdef BCM_CNIC
1724         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1725 #endif
1726         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1727
1728         /* Set "drop all" */
1729         bp->rx_mode = BNX2X_RX_MODE_NONE;
1730         bnx2x_set_storm_rx_mode(bp);
1731
1732         /* Stop Tx */
1733         bnx2x_tx_disable(bp);
1734
1735         del_timer_sync(&bp->timer);
1736
1737         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1738                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1739
1740         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1741
1742         /* Cleanup the chip if needed */
1743         if (unload_mode != UNLOAD_RECOVERY)
1744                 bnx2x_chip_cleanup(bp, unload_mode);
1745         else {
1746                 /* Disable HW interrupts, NAPI and Tx */
1747                 bnx2x_netif_stop(bp, 1);
1748
1749                 /* Release IRQs */
1750                 bnx2x_free_irq(bp);
1751         }
1752
1753         bp->port.pmf = 0;
1754
1755         /* Free SKBs, SGEs, TPA pool and driver internals */
1756         bnx2x_free_skbs(bp);
1757         for_each_rx_queue(bp, i)
1758                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1759
1760         bnx2x_free_mem(bp);
1761
1762         bp->state = BNX2X_STATE_CLOSED;
1763
1764         /* The last driver must disable a "close the gate" if there is no
1765          * parity attention or "process kill" pending.
1766          */
1767         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1768             bnx2x_reset_is_done(bp))
1769                 bnx2x_disable_close_the_gate(bp);
1770
1771         /* Reset MCP mail box sequence if there is on going recovery */
1772         if (unload_mode == UNLOAD_RECOVERY)
1773                 bp->fw_seq = 0;
1774
1775         return 0;
1776 }
1777
1778 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1779 {
1780         u16 pmcsr;
1781
1782         /* If there is no power capability, silently succeed */
1783         if (!bp->pm_cap) {
1784                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1785                 return 0;
1786         }
1787
1788         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1789
1790         switch (state) {
1791         case PCI_D0:
1792                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1793                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1794                                        PCI_PM_CTRL_PME_STATUS));
1795
1796                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1797                         /* delay required during transition out of D3hot */
1798                         msleep(20);
1799                 break;
1800
1801         case PCI_D3hot:
1802                 /* If there are other clients above don't
1803                    shut down the power */
1804                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1805                         return 0;
1806                 /* Don't shut down the power for emulation and FPGA */
1807                 if (CHIP_REV_IS_SLOW(bp))
1808                         return 0;
1809
1810                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1811                 pmcsr |= 3;
1812
1813                 if (bp->wol)
1814                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1815
1816                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1817                                       pmcsr);
1818
1819                 /* No more memory access after this point until
1820                 * device is brought back to D0.
1821                 */
1822                 break;
1823
1824         default:
1825                 return -EINVAL;
1826         }
1827         return 0;
1828 }
1829
1830 /*
1831  * net_device service functions
1832  */
1833 int bnx2x_poll(struct napi_struct *napi, int budget)
1834 {
1835         int work_done = 0;
1836         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1837                                                  napi);
1838         struct bnx2x *bp = fp->bp;
1839
1840         while (1) {
1841 #ifdef BNX2X_STOP_ON_ERROR
1842                 if (unlikely(bp->panic)) {
1843                         napi_complete(napi);
1844                         return 0;
1845                 }
1846 #endif
1847
1848                 if (bnx2x_has_tx_work(fp))
1849                         bnx2x_tx_int(fp);
1850
1851                 if (bnx2x_has_rx_work(fp)) {
1852                         work_done += bnx2x_rx_int(fp, budget - work_done);
1853
1854                         /* must not complete if we consumed full budget */
1855                         if (work_done >= budget)
1856                                 break;
1857                 }
1858
1859                 /* Fall out from the NAPI loop if needed */
1860                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1861 #ifdef BCM_CNIC
1862                         /* No need to update SB for FCoE L2 ring as long as
1863                          * it's connected to the default SB and the SB
1864                          * has been updated when NAPI was scheduled.
1865                          */
1866                         if (IS_FCOE_FP(fp)) {
1867                                 napi_complete(napi);
1868                                 break;
1869                         }
1870 #endif
1871
1872                         bnx2x_update_fpsb_idx(fp);
1873                         /* bnx2x_has_rx_work() reads the status block,
1874                          * thus we need to ensure that status block indices
1875                          * have been actually read (bnx2x_update_fpsb_idx)
1876                          * prior to this check (bnx2x_has_rx_work) so that
1877                          * we won't write the "newer" value of the status block
1878                          * to IGU (if there was a DMA right after
1879                          * bnx2x_has_rx_work and if there is no rmb, the memory
1880                          * reading (bnx2x_update_fpsb_idx) may be postponed
1881                          * to right before bnx2x_ack_sb). In this case there
1882                          * will never be another interrupt until there is
1883                          * another update of the status block, while there
1884                          * is still unhandled work.
1885                          */
1886                         rmb();
1887
1888                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1889                                 napi_complete(napi);
1890                                 /* Re-enable interrupts */
1891                                 DP(NETIF_MSG_HW,
1892                                    "Update index to %d\n", fp->fp_hc_idx);
1893                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1894                                              le16_to_cpu(fp->fp_hc_idx),
1895                                              IGU_INT_ENABLE, 1);
1896                                 break;
1897                         }
1898                 }
1899         }
1900
1901         return work_done;
1902 }
1903
1904 /* we split the first BD into headers and data BDs
1905  * to ease the pain of our fellow microcode engineers
1906  * we use one mapping for both BDs
1907  * So far this has only been observed to happen
1908  * in Other Operating Systems(TM)
1909  */
1910 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1911                                    struct bnx2x_fastpath *fp,
1912                                    struct sw_tx_bd *tx_buf,
1913                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1914                                    u16 bd_prod, int nbd)
1915 {
1916         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1917         struct eth_tx_bd *d_tx_bd;
1918         dma_addr_t mapping;
1919         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1920
1921         /* first fix first BD */
1922         h_tx_bd->nbd = cpu_to_le16(nbd);
1923         h_tx_bd->nbytes = cpu_to_le16(hlen);
1924
1925         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1926            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1927            h_tx_bd->addr_lo, h_tx_bd->nbd);
1928
1929         /* now get a new data BD
1930          * (after the pbd) and fill it */
1931         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1932         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1933
1934         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1935                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1936
1937         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1938         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1939         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1940
1941         /* this marks the BD as one that has no individual mapping */
1942         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1943
1944         DP(NETIF_MSG_TX_QUEUED,
1945            "TSO split data size is %d (%x:%x)\n",
1946            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1947
1948         /* update tx_bd */
1949         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1950
1951         return bd_prod;
1952 }
1953
1954 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1955 {
1956         if (fix > 0)
1957                 csum = (u16) ~csum_fold(csum_sub(csum,
1958                                 csum_partial(t_header - fix, fix, 0)));
1959
1960         else if (fix < 0)
1961                 csum = (u16) ~csum_fold(csum_add(csum,
1962                                 csum_partial(t_header, -fix, 0)));
1963
1964         return swab16(csum);
1965 }
1966
1967 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1968 {
1969         u32 rc;
1970
1971         if (skb->ip_summed != CHECKSUM_PARTIAL)
1972                 rc = XMIT_PLAIN;
1973
1974         else {
1975                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
1976                         rc = XMIT_CSUM_V6;
1977                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1978                                 rc |= XMIT_CSUM_TCP;
1979
1980                 } else {
1981                         rc = XMIT_CSUM_V4;
1982                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1983                                 rc |= XMIT_CSUM_TCP;
1984                 }
1985         }
1986
1987         if (skb_is_gso_v6(skb))
1988                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
1989         else if (skb_is_gso(skb))
1990                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
1991
1992         return rc;
1993 }
1994
1995 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1996 /* check if packet requires linearization (packet is too fragmented)
1997    no need to check fragmentation if page size > 8K (there will be no
1998    violation to FW restrictions) */
1999 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
2000                              u32 xmit_type)
2001 {
2002         int to_copy = 0;
2003         int hlen = 0;
2004         int first_bd_sz = 0;
2005
2006         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
2007         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
2008
2009                 if (xmit_type & XMIT_GSO) {
2010                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
2011                         /* Check if LSO packet needs to be copied:
2012                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
2013                         int wnd_size = MAX_FETCH_BD - 3;
2014                         /* Number of windows to check */
2015                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
2016                         int wnd_idx = 0;
2017                         int frag_idx = 0;
2018                         u32 wnd_sum = 0;
2019
2020                         /* Headers length */
2021                         hlen = (int)(skb_transport_header(skb) - skb->data) +
2022                                 tcp_hdrlen(skb);
2023
2024                         /* Amount of data (w/o headers) on linear part of SKB*/
2025                         first_bd_sz = skb_headlen(skb) - hlen;
2026
2027                         wnd_sum  = first_bd_sz;
2028
2029                         /* Calculate the first sum - it's special */
2030                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
2031                                 wnd_sum +=
2032                                         skb_shinfo(skb)->frags[frag_idx].size;
2033
2034                         /* If there was data on linear skb data - check it */
2035                         if (first_bd_sz > 0) {
2036                                 if (unlikely(wnd_sum < lso_mss)) {
2037                                         to_copy = 1;
2038                                         goto exit_lbl;
2039                                 }
2040
2041                                 wnd_sum -= first_bd_sz;
2042                         }
2043
2044                         /* Others are easier: run through the frag list and
2045                            check all windows */
2046                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
2047                                 wnd_sum +=
2048                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
2049
2050                                 if (unlikely(wnd_sum < lso_mss)) {
2051                                         to_copy = 1;
2052                                         break;
2053                                 }
2054                                 wnd_sum -=
2055                                         skb_shinfo(skb)->frags[wnd_idx].size;
2056                         }
2057                 } else {
2058                         /* in non-LSO too fragmented packet should always
2059                            be linearized */
2060                         to_copy = 1;
2061                 }
2062         }
2063
2064 exit_lbl:
2065         if (unlikely(to_copy))
2066                 DP(NETIF_MSG_TX_QUEUED,
2067                    "Linearization IS REQUIRED for %s packet. "
2068                    "num_frags %d  hlen %d  first_bd_sz %d\n",
2069                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
2070                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
2071
2072         return to_copy;
2073 }
2074 #endif
2075
2076 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
2077                                         u32 xmit_type)
2078 {
2079         *parsing_data |= (skb_shinfo(skb)->gso_size <<
2080                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
2081                               ETH_TX_PARSE_BD_E2_LSO_MSS;
2082         if ((xmit_type & XMIT_GSO_V6) &&
2083             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
2084                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
2085 }
2086
2087 /**
2088  * bnx2x_set_pbd_gso - update PBD in GSO case.
2089  *
2090  * @skb:        packet skb
2091  * @pbd:        parse BD
2092  * @xmit_type:  xmit flags
2093  */
2094 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
2095                                      struct eth_tx_parse_bd_e1x *pbd,
2096                                      u32 xmit_type)
2097 {
2098         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2099         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2100         pbd->tcp_flags = pbd_tcp_flags(skb);
2101
2102         if (xmit_type & XMIT_GSO_V4) {
2103                 pbd->ip_id = swab16(ip_hdr(skb)->id);
2104                 pbd->tcp_pseudo_csum =
2105                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2106                                                   ip_hdr(skb)->daddr,
2107                                                   0, IPPROTO_TCP, 0));
2108
2109         } else
2110                 pbd->tcp_pseudo_csum =
2111                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2112                                                 &ipv6_hdr(skb)->daddr,
2113                                                 0, IPPROTO_TCP, 0));
2114
2115         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2116 }
2117
2118 /**
2119  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
2120  *
2121  * @bp:                 driver handle
2122  * @skb:                packet skb
2123  * @parsing_data:       data to be updated
2124  * @xmit_type:          xmit flags
2125  *
2126  * 57712 related
2127  */
2128 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2129         u32 *parsing_data, u32 xmit_type)
2130 {
2131         *parsing_data |=
2132                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2133                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2134                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2135
2136         if (xmit_type & XMIT_CSUM_TCP) {
2137                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2138                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2139                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2140
2141                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2142         } else
2143                 /* We support checksum offload for TCP and UDP only.
2144                  * No need to pass the UDP header length - it's a constant.
2145                  */
2146                 return skb_transport_header(skb) +
2147                                 sizeof(struct udphdr) - skb->data;
2148 }
2149
2150 /**
2151  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
2152  *
2153  * @bp:         driver handle
2154  * @skb:        packet skb
2155  * @pbd:        parse BD to be updated
2156  * @xmit_type:  xmit flags
2157  */
2158 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2159         struct eth_tx_parse_bd_e1x *pbd,
2160         u32 xmit_type)
2161 {
2162         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2163
2164         /* for now NS flag is not used in Linux */
2165         pbd->global_data =
2166                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2167                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2168
2169         pbd->ip_hlen_w = (skb_transport_header(skb) -
2170                         skb_network_header(skb)) >> 1;
2171
2172         hlen += pbd->ip_hlen_w;
2173
2174         /* We support checksum offload for TCP and UDP only */
2175         if (xmit_type & XMIT_CSUM_TCP)
2176                 hlen += tcp_hdrlen(skb) / 2;
2177         else
2178                 hlen += sizeof(struct udphdr) / 2;
2179
2180         pbd->total_hlen_w = cpu_to_le16(hlen);
2181         hlen = hlen*2;
2182
2183         if (xmit_type & XMIT_CSUM_TCP) {
2184                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2185
2186         } else {
2187                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2188
2189                 DP(NETIF_MSG_TX_QUEUED,
2190                    "hlen %d  fix %d  csum before fix %x\n",
2191                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2192
2193                 /* HW bug: fixup the CSUM */
2194                 pbd->tcp_pseudo_csum =
2195                         bnx2x_csum_fix(skb_transport_header(skb),
2196                                        SKB_CS(skb), fix);
2197
2198                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2199                    pbd->tcp_pseudo_csum);
2200         }
2201
2202         return hlen;
2203 }
2204
2205 /* called with netif_tx_lock
2206  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2207  * netif_wake_queue()
2208  */
2209 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2210 {
2211         struct bnx2x *bp = netdev_priv(dev);
2212         struct bnx2x_fastpath *fp;
2213         struct netdev_queue *txq;
2214         struct sw_tx_bd *tx_buf;
2215         struct eth_tx_start_bd *tx_start_bd;
2216         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2217         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2218         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2219         u32 pbd_e2_parsing_data = 0;
2220         u16 pkt_prod, bd_prod;
2221         int nbd, fp_index;
2222         dma_addr_t mapping;
2223         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2224         int i;
2225         u8 hlen = 0;
2226         __le16 pkt_size = 0;
2227         struct ethhdr *eth;
2228         u8 mac_type = UNICAST_ADDRESS;
2229
2230 #ifdef BNX2X_STOP_ON_ERROR
2231         if (unlikely(bp->panic))
2232                 return NETDEV_TX_BUSY;
2233 #endif
2234
2235         fp_index = skb_get_queue_mapping(skb);
2236         txq = netdev_get_tx_queue(dev, fp_index);
2237
2238         fp = &bp->fp[fp_index];
2239
2240         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
2241                 fp->eth_q_stats.driver_xoff++;
2242                 netif_tx_stop_queue(txq);
2243                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2244                 return NETDEV_TX_BUSY;
2245         }
2246
2247         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
2248                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
2249            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2250            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2251
2252         eth = (struct ethhdr *)skb->data;
2253
2254         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2255         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2256                 if (is_broadcast_ether_addr(eth->h_dest))
2257                         mac_type = BROADCAST_ADDRESS;
2258                 else
2259                         mac_type = MULTICAST_ADDRESS;
2260         }
2261
2262 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2263         /* First, check if we need to linearize the skb (due to FW
2264            restrictions). No need to check fragmentation if page size > 8K
2265            (there will be no violation to FW restrictions) */
2266         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2267                 /* Statistics of linearization */
2268                 bp->lin_cnt++;
2269                 if (skb_linearize(skb) != 0) {
2270                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2271                            "silently dropping this SKB\n");
2272                         dev_kfree_skb_any(skb);
2273                         return NETDEV_TX_OK;
2274                 }
2275         }
2276 #endif
2277
2278         /*
2279         Please read carefully. First we use one BD which we mark as start,
2280         then we have a parsing info BD (used for TSO or xsum),
2281         and only then we have the rest of the TSO BDs.
2282         (don't forget to mark the last one as last,
2283         and to unmap only AFTER you write to the BD ...)
2284         And above all, all pdb sizes are in words - NOT DWORDS!
2285         */
2286
2287         pkt_prod = fp->tx_pkt_prod++;
2288         bd_prod = TX_BD(fp->tx_bd_prod);
2289
2290         /* get a tx_buf and first BD */
2291         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
2292         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
2293
2294         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2295         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2296                  mac_type);
2297
2298         /* header nbd */
2299         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2300
2301         /* remember the first BD of the packet */
2302         tx_buf->first_bd = fp->tx_bd_prod;
2303         tx_buf->skb = skb;
2304         tx_buf->flags = 0;
2305
2306         DP(NETIF_MSG_TX_QUEUED,
2307            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2308            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2309
2310         if (vlan_tx_tag_present(skb)) {
2311                 tx_start_bd->vlan_or_ethertype =
2312                     cpu_to_le16(vlan_tx_tag_get(skb));
2313                 tx_start_bd->bd_flags.as_bitfield |=
2314                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2315         } else
2316                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2317
2318         /* turn on parsing and get a BD */
2319         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2320
2321         if (xmit_type & XMIT_CSUM) {
2322                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2323
2324                 if (xmit_type & XMIT_CSUM_V4)
2325                         tx_start_bd->bd_flags.as_bitfield |=
2326                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2327                 else
2328                         tx_start_bd->bd_flags.as_bitfield |=
2329                                                 ETH_TX_BD_FLAGS_IPV6;
2330
2331                 if (!(xmit_type & XMIT_CSUM_TCP))
2332                         tx_start_bd->bd_flags.as_bitfield |=
2333                                                 ETH_TX_BD_FLAGS_IS_UDP;
2334         }
2335
2336         if (CHIP_IS_E2(bp)) {
2337                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2338                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2339                 /* Set PBD in checksum offload case */
2340                 if (xmit_type & XMIT_CSUM)
2341                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2342                                                      &pbd_e2_parsing_data,
2343                                                      xmit_type);
2344         } else {
2345                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2346                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2347                 /* Set PBD in checksum offload case */
2348                 if (xmit_type & XMIT_CSUM)
2349                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2350
2351         }
2352
2353         /* Map skb linear data for DMA */
2354         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2355                                  skb_headlen(skb), DMA_TO_DEVICE);
2356
2357         /* Setup the data pointer of the first BD of the packet */
2358         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2359         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2360         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2361         tx_start_bd->nbd = cpu_to_le16(nbd);
2362         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2363         pkt_size = tx_start_bd->nbytes;
2364
2365         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2366            "  nbytes %d  flags %x  vlan %x\n",
2367            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2368            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2369            tx_start_bd->bd_flags.as_bitfield,
2370            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2371
2372         if (xmit_type & XMIT_GSO) {
2373
2374                 DP(NETIF_MSG_TX_QUEUED,
2375                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2376                    skb->len, hlen, skb_headlen(skb),
2377                    skb_shinfo(skb)->gso_size);
2378
2379                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2380
2381                 if (unlikely(skb_headlen(skb) > hlen))
2382                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2383                                                  hlen, bd_prod, ++nbd);
2384                 if (CHIP_IS_E2(bp))
2385                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2386                                              xmit_type);
2387                 else
2388                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2389         }
2390
2391         /* Set the PBD's parsing_data field if not zero
2392          * (for the chips newer than 57711).
2393          */
2394         if (pbd_e2_parsing_data)
2395                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2396
2397         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2398
2399         /* Handle fragmented skb */
2400         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2401                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2402
2403                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2404                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2405                 if (total_pkt_bd == NULL)
2406                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2407
2408                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2409                                        frag->page_offset,
2410                                        frag->size, DMA_TO_DEVICE);
2411
2412                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2413                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2414                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2415                 le16_add_cpu(&pkt_size, frag->size);
2416
2417                 DP(NETIF_MSG_TX_QUEUED,
2418                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2419                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2420                    le16_to_cpu(tx_data_bd->nbytes));
2421         }
2422
2423         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2424
2425         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2426
2427         /* now send a tx doorbell, counting the next BD
2428          * if the packet contains or ends with it
2429          */
2430         if (TX_BD_POFF(bd_prod) < nbd)
2431                 nbd++;
2432
2433         if (total_pkt_bd != NULL)
2434                 total_pkt_bd->total_pkt_bytes = pkt_size;
2435
2436         if (pbd_e1x)
2437                 DP(NETIF_MSG_TX_QUEUED,
2438                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2439                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2440                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2441                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2442                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2443                     le16_to_cpu(pbd_e1x->total_hlen_w));
2444         if (pbd_e2)
2445                 DP(NETIF_MSG_TX_QUEUED,
2446                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2447                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2448                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2449                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2450                    pbd_e2->parsing_data);
2451         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2452
2453         /*
2454          * Make sure that the BD data is updated before updating the producer
2455          * since FW might read the BD right after the producer is updated.
2456          * This is only applicable for weak-ordered memory model archs such
2457          * as IA-64. The following barrier is also mandatory since FW will
2458          * assumes packets must have BDs.
2459          */
2460         wmb();
2461
2462         fp->tx_db.data.prod += nbd;
2463         barrier();
2464
2465         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2466
2467         mmiowb();
2468
2469         fp->tx_bd_prod += nbd;
2470
2471         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2472                 netif_tx_stop_queue(txq);
2473
2474                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2475                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2476                  * fp->bd_tx_cons */
2477                 smp_mb();
2478
2479                 fp->eth_q_stats.driver_xoff++;
2480                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2481                         netif_tx_wake_queue(txq);
2482         }
2483         fp->tx_pkt++;
2484
2485         return NETDEV_TX_OK;
2486 }
2487
2488 /* called with rtnl_lock */
2489 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2490 {
2491         struct sockaddr *addr = p;
2492         struct bnx2x *bp = netdev_priv(dev);
2493
2494         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2495                 return -EINVAL;
2496
2497         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2498         if (netif_running(dev))
2499                 bnx2x_set_eth_mac(bp, 1);
2500
2501         return 0;
2502 }
2503
2504 static void bnx2x_free_fp_mem_at(struct bnx2x *bp, int fp_index)
2505 {
2506         union host_hc_status_block *sb = &bnx2x_fp(bp, fp_index, status_blk);
2507         struct bnx2x_fastpath *fp = &bp->fp[fp_index];
2508
2509         /* Common */
2510 #ifdef BCM_CNIC
2511         if (IS_FCOE_IDX(fp_index)) {
2512                 memset(sb, 0, sizeof(union host_hc_status_block));
2513                 fp->status_blk_mapping = 0;
2514
2515         } else {
2516 #endif
2517                 /* status blocks */
2518                 if (CHIP_IS_E2(bp))
2519                         BNX2X_PCI_FREE(sb->e2_sb,
2520                                        bnx2x_fp(bp, fp_index,
2521                                                 status_blk_mapping),
2522                                        sizeof(struct host_hc_status_block_e2));
2523                 else
2524                         BNX2X_PCI_FREE(sb->e1x_sb,
2525                                        bnx2x_fp(bp, fp_index,
2526                                                 status_blk_mapping),
2527                                        sizeof(struct host_hc_status_block_e1x));
2528 #ifdef BCM_CNIC
2529         }
2530 #endif
2531         /* Rx */
2532         if (!skip_rx_queue(bp, fp_index)) {
2533                 bnx2x_free_rx_bds(fp);
2534
2535                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
2536                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_buf_ring));
2537                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_desc_ring),
2538                                bnx2x_fp(bp, fp_index, rx_desc_mapping),
2539                                sizeof(struct eth_rx_bd) * NUM_RX_BD);
2540
2541                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_comp_ring),
2542                                bnx2x_fp(bp, fp_index, rx_comp_mapping),
2543                                sizeof(struct eth_fast_path_rx_cqe) *
2544                                NUM_RCQ_BD);
2545
2546                 /* SGE ring */
2547                 BNX2X_FREE(bnx2x_fp(bp, fp_index, rx_page_ring));
2548                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, rx_sge_ring),
2549                                bnx2x_fp(bp, fp_index, rx_sge_mapping),
2550                                BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
2551         }
2552
2553         /* Tx */
2554         if (!skip_tx_queue(bp, fp_index)) {
2555                 /* fastpath tx rings: tx_buf tx_desc */
2556                 BNX2X_FREE(bnx2x_fp(bp, fp_index, tx_buf_ring));
2557                 BNX2X_PCI_FREE(bnx2x_fp(bp, fp_index, tx_desc_ring),
2558                                bnx2x_fp(bp, fp_index, tx_desc_mapping),
2559                                sizeof(union eth_tx_bd_types) * NUM_TX_BD);
2560         }
2561         /* end of fastpath */
2562 }
2563
2564 void bnx2x_free_fp_mem(struct bnx2x *bp)
2565 {
2566         int i;
2567         for_each_queue(bp, i)
2568                 bnx2x_free_fp_mem_at(bp, i);
2569 }
2570
2571 static inline void set_sb_shortcuts(struct bnx2x *bp, int index)
2572 {
2573         union host_hc_status_block status_blk = bnx2x_fp(bp, index, status_blk);
2574         if (CHIP_IS_E2(bp)) {
2575                 bnx2x_fp(bp, index, sb_index_values) =
2576                         (__le16 *)status_blk.e2_sb->sb.index_values;
2577                 bnx2x_fp(bp, index, sb_running_index) =
2578                         (__le16 *)status_blk.e2_sb->sb.running_index;
2579         } else {
2580                 bnx2x_fp(bp, index, sb_index_values) =
2581                         (__le16 *)status_blk.e1x_sb->sb.index_values;
2582                 bnx2x_fp(bp, index, sb_running_index) =
2583                         (__le16 *)status_blk.e1x_sb->sb.running_index;
2584         }
2585 }
2586
2587 static int bnx2x_alloc_fp_mem_at(struct bnx2x *bp, int index)
2588 {
2589         union host_hc_status_block *sb;
2590         struct bnx2x_fastpath *fp = &bp->fp[index];
2591         int ring_size = 0;
2592
2593         /* if rx_ring_size specified - use it */
2594         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
2595                            MAX_RX_AVAIL/bp->num_queues;
2596
2597         /* allocate at least number of buffers required by FW */
2598         rx_ring_size = max_t(int, fp->disable_tpa ? MIN_RX_SIZE_NONTPA :
2599                                                     MIN_RX_SIZE_TPA,
2600                                   rx_ring_size);
2601
2602         bnx2x_fp(bp, index, bp) = bp;
2603         bnx2x_fp(bp, index, index) = index;
2604
2605         /* Common */
2606         sb = &bnx2x_fp(bp, index, status_blk);
2607 #ifdef BCM_CNIC
2608         if (!IS_FCOE_IDX(index)) {
2609 #endif
2610                 /* status blocks */
2611                 if (CHIP_IS_E2(bp))
2612                         BNX2X_PCI_ALLOC(sb->e2_sb,
2613                                 &bnx2x_fp(bp, index, status_blk_mapping),
2614                                 sizeof(struct host_hc_status_block_e2));
2615                 else
2616                         BNX2X_PCI_ALLOC(sb->e1x_sb,
2617                                 &bnx2x_fp(bp, index, status_blk_mapping),
2618                             sizeof(struct host_hc_status_block_e1x));
2619 #ifdef BCM_CNIC
2620         }
2621 #endif
2622         set_sb_shortcuts(bp, index);
2623
2624         /* Tx */
2625         if (!skip_tx_queue(bp, index)) {
2626                 /* fastpath tx rings: tx_buf tx_desc */
2627                 BNX2X_ALLOC(bnx2x_fp(bp, index, tx_buf_ring),
2628                                 sizeof(struct sw_tx_bd) * NUM_TX_BD);
2629                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, tx_desc_ring),
2630                                 &bnx2x_fp(bp, index, tx_desc_mapping),
2631                                 sizeof(union eth_tx_bd_types) * NUM_TX_BD);
2632         }
2633
2634         /* Rx */
2635         if (!skip_rx_queue(bp, index)) {
2636                 /* fastpath rx rings: rx_buf rx_desc rx_comp */
2637                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_buf_ring),
2638                                 sizeof(struct sw_rx_bd) * NUM_RX_BD);
2639                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_desc_ring),
2640                                 &bnx2x_fp(bp, index, rx_desc_mapping),
2641                                 sizeof(struct eth_rx_bd) * NUM_RX_BD);
2642
2643                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_comp_ring),
2644                                 &bnx2x_fp(bp, index, rx_comp_mapping),
2645                                 sizeof(struct eth_fast_path_rx_cqe) *
2646                                 NUM_RCQ_BD);
2647
2648                 /* SGE ring */
2649                 BNX2X_ALLOC(bnx2x_fp(bp, index, rx_page_ring),
2650                                 sizeof(struct sw_rx_page) * NUM_RX_SGE);
2651                 BNX2X_PCI_ALLOC(bnx2x_fp(bp, index, rx_sge_ring),
2652                                 &bnx2x_fp(bp, index, rx_sge_mapping),
2653                                 BCM_PAGE_SIZE * NUM_RX_SGE_PAGES);
2654                 /* RX BD ring */
2655                 bnx2x_set_next_page_rx_bd(fp);
2656
2657                 /* CQ ring */
2658                 bnx2x_set_next_page_rx_cq(fp);
2659
2660                 /* BDs */
2661                 ring_size = bnx2x_alloc_rx_bds(fp, rx_ring_size);
2662                 if (ring_size < rx_ring_size)
2663                         goto alloc_mem_err;
2664         }
2665
2666         return 0;
2667
2668 /* handles low memory cases */
2669 alloc_mem_err:
2670         BNX2X_ERR("Unable to allocate full memory for queue %d (size %d)\n",
2671                                                 index, ring_size);
2672         /* FW will drop all packets if queue is not big enough,
2673          * In these cases we disable the queue
2674          * Min size diferent for TPA and non-TPA queues
2675          */
2676         if (ring_size < (fp->disable_tpa ?
2677                                 MIN_RX_SIZE_TPA : MIN_RX_SIZE_NONTPA)) {
2678                         /* release memory allocated for this queue */
2679                         bnx2x_free_fp_mem_at(bp, index);
2680                         return -ENOMEM;
2681         }
2682         return 0;
2683 }
2684
2685 int bnx2x_alloc_fp_mem(struct bnx2x *bp)
2686 {
2687         int i;
2688
2689         /**
2690          * 1. Allocate FP for leading - fatal if error
2691          * 2. {CNIC} Allocate FCoE FP - fatal if error
2692          * 3. Allocate RSS - fix number of queues if error
2693          */
2694
2695         /* leading */
2696         if (bnx2x_alloc_fp_mem_at(bp, 0))
2697                 return -ENOMEM;
2698 #ifdef BCM_CNIC
2699         /* FCoE */
2700         if (bnx2x_alloc_fp_mem_at(bp, FCOE_IDX))
2701                 return -ENOMEM;
2702 #endif
2703         /* RSS */
2704         for_each_nondefault_eth_queue(bp, i)
2705                 if (bnx2x_alloc_fp_mem_at(bp, i))
2706                         break;
2707
2708         /* handle memory failures */
2709         if (i != BNX2X_NUM_ETH_QUEUES(bp)) {
2710                 int delta = BNX2X_NUM_ETH_QUEUES(bp) - i;
2711
2712                 WARN_ON(delta < 0);
2713 #ifdef BCM_CNIC
2714                 /**
2715                  * move non eth FPs next to last eth FP
2716                  * must be done in that order
2717                  * FCOE_IDX < FWD_IDX < OOO_IDX
2718                  */
2719
2720                 /* move FCoE fp */
2721                 bnx2x_move_fp(bp, FCOE_IDX, FCOE_IDX - delta);
2722 #endif
2723                 bp->num_queues -= delta;
2724                 BNX2X_ERR("Adjusted num of queues from %d to %d\n",
2725                           bp->num_queues + delta, bp->num_queues);
2726         }
2727
2728         return 0;
2729 }
2730
2731 static int bnx2x_setup_irqs(struct bnx2x *bp)
2732 {
2733         int rc = 0;
2734         if (bp->flags & USING_MSIX_FLAG) {
2735                 rc = bnx2x_req_msix_irqs(bp);
2736                 if (rc)
2737                         return rc;
2738         } else {
2739                 bnx2x_ack_int(bp);
2740                 rc = bnx2x_req_irq(bp);
2741                 if (rc) {
2742                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2743                         return rc;
2744                 }
2745                 if (bp->flags & USING_MSI_FLAG) {
2746                         bp->dev->irq = bp->pdev->irq;
2747                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2748                                bp->pdev->irq);
2749                 }
2750         }
2751
2752         return 0;
2753 }
2754
2755 void bnx2x_free_mem_bp(struct bnx2x *bp)
2756 {
2757         kfree(bp->fp);
2758         kfree(bp->msix_table);
2759         kfree(bp->ilt);
2760 }
2761
2762 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2763 {
2764         struct bnx2x_fastpath *fp;
2765         struct msix_entry *tbl;
2766         struct bnx2x_ilt *ilt;
2767
2768         /* fp array */
2769         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2770         if (!fp)
2771                 goto alloc_err;
2772         bp->fp = fp;
2773
2774         /* msix table */
2775         tbl = kzalloc((FP_SB_COUNT(bp->l2_cid_count) + 1) * sizeof(*tbl),
2776                                   GFP_KERNEL);
2777         if (!tbl)
2778                 goto alloc_err;
2779         bp->msix_table = tbl;
2780
2781         /* ilt */
2782         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2783         if (!ilt)
2784                 goto alloc_err;
2785         bp->ilt = ilt;
2786
2787         return 0;
2788 alloc_err:
2789         bnx2x_free_mem_bp(bp);
2790         return -ENOMEM;
2791
2792 }
2793
2794 static int bnx2x_reload_if_running(struct net_device *dev)
2795 {
2796         struct bnx2x *bp = netdev_priv(dev);
2797
2798         if (unlikely(!netif_running(dev)))
2799                 return 0;
2800
2801         bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2802         return bnx2x_nic_load(bp, LOAD_NORMAL);
2803 }
2804
2805 /* called with rtnl_lock */
2806 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2807 {
2808         struct bnx2x *bp = netdev_priv(dev);
2809
2810         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2811                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2812                 return -EAGAIN;
2813         }
2814
2815         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2816             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2817                 return -EINVAL;
2818
2819         /* This does not race with packet allocation
2820          * because the actual alloc size is
2821          * only updated as part of load
2822          */
2823         dev->mtu = new_mtu;
2824
2825         return bnx2x_reload_if_running(dev);
2826 }
2827
2828 u32 bnx2x_fix_features(struct net_device *dev, u32 features)
2829 {
2830         struct bnx2x *bp = netdev_priv(dev);
2831
2832         /* TPA requires Rx CSUM offloading */
2833         if (!(features & NETIF_F_RXCSUM) || bp->disable_tpa)
2834                 features &= ~NETIF_F_LRO;
2835
2836         return features;
2837 }
2838
2839 int bnx2x_set_features(struct net_device *dev, u32 features)
2840 {
2841         struct bnx2x *bp = netdev_priv(dev);
2842         u32 flags = bp->flags;
2843         bool bnx2x_reload = false;
2844
2845         if (features & NETIF_F_LRO)
2846                 flags |= TPA_ENABLE_FLAG;
2847         else
2848                 flags &= ~TPA_ENABLE_FLAG;
2849
2850         if (features & NETIF_F_LOOPBACK) {
2851                 if (bp->link_params.loopback_mode != LOOPBACK_BMAC) {
2852                         bp->link_params.loopback_mode = LOOPBACK_BMAC;
2853                         bnx2x_reload = true;
2854                 }
2855         } else {
2856                 if (bp->link_params.loopback_mode != LOOPBACK_NONE) {
2857                         bp->link_params.loopback_mode = LOOPBACK_NONE;
2858                         bnx2x_reload = true;
2859                 }
2860         }
2861
2862         if (flags ^ bp->flags) {
2863                 bp->flags = flags;
2864                 bnx2x_reload = true;
2865         }
2866
2867         if (bnx2x_reload) {
2868                 if (bp->recovery_state == BNX2X_RECOVERY_DONE)
2869                         return bnx2x_reload_if_running(dev);
2870                 /* else: bnx2x_nic_load() will be called at end of recovery */
2871         }
2872
2873         return 0;
2874 }
2875
2876 void bnx2x_tx_timeout(struct net_device *dev)
2877 {
2878         struct bnx2x *bp = netdev_priv(dev);
2879
2880 #ifdef BNX2X_STOP_ON_ERROR
2881         if (!bp->panic)
2882                 bnx2x_panic();
2883 #endif
2884         /* This allows the netif to be shutdown gracefully before resetting */
2885         schedule_delayed_work(&bp->reset_task, 0);
2886 }
2887
2888 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2889 {
2890         struct net_device *dev = pci_get_drvdata(pdev);
2891         struct bnx2x *bp;
2892
2893         if (!dev) {
2894                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2895                 return -ENODEV;
2896         }
2897         bp = netdev_priv(dev);
2898
2899         rtnl_lock();
2900
2901         pci_save_state(pdev);
2902
2903         if (!netif_running(dev)) {
2904                 rtnl_unlock();
2905                 return 0;
2906         }
2907
2908         netif_device_detach(dev);
2909
2910         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2911
2912         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2913
2914         rtnl_unlock();
2915
2916         return 0;
2917 }
2918
2919 int bnx2x_resume(struct pci_dev *pdev)
2920 {
2921         struct net_device *dev = pci_get_drvdata(pdev);
2922         struct bnx2x *bp;
2923         int rc;
2924
2925         if (!dev) {
2926                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2927                 return -ENODEV;
2928         }
2929         bp = netdev_priv(dev);
2930
2931         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2932                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2933                 return -EAGAIN;
2934         }
2935
2936         rtnl_lock();
2937
2938         pci_restore_state(pdev);
2939
2940         if (!netif_running(dev)) {
2941                 rtnl_unlock();
2942                 return 0;
2943         }
2944
2945         bnx2x_set_power_state(bp, PCI_D0);
2946         netif_device_attach(dev);
2947
2948         /* Since the chip was reset, clear the FW sequence number */
2949         bp->fw_seq = 0;
2950         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2951
2952         rtnl_unlock();
2953
2954         return rc;
2955 }