drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <linux/if_vlan.h>
  20 #include <linux/ip.h>
  21 #include <net/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #include "bnx2x_init.h"
  27
  28 static int bnx2x_setup_irqs(struct bnx2x *bp);
  29
  30 /* free skb in the packet ring at pos idx
  31  * return idx of last bd freed
  32  */
  33 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  34                              u16 idx)
  35 {
  36         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  37         struct eth_tx_start_bd *tx_start_bd;
  38         struct eth_tx_bd *tx_data_bd;
  39         struct sk_buff *skb = tx_buf->skb;
  40         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  41         int nbd;
  42
  43         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  44         prefetch(&skb->end);
  45
  46         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  47            idx, tx_buf, skb);
  48
  49         /* unmap first bd */
  50         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  51         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  52         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  53                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  54
  55         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  56 #ifdef BNX2X_STOP_ON_ERROR
  57         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  58                 BNX2X_ERR("BAD nbd!\n");
  59                 bnx2x_panic();
  60         }
  61 #endif
  62         new_cons = nbd + tx_buf->first_bd;
  63
  64         /* Get the next bd */
  65         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  66
  67         /* Skip a parse bd... */
  68         --nbd;
  69         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  70
  71         /* ...and the TSO split header bd since they have no mapping */
  72         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  73                 --nbd;
  74                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  75         }
  76
  77         /* now free frags */
  78         while (nbd > 0) {
  79
  80                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  81                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  82                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  83                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  84                 if (--nbd)
  85                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  86         }
  87
  88         /* release skb */
  89         WARN_ON(!skb);
  90         dev_kfree_skb(skb);
  91         tx_buf->first_bd = 0;
  92         tx_buf->skb = NULL;
  93
  94         return new_cons;
  95 }
  96
  97 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
  98 {
  99         struct bnx2x *bp = fp->bp;
 100         struct netdev_queue *txq;
 101         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 102
 103 #ifdef BNX2X_STOP_ON_ERROR
 104         if (unlikely(bp->panic))
 105                 return -1;
 106 #endif
 107
 108         txq = netdev_get_tx_queue(bp->dev, fp->index);
 109         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 110         sw_cons = fp->tx_pkt_cons;
 111
 112         while (sw_cons != hw_cons) {
 113                 u16 pkt_cons;
 114
 115                 pkt_cons = TX_BD(sw_cons);
 116
 117                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 118                                       " pkt_cons %u\n",
 119                    fp->index, hw_cons, sw_cons, pkt_cons);
 120
 121                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 122                 sw_cons++;
 123         }
 124
 125         fp->tx_pkt_cons = sw_cons;
 126         fp->tx_bd_cons = bd_cons;
 127
 128         /* Need to make the tx_bd_cons update visible to start_xmit()
 129          * before checking for netif_tx_queue_stopped().  Without the
 130          * memory barrier, there is a small possibility that
 131          * start_xmit() will miss it and cause the queue to be stopped
 132          * forever.
 133          */
 134         smp_mb();
 135
 136         if (unlikely(netif_tx_queue_stopped(txq))) {
 137                 /* Taking tx_lock() is needed to prevent reenabling the queue
 138                  * while it's empty. This could have happen if rx_action() gets
 139                  * suspended in bnx2x_tx_int() after the condition before
 140                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 141                  *
 142                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 143                  * sends some packets consuming the whole queue again->
 144                  * stops the queue
 145                  */
 146
 147                 __netif_tx_lock(txq, smp_processor_id());
 148
 149                 if ((netif_tx_queue_stopped(txq)) &&
 150                     (bp->state == BNX2X_STATE_OPEN) &&
 151                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 152                         netif_tx_wake_queue(txq);
 153
 154                 __netif_tx_unlock(txq);
 155         }
 156         return 0;
 157 }
 158
 159 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 160                                              u16 idx)
 161 {
 162         u16 last_max = fp->last_max_sge;
 163
 164         if (SUB_S16(idx, last_max) > 0)
 165                 fp->last_max_sge = idx;
 166 }
 167
 168 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 169                                   struct eth_fast_path_rx_cqe *fp_cqe)
 170 {
 171         struct bnx2x *bp = fp->bp;
 172         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 173                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 174                       SGE_PAGE_SHIFT;
 175         u16 last_max, last_elem, first_elem;
 176         u16 delta = 0;
 177         u16 i;
 178
 179         if (!sge_len)
 180                 return;
 181
 182         /* First mark all used pages */
 183         for (i = 0; i < sge_len; i++)
 184                 SGE_MASK_CLEAR_BIT(fp,
 185                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 186
 187         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 188            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 189
 190         /* Here we assume that the last SGE index is the biggest */
 191         prefetch((void *)(fp->sge_mask));
 192         bnx2x_update_last_max_sge(fp,
 193                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 194
 195         last_max = RX_SGE(fp->last_max_sge);
 196         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 197         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 198
 199         /* If ring is not full */
 200         if (last_elem + 1 != first_elem)
 201                 last_elem++;
 202
 203         /* Now update the prod */
 204         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 205                 if (likely(fp->sge_mask[i]))
 206                         break;
 207
 208                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 209                 delta += RX_SGE_MASK_ELEM_SZ;
 210         }
 211
 212         if (delta > 0) {
 213                 fp->rx_sge_prod += delta;
 214                 /* clear page-end entries */
 215                 bnx2x_clear_sge_mask_next_elems(fp);
 216         }
 217
 218         DP(NETIF_MSG_RX_STATUS,
 219            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 220            fp->last_max_sge, fp->rx_sge_prod);
 221 }
 222
 223 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 224                             struct sk_buff *skb, u16 cons, u16 prod)
 225 {
 226         struct bnx2x *bp = fp->bp;
 227         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 228         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 229         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 230         dma_addr_t mapping;
 231
 232         /* move empty skb from pool to prod and map it */
 233         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 234         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 235                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 236         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 237
 238         /* move partial skb from cons to pool (don't unmap yet) */
 239         fp->tpa_pool[queue] = *cons_rx_buf;
 240
 241         /* mark bin state as start - print error if current state != stop */
 242         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 243                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 244
 245         fp->tpa_state[queue] = BNX2X_TPA_START;
 246
 247         /* point prod_bd to new skb */
 248         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 249         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 250
 251 #ifdef BNX2X_STOP_ON_ERROR
 252         fp->tpa_queue_used |= (1 << queue);
 253 #ifdef _ASM_GENERIC_INT_L64_H
 254         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 255 #else
 256         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 257 #endif
 258            fp->tpa_queue_used);
 259 #endif
 260 }
 261
 262 /* Timestamp option length allowed for TPA aggregation:
 263  *
 264  *              nop nop kind length echo val
 265  */
 266 #define TPA_TSTAMP_OPT_LEN      12
 267 /**
 268  * Calculate the approximate value of the MSS for this
 269  * aggregation using the first packet of it.
 270  *
 271  * @param bp
 272  * @param parsing_flags Parsing flags from the START CQE
 273  * @param len_on_bd Total length of the first packet for the
 274  *                   aggregation.
 275  */
 276 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 277                                     u16 len_on_bd)
 278 {
 279         /* TPA arrgregation won't have an IP options and TCP options
 280          * other than timestamp.
 281          */
 282         u16 hdrs_len = ETH_HLEN + sizeof(struct iphdr) + sizeof(struct tcphdr);
 283
 284
 285         /* Check if there was a TCP timestamp, if there is it's will
 286          * always be 12 bytes length: nop nop kind length echo val.
 287          *
 288          * Otherwise FW would close the aggregation.
 289          */
 290         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 291                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 292
 293         return len_on_bd - hdrs_len;
 294 }
 295
 296 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 297                                struct sk_buff *skb,
 298                                struct eth_fast_path_rx_cqe *fp_cqe,
 299                                u16 cqe_idx, u16 parsing_flags)
 300 {
 301         struct sw_rx_page *rx_pg, old_rx_pg;
 302         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 303         u32 i, frag_len, frag_size, pages;
 304         int err;
 305         int j;
 306
 307         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 308         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 309
 310         /* This is needed in order to enable forwarding support */
 311         if (frag_size)
 312                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp, parsing_flags,
 313                                                               len_on_bd);
 314
 315 #ifdef BNX2X_STOP_ON_ERROR
 316         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 317                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 318                           pages, cqe_idx);
 319                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 320                           fp_cqe->pkt_len, len_on_bd);
 321                 bnx2x_panic();
 322                 return -EINVAL;
 323         }
 324 #endif
 325
 326         /* Run through the SGL and compose the fragmented skb */
 327         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 328                 u16 sge_idx =
 329                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
 330
 331                 /* FW gives the indices of the SGE as if the ring is an array
 332                    (meaning that "next" element will consume 2 indices) */
 333                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 334                 rx_pg = &fp->rx_page_ring[sge_idx];
 335                 old_rx_pg = *rx_pg;
 336
 337                 /* If we fail to allocate a substitute page, we simply stop
 338                    where we are and drop the whole packet */
 339                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 340                 if (unlikely(err)) {
 341                         fp->eth_q_stats.rx_skb_alloc_failed++;
 342                         return err;
 343                 }
 344
 345                 /* Unmap the page as we r going to pass it to the stack */
 346                 dma_unmap_page(&bp->pdev->dev,
 347                                dma_unmap_addr(&old_rx_pg, mapping),
 348                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 349
 350                 /* Add one frag and update the appropriate fields in the skb */
 351                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 352
 353                 skb->data_len += frag_len;
 354                 skb->truesize += frag_len;
 355                 skb->len += frag_len;
 356
 357                 frag_size -= frag_len;
 358         }
 359
 360         return 0;
 361 }
 362
 363 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 364                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 365                            u16 cqe_idx)
 366 {
 367         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 368         struct sk_buff *skb = rx_buf->skb;
 369         /* alloc new skb */
 370         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 371
 372         /* Unmap skb in the pool anyway, as we are going to change
 373            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 374            fails. */
 375         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 376                          fp->rx_buf_size, DMA_FROM_DEVICE);
 377
 378         if (likely(new_skb)) {
 379                 /* fix ip xsum and give it to the stack */
 380                 /* (no need to map the new skb) */
 381                 u16 parsing_flags =
 382                         le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags);
 383
 384                 prefetch(skb);
 385                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 386
 387 #ifdef BNX2X_STOP_ON_ERROR
 388                 if (pad + len > fp->rx_buf_size) {
 389                         BNX2X_ERR("skb_put is about to fail...  "
 390                                   "pad %d  len %d  rx_buf_size %d\n",
 391                                   pad, len, fp->rx_buf_size);
 392                         bnx2x_panic();
 393                         return;
 394                 }
 395 #endif
 396
 397                 skb_reserve(skb, pad);
 398                 skb_put(skb, len);
 399
 400                 skb->protocol = eth_type_trans(skb, bp->dev);
 401                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 402
 403                 {
 404                         struct iphdr *iph;
 405
 406                         iph = (struct iphdr *)skb->data;
 407                         iph->check = 0;
 408                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 409                 }
 410
 411                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 412                                          &cqe->fast_path_cqe, cqe_idx,
 413                                          parsing_flags)) {
 414                         if (parsing_flags & PARSING_FLAGS_VLAN)
 415                                 __vlan_hwaccel_put_tag(skb,
 416                                                  le16_to_cpu(cqe->fast_path_cqe.
 417                                                              vlan_tag));
 418                         napi_gro_receive(&fp->napi, skb);
 419                 } else {
 420                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 421                            " - dropping packet!\n");
 422                         dev_kfree_skb(skb);
 423                 }
 424
 425
 426                 /* put new skb in bin */
 427                 fp->tpa_pool[queue].skb = new_skb;
 428
 429         } else {
 430                 /* else drop the packet and keep the buffer in the bin */
 431                 DP(NETIF_MSG_RX_STATUS,
 432                    "Failed to allocate new skb - dropping packet!\n");
 433                 fp->eth_q_stats.rx_skb_alloc_failed++;
 434         }
 435
 436         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 437 }
 438
 439 /* Set Toeplitz hash value in the skb using the value from the
 440  * CQE (calculated by HW).
 441  */
 442 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 443                                         struct sk_buff *skb)
 444 {
 445         /* Set Toeplitz hash from CQE */
 446         if ((bp->dev->features & NETIF_F_RXHASH) &&
 447             (cqe->fast_path_cqe.status_flags &
 448              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 449                 skb->rxhash =
 450                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 451 }
 452
 453 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 454 {
 455         struct bnx2x *bp = fp->bp;
 456         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 457         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 458         int rx_pkt = 0;
 459
 460 #ifdef BNX2X_STOP_ON_ERROR
 461         if (unlikely(bp->panic))
 462                 return 0;
 463 #endif
 464
 465         /* CQ "next element" is of the size of the regular element,
 466            that's why it's ok here */
 467         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 468         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 469                 hw_comp_cons++;
 470
 471         bd_cons = fp->rx_bd_cons;
 472         bd_prod = fp->rx_bd_prod;
 473         bd_prod_fw = bd_prod;
 474         sw_comp_cons = fp->rx_comp_cons;
 475         sw_comp_prod = fp->rx_comp_prod;
 476
 477         /* Memory barrier necessary as speculative reads of the rx
 478          * buffer can be ahead of the index in the status block
 479          */
 480         rmb();
 481
 482         DP(NETIF_MSG_RX_STATUS,
 483            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 484            fp->index, hw_comp_cons, sw_comp_cons);
 485
 486         while (sw_comp_cons != hw_comp_cons) {
 487                 struct sw_rx_bd *rx_buf = NULL;
 488                 struct sk_buff *skb;
 489                 union eth_rx_cqe *cqe;
 490                 u8 cqe_fp_flags;
 491                 u16 len, pad;
 492
 493                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 494                 bd_prod = RX_BD(bd_prod);
 495                 bd_cons = RX_BD(bd_cons);
 496
 497                 /* Prefetch the page containing the BD descriptor
 498                    at producer's index. It will be needed when new skb is
 499                    allocated */
 500                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 501                                              (&fp->rx_desc_ring[bd_prod])) -
 502                                   PAGE_SIZE + 1));
 503
 504                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 505                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 506
 507                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 508                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 509                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 510                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 511                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 512                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 513
 514                 /* is this a slowpath msg? */
 515                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 516                         bnx2x_sp_event(fp, cqe);
 517                         goto next_cqe;
 518
 519                 /* this is an rx packet */
 520                 } else {
 521                         rx_buf = &fp->rx_buf_ring[bd_cons];
 522                         skb = rx_buf->skb;
 523                         prefetch(skb);
 524                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 525                         pad = cqe->fast_path_cqe.placement_offset;
 526
 527                         /* - If CQE is marked both TPA_START and TPA_END it is
 528                          *   a non-TPA CQE.
 529                          * - FP CQE will always have either TPA_START or/and
 530                          *   TPA_STOP flags set.
 531                          */
 532                         if ((!fp->disable_tpa) &&
 533                             (TPA_TYPE(cqe_fp_flags) !=
 534                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 535                                 u16 queue = cqe->fast_path_cqe.queue_index;
 536
 537                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 538                                         DP(NETIF_MSG_RX_STATUS,
 539                                            "calling tpa_start on queue %d\n",
 540                                            queue);
 541
 542                                         bnx2x_tpa_start(fp, queue, skb,
 543                                                         bd_cons, bd_prod);
 544
 545                                         /* Set Toeplitz hash for an LRO skb */
 546                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 547
 548                                         goto next_rx;
 549                                 } else { /* TPA_STOP */
 550                                         DP(NETIF_MSG_RX_STATUS,
 551                                            "calling tpa_stop on queue %d\n",
 552                                            queue);
 553
 554                                         if (!BNX2X_RX_SUM_FIX(cqe))
 555                                                 BNX2X_ERR("STOP on none TCP "
 556                                                           "data\n");
 557
 558                                         /* This is a size of the linear data
 559                                            on this skb */
 560                                         len = le16_to_cpu(cqe->fast_path_cqe.
 561                                                                 len_on_bd);
 562                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 563                                                     len, cqe, comp_ring_cons);
 564 #ifdef BNX2X_STOP_ON_ERROR
 565                                         if (bp->panic)
 566                                                 return 0;
 567 #endif
 568
 569                                         bnx2x_update_sge_prod(fp,
 570                                                         &cqe->fast_path_cqe);
 571                                         goto next_cqe;
 572                                 }
 573                         }
 574
 575                         dma_sync_single_for_device(&bp->pdev->dev,
 576                                         dma_unmap_addr(rx_buf, mapping),
 577                                                    pad + RX_COPY_THRESH,
 578                                                    DMA_FROM_DEVICE);
 579                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 580
 581                         /* is this an error packet? */
 582                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 583                                 DP(NETIF_MSG_RX_ERR,
 584                                    "ERROR  flags %x  rx packet %u\n",
 585                                    cqe_fp_flags, sw_comp_cons);
 586                                 fp->eth_q_stats.rx_err_discard_pkt++;
 587                                 goto reuse_rx;
 588                         }
 589
 590                         /* Since we don't have a jumbo ring
 591                          * copy small packets if mtu > 1500
 592                          */
 593                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 594                             (len <= RX_COPY_THRESH)) {
 595                                 struct sk_buff *new_skb;
 596
 597                                 new_skb = netdev_alloc_skb(bp->dev,
 598                                                            len + pad);
 599                                 if (new_skb == NULL) {
 600                                         DP(NETIF_MSG_RX_ERR,
 601                                            "ERROR  packet dropped "
 602                                            "because of alloc failure\n");
 603                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 604                                         goto reuse_rx;
 605                                 }
 606
 607                                 /* aligned copy */
 608                                 skb_copy_from_linear_data_offset(skb, pad,
 609                                                     new_skb->data + pad, len);
 610                                 skb_reserve(new_skb, pad);
 611                                 skb_put(new_skb, len);
 612
 613                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 614
 615                                 skb = new_skb;
 616
 617                         } else
 618                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 619                                 dma_unmap_single(&bp->pdev->dev,
 620                                         dma_unmap_addr(rx_buf, mapping),
 621                                                  fp->rx_buf_size,
 622                                                  DMA_FROM_DEVICE);
 623                                 skb_reserve(skb, pad);
 624                                 skb_put(skb, len);
 625
 626                         } else {
 627                                 DP(NETIF_MSG_RX_ERR,
 628                                    "ERROR  packet dropped because "
 629                                    "of alloc failure\n");
 630                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 631 reuse_rx:
 632                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 633                                 goto next_rx;
 634                         }
 635
 636                         skb->protocol = eth_type_trans(skb, bp->dev);
 637
 638                         /* Set Toeplitz hash for a none-LRO skb */
 639                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 640
 641                         skb_checksum_none_assert(skb);
 642
 643                         if (bp->rx_csum) {
 644                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 645                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 646                                 else
 647                                         fp->eth_q_stats.hw_csum_err++;
 648                         }
 649                 }
 650
 651                 skb_record_rx_queue(skb, fp->index);
 652
 653                 if (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 654                      PARSING_FLAGS_VLAN)
 655                         __vlan_hwaccel_put_tag(skb,
 656                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
 657                 napi_gro_receive(&fp->napi, skb);
 658
 659
 660 next_rx:
 661                 rx_buf->skb = NULL;
 662
 663                 bd_cons = NEXT_RX_IDX(bd_cons);
 664                 bd_prod = NEXT_RX_IDX(bd_prod);
 665                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 666                 rx_pkt++;
 667 next_cqe:
 668                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 669                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 670
 671                 if (rx_pkt == budget)
 672                         break;
 673         } /* while */
 674
 675         fp->rx_bd_cons = bd_cons;
 676         fp->rx_bd_prod = bd_prod_fw;
 677         fp->rx_comp_cons = sw_comp_cons;
 678         fp->rx_comp_prod = sw_comp_prod;
 679
 680         /* Update producers */
 681         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 682                              fp->rx_sge_prod);
 683
 684         fp->rx_pkt += rx_pkt;
 685         fp->rx_calls++;
 686
 687         return rx_pkt;
 688 }
 689
 690 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 691 {
 692         struct bnx2x_fastpath *fp = fp_cookie;
 693         struct bnx2x *bp = fp->bp;
 694
 695         /* Return here if interrupt is disabled */
 696         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 697                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 698                 return IRQ_HANDLED;
 699         }
 700
 701         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 702                          "[fp %d fw_sd %d igusb %d]\n",
 703            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 704         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 705
 706 #ifdef BNX2X_STOP_ON_ERROR
 707         if (unlikely(bp->panic))
 708                 return IRQ_HANDLED;
 709 #endif
 710
 711         /* Handle Rx and Tx according to MSI-X vector */
 712         prefetch(fp->rx_cons_sb);
 713         prefetch(fp->tx_cons_sb);
 714         prefetch(&fp->sb_running_index[SM_RX_ID]);
 715         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 716
 717         return IRQ_HANDLED;
 718 }
 719
 720 /* HW Lock for shared dual port PHYs */
 721 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 722 {
 723         mutex_lock(&bp->port.phy_mutex);
 724
 725         if (bp->port.need_hw_lock)
 726                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 727 }
 728
 729 void bnx2x_release_phy_lock(struct bnx2x *bp)
 730 {
 731         if (bp->port.need_hw_lock)
 732                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 733
 734         mutex_unlock(&bp->port.phy_mutex);
 735 }
 736
 737 /* calculates MF speed according to current linespeed and MF configuration */
 738 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 739 {
 740         u16 line_speed = bp->link_vars.line_speed;
 741         if (IS_MF(bp)) {
 742                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 743                                                    bp->mf_config[BP_VN(bp)]);
 744
 745                 /* Calculate the current MAX line speed limit for the MF
 746                  * devices
 747                  */
 748                 if (IS_MF_SI(bp))
 749                         line_speed = (line_speed * maxCfg) / 100;
 750                 else { /* SD mode */
 751                         u16 vn_max_rate = maxCfg * 100;
 752
 753                         if (vn_max_rate < line_speed)
 754                                 line_speed = vn_max_rate;
 755                 }
 756         }
 757
 758         return line_speed;
 759 }
 760
 761 void bnx2x_link_report(struct bnx2x *bp)
 762 {
 763         if (bp->flags & MF_FUNC_DIS) {
 764                 netif_carrier_off(bp->dev);
 765                 netdev_err(bp->dev, "NIC Link is Down\n");
 766                 return;
 767         }
 768
 769         if (bp->link_vars.link_up) {
 770                 u16 line_speed;
 771
 772                 if (bp->state == BNX2X_STATE_OPEN)
 773                         netif_carrier_on(bp->dev);
 774                 netdev_info(bp->dev, "NIC Link is Up, ");
 775
 776                 line_speed = bnx2x_get_mf_speed(bp);
 777
 778                 pr_cont("%d Mbps ", line_speed);
 779
 780                 if (bp->link_vars.duplex == DUPLEX_FULL)
 781                         pr_cont("full duplex");
 782                 else
 783                         pr_cont("half duplex");
 784
 785                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 786                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 787                                 pr_cont(", receive ");
 788                                 if (bp->link_vars.flow_ctrl &
 789                                     BNX2X_FLOW_CTRL_TX)
 790                                         pr_cont("& transmit ");
 791                         } else {
 792                                 pr_cont(", transmit ");
 793                         }
 794                         pr_cont("flow control ON");
 795                 }
 796                 pr_cont("\n");
 797
 798         } else { /* link_down */
 799                 netif_carrier_off(bp->dev);
 800                 netdev_err(bp->dev, "NIC Link is Down\n");
 801         }
 802 }
 803
 804 /* Returns the number of actually allocated BDs */
 805 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
 806                                       int rx_ring_size)
 807 {
 808         struct bnx2x *bp = fp->bp;
 809         u16 ring_prod, cqe_ring_prod;
 810         int i;
 811
 812         fp->rx_comp_cons = 0;
 813         cqe_ring_prod = ring_prod = 0;
 814         for (i = 0; i < rx_ring_size; i++) {
 815                 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 816                         BNX2X_ERR("was only able to allocate "
 817                                   "%d rx skbs on queue[%d]\n", i, fp->index);
 818                         fp->eth_q_stats.rx_skb_alloc_failed++;
 819                         break;
 820                 }
 821                 ring_prod = NEXT_RX_IDX(ring_prod);
 822                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 823                 WARN_ON(ring_prod <= i);
 824         }
 825
 826         fp->rx_bd_prod = ring_prod;
 827         /* Limit the CQE producer by the CQE ring size */
 828         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 829                                cqe_ring_prod);
 830         fp->rx_pkt = fp->rx_calls = 0;
 831
 832         return i;
 833 }
 834
 835 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath *fp)
 836 {
 837         struct bnx2x *bp = fp->bp;
 838         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
 839                                               MAX_RX_AVAIL/bp->num_queues;
 840
 841         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
 842
 843         bnx2x_alloc_rx_bds(fp, rx_ring_size);
 844
 845         /* Warning!
 846          * this will generate an interrupt (to the TSTORM)
 847          * must only be done after chip is initialized
 848          */
 849         bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
 850                              fp->rx_sge_prod);
 851 }
 852
 853 void bnx2x_init_rx_rings(struct bnx2x *bp)
 854 {
 855         int func = BP_FUNC(bp);
 856         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 857                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 858         u16 ring_prod;
 859         int i, j;
 860
 861         for_each_rx_queue(bp, j) {
 862                 struct bnx2x_fastpath *fp = &bp->fp[j];
 863
 864                 DP(NETIF_MSG_IFUP,
 865                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
 866
 867                 if (!fp->disable_tpa) {
 868                         for (i = 0; i < max_agg_queues; i++) {
 869                                 fp->tpa_pool[i].skb =
 870                                    netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 871                                 if (!fp->tpa_pool[i].skb) {
 872                                         BNX2X_ERR("Failed to allocate TPA "
 873                                                   "skb pool for queue[%d] - "
 874                                                   "disabling TPA on this "
 875                                                   "queue!\n", j);
 876                                         bnx2x_free_tpa_pool(bp, fp, i);
 877                                         fp->disable_tpa = 1;
 878                                         break;
 879                                 }
 880                                 dma_unmap_addr_set((struct sw_rx_bd *)
 881                                                         &bp->fp->tpa_pool[i],
 882                                                    mapping, 0);
 883                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 884                         }
 885
 886                         /* "next page" elements initialization */
 887                         bnx2x_set_next_page_sgl(fp);
 888
 889                         /* set SGEs bit mask */
 890                         bnx2x_init_sge_ring_bit_mask(fp);
 891
 892                         /* Allocate SGEs and initialize the ring elements */
 893                         for (i = 0, ring_prod = 0;
 894                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 895
 896                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 897                                         BNX2X_ERR("was only able to allocate "
 898                                                   "%d rx sges\n", i);
 899                                         BNX2X_ERR("disabling TPA for"
 900                                                   " queue[%d]\n", j);
 901                                         /* Cleanup already allocated elements */
 902                                         bnx2x_free_rx_sge_range(bp,
 903                                                                 fp, ring_prod);
 904                                         bnx2x_free_tpa_pool(bp,
 905                                                             fp, max_agg_queues);
 906                                         fp->disable_tpa = 1;
 907                                         ring_prod = 0;
 908                                         break;
 909                                 }
 910                                 ring_prod = NEXT_SGE_IDX(ring_prod);
 911                         }
 912
 913                         fp->rx_sge_prod = ring_prod;
 914                 }
 915         }
 916
 917         for_each_rx_queue(bp, j) {
 918                 struct bnx2x_fastpath *fp = &bp->fp[j];
 919
 920                 fp->rx_bd_cons = 0;
 921
 922                 bnx2x_set_next_page_rx_bd(fp);
 923
 924                 /* CQ ring */
 925                 bnx2x_set_next_page_rx_cq(fp);
 926
 927                 /* Allocate BDs and initialize BD ring */
 928                 bnx2x_alloc_rx_bd_ring(fp);
 929
 930                 if (j != 0)
 931                         continue;
 932
 933                 if (!CHIP_IS_E2(bp)) {
 934                         REG_WR(bp, BAR_USTRORM_INTMEM +
 935                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 936                                U64_LO(fp->rx_comp_mapping));
 937                         REG_WR(bp, BAR_USTRORM_INTMEM +
 938                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 939                                U64_HI(fp->rx_comp_mapping));
 940                 }
 941         }
 942 }
 943
 944 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 945 {
 946         int i;
 947
 948         for_each_tx_queue(bp, i) {
 949                 struct bnx2x_fastpath *fp = &bp->fp[i];
 950
 951                 u16 bd_cons = fp->tx_bd_cons;
 952                 u16 sw_prod = fp->tx_pkt_prod;
 953                 u16 sw_cons = fp->tx_pkt_cons;
 954
 955                 while (sw_cons != sw_prod) {
 956                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 957                         sw_cons++;
 958                 }
 959         }
 960 }
 961
 962 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 963 {
 964         int i, j;
 965
 966         for_each_rx_queue(bp, j) {
 967                 struct bnx2x_fastpath *fp = &bp->fp[j];
 968
 969                 for (i = 0; i < NUM_RX_BD; i++) {
 970                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 971                         struct sk_buff *skb = rx_buf->skb;
 972
 973                         if (skb == NULL)
 974                                 continue;
 975
 976                         dma_unmap_single(&bp->pdev->dev,
 977                                          dma_unmap_addr(rx_buf, mapping),
 978                                          fp->rx_buf_size, DMA_FROM_DEVICE);
 979
 980                         rx_buf->skb = NULL;
 981                         dev_kfree_skb(skb);
 982                 }
 983                 if (!fp->disable_tpa)
 984                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 985                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 986                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 987         }
 988 }
 989
 990 void bnx2x_free_skbs(struct bnx2x *bp)
 991 {
 992         bnx2x_free_tx_skbs(bp);
 993         bnx2x_free_rx_skbs(bp);
 994 }
 995
 996 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
 997 {
 998         /* load old values */
 999         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1000
1001         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1002                 /* leave all but MAX value */
1003                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1004
1005                 /* set new MAX value */
1006                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1007                                 & FUNC_MF_CFG_MAX_BW_MASK;
1008
1009                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1010         }
1011 }
1012
1013 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
1014 {
1015         int i, offset = 1;
1016
1017         free_irq(bp->msix_table[0].vector, bp->dev);
1018         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1019            bp->msix_table[0].vector);
1020
1021 #ifdef BCM_CNIC
1022         offset++;
1023 #endif
1024         for_each_eth_queue(bp, i) {
1025                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
1026                    "state %x\n", i, bp->msix_table[i + offset].vector,
1027                    bnx2x_fp(bp, i, state));
1028
1029                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
1030         }
1031 }
1032
1033 void bnx2x_free_irq(struct bnx2x *bp)
1034 {
1035         if (bp->flags & USING_MSIX_FLAG)
1036                 bnx2x_free_msix_irqs(bp);
1037         else if (bp->flags & USING_MSI_FLAG)
1038                 free_irq(bp->pdev->irq, bp->dev);
1039         else
1040                 free_irq(bp->pdev->irq, bp->dev);
1041 }
1042
1043 int bnx2x_enable_msix(struct bnx2x *bp)
1044 {
1045         int msix_vec = 0, i, rc, req_cnt;
1046
1047         bp->msix_table[msix_vec].entry = msix_vec;
1048         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1049            bp->msix_table[0].entry);
1050         msix_vec++;
1051
1052 #ifdef BCM_CNIC
1053         bp->msix_table[msix_vec].entry = msix_vec;
1054         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1055            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1056         msix_vec++;
1057 #endif
1058         for_each_eth_queue(bp, i) {
1059                 bp->msix_table[msix_vec].entry = msix_vec;
1060                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1061                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1062                 msix_vec++;
1063         }
1064
1065         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1066
1067         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1068
1069         /*
1070          * reconfigure number of tx/rx queues according to available
1071          * MSI-X vectors
1072          */
1073         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1074                 /* how less vectors we will have? */
1075                 int diff = req_cnt - rc;
1076
1077                 DP(NETIF_MSG_IFUP,
1078                    "Trying to use less MSI-X vectors: %d\n", rc);
1079
1080                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1081
1082                 if (rc) {
1083                         DP(NETIF_MSG_IFUP,
1084                            "MSI-X is not attainable  rc %d\n", rc);
1085                         return rc;
1086                 }
1087                 /*
1088                  * decrease number of queues by number of unallocated entries
1089                  */
1090                 bp->num_queues -= diff;
1091
1092                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1093                                   bp->num_queues);
1094         } else if (rc) {
1095                 /* fall to INTx if not enough memory */
1096                 if (rc == -ENOMEM)
1097                         bp->flags |= DISABLE_MSI_FLAG;
1098                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1099                 return rc;
1100         }
1101
1102         bp->flags |= USING_MSIX_FLAG;
1103
1104         return 0;
1105 }
1106
1107 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1108 {
1109         int i, rc, offset = 1;
1110
1111         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1112                          bp->dev->name, bp->dev);
1113         if (rc) {
1114                 BNX2X_ERR("request sp irq failed\n");
1115                 return -EBUSY;
1116         }
1117
1118 #ifdef BCM_CNIC
1119         offset++;
1120 #endif
1121         for_each_eth_queue(bp, i) {
1122                 struct bnx2x_fastpath *fp = &bp->fp[i];
1123                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1124                          bp->dev->name, i);
1125
1126                 rc = request_irq(bp->msix_table[offset].vector,
1127                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1128                 if (rc) {
1129                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1130                         bnx2x_free_msix_irqs(bp);
1131                         return -EBUSY;
1132                 }
1133
1134                 offset++;
1135                 fp->state = BNX2X_FP_STATE_IRQ;
1136         }
1137
1138         i = BNX2X_NUM_ETH_QUEUES(bp);
1139         offset = 1 + CNIC_CONTEXT_USE;
1140         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1141                " ... fp[%d] %d\n",
1142                bp->msix_table[0].vector,
1143                0, bp->msix_table[offset].vector,
1144                i - 1, bp->msix_table[offset + i - 1].vector);
1145
1146         return 0;
1147 }
1148
1149 int bnx2x_enable_msi(struct bnx2x *bp)
1150 {
1151         int rc;
1152
1153         rc = pci_enable_msi(bp->pdev);
1154         if (rc) {
1155                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1156                 return -1;
1157         }
1158         bp->flags |= USING_MSI_FLAG;
1159
1160         return 0;
1161 }
1162
1163 static int bnx2x_req_irq(struct bnx2x *bp)
1164 {
1165         unsigned long flags;
1166         int rc;
1167
1168         if (bp->flags & USING_MSI_FLAG)
1169                 flags = 0;
1170         else
1171                 flags = IRQF_SHARED;
1172
1173         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1174                          bp->dev->name, bp->dev);
1175         if (!rc)
1176                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1177
1178         return rc;
1179 }
1180
1181 static void bnx2x_napi_enable(struct bnx2x *bp)
1182 {
1183         int i;
1184
1185         for_each_napi_queue(bp, i)
1186                 napi_enable(&bnx2x_fp(bp, i, napi));
1187 }
1188
1189 static void bnx2x_napi_disable(struct bnx2x *bp)
1190 {
1191         int i;
1192
1193         for_each_napi_queue(bp, i)
1194                 napi_disable(&bnx2x_fp(bp, i, napi));
1195 }
1196
1197 void bnx2x_netif_start(struct bnx2x *bp)
1198 {
1199         int intr_sem;
1200
1201         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1202         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1203
1204         if (intr_sem) {
1205                 if (netif_running(bp->dev)) {
1206                         bnx2x_napi_enable(bp);
1207                         bnx2x_int_enable(bp);
1208                         if (bp->state == BNX2X_STATE_OPEN)
1209                                 netif_tx_wake_all_queues(bp->dev);
1210                 }
1211         }
1212 }
1213
1214 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1215 {
1216         bnx2x_int_disable_sync(bp, disable_hw);
1217         bnx2x_napi_disable(bp);
1218         netif_tx_disable(bp->dev);
1219 }
1220
1221 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1222 {
1223 #ifdef BCM_CNIC
1224         struct bnx2x *bp = netdev_priv(dev);
1225         if (NO_FCOE(bp))
1226                 return skb_tx_hash(dev, skb);
1227         else {
1228                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1229                 u16 ether_type = ntohs(hdr->h_proto);
1230
1231                 /* Skip VLAN tag if present */
1232                 if (ether_type == ETH_P_8021Q) {
1233                         struct vlan_ethhdr *vhdr =
1234                                 (struct vlan_ethhdr *)skb->data;
1235
1236                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1237                 }
1238
1239                 /* If ethertype is FCoE or FIP - use FCoE ring */
1240                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1241                         return bnx2x_fcoe(bp, index);
1242         }
1243 #endif
1244         /* Select a none-FCoE queue:  if FCoE is enabled, exclude FCoE L2 ring
1245          */
1246         return __skb_tx_hash(dev, skb,
1247                         dev->real_num_tx_queues - FCOE_CONTEXT_USE);
1248 }
1249
1250 void bnx2x_set_num_queues(struct bnx2x *bp)
1251 {
1252         switch (bp->multi_mode) {
1253         case ETH_RSS_MODE_DISABLED:
1254                 bp->num_queues = 1;
1255                 break;
1256         case ETH_RSS_MODE_REGULAR:
1257                 bp->num_queues = bnx2x_calc_num_queues(bp);
1258                 break;
1259
1260         default:
1261                 bp->num_queues = 1;
1262                 break;
1263         }
1264
1265         /* Add special queues */
1266         bp->num_queues += NONE_ETH_CONTEXT_USE;
1267 }
1268
1269 #ifdef BCM_CNIC
1270 static inline void bnx2x_set_fcoe_eth_macs(struct bnx2x *bp)
1271 {
1272         if (!NO_FCOE(bp)) {
1273                 if (!IS_MF_SD(bp))
1274                         bnx2x_set_fip_eth_mac_addr(bp, 1);
1275                 bnx2x_set_all_enode_macs(bp, 1);
1276                 bp->flags |= FCOE_MACS_SET;
1277         }
1278 }
1279 #endif
1280
1281 static void bnx2x_release_firmware(struct bnx2x *bp)
1282 {
1283         kfree(bp->init_ops_offsets);
1284         kfree(bp->init_ops);
1285         kfree(bp->init_data);
1286         release_firmware(bp->firmware);
1287 }
1288
1289 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1290 {
1291         int rc, num = bp->num_queues;
1292
1293 #ifdef BCM_CNIC
1294         if (NO_FCOE(bp))
1295                 num -= FCOE_CONTEXT_USE;
1296
1297 #endif
1298         netif_set_real_num_tx_queues(bp->dev, num);
1299         rc = netif_set_real_num_rx_queues(bp->dev, num);
1300         return rc;
1301 }
1302
1303 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1304 {
1305         int i;
1306
1307         for_each_queue(bp, i) {
1308                 struct bnx2x_fastpath *fp = &bp->fp[i];
1309
1310                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1311                 if (IS_FCOE_IDX(i))
1312                         /*
1313                          * Although there are no IP frames expected to arrive to
1314                          * this ring we still want to add an
1315                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1316                          * overrun attack.
1317                          */
1318                         fp->rx_buf_size =
1319                                 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1320                                 BNX2X_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1321                 else
1322                         fp->rx_buf_size =
1323                                 bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
1324                                 IP_HEADER_ALIGNMENT_PADDING;
1325         }
1326 }
1327
1328 /* must be called with rtnl_lock */
1329 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1330 {
1331         u32 load_code;
1332         int i, rc;
1333
1334         /* Set init arrays */
1335         rc = bnx2x_init_firmware(bp);
1336         if (rc) {
1337                 BNX2X_ERR("Error loading firmware\n");
1338                 return rc;
1339         }
1340
1341 #ifdef BNX2X_STOP_ON_ERROR
1342         if (unlikely(bp->panic))
1343                 return -EPERM;
1344 #endif
1345
1346         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1347
1348         /* must be called before memory allocation and HW init */
1349         bnx2x_ilt_set_info(bp);
1350
1351         /* Set the receive queues buffer size */
1352         bnx2x_set_rx_buf_size(bp);
1353
1354         if (bnx2x_alloc_mem(bp))
1355                 return -ENOMEM;
1356
1357         rc = bnx2x_set_real_num_queues(bp);
1358         if (rc) {
1359                 BNX2X_ERR("Unable to set real_num_queues\n");
1360                 goto load_error0;
1361         }
1362
1363         for_each_queue(bp, i)
1364                 bnx2x_fp(bp, i, disable_tpa) =
1365                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1366
1367 #ifdef BCM_CNIC
1368         /* We don't want TPA on FCoE L2 ring */
1369         bnx2x_fcoe(bp, disable_tpa) = 1;
1370 #endif
1371         bnx2x_napi_enable(bp);
1372
1373         /* Send LOAD_REQUEST command to MCP
1374            Returns the type of LOAD command:
1375            if it is the first port to be initialized
1376            common blocks should be initialized, otherwise - not
1377         */
1378         if (!BP_NOMCP(bp)) {
1379                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1380                 if (!load_code) {
1381                         BNX2X_ERR("MCP response failure, aborting\n");
1382                         rc = -EBUSY;
1383                         goto load_error1;
1384                 }
1385                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1386                         rc = -EBUSY; /* other port in diagnostic mode */
1387                         goto load_error1;
1388                 }
1389
1390         } else {
1391                 int path = BP_PATH(bp);
1392                 int port = BP_PORT(bp);
1393
1394                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1395                    path, load_count[path][0], load_count[path][1],
1396                    load_count[path][2]);
1397                 load_count[path][0]++;
1398                 load_count[path][1 + port]++;
1399                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1400                    path, load_count[path][0], load_count[path][1],
1401                    load_count[path][2]);
1402                 if (load_count[path][0] == 1)
1403                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1404                 else if (load_count[path][1 + port] == 1)
1405                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1406                 else
1407                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1408         }
1409
1410         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1411             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1412             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1413                 bp->port.pmf = 1;
1414         else
1415                 bp->port.pmf = 0;
1416         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1417
1418         /* Initialize HW */
1419         rc = bnx2x_init_hw(bp, load_code);
1420         if (rc) {
1421                 BNX2X_ERR("HW init failed, aborting\n");
1422                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1423                 goto load_error2;
1424         }
1425
1426         /* Connect to IRQs */
1427         rc = bnx2x_setup_irqs(bp);
1428         if (rc) {
1429                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1430                 goto load_error2;
1431         }
1432
1433         /* Setup NIC internals and enable interrupts */
1434         bnx2x_nic_init(bp, load_code);
1435
1436         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1437             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1438             (bp->common.shmem2_base))
1439                 SHMEM2_WR(bp, dcc_support,
1440                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1441                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1442
1443         /* Send LOAD_DONE command to MCP */
1444         if (!BP_NOMCP(bp)) {
1445                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1446                 if (!load_code) {
1447                         BNX2X_ERR("MCP response failure, aborting\n");
1448                         rc = -EBUSY;
1449                         goto load_error3;
1450                 }
1451         }
1452
1453         bnx2x_dcbx_init(bp);
1454
1455         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1456
1457         rc = bnx2x_func_start(bp);
1458         if (rc) {
1459                 BNX2X_ERR("Function start failed!\n");
1460 #ifndef BNX2X_STOP_ON_ERROR
1461                 goto load_error3;
1462 #else
1463                 bp->panic = 1;
1464                 return -EBUSY;
1465 #endif
1466         }
1467
1468         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1469         if (rc) {
1470                 BNX2X_ERR("Setup leading failed!\n");
1471 #ifndef BNX2X_STOP_ON_ERROR
1472                 goto load_error3;
1473 #else
1474                 bp->panic = 1;
1475                 return -EBUSY;
1476 #endif
1477         }
1478
1479         if (!CHIP_IS_E1(bp) &&
1480             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1481                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1482                 bp->flags |= MF_FUNC_DIS;
1483         }
1484
1485 #ifdef BCM_CNIC
1486         /* Enable Timer scan */
1487         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1488 #endif
1489
1490         for_each_nondefault_queue(bp, i) {
1491                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1492                 if (rc)
1493 #ifdef BCM_CNIC
1494                         goto load_error4;
1495 #else
1496                         goto load_error3;
1497 #endif
1498         }
1499
1500         /* Now when Clients are configured we are ready to work */
1501         bp->state = BNX2X_STATE_OPEN;
1502
1503 #ifdef BCM_CNIC
1504         bnx2x_set_fcoe_eth_macs(bp);
1505 #endif
1506
1507         bnx2x_set_eth_mac(bp, 1);
1508
1509         /* Clear MC configuration */
1510         if (CHIP_IS_E1(bp))
1511                 bnx2x_invalidate_e1_mc_list(bp);
1512         else
1513                 bnx2x_invalidate_e1h_mc_list(bp);
1514
1515         /* Clear UC lists configuration */
1516         bnx2x_invalidate_uc_list(bp);
1517
1518         if (bp->pending_max) {
1519                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1520                 bp->pending_max = 0;
1521         }
1522
1523         if (bp->port.pmf)
1524                 bnx2x_initial_phy_init(bp, load_mode);
1525
1526         /* Initialize Rx filtering */
1527         bnx2x_set_rx_mode(bp->dev);
1528
1529         /* Start fast path */
1530         switch (load_mode) {
1531         case LOAD_NORMAL:
1532                 /* Tx queue should be only reenabled */
1533                 netif_tx_wake_all_queues(bp->dev);
1534                 /* Initialize the receive filter. */
1535                 break;
1536
1537         case LOAD_OPEN:
1538                 netif_tx_start_all_queues(bp->dev);
1539                 smp_mb__after_clear_bit();
1540                 break;
1541
1542         case LOAD_DIAG:
1543                 bp->state = BNX2X_STATE_DIAG;
1544                 break;
1545
1546         default:
1547                 break;
1548         }
1549
1550         if (!bp->port.pmf)
1551                 bnx2x__link_status_update(bp);
1552
1553         /* start the timer */
1554         mod_timer(&bp->timer, jiffies + bp->current_interval);
1555
1556 #ifdef BCM_CNIC
1557         bnx2x_setup_cnic_irq_info(bp);
1558         if (bp->state == BNX2X_STATE_OPEN)
1559                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1560 #endif
1561         bnx2x_inc_load_cnt(bp);
1562
1563         bnx2x_release_firmware(bp);
1564
1565         return 0;
1566
1567 #ifdef BCM_CNIC
1568 load_error4:
1569         /* Disable Timer scan */
1570         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1571 #endif
1572 load_error3:
1573         bnx2x_int_disable_sync(bp, 1);
1574
1575         /* Free SKBs, SGEs, TPA pool and driver internals */
1576         bnx2x_free_skbs(bp);
1577         for_each_rx_queue(bp, i)
1578                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1579
1580         /* Release IRQs */
1581         bnx2x_free_irq(bp);
1582 load_error2:
1583         if (!BP_NOMCP(bp)) {
1584                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1585                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1586         }
1587
1588         bp->port.pmf = 0;
1589 load_error1:
1590         bnx2x_napi_disable(bp);
1591 load_error0:
1592         bnx2x_free_mem(bp);
1593
1594         bnx2x_release_firmware(bp);
1595
1596         return rc;
1597 }
1598
1599 /* must be called with rtnl_lock */
1600 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1601 {
1602         int i;
1603
1604         if (bp->state == BNX2X_STATE_CLOSED) {
1605                 /* Interface has been removed - nothing to recover */
1606                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1607                 bp->is_leader = 0;
1608                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1609                 smp_wmb();
1610
1611                 return -EINVAL;
1612         }
1613
1614 #ifdef BCM_CNIC
1615         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1616 #endif
1617         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1618
1619         /* Set "drop all" */
1620         bp->rx_mode = BNX2X_RX_MODE_NONE;
1621         bnx2x_set_storm_rx_mode(bp);
1622
1623         /* Stop Tx */
1624         bnx2x_tx_disable(bp);
1625
1626         del_timer_sync(&bp->timer);
1627
1628         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1629                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1630
1631         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1632
1633         /* Cleanup the chip if needed */
1634         if (unload_mode != UNLOAD_RECOVERY)
1635                 bnx2x_chip_cleanup(bp, unload_mode);
1636         else {
1637                 /* Disable HW interrupts, NAPI and Tx */
1638                 bnx2x_netif_stop(bp, 1);
1639
1640                 /* Release IRQs */
1641                 bnx2x_free_irq(bp);
1642         }
1643
1644         bp->port.pmf = 0;
1645
1646         /* Free SKBs, SGEs, TPA pool and driver internals */
1647         bnx2x_free_skbs(bp);
1648         for_each_rx_queue(bp, i)
1649                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1650
1651         bnx2x_free_mem(bp);
1652
1653         bp->state = BNX2X_STATE_CLOSED;
1654
1655         /* The last driver must disable a "close the gate" if there is no
1656          * parity attention or "process kill" pending.
1657          */
1658         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1659             bnx2x_reset_is_done(bp))
1660                 bnx2x_disable_close_the_gate(bp);
1661
1662         /* Reset MCP mail box sequence if there is on going recovery */
1663         if (unload_mode == UNLOAD_RECOVERY)
1664                 bp->fw_seq = 0;
1665
1666         return 0;
1667 }
1668
1669 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1670 {
1671         u16 pmcsr;
1672
1673         /* If there is no power capability, silently succeed */
1674         if (!bp->pm_cap) {
1675                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1676                 return 0;
1677         }
1678
1679         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1680
1681         switch (state) {
1682         case PCI_D0:
1683                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1684                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1685                                        PCI_PM_CTRL_PME_STATUS));
1686
1687                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1688                         /* delay required during transition out of D3hot */
1689                         msleep(20);
1690                 break;
1691
1692         case PCI_D3hot:
1693                 /* If there are other clients above don't
1694                    shut down the power */
1695                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1696                         return 0;
1697                 /* Don't shut down the power for emulation and FPGA */
1698                 if (CHIP_REV_IS_SLOW(bp))
1699                         return 0;
1700
1701                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1702                 pmcsr |= 3;
1703
1704                 if (bp->wol)
1705                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1706
1707                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1708                                       pmcsr);
1709
1710                 /* No more memory access after this point until
1711                 * device is brought back to D0.
1712                 */
1713                 break;
1714
1715         default:
1716                 return -EINVAL;
1717         }
1718         return 0;
1719 }
1720
1721 /*
1722  * net_device service functions
1723  */
1724 int bnx2x_poll(struct napi_struct *napi, int budget)
1725 {
1726         int work_done = 0;
1727         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1728                                                  napi);
1729         struct bnx2x *bp = fp->bp;
1730
1731         while (1) {
1732 #ifdef BNX2X_STOP_ON_ERROR
1733                 if (unlikely(bp->panic)) {
1734                         napi_complete(napi);
1735                         return 0;
1736                 }
1737 #endif
1738
1739                 if (bnx2x_has_tx_work(fp))
1740                         bnx2x_tx_int(fp);
1741
1742                 if (bnx2x_has_rx_work(fp)) {
1743                         work_done += bnx2x_rx_int(fp, budget - work_done);
1744
1745                         /* must not complete if we consumed full budget */
1746                         if (work_done >= budget)
1747                                 break;
1748                 }
1749
1750                 /* Fall out from the NAPI loop if needed */
1751                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1752 #ifdef BCM_CNIC
1753                         /* No need to update SB for FCoE L2 ring as long as
1754                          * it's connected to the default SB and the SB
1755                          * has been updated when NAPI was scheduled.
1756                          */
1757                         if (IS_FCOE_FP(fp)) {
1758                                 napi_complete(napi);
1759                                 break;
1760                         }
1761 #endif
1762
1763                         bnx2x_update_fpsb_idx(fp);
1764                         /* bnx2x_has_rx_work() reads the status block,
1765                          * thus we need to ensure that status block indices
1766                          * have been actually read (bnx2x_update_fpsb_idx)
1767                          * prior to this check (bnx2x_has_rx_work) so that
1768                          * we won't write the "newer" value of the status block
1769                          * to IGU (if there was a DMA right after
1770                          * bnx2x_has_rx_work and if there is no rmb, the memory
1771                          * reading (bnx2x_update_fpsb_idx) may be postponed
1772                          * to right before bnx2x_ack_sb). In this case there
1773                          * will never be another interrupt until there is
1774                          * another update of the status block, while there
1775                          * is still unhandled work.
1776                          */
1777                         rmb();
1778
1779                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1780                                 napi_complete(napi);
1781                                 /* Re-enable interrupts */
1782                                 DP(NETIF_MSG_HW,
1783                                    "Update index to %d\n", fp->fp_hc_idx);
1784                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1785                                              le16_to_cpu(fp->fp_hc_idx),
1786                                              IGU_INT_ENABLE, 1);
1787                                 break;
1788                         }
1789                 }
1790         }
1791
1792         return work_done;
1793 }
1794
1795 /* we split the first BD into headers and data BDs
1796  * to ease the pain of our fellow microcode engineers
1797  * we use one mapping for both BDs
1798  * So far this has only been observed to happen
1799  * in Other Operating Systems(TM)
1800  */
1801 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1802                                    struct bnx2x_fastpath *fp,
1803                                    struct sw_tx_bd *tx_buf,
1804                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1805                                    u16 bd_prod, int nbd)
1806 {
1807         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1808         struct eth_tx_bd *d_tx_bd;
1809         dma_addr_t mapping;
1810         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1811
1812         /* first fix first BD */
1813         h_tx_bd->nbd = cpu_to_le16(nbd);
1814         h_tx_bd->nbytes = cpu_to_le16(hlen);
1815
1816         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1817            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1818            h_tx_bd->addr_lo, h_tx_bd->nbd);
1819
1820         /* now get a new data BD
1821          * (after the pbd) and fill it */
1822         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1823         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1824
1825         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1826                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1827
1828         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1829         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1830         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1831
1832         /* this marks the BD as one that has no individual mapping */
1833         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1834
1835         DP(NETIF_MSG_TX_QUEUED,
1836            "TSO split data size is %d (%x:%x)\n",
1837            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1838
1839         /* update tx_bd */
1840         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1841
1842         return bd_prod;
1843 }
1844
1845 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1846 {
1847         if (fix > 0)
1848                 csum = (u16) ~csum_fold(csum_sub(csum,
1849                                 csum_partial(t_header - fix, fix, 0)));
1850
1851         else if (fix < 0)
1852                 csum = (u16) ~csum_fold(csum_add(csum,
1853                                 csum_partial(t_header, -fix, 0)));
1854
1855         return swab16(csum);
1856 }
1857
1858 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1859 {
1860         u32 rc;
1861
1862         if (skb->ip_summed != CHECKSUM_PARTIAL)
1863                 rc = XMIT_PLAIN;
1864
1865         else {
1866                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
1867                         rc = XMIT_CSUM_V6;
1868                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1869                                 rc |= XMIT_CSUM_TCP;
1870
1871                 } else {
1872                         rc = XMIT_CSUM_V4;
1873                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1874                                 rc |= XMIT_CSUM_TCP;
1875                 }
1876         }
1877
1878         if (skb_is_gso_v6(skb))
1879                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
1880         else if (skb_is_gso(skb))
1881                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
1882
1883         return rc;
1884 }
1885
1886 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1887 /* check if packet requires linearization (packet is too fragmented)
1888    no need to check fragmentation if page size > 8K (there will be no
1889    violation to FW restrictions) */
1890 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1891                              u32 xmit_type)
1892 {
1893         int to_copy = 0;
1894         int hlen = 0;
1895         int first_bd_sz = 0;
1896
1897         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1898         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1899
1900                 if (xmit_type & XMIT_GSO) {
1901                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1902                         /* Check if LSO packet needs to be copied:
1903                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1904                         int wnd_size = MAX_FETCH_BD - 3;
1905                         /* Number of windows to check */
1906                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1907                         int wnd_idx = 0;
1908                         int frag_idx = 0;
1909                         u32 wnd_sum = 0;
1910
1911                         /* Headers length */
1912                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1913                                 tcp_hdrlen(skb);
1914
1915                         /* Amount of data (w/o headers) on linear part of SKB*/
1916                         first_bd_sz = skb_headlen(skb) - hlen;
1917
1918                         wnd_sum  = first_bd_sz;
1919
1920                         /* Calculate the first sum - it's special */
1921                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1922                                 wnd_sum +=
1923                                         skb_shinfo(skb)->frags[frag_idx].size;
1924
1925                         /* If there was data on linear skb data - check it */
1926                         if (first_bd_sz > 0) {
1927                                 if (unlikely(wnd_sum < lso_mss)) {
1928                                         to_copy = 1;
1929                                         goto exit_lbl;
1930                                 }
1931
1932                                 wnd_sum -= first_bd_sz;
1933                         }
1934
1935                         /* Others are easier: run through the frag list and
1936                            check all windows */
1937                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1938                                 wnd_sum +=
1939                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1940
1941                                 if (unlikely(wnd_sum < lso_mss)) {
1942                                         to_copy = 1;
1943                                         break;
1944                                 }
1945                                 wnd_sum -=
1946                                         skb_shinfo(skb)->frags[wnd_idx].size;
1947                         }
1948                 } else {
1949                         /* in non-LSO too fragmented packet should always
1950                            be linearized */
1951                         to_copy = 1;
1952                 }
1953         }
1954
1955 exit_lbl:
1956         if (unlikely(to_copy))
1957                 DP(NETIF_MSG_TX_QUEUED,
1958                    "Linearization IS REQUIRED for %s packet. "
1959                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1960                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1961                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1962
1963         return to_copy;
1964 }
1965 #endif
1966
1967 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
1968                                         u32 xmit_type)
1969 {
1970         *parsing_data |= (skb_shinfo(skb)->gso_size <<
1971                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
1972                               ETH_TX_PARSE_BD_E2_LSO_MSS;
1973         if ((xmit_type & XMIT_GSO_V6) &&
1974             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
1975                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
1976 }
1977
1978 /**
1979  * Update PBD in GSO case.
1980  *
1981  * @param skb
1982  * @param tx_start_bd
1983  * @param pbd
1984  * @param xmit_type
1985  */
1986 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
1987                                      struct eth_tx_parse_bd_e1x *pbd,
1988                                      u32 xmit_type)
1989 {
1990         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1991         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1992         pbd->tcp_flags = pbd_tcp_flags(skb);
1993
1994         if (xmit_type & XMIT_GSO_V4) {
1995                 pbd->ip_id = swab16(ip_hdr(skb)->id);
1996                 pbd->tcp_pseudo_csum =
1997                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1998                                                   ip_hdr(skb)->daddr,
1999                                                   0, IPPROTO_TCP, 0));
2000
2001         } else
2002                 pbd->tcp_pseudo_csum =
2003                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2004                                                 &ipv6_hdr(skb)->daddr,
2005                                                 0, IPPROTO_TCP, 0));
2006
2007         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2008 }
2009
2010 /**
2011  *
2012  * @param skb
2013  * @param tx_start_bd
2014  * @param pbd_e2
2015  * @param xmit_type
2016  *
2017  * @return header len
2018  */
2019 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2020         u32 *parsing_data, u32 xmit_type)
2021 {
2022         *parsing_data |= ((tcp_hdrlen(skb)/4) <<
2023                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2024                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2025
2026         *parsing_data |= ((((u8 *)tcp_hdr(skb) - skb->data) / 2) <<
2027                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2028                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2029
2030         return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2031 }
2032
2033 /**
2034  *
2035  * @param skb
2036  * @param tx_start_bd
2037  * @param pbd
2038  * @param xmit_type
2039  *
2040  * @return Header length
2041  */
2042 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2043         struct eth_tx_parse_bd_e1x *pbd,
2044         u32 xmit_type)
2045 {
2046         u8 hlen = (skb_network_header(skb) - skb->data) / 2;
2047
2048         /* for now NS flag is not used in Linux */
2049         pbd->global_data =
2050                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2051                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2052
2053         pbd->ip_hlen_w = (skb_transport_header(skb) -
2054                         skb_network_header(skb)) / 2;
2055
2056         hlen += pbd->ip_hlen_w + tcp_hdrlen(skb) / 2;
2057
2058         pbd->total_hlen_w = cpu_to_le16(hlen);
2059         hlen = hlen*2;
2060
2061         if (xmit_type & XMIT_CSUM_TCP) {
2062                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2063
2064         } else {
2065                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2066
2067                 DP(NETIF_MSG_TX_QUEUED,
2068                    "hlen %d  fix %d  csum before fix %x\n",
2069                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2070
2071                 /* HW bug: fixup the CSUM */
2072                 pbd->tcp_pseudo_csum =
2073                         bnx2x_csum_fix(skb_transport_header(skb),
2074                                        SKB_CS(skb), fix);
2075
2076                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2077                    pbd->tcp_pseudo_csum);
2078         }
2079
2080         return hlen;
2081 }
2082
2083 /* called with netif_tx_lock
2084  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2085  * netif_wake_queue()
2086  */
2087 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2088 {
2089         struct bnx2x *bp = netdev_priv(dev);
2090         struct bnx2x_fastpath *fp;
2091         struct netdev_queue *txq;
2092         struct sw_tx_bd *tx_buf;
2093         struct eth_tx_start_bd *tx_start_bd;
2094         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2095         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2096         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2097         u32 pbd_e2_parsing_data = 0;
2098         u16 pkt_prod, bd_prod;
2099         int nbd, fp_index;
2100         dma_addr_t mapping;
2101         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2102         int i;
2103         u8 hlen = 0;
2104         __le16 pkt_size = 0;
2105         struct ethhdr *eth;
2106         u8 mac_type = UNICAST_ADDRESS;
2107
2108 #ifdef BNX2X_STOP_ON_ERROR
2109         if (unlikely(bp->panic))
2110                 return NETDEV_TX_BUSY;
2111 #endif
2112
2113         fp_index = skb_get_queue_mapping(skb);
2114         txq = netdev_get_tx_queue(dev, fp_index);
2115
2116         fp = &bp->fp[fp_index];
2117
2118         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
2119                 fp->eth_q_stats.driver_xoff++;
2120                 netif_tx_stop_queue(txq);
2121                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2122                 return NETDEV_TX_BUSY;
2123         }
2124
2125         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
2126                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
2127            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2128            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2129
2130         eth = (struct ethhdr *)skb->data;
2131
2132         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2133         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2134                 if (is_broadcast_ether_addr(eth->h_dest))
2135                         mac_type = BROADCAST_ADDRESS;
2136                 else
2137                         mac_type = MULTICAST_ADDRESS;
2138         }
2139
2140 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2141         /* First, check if we need to linearize the skb (due to FW
2142            restrictions). No need to check fragmentation if page size > 8K
2143            (there will be no violation to FW restrictions) */
2144         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2145                 /* Statistics of linearization */
2146                 bp->lin_cnt++;
2147                 if (skb_linearize(skb) != 0) {
2148                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2149                            "silently dropping this SKB\n");
2150                         dev_kfree_skb_any(skb);
2151                         return NETDEV_TX_OK;
2152                 }
2153         }
2154 #endif
2155
2156         /*
2157         Please read carefully. First we use one BD which we mark as start,
2158         then we have a parsing info BD (used for TSO or xsum),
2159         and only then we have the rest of the TSO BDs.
2160         (don't forget to mark the last one as last,
2161         and to unmap only AFTER you write to the BD ...)
2162         And above all, all pdb sizes are in words - NOT DWORDS!
2163         */
2164
2165         pkt_prod = fp->tx_pkt_prod++;
2166         bd_prod = TX_BD(fp->tx_bd_prod);
2167
2168         /* get a tx_buf and first BD */
2169         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
2170         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
2171
2172         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2173         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2174                  mac_type);
2175
2176         /* header nbd */
2177         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2178
2179         /* remember the first BD of the packet */
2180         tx_buf->first_bd = fp->tx_bd_prod;
2181         tx_buf->skb = skb;
2182         tx_buf->flags = 0;
2183
2184         DP(NETIF_MSG_TX_QUEUED,
2185            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2186            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2187
2188         if (vlan_tx_tag_present(skb)) {
2189                 tx_start_bd->vlan_or_ethertype =
2190                     cpu_to_le16(vlan_tx_tag_get(skb));
2191                 tx_start_bd->bd_flags.as_bitfield |=
2192                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2193         } else
2194                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2195
2196         /* turn on parsing and get a BD */
2197         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2198
2199         if (xmit_type & XMIT_CSUM) {
2200                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2201
2202                 if (xmit_type & XMIT_CSUM_V4)
2203                         tx_start_bd->bd_flags.as_bitfield |=
2204                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2205                 else
2206                         tx_start_bd->bd_flags.as_bitfield |=
2207                                                 ETH_TX_BD_FLAGS_IPV6;
2208
2209                 if (!(xmit_type & XMIT_CSUM_TCP))
2210                         tx_start_bd->bd_flags.as_bitfield |=
2211                                                 ETH_TX_BD_FLAGS_IS_UDP;
2212         }
2213
2214         if (CHIP_IS_E2(bp)) {
2215                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2216                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2217                 /* Set PBD in checksum offload case */
2218                 if (xmit_type & XMIT_CSUM)
2219                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2220                                                      &pbd_e2_parsing_data,
2221                                                      xmit_type);
2222         } else {
2223                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2224                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2225                 /* Set PBD in checksum offload case */
2226                 if (xmit_type & XMIT_CSUM)
2227                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2228
2229         }
2230
2231         /* Map skb linear data for DMA */
2232         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2233                                  skb_headlen(skb), DMA_TO_DEVICE);
2234
2235         /* Setup the data pointer of the first BD of the packet */
2236         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2237         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2238         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2239         tx_start_bd->nbd = cpu_to_le16(nbd);
2240         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2241         pkt_size = tx_start_bd->nbytes;
2242
2243         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2244            "  nbytes %d  flags %x  vlan %x\n",
2245            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2246            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2247            tx_start_bd->bd_flags.as_bitfield,
2248            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2249
2250         if (xmit_type & XMIT_GSO) {
2251
2252                 DP(NETIF_MSG_TX_QUEUED,
2253                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2254                    skb->len, hlen, skb_headlen(skb),
2255                    skb_shinfo(skb)->gso_size);
2256
2257                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2258
2259                 if (unlikely(skb_headlen(skb) > hlen))
2260                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2261                                                  hlen, bd_prod, ++nbd);
2262                 if (CHIP_IS_E2(bp))
2263                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2264                                              xmit_type);
2265                 else
2266                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2267         }
2268
2269         /* Set the PBD's parsing_data field if not zero
2270          * (for the chips newer than 57711).
2271          */
2272         if (pbd_e2_parsing_data)
2273                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2274
2275         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2276
2277         /* Handle fragmented skb */
2278         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2279                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2280
2281                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2282                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2283                 if (total_pkt_bd == NULL)
2284                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2285
2286                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2287                                        frag->page_offset,
2288                                        frag->size, DMA_TO_DEVICE);
2289
2290                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2291                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2292                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2293                 le16_add_cpu(&pkt_size, frag->size);
2294
2295                 DP(NETIF_MSG_TX_QUEUED,
2296                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2297                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2298                    le16_to_cpu(tx_data_bd->nbytes));
2299         }
2300
2301         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2302
2303         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2304
2305         /* now send a tx doorbell, counting the next BD
2306          * if the packet contains or ends with it
2307          */
2308         if (TX_BD_POFF(bd_prod) < nbd)
2309                 nbd++;
2310
2311         if (total_pkt_bd != NULL)
2312                 total_pkt_bd->total_pkt_bytes = pkt_size;
2313
2314         if (pbd_e1x)
2315                 DP(NETIF_MSG_TX_QUEUED,
2316                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2317                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2318                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2319                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2320                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2321                     le16_to_cpu(pbd_e1x->total_hlen_w));
2322         if (pbd_e2)
2323                 DP(NETIF_MSG_TX_QUEUED,
2324                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2325                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2326                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2327                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2328                    pbd_e2->parsing_data);
2329         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2330
2331         /*
2332          * Make sure that the BD data is updated before updating the producer
2333          * since FW might read the BD right after the producer is updated.
2334          * This is only applicable for weak-ordered memory model archs such
2335          * as IA-64. The following barrier is also mandatory since FW will
2336          * assumes packets must have BDs.
2337          */
2338         wmb();
2339
2340         fp->tx_db.data.prod += nbd;
2341         barrier();
2342
2343         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2344
2345         mmiowb();
2346
2347         fp->tx_bd_prod += nbd;
2348
2349         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2350                 netif_tx_stop_queue(txq);
2351
2352                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2353                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2354                  * fp->bd_tx_cons */
2355                 smp_mb();
2356
2357                 fp->eth_q_stats.driver_xoff++;
2358                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2359                         netif_tx_wake_queue(txq);
2360         }
2361         fp->tx_pkt++;
2362
2363         return NETDEV_TX_OK;
2364 }
2365
2366 /* called with rtnl_lock */
2367 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2368 {
2369         struct sockaddr *addr = p;
2370         struct bnx2x *bp = netdev_priv(dev);
2371
2372         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2373                 return -EINVAL;
2374
2375         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2376         if (netif_running(dev))
2377                 bnx2x_set_eth_mac(bp, 1);
2378
2379         return 0;
2380 }
2381
2382
2383 static int bnx2x_setup_irqs(struct bnx2x *bp)
2384 {
2385         int rc = 0;
2386         if (bp->flags & USING_MSIX_FLAG) {
2387                 rc = bnx2x_req_msix_irqs(bp);
2388                 if (rc)
2389                         return rc;
2390         } else {
2391                 bnx2x_ack_int(bp);
2392                 rc = bnx2x_req_irq(bp);
2393                 if (rc) {
2394                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2395                         return rc;
2396                 }
2397                 if (bp->flags & USING_MSI_FLAG) {
2398                         bp->dev->irq = bp->pdev->irq;
2399                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2400                                bp->pdev->irq);
2401                 }
2402         }
2403
2404         return 0;
2405 }
2406
2407 void bnx2x_free_mem_bp(struct bnx2x *bp)
2408 {
2409         kfree(bp->fp);
2410         kfree(bp->msix_table);
2411         kfree(bp->ilt);
2412 }
2413
2414 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2415 {
2416         struct bnx2x_fastpath *fp;
2417         struct msix_entry *tbl;
2418         struct bnx2x_ilt *ilt;
2419
2420         /* fp array */
2421         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2422         if (!fp)
2423                 goto alloc_err;
2424         bp->fp = fp;
2425
2426         /* msix table */
2427         tbl = kzalloc((FP_SB_COUNT(bp->l2_cid_count) + 1) * sizeof(*tbl),
2428                                   GFP_KERNEL);
2429         if (!tbl)
2430                 goto alloc_err;
2431         bp->msix_table = tbl;
2432
2433         /* ilt */
2434         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2435         if (!ilt)
2436                 goto alloc_err;
2437         bp->ilt = ilt;
2438
2439         return 0;
2440 alloc_err:
2441         bnx2x_free_mem_bp(bp);
2442         return -ENOMEM;
2443
2444 }
2445
2446 /* called with rtnl_lock */
2447 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2448 {
2449         struct bnx2x *bp = netdev_priv(dev);
2450         int rc = 0;
2451
2452         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2453                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2454                 return -EAGAIN;
2455         }
2456
2457         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2458             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2459                 return -EINVAL;
2460
2461         /* This does not race with packet allocation
2462          * because the actual alloc size is
2463          * only updated as part of load
2464          */
2465         dev->mtu = new_mtu;
2466
2467         if (netif_running(dev)) {
2468                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2469                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2470         }
2471
2472         return rc;
2473 }
2474
2475 void bnx2x_tx_timeout(struct net_device *dev)
2476 {
2477         struct bnx2x *bp = netdev_priv(dev);
2478
2479 #ifdef BNX2X_STOP_ON_ERROR
2480         if (!bp->panic)
2481                 bnx2x_panic();
2482 #endif
2483         /* This allows the netif to be shutdown gracefully before resetting */
2484         schedule_delayed_work(&bp->reset_task, 0);
2485 }
2486
2487 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2488 {
2489         struct net_device *dev = pci_get_drvdata(pdev);
2490         struct bnx2x *bp;
2491
2492         if (!dev) {
2493                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2494                 return -ENODEV;
2495         }
2496         bp = netdev_priv(dev);
2497
2498         rtnl_lock();
2499
2500         pci_save_state(pdev);
2501
2502         if (!netif_running(dev)) {
2503                 rtnl_unlock();
2504                 return 0;
2505         }
2506
2507         netif_device_detach(dev);
2508
2509         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2510
2511         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2512
2513         rtnl_unlock();
2514
2515         return 0;
2516 }
2517
2518 int bnx2x_resume(struct pci_dev *pdev)
2519 {
2520         struct net_device *dev = pci_get_drvdata(pdev);
2521         struct bnx2x *bp;
2522         int rc;
2523
2524         if (!dev) {
2525                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2526                 return -ENODEV;
2527         }
2528         bp = netdev_priv(dev);
2529
2530         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2531                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2532                 return -EAGAIN;
2533         }
2534
2535         rtnl_lock();
2536
2537         pci_restore_state(pdev);
2538
2539         if (!netif_running(dev)) {
2540                 rtnl_unlock();
2541                 return 0;
2542         }
2543
2544         bnx2x_set_power_state(bp, PCI_D0);
2545         netif_device_attach(dev);
2546
2547         /* Since the chip was reset, clear the FW sequence number */
2548         bp->fw_seq = 0;
2549         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2550
2551         rtnl_unlock();
2552
2553         return rc;
2554 }