drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <linux/if_vlan.h>
  20 #include <linux/ip.h>
  21 #include <net/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #include "bnx2x_init.h"
  27
  28 static int bnx2x_setup_irqs(struct bnx2x *bp);
  29
  30 /* free skb in the packet ring at pos idx
  31  * return idx of last bd freed
  32  */
  33 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  34                              u16 idx)
  35 {
  36         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  37         struct eth_tx_start_bd *tx_start_bd;
  38         struct eth_tx_bd *tx_data_bd;
  39         struct sk_buff *skb = tx_buf->skb;
  40         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  41         int nbd;
  42
  43         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  44         prefetch(&skb->end);
  45
  46         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  47            idx, tx_buf, skb);
  48
  49         /* unmap first bd */
  50         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  51         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  52         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  53                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  54
  55         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  56 #ifdef BNX2X_STOP_ON_ERROR
  57         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  58                 BNX2X_ERR("BAD nbd!\n");
  59                 bnx2x_panic();
  60         }
  61 #endif
  62         new_cons = nbd + tx_buf->first_bd;
  63
  64         /* Get the next bd */
  65         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  66
  67         /* Skip a parse bd... */
  68         --nbd;
  69         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  70
  71         /* ...and the TSO split header bd since they have no mapping */
  72         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  73                 --nbd;
  74                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  75         }
  76
  77         /* now free frags */
  78         while (nbd > 0) {
  79
  80                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  81                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  82                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  83                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  84                 if (--nbd)
  85                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  86         }
  87
  88         /* release skb */
  89         WARN_ON(!skb);
  90         dev_kfree_skb(skb);
  91         tx_buf->first_bd = 0;
  92         tx_buf->skb = NULL;
  93
  94         return new_cons;
  95 }
  96
  97 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
  98 {
  99         struct bnx2x *bp = fp->bp;
 100         struct netdev_queue *txq;
 101         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 102
 103 #ifdef BNX2X_STOP_ON_ERROR
 104         if (unlikely(bp->panic))
 105                 return -1;
 106 #endif
 107
 108         txq = netdev_get_tx_queue(bp->dev, fp->index);
 109         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 110         sw_cons = fp->tx_pkt_cons;
 111
 112         while (sw_cons != hw_cons) {
 113                 u16 pkt_cons;
 114
 115                 pkt_cons = TX_BD(sw_cons);
 116
 117                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 118                                       " pkt_cons %u\n",
 119                    fp->index, hw_cons, sw_cons, pkt_cons);
 120
 121                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 122                 sw_cons++;
 123         }
 124
 125         fp->tx_pkt_cons = sw_cons;
 126         fp->tx_bd_cons = bd_cons;
 127
 128         /* Need to make the tx_bd_cons update visible to start_xmit()
 129          * before checking for netif_tx_queue_stopped().  Without the
 130          * memory barrier, there is a small possibility that
 131          * start_xmit() will miss it and cause the queue to be stopped
 132          * forever.
 133          */
 134         smp_mb();
 135
 136         if (unlikely(netif_tx_queue_stopped(txq))) {
 137                 /* Taking tx_lock() is needed to prevent reenabling the queue
 138                  * while it's empty. This could have happen if rx_action() gets
 139                  * suspended in bnx2x_tx_int() after the condition before
 140                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 141                  *
 142                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 143                  * sends some packets consuming the whole queue again->
 144                  * stops the queue
 145                  */
 146
 147                 __netif_tx_lock(txq, smp_processor_id());
 148
 149                 if ((netif_tx_queue_stopped(txq)) &&
 150                     (bp->state == BNX2X_STATE_OPEN) &&
 151                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 152                         netif_tx_wake_queue(txq);
 153
 154                 __netif_tx_unlock(txq);
 155         }
 156         return 0;
 157 }
 158
 159 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 160                                              u16 idx)
 161 {
 162         u16 last_max = fp->last_max_sge;
 163
 164         if (SUB_S16(idx, last_max) > 0)
 165                 fp->last_max_sge = idx;
 166 }
 167
 168 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 169                                   struct eth_fast_path_rx_cqe *fp_cqe)
 170 {
 171         struct bnx2x *bp = fp->bp;
 172         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 173                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 174                       SGE_PAGE_SHIFT;
 175         u16 last_max, last_elem, first_elem;
 176         u16 delta = 0;
 177         u16 i;
 178
 179         if (!sge_len)
 180                 return;
 181
 182         /* First mark all used pages */
 183         for (i = 0; i < sge_len; i++)
 184                 SGE_MASK_CLEAR_BIT(fp,
 185                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 186
 187         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 188            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 189
 190         /* Here we assume that the last SGE index is the biggest */
 191         prefetch((void *)(fp->sge_mask));
 192         bnx2x_update_last_max_sge(fp,
 193                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 194
 195         last_max = RX_SGE(fp->last_max_sge);
 196         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 197         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 198
 199         /* If ring is not full */
 200         if (last_elem + 1 != first_elem)
 201                 last_elem++;
 202
 203         /* Now update the prod */
 204         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 205                 if (likely(fp->sge_mask[i]))
 206                         break;
 207
 208                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 209                 delta += RX_SGE_MASK_ELEM_SZ;
 210         }
 211
 212         if (delta > 0) {
 213                 fp->rx_sge_prod += delta;
 214                 /* clear page-end entries */
 215                 bnx2x_clear_sge_mask_next_elems(fp);
 216         }
 217
 218         DP(NETIF_MSG_RX_STATUS,
 219            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 220            fp->last_max_sge, fp->rx_sge_prod);
 221 }
 222
 223 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 224                             struct sk_buff *skb, u16 cons, u16 prod)
 225 {
 226         struct bnx2x *bp = fp->bp;
 227         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 228         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 229         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 230         dma_addr_t mapping;
 231
 232         /* move empty skb from pool to prod and map it */
 233         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 234         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 235                                  fp->rx_buf_size, DMA_FROM_DEVICE);
 236         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 237
 238         /* move partial skb from cons to pool (don't unmap yet) */
 239         fp->tpa_pool[queue] = *cons_rx_buf;
 240
 241         /* mark bin state as start - print error if current state != stop */
 242         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 243                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 244
 245         fp->tpa_state[queue] = BNX2X_TPA_START;
 246
 247         /* point prod_bd to new skb */
 248         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 249         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 250
 251 #ifdef BNX2X_STOP_ON_ERROR
 252         fp->tpa_queue_used |= (1 << queue);
 253 #ifdef _ASM_GENERIC_INT_L64_H
 254         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 255 #else
 256         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 257 #endif
 258            fp->tpa_queue_used);
 259 #endif
 260 }
 261
 262 /* Timestamp option length allowed for TPA aggregation:
 263  *
 264  *              nop nop kind length echo val
 265  */
 266 #define TPA_TSTAMP_OPT_LEN      12
 267 /**
 268  * Calculate the approximate value of the MSS for this
 269  * aggregation using the first packet of it.
 270  *
 271  * @param bp
 272  * @param parsing_flags Parsing flags from the START CQE
 273  * @param len_on_bd Total length of the first packet for the
 274  *                   aggregation.
 275  */
 276 static inline u16 bnx2x_set_lro_mss(struct bnx2x *bp, u16 parsing_flags,
 277                                     u16 len_on_bd)
 278 {
 279         /* TPA arrgregation won't have an IP options and TCP options
 280          * other than timestamp.
 281          */
 282         u16 hdrs_len = ETH_HLEN + sizeof(struct iphdr) + sizeof(struct tcphdr);
 283
 284
 285         /* Check if there was a TCP timestamp, if there is it's will
 286          * always be 12 bytes length: nop nop kind length echo val.
 287          *
 288          * Otherwise FW would close the aggregation.
 289          */
 290         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
 291                 hdrs_len += TPA_TSTAMP_OPT_LEN;
 292
 293         return len_on_bd - hdrs_len;
 294 }
 295
 296 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 297                                struct sk_buff *skb,
 298                                struct eth_fast_path_rx_cqe *fp_cqe,
 299                                u16 cqe_idx, u16 parsing_flags)
 300 {
 301         struct sw_rx_page *rx_pg, old_rx_pg;
 302         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 303         u32 i, frag_len, frag_size, pages;
 304         int err;
 305         int j;
 306
 307         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 308         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 309
 310         /* This is needed in order to enable forwarding support */
 311         if (frag_size)
 312                 skb_shinfo(skb)->gso_size = bnx2x_set_lro_mss(bp, parsing_flags,
 313                                                               len_on_bd);
 314
 315 #ifdef BNX2X_STOP_ON_ERROR
 316         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 317                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 318                           pages, cqe_idx);
 319                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 320                           fp_cqe->pkt_len, len_on_bd);
 321                 bnx2x_panic();
 322                 return -EINVAL;
 323         }
 324 #endif
 325
 326         /* Run through the SGL and compose the fragmented skb */
 327         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 328                 u16 sge_idx =
 329                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
 330
 331                 /* FW gives the indices of the SGE as if the ring is an array
 332                    (meaning that "next" element will consume 2 indices) */
 333                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 334                 rx_pg = &fp->rx_page_ring[sge_idx];
 335                 old_rx_pg = *rx_pg;
 336
 337                 /* If we fail to allocate a substitute page, we simply stop
 338                    where we are and drop the whole packet */
 339                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 340                 if (unlikely(err)) {
 341                         fp->eth_q_stats.rx_skb_alloc_failed++;
 342                         return err;
 343                 }
 344
 345                 /* Unmap the page as we r going to pass it to the stack */
 346                 dma_unmap_page(&bp->pdev->dev,
 347                                dma_unmap_addr(&old_rx_pg, mapping),
 348                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 349
 350                 /* Add one frag and update the appropriate fields in the skb */
 351                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 352
 353                 skb->data_len += frag_len;
 354                 skb->truesize += frag_len;
 355                 skb->len += frag_len;
 356
 357                 frag_size -= frag_len;
 358         }
 359
 360         return 0;
 361 }
 362
 363 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 364                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 365                            u16 cqe_idx)
 366 {
 367         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 368         struct sk_buff *skb = rx_buf->skb;
 369         /* alloc new skb */
 370         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 371
 372         /* Unmap skb in the pool anyway, as we are going to change
 373            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 374            fails. */
 375         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 376                          fp->rx_buf_size, DMA_FROM_DEVICE);
 377
 378         if (likely(new_skb)) {
 379                 /* fix ip xsum and give it to the stack */
 380                 /* (no need to map the new skb) */
 381                 u16 parsing_flags =
 382                         le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags);
 383
 384                 prefetch(skb);
 385                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 386
 387 #ifdef BNX2X_STOP_ON_ERROR
 388                 if (pad + len > fp->rx_buf_size) {
 389                         BNX2X_ERR("skb_put is about to fail...  "
 390                                   "pad %d  len %d  rx_buf_size %d\n",
 391                                   pad, len, fp->rx_buf_size);
 392                         bnx2x_panic();
 393                         return;
 394                 }
 395 #endif
 396
 397                 skb_reserve(skb, pad);
 398                 skb_put(skb, len);
 399
 400                 skb->protocol = eth_type_trans(skb, bp->dev);
 401                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 402
 403                 {
 404                         struct iphdr *iph;
 405
 406                         iph = (struct iphdr *)skb->data;
 407                         iph->check = 0;
 408                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 409                 }
 410
 411                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 412                                          &cqe->fast_path_cqe, cqe_idx,
 413                                          parsing_flags)) {
 414                         if (parsing_flags & PARSING_FLAGS_VLAN)
 415                                 __vlan_hwaccel_put_tag(skb,
 416                                                  le16_to_cpu(cqe->fast_path_cqe.
 417                                                              vlan_tag));
 418                         napi_gro_receive(&fp->napi, skb);
 419                 } else {
 420                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 421                            " - dropping packet!\n");
 422                         dev_kfree_skb(skb);
 423                 }
 424
 425
 426                 /* put new skb in bin */
 427                 fp->tpa_pool[queue].skb = new_skb;
 428
 429         } else {
 430                 /* else drop the packet and keep the buffer in the bin */
 431                 DP(NETIF_MSG_RX_STATUS,
 432                    "Failed to allocate new skb - dropping packet!\n");
 433                 fp->eth_q_stats.rx_skb_alloc_failed++;
 434         }
 435
 436         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 437 }
 438
 439 /* Set Toeplitz hash value in the skb using the value from the
 440  * CQE (calculated by HW).
 441  */
 442 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 443                                         struct sk_buff *skb)
 444 {
 445         /* Set Toeplitz hash from CQE */
 446         if ((bp->dev->features & NETIF_F_RXHASH) &&
 447             (cqe->fast_path_cqe.status_flags &
 448              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 449                 skb->rxhash =
 450                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 451 }
 452
 453 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 454 {
 455         struct bnx2x *bp = fp->bp;
 456         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 457         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 458         int rx_pkt = 0;
 459
 460 #ifdef BNX2X_STOP_ON_ERROR
 461         if (unlikely(bp->panic))
 462                 return 0;
 463 #endif
 464
 465         /* CQ "next element" is of the size of the regular element,
 466            that's why it's ok here */
 467         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 468         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 469                 hw_comp_cons++;
 470
 471         bd_cons = fp->rx_bd_cons;
 472         bd_prod = fp->rx_bd_prod;
 473         bd_prod_fw = bd_prod;
 474         sw_comp_cons = fp->rx_comp_cons;
 475         sw_comp_prod = fp->rx_comp_prod;
 476
 477         /* Memory barrier necessary as speculative reads of the rx
 478          * buffer can be ahead of the index in the status block
 479          */
 480         rmb();
 481
 482         DP(NETIF_MSG_RX_STATUS,
 483            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 484            fp->index, hw_comp_cons, sw_comp_cons);
 485
 486         while (sw_comp_cons != hw_comp_cons) {
 487                 struct sw_rx_bd *rx_buf = NULL;
 488                 struct sk_buff *skb;
 489                 union eth_rx_cqe *cqe;
 490                 u8 cqe_fp_flags;
 491                 u16 len, pad;
 492
 493                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 494                 bd_prod = RX_BD(bd_prod);
 495                 bd_cons = RX_BD(bd_cons);
 496
 497                 /* Prefetch the page containing the BD descriptor
 498                    at producer's index. It will be needed when new skb is
 499                    allocated */
 500                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 501                                              (&fp->rx_desc_ring[bd_prod])) -
 502                                   PAGE_SIZE + 1));
 503
 504                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 505                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 506
 507                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 508                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 509                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 510                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 511                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 512                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 513
 514                 /* is this a slowpath msg? */
 515                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 516                         bnx2x_sp_event(fp, cqe);
 517                         goto next_cqe;
 518
 519                 /* this is an rx packet */
 520                 } else {
 521                         rx_buf = &fp->rx_buf_ring[bd_cons];
 522                         skb = rx_buf->skb;
 523                         prefetch(skb);
 524                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 525                         pad = cqe->fast_path_cqe.placement_offset;
 526
 527                         /* - If CQE is marked both TPA_START and TPA_END it is
 528                          *   a non-TPA CQE.
 529                          * - FP CQE will always have either TPA_START or/and
 530                          *   TPA_STOP flags set.
 531                          */
 532                         if ((!fp->disable_tpa) &&
 533                             (TPA_TYPE(cqe_fp_flags) !=
 534                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 535                                 u16 queue = cqe->fast_path_cqe.queue_index;
 536
 537                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 538                                         DP(NETIF_MSG_RX_STATUS,
 539                                            "calling tpa_start on queue %d\n",
 540                                            queue);
 541
 542                                         bnx2x_tpa_start(fp, queue, skb,
 543                                                         bd_cons, bd_prod);
 544
 545                                         /* Set Toeplitz hash for an LRO skb */
 546                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 547
 548                                         goto next_rx;
 549                                 } else { /* TPA_STOP */
 550                                         DP(NETIF_MSG_RX_STATUS,
 551                                            "calling tpa_stop on queue %d\n",
 552                                            queue);
 553
 554                                         if (!BNX2X_RX_SUM_FIX(cqe))
 555                                                 BNX2X_ERR("STOP on none TCP "
 556                                                           "data\n");
 557
 558                                         /* This is a size of the linear data
 559                                            on this skb */
 560                                         len = le16_to_cpu(cqe->fast_path_cqe.
 561                                                                 len_on_bd);
 562                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 563                                                     len, cqe, comp_ring_cons);
 564 #ifdef BNX2X_STOP_ON_ERROR
 565                                         if (bp->panic)
 566                                                 return 0;
 567 #endif
 568
 569                                         bnx2x_update_sge_prod(fp,
 570                                                         &cqe->fast_path_cqe);
 571                                         goto next_cqe;
 572                                 }
 573                         }
 574
 575                         dma_sync_single_for_device(&bp->pdev->dev,
 576                                         dma_unmap_addr(rx_buf, mapping),
 577                                                    pad + RX_COPY_THRESH,
 578                                                    DMA_FROM_DEVICE);
 579                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 580
 581                         /* is this an error packet? */
 582                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 583                                 DP(NETIF_MSG_RX_ERR,
 584                                    "ERROR  flags %x  rx packet %u\n",
 585                                    cqe_fp_flags, sw_comp_cons);
 586                                 fp->eth_q_stats.rx_err_discard_pkt++;
 587                                 goto reuse_rx;
 588                         }
 589
 590                         /* Since we don't have a jumbo ring
 591                          * copy small packets if mtu > 1500
 592                          */
 593                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 594                             (len <= RX_COPY_THRESH)) {
 595                                 struct sk_buff *new_skb;
 596
 597                                 new_skb = netdev_alloc_skb(bp->dev,
 598                                                            len + pad);
 599                                 if (new_skb == NULL) {
 600                                         DP(NETIF_MSG_RX_ERR,
 601                                            "ERROR  packet dropped "
 602                                            "because of alloc failure\n");
 603                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 604                                         goto reuse_rx;
 605                                 }
 606
 607                                 /* aligned copy */
 608                                 skb_copy_from_linear_data_offset(skb, pad,
 609                                                     new_skb->data + pad, len);
 610                                 skb_reserve(new_skb, pad);
 611                                 skb_put(new_skb, len);
 612
 613                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 614
 615                                 skb = new_skb;
 616
 617                         } else
 618                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 619                                 dma_unmap_single(&bp->pdev->dev,
 620                                         dma_unmap_addr(rx_buf, mapping),
 621                                                  fp->rx_buf_size,
 622                                                  DMA_FROM_DEVICE);
 623                                 skb_reserve(skb, pad);
 624                                 skb_put(skb, len);
 625
 626                         } else {
 627                                 DP(NETIF_MSG_RX_ERR,
 628                                    "ERROR  packet dropped because "
 629                                    "of alloc failure\n");
 630                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 631 reuse_rx:
 632                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 633                                 goto next_rx;
 634                         }
 635
 636                         skb->protocol = eth_type_trans(skb, bp->dev);
 637
 638                         /* Set Toeplitz hash for a none-LRO skb */
 639                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 640
 641                         skb_checksum_none_assert(skb);
 642
 643                         if (bp->rx_csum) {
 644                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 645                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 646                                 else
 647                                         fp->eth_q_stats.hw_csum_err++;
 648                         }
 649                 }
 650
 651                 skb_record_rx_queue(skb, fp->index);
 652
 653                 if (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 654                      PARSING_FLAGS_VLAN)
 655                         __vlan_hwaccel_put_tag(skb,
 656                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
 657                 napi_gro_receive(&fp->napi, skb);
 658
 659
 660 next_rx:
 661                 rx_buf->skb = NULL;
 662
 663                 bd_cons = NEXT_RX_IDX(bd_cons);
 664                 bd_prod = NEXT_RX_IDX(bd_prod);
 665                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 666                 rx_pkt++;
 667 next_cqe:
 668                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 669                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 670
 671                 if (rx_pkt == budget)
 672                         break;
 673         } /* while */
 674
 675         fp->rx_bd_cons = bd_cons;
 676         fp->rx_bd_prod = bd_prod_fw;
 677         fp->rx_comp_cons = sw_comp_cons;
 678         fp->rx_comp_prod = sw_comp_prod;
 679
 680         /* Update producers */
 681         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 682                              fp->rx_sge_prod);
 683
 684         fp->rx_pkt += rx_pkt;
 685         fp->rx_calls++;
 686
 687         return rx_pkt;
 688 }
 689
 690 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 691 {
 692         struct bnx2x_fastpath *fp = fp_cookie;
 693         struct bnx2x *bp = fp->bp;
 694
 695         /* Return here if interrupt is disabled */
 696         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 697                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 698                 return IRQ_HANDLED;
 699         }
 700
 701         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 702                          "[fp %d fw_sd %d igusb %d]\n",
 703            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 704         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 705
 706 #ifdef BNX2X_STOP_ON_ERROR
 707         if (unlikely(bp->panic))
 708                 return IRQ_HANDLED;
 709 #endif
 710
 711         /* Handle Rx and Tx according to MSI-X vector */
 712         prefetch(fp->rx_cons_sb);
 713         prefetch(fp->tx_cons_sb);
 714         prefetch(&fp->sb_running_index[SM_RX_ID]);
 715         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 716
 717         return IRQ_HANDLED;
 718 }
 719
 720 /* HW Lock for shared dual port PHYs */
 721 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 722 {
 723         mutex_lock(&bp->port.phy_mutex);
 724
 725         if (bp->port.need_hw_lock)
 726                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 727 }
 728
 729 void bnx2x_release_phy_lock(struct bnx2x *bp)
 730 {
 731         if (bp->port.need_hw_lock)
 732                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 733
 734         mutex_unlock(&bp->port.phy_mutex);
 735 }
 736
 737 /* calculates MF speed according to current linespeed and MF configuration */
 738 u16 bnx2x_get_mf_speed(struct bnx2x *bp)
 739 {
 740         u16 line_speed = bp->link_vars.line_speed;
 741         if (IS_MF(bp)) {
 742                 u16 maxCfg = bnx2x_extract_max_cfg(bp,
 743                                                    bp->mf_config[BP_VN(bp)]);
 744
 745                 /* Calculate the current MAX line speed limit for the MF
 746                  * devices
 747                  */
 748                 if (IS_MF_SI(bp))
 749                         line_speed = (line_speed * maxCfg) / 100;
 750                 else { /* SD mode */
 751                         u16 vn_max_rate = maxCfg * 100;
 752
 753                         if (vn_max_rate < line_speed)
 754                                 line_speed = vn_max_rate;
 755                 }
 756         }
 757
 758         return line_speed;
 759 }
 760
 761 void bnx2x_link_report(struct bnx2x *bp)
 762 {
 763         if (bp->flags & MF_FUNC_DIS) {
 764                 netif_carrier_off(bp->dev);
 765                 netdev_err(bp->dev, "NIC Link is Down\n");
 766                 return;
 767         }
 768
 769         if (bp->link_vars.link_up) {
 770                 u16 line_speed;
 771
 772                 if (bp->state == BNX2X_STATE_OPEN)
 773                         netif_carrier_on(bp->dev);
 774                 netdev_info(bp->dev, "NIC Link is Up, ");
 775
 776                 line_speed = bnx2x_get_mf_speed(bp);
 777
 778                 pr_cont("%d Mbps ", line_speed);
 779
 780                 if (bp->link_vars.duplex == DUPLEX_FULL)
 781                         pr_cont("full duplex");
 782                 else
 783                         pr_cont("half duplex");
 784
 785                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 786                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 787                                 pr_cont(", receive ");
 788                                 if (bp->link_vars.flow_ctrl &
 789                                     BNX2X_FLOW_CTRL_TX)
 790                                         pr_cont("& transmit ");
 791                         } else {
 792                                 pr_cont(", transmit ");
 793                         }
 794                         pr_cont("flow control ON");
 795                 }
 796                 pr_cont("\n");
 797
 798         } else { /* link_down */
 799                 netif_carrier_off(bp->dev);
 800                 netdev_err(bp->dev, "NIC Link is Down\n");
 801         }
 802 }
 803
 804 /* Returns the number of actually allocated BDs */
 805 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
 806                                       int rx_ring_size)
 807 {
 808         struct bnx2x *bp = fp->bp;
 809         u16 ring_prod, cqe_ring_prod;
 810         int i;
 811
 812         fp->rx_comp_cons = 0;
 813         cqe_ring_prod = ring_prod = 0;
 814         for (i = 0; i < rx_ring_size; i++) {
 815                 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 816                         BNX2X_ERR("was only able to allocate "
 817                                   "%d rx skbs on queue[%d]\n", i, fp->index);
 818                         fp->eth_q_stats.rx_skb_alloc_failed++;
 819                         break;
 820                 }
 821                 ring_prod = NEXT_RX_IDX(ring_prod);
 822                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 823                 WARN_ON(ring_prod <= i);
 824         }
 825
 826         fp->rx_bd_prod = ring_prod;
 827         /* Limit the CQE producer by the CQE ring size */
 828         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 829                                cqe_ring_prod);
 830         fp->rx_pkt = fp->rx_calls = 0;
 831
 832         return i;
 833 }
 834
 835 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath *fp)
 836 {
 837         struct bnx2x *bp = fp->bp;
 838         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
 839                                               MAX_RX_AVAIL/bp->num_queues;
 840
 841         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
 842
 843         bnx2x_alloc_rx_bds(fp, rx_ring_size);
 844
 845         /* Warning!
 846          * this will generate an interrupt (to the TSTORM)
 847          * must only be done after chip is initialized
 848          */
 849         bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
 850                              fp->rx_sge_prod);
 851 }
 852
 853 void bnx2x_init_rx_rings(struct bnx2x *bp)
 854 {
 855         int func = BP_FUNC(bp);
 856         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 857                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 858         u16 ring_prod;
 859         int i, j;
 860
 861         for_each_rx_queue(bp, j) {
 862                 struct bnx2x_fastpath *fp = &bp->fp[j];
 863
 864                 DP(NETIF_MSG_IFUP,
 865                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
 866
 867                 if (!fp->disable_tpa) {
 868                         for (i = 0; i < max_agg_queues; i++) {
 869                                 fp->tpa_pool[i].skb =
 870                                    netdev_alloc_skb(bp->dev, fp->rx_buf_size);
 871                                 if (!fp->tpa_pool[i].skb) {
 872                                         BNX2X_ERR("Failed to allocate TPA "
 873                                                   "skb pool for queue[%d] - "
 874                                                   "disabling TPA on this "
 875                                                   "queue!\n", j);
 876                                         bnx2x_free_tpa_pool(bp, fp, i);
 877                                         fp->disable_tpa = 1;
 878                                         break;
 879                                 }
 880                                 dma_unmap_addr_set((struct sw_rx_bd *)
 881                                                         &bp->fp->tpa_pool[i],
 882                                                    mapping, 0);
 883                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 884                         }
 885
 886                         /* "next page" elements initialization */
 887                         bnx2x_set_next_page_sgl(fp);
 888
 889                         /* set SGEs bit mask */
 890                         bnx2x_init_sge_ring_bit_mask(fp);
 891
 892                         /* Allocate SGEs and initialize the ring elements */
 893                         for (i = 0, ring_prod = 0;
 894                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 895
 896                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 897                                         BNX2X_ERR("was only able to allocate "
 898                                                   "%d rx sges\n", i);
 899                                         BNX2X_ERR("disabling TPA for"
 900                                                   " queue[%d]\n", j);
 901                                         /* Cleanup already allocated elements */
 902                                         bnx2x_free_rx_sge_range(bp,
 903                                                                 fp, ring_prod);
 904                                         bnx2x_free_tpa_pool(bp,
 905                                                             fp, max_agg_queues);
 906                                         fp->disable_tpa = 1;
 907                                         ring_prod = 0;
 908                                         break;
 909                                 }
 910                                 ring_prod = NEXT_SGE_IDX(ring_prod);
 911                         }
 912
 913                         fp->rx_sge_prod = ring_prod;
 914                 }
 915         }
 916
 917         for_each_rx_queue(bp, j) {
 918                 struct bnx2x_fastpath *fp = &bp->fp[j];
 919
 920                 fp->rx_bd_cons = 0;
 921
 922                 bnx2x_set_next_page_rx_bd(fp);
 923
 924                 /* CQ ring */
 925                 bnx2x_set_next_page_rx_cq(fp);
 926
 927                 /* Allocate BDs and initialize BD ring */
 928                 bnx2x_alloc_rx_bd_ring(fp);
 929
 930                 if (j != 0)
 931                         continue;
 932
 933                 if (!CHIP_IS_E2(bp)) {
 934                         REG_WR(bp, BAR_USTRORM_INTMEM +
 935                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 936                                U64_LO(fp->rx_comp_mapping));
 937                         REG_WR(bp, BAR_USTRORM_INTMEM +
 938                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 939                                U64_HI(fp->rx_comp_mapping));
 940                 }
 941         }
 942 }
 943
 944 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 945 {
 946         int i;
 947
 948         for_each_tx_queue(bp, i) {
 949                 struct bnx2x_fastpath *fp = &bp->fp[i];
 950
 951                 u16 bd_cons = fp->tx_bd_cons;
 952                 u16 sw_prod = fp->tx_pkt_prod;
 953                 u16 sw_cons = fp->tx_pkt_cons;
 954
 955                 while (sw_cons != sw_prod) {
 956                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 957                         sw_cons++;
 958                 }
 959         }
 960 }
 961
 962 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 963 {
 964         int i, j;
 965
 966         for_each_rx_queue(bp, j) {
 967                 struct bnx2x_fastpath *fp = &bp->fp[j];
 968
 969                 for (i = 0; i < NUM_RX_BD; i++) {
 970                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 971                         struct sk_buff *skb = rx_buf->skb;
 972
 973                         if (skb == NULL)
 974                                 continue;
 975
 976                         dma_unmap_single(&bp->pdev->dev,
 977                                          dma_unmap_addr(rx_buf, mapping),
 978                                          fp->rx_buf_size, DMA_FROM_DEVICE);
 979
 980                         rx_buf->skb = NULL;
 981                         dev_kfree_skb(skb);
 982                 }
 983                 if (!fp->disable_tpa)
 984                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 985                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 986                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 987         }
 988 }
 989
 990 void bnx2x_free_skbs(struct bnx2x *bp)
 991 {
 992         bnx2x_free_tx_skbs(bp);
 993         bnx2x_free_rx_skbs(bp);
 994 }
 995
 996 void bnx2x_update_max_mf_config(struct bnx2x *bp, u32 value)
 997 {
 998         /* load old values */
 999         u32 mf_cfg = bp->mf_config[BP_VN(bp)];
1000
1001         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1002                 /* leave all but MAX value */
1003                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1004
1005                 /* set new MAX value */
1006                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1007                                 & FUNC_MF_CFG_MAX_BW_MASK;
1008
1009                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1010         }
1011 }
1012
1013 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
1014 {
1015         int i, offset = 1;
1016
1017         free_irq(bp->msix_table[0].vector, bp->dev);
1018         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1019            bp->msix_table[0].vector);
1020
1021 #ifdef BCM_CNIC
1022         offset++;
1023 #endif
1024         for_each_eth_queue(bp, i) {
1025                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
1026                    "state %x\n", i, bp->msix_table[i + offset].vector,
1027                    bnx2x_fp(bp, i, state));
1028
1029                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
1030         }
1031 }
1032
1033 void bnx2x_free_irq(struct bnx2x *bp)
1034 {
1035         if (bp->flags & USING_MSIX_FLAG)
1036                 bnx2x_free_msix_irqs(bp);
1037         else if (bp->flags & USING_MSI_FLAG)
1038                 free_irq(bp->pdev->irq, bp->dev);
1039         else
1040                 free_irq(bp->pdev->irq, bp->dev);
1041 }
1042
1043 int bnx2x_enable_msix(struct bnx2x *bp)
1044 {
1045         int msix_vec = 0, i, rc, req_cnt;
1046
1047         bp->msix_table[msix_vec].entry = msix_vec;
1048         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1049            bp->msix_table[0].entry);
1050         msix_vec++;
1051
1052 #ifdef BCM_CNIC
1053         bp->msix_table[msix_vec].entry = msix_vec;
1054         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1055            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1056         msix_vec++;
1057 #endif
1058         for_each_eth_queue(bp, i) {
1059                 bp->msix_table[msix_vec].entry = msix_vec;
1060                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1061                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1062                 msix_vec++;
1063         }
1064
1065         req_cnt = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1066
1067         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1068
1069         /*
1070          * reconfigure number of tx/rx queues according to available
1071          * MSI-X vectors
1072          */
1073         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1074                 /* how less vectors we will have? */
1075                 int diff = req_cnt - rc;
1076
1077                 DP(NETIF_MSG_IFUP,
1078                    "Trying to use less MSI-X vectors: %d\n", rc);
1079
1080                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1081
1082                 if (rc) {
1083                         DP(NETIF_MSG_IFUP,
1084                            "MSI-X is not attainable  rc %d\n", rc);
1085                         return rc;
1086                 }
1087                 /*
1088                  * decrease number of queues by number of unallocated entries
1089                  */
1090                 bp->num_queues -= diff;
1091
1092                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1093                                   bp->num_queues);
1094         } else if (rc) {
1095                 /* fall to INTx if not enough memory */
1096                 if (rc == -ENOMEM)
1097                         bp->flags |= DISABLE_MSI_FLAG;
1098                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1099                 return rc;
1100         }
1101
1102         bp->flags |= USING_MSIX_FLAG;
1103
1104         return 0;
1105 }
1106
1107 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1108 {
1109         int i, rc, offset = 1;
1110
1111         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1112                          bp->dev->name, bp->dev);
1113         if (rc) {
1114                 BNX2X_ERR("request sp irq failed\n");
1115                 return -EBUSY;
1116         }
1117
1118 #ifdef BCM_CNIC
1119         offset++;
1120 #endif
1121         for_each_eth_queue(bp, i) {
1122                 struct bnx2x_fastpath *fp = &bp->fp[i];
1123                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1124                          bp->dev->name, i);
1125
1126                 rc = request_irq(bp->msix_table[offset].vector,
1127                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1128                 if (rc) {
1129                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1130                         bnx2x_free_msix_irqs(bp);
1131                         return -EBUSY;
1132                 }
1133
1134                 offset++;
1135                 fp->state = BNX2X_FP_STATE_IRQ;
1136         }
1137
1138         i = BNX2X_NUM_ETH_QUEUES(bp);
1139         offset = 1 + CNIC_CONTEXT_USE;
1140         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1141                " ... fp[%d] %d\n",
1142                bp->msix_table[0].vector,
1143                0, bp->msix_table[offset].vector,
1144                i - 1, bp->msix_table[offset + i - 1].vector);
1145
1146         return 0;
1147 }
1148
1149 int bnx2x_enable_msi(struct bnx2x *bp)
1150 {
1151         int rc;
1152
1153         rc = pci_enable_msi(bp->pdev);
1154         if (rc) {
1155                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1156                 return -1;
1157         }
1158         bp->flags |= USING_MSI_FLAG;
1159
1160         return 0;
1161 }
1162
1163 static int bnx2x_req_irq(struct bnx2x *bp)
1164 {
1165         unsigned long flags;
1166         int rc;
1167
1168         if (bp->flags & USING_MSI_FLAG)
1169                 flags = 0;
1170         else
1171                 flags = IRQF_SHARED;
1172
1173         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1174                          bp->dev->name, bp->dev);
1175         if (!rc)
1176                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1177
1178         return rc;
1179 }
1180
1181 static void bnx2x_napi_enable(struct bnx2x *bp)
1182 {
1183         int i;
1184
1185         for_each_napi_queue(bp, i)
1186                 napi_enable(&bnx2x_fp(bp, i, napi));
1187 }
1188
1189 static void bnx2x_napi_disable(struct bnx2x *bp)
1190 {
1191         int i;
1192
1193         for_each_napi_queue(bp, i)
1194                 napi_disable(&bnx2x_fp(bp, i, napi));
1195 }
1196
1197 void bnx2x_netif_start(struct bnx2x *bp)
1198 {
1199         int intr_sem;
1200
1201         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1202         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1203
1204         if (intr_sem) {
1205                 if (netif_running(bp->dev)) {
1206                         bnx2x_napi_enable(bp);
1207                         bnx2x_int_enable(bp);
1208                         if (bp->state == BNX2X_STATE_OPEN)
1209                                 netif_tx_wake_all_queues(bp->dev);
1210                 }
1211         }
1212 }
1213
1214 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1215 {
1216         bnx2x_int_disable_sync(bp, disable_hw);
1217         bnx2x_napi_disable(bp);
1218         netif_tx_disable(bp->dev);
1219 }
1220
1221 u16 bnx2x_select_queue(struct net_device *dev, struct sk_buff *skb)
1222 {
1223 #ifdef BCM_CNIC
1224         struct bnx2x *bp = netdev_priv(dev);
1225         if (NO_FCOE(bp))
1226                 return skb_tx_hash(dev, skb);
1227         else {
1228                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1229                 u16 ether_type = ntohs(hdr->h_proto);
1230
1231                 /* Skip VLAN tag if present */
1232                 if (ether_type == ETH_P_8021Q) {
1233                         struct vlan_ethhdr *vhdr =
1234                                 (struct vlan_ethhdr *)skb->data;
1235
1236                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1237                 }
1238
1239                 /* If ethertype is FCoE or FIP - use FCoE ring */
1240                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1241                         return bnx2x_fcoe(bp, index);
1242         }
1243 #endif
1244         /* Select a none-FCoE queue:  if FCoE is enabled, exclude FCoE L2 ring
1245          */
1246         return __skb_tx_hash(dev, skb,
1247                         dev->real_num_tx_queues - FCOE_CONTEXT_USE);
1248 }
1249
1250 void bnx2x_set_num_queues(struct bnx2x *bp)
1251 {
1252         switch (bp->multi_mode) {
1253         case ETH_RSS_MODE_DISABLED:
1254                 bp->num_queues = 1;
1255                 break;
1256         case ETH_RSS_MODE_REGULAR:
1257                 bp->num_queues = bnx2x_calc_num_queues(bp);
1258                 break;
1259
1260         default:
1261                 bp->num_queues = 1;
1262                 break;
1263         }
1264
1265         /* Add special queues */
1266         bp->num_queues += NONE_ETH_CONTEXT_USE;
1267 }
1268
1269 #ifdef BCM_CNIC
1270 static inline void bnx2x_set_fcoe_eth_macs(struct bnx2x *bp)
1271 {
1272         if (!NO_FCOE(bp)) {
1273                 if (!IS_MF_SD(bp))
1274                         bnx2x_set_fip_eth_mac_addr(bp, 1);
1275                 bnx2x_set_all_enode_macs(bp, 1);
1276                 bp->flags |= FCOE_MACS_SET;
1277         }
1278 }
1279 #endif
1280
1281 static void bnx2x_release_firmware(struct bnx2x *bp)
1282 {
1283         kfree(bp->init_ops_offsets);
1284         kfree(bp->init_ops);
1285         kfree(bp->init_data);
1286         release_firmware(bp->firmware);
1287 }
1288
1289 static inline int bnx2x_set_real_num_queues(struct bnx2x *bp)
1290 {
1291         int rc, num = bp->num_queues;
1292
1293 #ifdef BCM_CNIC
1294         if (NO_FCOE(bp))
1295                 num -= FCOE_CONTEXT_USE;
1296
1297 #endif
1298         netif_set_real_num_tx_queues(bp->dev, num);
1299         rc = netif_set_real_num_rx_queues(bp->dev, num);
1300         return rc;
1301 }
1302
1303 static inline void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1304 {
1305         int i;
1306
1307         for_each_queue(bp, i) {
1308                 struct bnx2x_fastpath *fp = &bp->fp[i];
1309
1310                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
1311                 if (IS_FCOE_IDX(i))
1312                         /*
1313                          * Although there are no IP frames expected to arrive to
1314                          * this ring we still want to add an
1315                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
1316                          * overrun attack.
1317                          */
1318                         fp->rx_buf_size =
1319                                 BNX2X_FCOE_MINI_JUMBO_MTU + ETH_OVREHEAD +
1320                                 BNX2X_RX_ALIGN + IP_HEADER_ALIGNMENT_PADDING;
1321                 else
1322                         fp->rx_buf_size =
1323                                 bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
1324                                 IP_HEADER_ALIGNMENT_PADDING;
1325         }
1326 }
1327
1328 /* must be called with rtnl_lock */
1329 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1330 {
1331         u32 load_code;
1332         int i, rc;
1333
1334         /* Set init arrays */
1335         rc = bnx2x_init_firmware(bp);
1336         if (rc) {
1337                 BNX2X_ERR("Error loading firmware\n");
1338                 return rc;
1339         }
1340
1341 #ifdef BNX2X_STOP_ON_ERROR
1342         if (unlikely(bp->panic))
1343                 return -EPERM;
1344 #endif
1345
1346         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1347
1348         /* must be called before memory allocation and HW init */
1349         bnx2x_ilt_set_info(bp);
1350
1351         /* Set the receive queues buffer size */
1352         bnx2x_set_rx_buf_size(bp);
1353
1354         if (bnx2x_alloc_mem(bp))
1355                 return -ENOMEM;
1356
1357         rc = bnx2x_set_real_num_queues(bp);
1358         if (rc) {
1359                 BNX2X_ERR("Unable to set real_num_queues\n");
1360                 goto load_error0;
1361         }
1362
1363         for_each_queue(bp, i)
1364                 bnx2x_fp(bp, i, disable_tpa) =
1365                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1366
1367 #ifdef BCM_CNIC
1368         /* We don't want TPA on FCoE L2 ring */
1369         bnx2x_fcoe(bp, disable_tpa) = 1;
1370 #endif
1371         bnx2x_napi_enable(bp);
1372
1373         /* Send LOAD_REQUEST command to MCP
1374            Returns the type of LOAD command:
1375            if it is the first port to be initialized
1376            common blocks should be initialized, otherwise - not
1377         */
1378         if (!BP_NOMCP(bp)) {
1379                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1380                 if (!load_code) {
1381                         BNX2X_ERR("MCP response failure, aborting\n");
1382                         rc = -EBUSY;
1383                         goto load_error1;
1384                 }
1385                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1386                         rc = -EBUSY; /* other port in diagnostic mode */
1387                         goto load_error1;
1388                 }
1389
1390         } else {
1391                 int path = BP_PATH(bp);
1392                 int port = BP_PORT(bp);
1393
1394                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1395                    path, load_count[path][0], load_count[path][1],
1396                    load_count[path][2]);
1397                 load_count[path][0]++;
1398                 load_count[path][1 + port]++;
1399                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1400                    path, load_count[path][0], load_count[path][1],
1401                    load_count[path][2]);
1402                 if (load_count[path][0] == 1)
1403                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1404                 else if (load_count[path][1 + port] == 1)
1405                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1406                 else
1407                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1408         }
1409
1410         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1411             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1412             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1413                 bp->port.pmf = 1;
1414         else
1415                 bp->port.pmf = 0;
1416         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1417
1418         /* Initialize HW */
1419         rc = bnx2x_init_hw(bp, load_code);
1420         if (rc) {
1421                 BNX2X_ERR("HW init failed, aborting\n");
1422                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1423                 goto load_error2;
1424         }
1425
1426         /* Connect to IRQs */
1427         rc = bnx2x_setup_irqs(bp);
1428         if (rc) {
1429                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1430                 goto load_error2;
1431         }
1432
1433         /* Setup NIC internals and enable interrupts */
1434         bnx2x_nic_init(bp, load_code);
1435
1436         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1437             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1438             (bp->common.shmem2_base))
1439                 SHMEM2_WR(bp, dcc_support,
1440                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1441                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1442
1443         /* Send LOAD_DONE command to MCP */
1444         if (!BP_NOMCP(bp)) {
1445                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1446                 if (!load_code) {
1447                         BNX2X_ERR("MCP response failure, aborting\n");
1448                         rc = -EBUSY;
1449                         goto load_error3;
1450                 }
1451         }
1452
1453         bnx2x_dcbx_init(bp);
1454
1455         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1456
1457         rc = bnx2x_func_start(bp);
1458         if (rc) {
1459                 BNX2X_ERR("Function start failed!\n");
1460 #ifndef BNX2X_STOP_ON_ERROR
1461                 goto load_error3;
1462 #else
1463                 bp->panic = 1;
1464                 return -EBUSY;
1465 #endif
1466         }
1467
1468         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1469         if (rc) {
1470                 BNX2X_ERR("Setup leading failed!\n");
1471 #ifndef BNX2X_STOP_ON_ERROR
1472                 goto load_error3;
1473 #else
1474                 bp->panic = 1;
1475                 return -EBUSY;
1476 #endif
1477         }
1478
1479         if (!CHIP_IS_E1(bp) &&
1480             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1481                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1482                 bp->flags |= MF_FUNC_DIS;
1483         }
1484
1485 #ifdef BCM_CNIC
1486         /* Enable Timer scan */
1487         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1488 #endif
1489
1490         for_each_nondefault_queue(bp, i) {
1491                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1492                 if (rc)
1493 #ifdef BCM_CNIC
1494                         goto load_error4;
1495 #else
1496                         goto load_error3;
1497 #endif
1498         }
1499
1500         /* Now when Clients are configured we are ready to work */
1501         bp->state = BNX2X_STATE_OPEN;
1502
1503 #ifdef BCM_CNIC
1504         bnx2x_set_fcoe_eth_macs(bp);
1505 #endif
1506
1507         bnx2x_set_eth_mac(bp, 1);
1508
1509         /* Clear MC configuration */
1510         if (CHIP_IS_E1(bp))
1511                 bnx2x_invalidate_e1_mc_list(bp);
1512         else
1513                 bnx2x_invalidate_e1h_mc_list(bp);
1514
1515         /* Clear UC lists configuration */
1516         bnx2x_invalidate_uc_list(bp);
1517
1518         if (bp->pending_max) {
1519                 bnx2x_update_max_mf_config(bp, bp->pending_max);
1520                 bp->pending_max = 0;
1521         }
1522
1523         if (bp->port.pmf)
1524                 bnx2x_initial_phy_init(bp, load_mode);
1525
1526         /* Initialize Rx filtering */
1527         bnx2x_set_rx_mode(bp->dev);
1528
1529         /* Start fast path */
1530         switch (load_mode) {
1531         case LOAD_NORMAL:
1532                 /* Tx queue should be only reenabled */
1533                 netif_tx_wake_all_queues(bp->dev);
1534                 /* Initialize the receive filter. */
1535                 break;
1536
1537         case LOAD_OPEN:
1538                 netif_tx_start_all_queues(bp->dev);
1539                 smp_mb__after_clear_bit();
1540                 break;
1541
1542         case LOAD_DIAG:
1543                 bp->state = BNX2X_STATE_DIAG;
1544                 break;
1545
1546         default:
1547                 break;
1548         }
1549
1550         if (!bp->port.pmf)
1551                 bnx2x__link_status_update(bp);
1552
1553         /* start the timer */
1554         mod_timer(&bp->timer, jiffies + bp->current_interval);
1555
1556 #ifdef BCM_CNIC
1557         bnx2x_setup_cnic_irq_info(bp);
1558         if (bp->state == BNX2X_STATE_OPEN)
1559                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1560 #endif
1561         bnx2x_inc_load_cnt(bp);
1562
1563         bnx2x_release_firmware(bp);
1564
1565         return 0;
1566
1567 #ifdef BCM_CNIC
1568 load_error4:
1569         /* Disable Timer scan */
1570         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1571 #endif
1572 load_error3:
1573         bnx2x_int_disable_sync(bp, 1);
1574
1575         /* Free SKBs, SGEs, TPA pool and driver internals */
1576         bnx2x_free_skbs(bp);
1577         for_each_rx_queue(bp, i)
1578                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1579
1580         /* Release IRQs */
1581         bnx2x_free_irq(bp);
1582 load_error2:
1583         if (!BP_NOMCP(bp)) {
1584                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1585                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1586         }
1587
1588         bp->port.pmf = 0;
1589 load_error1:
1590         bnx2x_napi_disable(bp);
1591 load_error0:
1592         bnx2x_free_mem(bp);
1593
1594         bnx2x_release_firmware(bp);
1595
1596         return rc;
1597 }
1598
1599 /* must be called with rtnl_lock */
1600 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1601 {
1602         int i;
1603
1604         if (bp->state == BNX2X_STATE_CLOSED) {
1605                 /* Interface has been removed - nothing to recover */
1606                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1607                 bp->is_leader = 0;
1608                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1609                 smp_wmb();
1610
1611                 return -EINVAL;
1612         }
1613
1614 #ifdef BCM_CNIC
1615         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1616 #endif
1617         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1618
1619         /* Set "drop all" */
1620         bp->rx_mode = BNX2X_RX_MODE_NONE;
1621         bnx2x_set_storm_rx_mode(bp);
1622
1623         /* Stop Tx */
1624         bnx2x_tx_disable(bp);
1625
1626         del_timer_sync(&bp->timer);
1627
1628         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1629                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1630
1631         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1632
1633         /* Cleanup the chip if needed */
1634         if (unload_mode != UNLOAD_RECOVERY)
1635                 bnx2x_chip_cleanup(bp, unload_mode);
1636         else {
1637                 /* Disable HW interrupts, NAPI and Tx */
1638                 bnx2x_netif_stop(bp, 1);
1639
1640                 /* Release IRQs */
1641                 bnx2x_free_irq(bp);
1642         }
1643
1644         bp->port.pmf = 0;
1645
1646         /* Free SKBs, SGEs, TPA pool and driver internals */
1647         bnx2x_free_skbs(bp);
1648         for_each_rx_queue(bp, i)
1649                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1650
1651         bnx2x_free_mem(bp);
1652
1653         bp->state = BNX2X_STATE_CLOSED;
1654
1655         /* The last driver must disable a "close the gate" if there is no
1656          * parity attention or "process kill" pending.
1657          */
1658         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1659             bnx2x_reset_is_done(bp))
1660                 bnx2x_disable_close_the_gate(bp);
1661
1662         /* Reset MCP mail box sequence if there is on going recovery */
1663         if (unload_mode == UNLOAD_RECOVERY)
1664                 bp->fw_seq = 0;
1665
1666         return 0;
1667 }
1668
1669 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1670 {
1671         u16 pmcsr;
1672
1673         /* If there is no power capability, silently succeed */
1674         if (!bp->pm_cap) {
1675                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1676                 return 0;
1677         }
1678
1679         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1680
1681         switch (state) {
1682         case PCI_D0:
1683                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1684                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1685                                        PCI_PM_CTRL_PME_STATUS));
1686
1687                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1688                         /* delay required during transition out of D3hot */
1689                         msleep(20);
1690                 break;
1691
1692         case PCI_D3hot:
1693                 /* If there are other clients above don't
1694                    shut down the power */
1695                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1696                         return 0;
1697                 /* Don't shut down the power for emulation and FPGA */
1698                 if (CHIP_REV_IS_SLOW(bp))
1699                         return 0;
1700
1701                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1702                 pmcsr |= 3;
1703
1704                 if (bp->wol)
1705                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1706
1707                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1708                                       pmcsr);
1709
1710                 /* No more memory access after this point until
1711                 * device is brought back to D0.
1712                 */
1713                 break;
1714
1715         default:
1716                 return -EINVAL;
1717         }
1718         return 0;
1719 }
1720
1721 /*
1722  * net_device service functions
1723  */
1724 int bnx2x_poll(struct napi_struct *napi, int budget)
1725 {
1726         int work_done = 0;
1727         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1728                                                  napi);
1729         struct bnx2x *bp = fp->bp;
1730
1731         while (1) {
1732 #ifdef BNX2X_STOP_ON_ERROR
1733                 if (unlikely(bp->panic)) {
1734                         napi_complete(napi);
1735                         return 0;
1736                 }
1737 #endif
1738
1739                 if (bnx2x_has_tx_work(fp))
1740                         bnx2x_tx_int(fp);
1741
1742                 if (bnx2x_has_rx_work(fp)) {
1743                         work_done += bnx2x_rx_int(fp, budget - work_done);
1744
1745                         /* must not complete if we consumed full budget */
1746                         if (work_done >= budget)
1747                                 break;
1748                 }
1749
1750                 /* Fall out from the NAPI loop if needed */
1751                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1752 #ifdef BCM_CNIC
1753                         /* No need to update SB for FCoE L2 ring as long as
1754                          * it's connected to the default SB and the SB
1755                          * has been updated when NAPI was scheduled.
1756                          */
1757                         if (IS_FCOE_FP(fp)) {
1758                                 napi_complete(napi);
1759                                 break;
1760                         }
1761 #endif
1762
1763                         bnx2x_update_fpsb_idx(fp);
1764                         /* bnx2x_has_rx_work() reads the status block,
1765                          * thus we need to ensure that status block indices
1766                          * have been actually read (bnx2x_update_fpsb_idx)
1767                          * prior to this check (bnx2x_has_rx_work) so that
1768                          * we won't write the "newer" value of the status block
1769                          * to IGU (if there was a DMA right after
1770                          * bnx2x_has_rx_work and if there is no rmb, the memory
1771                          * reading (bnx2x_update_fpsb_idx) may be postponed
1772                          * to right before bnx2x_ack_sb). In this case there
1773                          * will never be another interrupt until there is
1774                          * another update of the status block, while there
1775                          * is still unhandled work.
1776                          */
1777                         rmb();
1778
1779                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1780                                 napi_complete(napi);
1781                                 /* Re-enable interrupts */
1782                                 DP(NETIF_MSG_HW,
1783                                    "Update index to %d\n", fp->fp_hc_idx);
1784                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1785                                              le16_to_cpu(fp->fp_hc_idx),
1786                                              IGU_INT_ENABLE, 1);
1787                                 break;
1788                         }
1789                 }
1790         }
1791
1792         return work_done;
1793 }
1794
1795 /* we split the first BD into headers and data BDs
1796  * to ease the pain of our fellow microcode engineers
1797  * we use one mapping for both BDs
1798  * So far this has only been observed to happen
1799  * in Other Operating Systems(TM)
1800  */
1801 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1802                                    struct bnx2x_fastpath *fp,
1803                                    struct sw_tx_bd *tx_buf,
1804                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1805                                    u16 bd_prod, int nbd)
1806 {
1807         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1808         struct eth_tx_bd *d_tx_bd;
1809         dma_addr_t mapping;
1810         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1811
1812         /* first fix first BD */
1813         h_tx_bd->nbd = cpu_to_le16(nbd);
1814         h_tx_bd->nbytes = cpu_to_le16(hlen);
1815
1816         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1817            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1818            h_tx_bd->addr_lo, h_tx_bd->nbd);
1819
1820         /* now get a new data BD
1821          * (after the pbd) and fill it */
1822         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1823         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1824
1825         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1826                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1827
1828         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1829         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1830         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1831
1832         /* this marks the BD as one that has no individual mapping */
1833         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1834
1835         DP(NETIF_MSG_TX_QUEUED,
1836            "TSO split data size is %d (%x:%x)\n",
1837            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1838
1839         /* update tx_bd */
1840         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1841
1842         return bd_prod;
1843 }
1844
1845 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1846 {
1847         if (fix > 0)
1848                 csum = (u16) ~csum_fold(csum_sub(csum,
1849                                 csum_partial(t_header - fix, fix, 0)));
1850
1851         else if (fix < 0)
1852                 csum = (u16) ~csum_fold(csum_add(csum,
1853                                 csum_partial(t_header, -fix, 0)));
1854
1855         return swab16(csum);
1856 }
1857
1858 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1859 {
1860         u32 rc;
1861
1862         if (skb->ip_summed != CHECKSUM_PARTIAL)
1863                 rc = XMIT_PLAIN;
1864
1865         else {
1866                 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
1867                         rc = XMIT_CSUM_V6;
1868                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1869                                 rc |= XMIT_CSUM_TCP;
1870
1871                 } else {
1872                         rc = XMIT_CSUM_V4;
1873                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1874                                 rc |= XMIT_CSUM_TCP;
1875                 }
1876         }
1877
1878         if (skb_is_gso_v6(skb))
1879                 rc |= XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6;
1880         else if (skb_is_gso(skb))
1881                 rc |= XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP;
1882
1883         return rc;
1884 }
1885
1886 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1887 /* check if packet requires linearization (packet is too fragmented)
1888    no need to check fragmentation if page size > 8K (there will be no
1889    violation to FW restrictions) */
1890 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1891                              u32 xmit_type)
1892 {
1893         int to_copy = 0;
1894         int hlen = 0;
1895         int first_bd_sz = 0;
1896
1897         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1898         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1899
1900                 if (xmit_type & XMIT_GSO) {
1901                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1902                         /* Check if LSO packet needs to be copied:
1903                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1904                         int wnd_size = MAX_FETCH_BD - 3;
1905                         /* Number of windows to check */
1906                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1907                         int wnd_idx = 0;
1908                         int frag_idx = 0;
1909                         u32 wnd_sum = 0;
1910
1911                         /* Headers length */
1912                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1913                                 tcp_hdrlen(skb);
1914
1915                         /* Amount of data (w/o headers) on linear part of SKB*/
1916                         first_bd_sz = skb_headlen(skb) - hlen;
1917
1918                         wnd_sum  = first_bd_sz;
1919
1920                         /* Calculate the first sum - it's special */
1921                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1922                                 wnd_sum +=
1923                                         skb_shinfo(skb)->frags[frag_idx].size;
1924
1925                         /* If there was data on linear skb data - check it */
1926                         if (first_bd_sz > 0) {
1927                                 if (unlikely(wnd_sum < lso_mss)) {
1928                                         to_copy = 1;
1929                                         goto exit_lbl;
1930                                 }
1931
1932                                 wnd_sum -= first_bd_sz;
1933                         }
1934
1935                         /* Others are easier: run through the frag list and
1936                            check all windows */
1937                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1938                                 wnd_sum +=
1939                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1940
1941                                 if (unlikely(wnd_sum < lso_mss)) {
1942                                         to_copy = 1;
1943                                         break;
1944                                 }
1945                                 wnd_sum -=
1946                                         skb_shinfo(skb)->frags[wnd_idx].size;
1947                         }
1948                 } else {
1949                         /* in non-LSO too fragmented packet should always
1950                            be linearized */
1951                         to_copy = 1;
1952                 }
1953         }
1954
1955 exit_lbl:
1956         if (unlikely(to_copy))
1957                 DP(NETIF_MSG_TX_QUEUED,
1958                    "Linearization IS REQUIRED for %s packet. "
1959                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1960                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1961                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1962
1963         return to_copy;
1964 }
1965 #endif
1966
1967 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb, u32 *parsing_data,
1968                                         u32 xmit_type)
1969 {
1970         *parsing_data |= (skb_shinfo(skb)->gso_size <<
1971                               ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT) &
1972                               ETH_TX_PARSE_BD_E2_LSO_MSS;
1973         if ((xmit_type & XMIT_GSO_V6) &&
1974             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
1975                 *parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
1976 }
1977
1978 /**
1979  * Update PBD in GSO case.
1980  *
1981  * @param skb
1982  * @param tx_start_bd
1983  * @param pbd
1984  * @param xmit_type
1985  */
1986 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
1987                                      struct eth_tx_parse_bd_e1x *pbd,
1988                                      u32 xmit_type)
1989 {
1990         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1991         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1992         pbd->tcp_flags = pbd_tcp_flags(skb);
1993
1994         if (xmit_type & XMIT_GSO_V4) {
1995                 pbd->ip_id = swab16(ip_hdr(skb)->id);
1996                 pbd->tcp_pseudo_csum =
1997                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1998                                                   ip_hdr(skb)->daddr,
1999                                                   0, IPPROTO_TCP, 0));
2000
2001         } else
2002                 pbd->tcp_pseudo_csum =
2003                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2004                                                 &ipv6_hdr(skb)->daddr,
2005                                                 0, IPPROTO_TCP, 0));
2006
2007         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
2008 }
2009
2010 /**
2011  *
2012  * @param skb
2013  * @param tx_start_bd
2014  * @param pbd_e2
2015  * @param xmit_type
2016  *
2017  * @return header len
2018  */
2019 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
2020         u32 *parsing_data, u32 xmit_type)
2021 {
2022         *parsing_data |=
2023                         ((((u8 *)skb_transport_header(skb) - skb->data) >> 1) <<
2024                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT) &
2025                         ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W;
2026
2027         if (xmit_type & XMIT_CSUM_TCP) {
2028                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
2029                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
2030                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
2031
2032                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
2033         } else
2034                 /* We support checksum offload for TCP and UDP only.
2035                  * No need to pass the UDP header length - it's a constant.
2036                  */
2037                 return skb_transport_header(skb) +
2038                                 sizeof(struct udphdr) - skb->data;
2039 }
2040
2041 /**
2042  *
2043  * @param skb
2044  * @param tx_start_bd
2045  * @param pbd
2046  * @param xmit_type
2047  *
2048  * @return Header length
2049  */
2050 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
2051         struct eth_tx_parse_bd_e1x *pbd,
2052         u32 xmit_type)
2053 {
2054         u8 hlen = (skb_network_header(skb) - skb->data) >> 1;
2055
2056         /* for now NS flag is not used in Linux */
2057         pbd->global_data =
2058                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
2059                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
2060
2061         pbd->ip_hlen_w = (skb_transport_header(skb) -
2062                         skb_network_header(skb)) >> 1;
2063
2064         hlen += pbd->ip_hlen_w;
2065
2066         /* We support checksum offload for TCP and UDP only */
2067         if (xmit_type & XMIT_CSUM_TCP)
2068                 hlen += tcp_hdrlen(skb) / 2;
2069         else
2070                 hlen += sizeof(struct udphdr) / 2;
2071
2072         pbd->total_hlen_w = cpu_to_le16(hlen);
2073         hlen = hlen*2;
2074
2075         if (xmit_type & XMIT_CSUM_TCP) {
2076                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
2077
2078         } else {
2079                 s8 fix = SKB_CS_OFF(skb); /* signed! */
2080
2081                 DP(NETIF_MSG_TX_QUEUED,
2082                    "hlen %d  fix %d  csum before fix %x\n",
2083                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
2084
2085                 /* HW bug: fixup the CSUM */
2086                 pbd->tcp_pseudo_csum =
2087                         bnx2x_csum_fix(skb_transport_header(skb),
2088                                        SKB_CS(skb), fix);
2089
2090                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2091                    pbd->tcp_pseudo_csum);
2092         }
2093
2094         return hlen;
2095 }
2096
2097 /* called with netif_tx_lock
2098  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
2099  * netif_wake_queue()
2100  */
2101 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
2102 {
2103         struct bnx2x *bp = netdev_priv(dev);
2104         struct bnx2x_fastpath *fp;
2105         struct netdev_queue *txq;
2106         struct sw_tx_bd *tx_buf;
2107         struct eth_tx_start_bd *tx_start_bd;
2108         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
2109         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
2110         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
2111         u32 pbd_e2_parsing_data = 0;
2112         u16 pkt_prod, bd_prod;
2113         int nbd, fp_index;
2114         dma_addr_t mapping;
2115         u32 xmit_type = bnx2x_xmit_type(bp, skb);
2116         int i;
2117         u8 hlen = 0;
2118         __le16 pkt_size = 0;
2119         struct ethhdr *eth;
2120         u8 mac_type = UNICAST_ADDRESS;
2121
2122 #ifdef BNX2X_STOP_ON_ERROR
2123         if (unlikely(bp->panic))
2124                 return NETDEV_TX_BUSY;
2125 #endif
2126
2127         fp_index = skb_get_queue_mapping(skb);
2128         txq = netdev_get_tx_queue(dev, fp_index);
2129
2130         fp = &bp->fp[fp_index];
2131
2132         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
2133                 fp->eth_q_stats.driver_xoff++;
2134                 netif_tx_stop_queue(txq);
2135                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
2136                 return NETDEV_TX_BUSY;
2137         }
2138
2139         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
2140                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
2141            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
2142            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
2143
2144         eth = (struct ethhdr *)skb->data;
2145
2146         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
2147         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
2148                 if (is_broadcast_ether_addr(eth->h_dest))
2149                         mac_type = BROADCAST_ADDRESS;
2150                 else
2151                         mac_type = MULTICAST_ADDRESS;
2152         }
2153
2154 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
2155         /* First, check if we need to linearize the skb (due to FW
2156            restrictions). No need to check fragmentation if page size > 8K
2157            (there will be no violation to FW restrictions) */
2158         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
2159                 /* Statistics of linearization */
2160                 bp->lin_cnt++;
2161                 if (skb_linearize(skb) != 0) {
2162                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
2163                            "silently dropping this SKB\n");
2164                         dev_kfree_skb_any(skb);
2165                         return NETDEV_TX_OK;
2166                 }
2167         }
2168 #endif
2169
2170         /*
2171         Please read carefully. First we use one BD which we mark as start,
2172         then we have a parsing info BD (used for TSO or xsum),
2173         and only then we have the rest of the TSO BDs.
2174         (don't forget to mark the last one as last,
2175         and to unmap only AFTER you write to the BD ...)
2176         And above all, all pdb sizes are in words - NOT DWORDS!
2177         */
2178
2179         pkt_prod = fp->tx_pkt_prod++;
2180         bd_prod = TX_BD(fp->tx_bd_prod);
2181
2182         /* get a tx_buf and first BD */
2183         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
2184         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
2185
2186         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2187         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2188                  mac_type);
2189
2190         /* header nbd */
2191         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2192
2193         /* remember the first BD of the packet */
2194         tx_buf->first_bd = fp->tx_bd_prod;
2195         tx_buf->skb = skb;
2196         tx_buf->flags = 0;
2197
2198         DP(NETIF_MSG_TX_QUEUED,
2199            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2200            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2201
2202         if (vlan_tx_tag_present(skb)) {
2203                 tx_start_bd->vlan_or_ethertype =
2204                     cpu_to_le16(vlan_tx_tag_get(skb));
2205                 tx_start_bd->bd_flags.as_bitfield |=
2206                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2207         } else
2208                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2209
2210         /* turn on parsing and get a BD */
2211         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2212
2213         if (xmit_type & XMIT_CSUM) {
2214                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2215
2216                 if (xmit_type & XMIT_CSUM_V4)
2217                         tx_start_bd->bd_flags.as_bitfield |=
2218                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2219                 else
2220                         tx_start_bd->bd_flags.as_bitfield |=
2221                                                 ETH_TX_BD_FLAGS_IPV6;
2222
2223                 if (!(xmit_type & XMIT_CSUM_TCP))
2224                         tx_start_bd->bd_flags.as_bitfield |=
2225                                                 ETH_TX_BD_FLAGS_IS_UDP;
2226         }
2227
2228         if (CHIP_IS_E2(bp)) {
2229                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2230                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2231                 /* Set PBD in checksum offload case */
2232                 if (xmit_type & XMIT_CSUM)
2233                         hlen = bnx2x_set_pbd_csum_e2(bp, skb,
2234                                                      &pbd_e2_parsing_data,
2235                                                      xmit_type);
2236         } else {
2237                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2238                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2239                 /* Set PBD in checksum offload case */
2240                 if (xmit_type & XMIT_CSUM)
2241                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2242
2243         }
2244
2245         /* Map skb linear data for DMA */
2246         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2247                                  skb_headlen(skb), DMA_TO_DEVICE);
2248
2249         /* Setup the data pointer of the first BD of the packet */
2250         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2251         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2252         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2253         tx_start_bd->nbd = cpu_to_le16(nbd);
2254         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2255         pkt_size = tx_start_bd->nbytes;
2256
2257         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2258            "  nbytes %d  flags %x  vlan %x\n",
2259            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2260            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2261            tx_start_bd->bd_flags.as_bitfield,
2262            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2263
2264         if (xmit_type & XMIT_GSO) {
2265
2266                 DP(NETIF_MSG_TX_QUEUED,
2267                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2268                    skb->len, hlen, skb_headlen(skb),
2269                    skb_shinfo(skb)->gso_size);
2270
2271                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2272
2273                 if (unlikely(skb_headlen(skb) > hlen))
2274                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2275                                                  hlen, bd_prod, ++nbd);
2276                 if (CHIP_IS_E2(bp))
2277                         bnx2x_set_pbd_gso_e2(skb, &pbd_e2_parsing_data,
2278                                              xmit_type);
2279                 else
2280                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2281         }
2282
2283         /* Set the PBD's parsing_data field if not zero
2284          * (for the chips newer than 57711).
2285          */
2286         if (pbd_e2_parsing_data)
2287                 pbd_e2->parsing_data = cpu_to_le32(pbd_e2_parsing_data);
2288
2289         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2290
2291         /* Handle fragmented skb */
2292         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2293                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2294
2295                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2296                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2297                 if (total_pkt_bd == NULL)
2298                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2299
2300                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2301                                        frag->page_offset,
2302                                        frag->size, DMA_TO_DEVICE);
2303
2304                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2305                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2306                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2307                 le16_add_cpu(&pkt_size, frag->size);
2308
2309                 DP(NETIF_MSG_TX_QUEUED,
2310                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2311                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2312                    le16_to_cpu(tx_data_bd->nbytes));
2313         }
2314
2315         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2316
2317         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2318
2319         /* now send a tx doorbell, counting the next BD
2320          * if the packet contains or ends with it
2321          */
2322         if (TX_BD_POFF(bd_prod) < nbd)
2323                 nbd++;
2324
2325         if (total_pkt_bd != NULL)
2326                 total_pkt_bd->total_pkt_bytes = pkt_size;
2327
2328         if (pbd_e1x)
2329                 DP(NETIF_MSG_TX_QUEUED,
2330                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2331                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2332                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2333                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2334                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2335                     le16_to_cpu(pbd_e1x->total_hlen_w));
2336         if (pbd_e2)
2337                 DP(NETIF_MSG_TX_QUEUED,
2338                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2339                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2340                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2341                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2342                    pbd_e2->parsing_data);
2343         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2344
2345         /*
2346          * Make sure that the BD data is updated before updating the producer
2347          * since FW might read the BD right after the producer is updated.
2348          * This is only applicable for weak-ordered memory model archs such
2349          * as IA-64. The following barrier is also mandatory since FW will
2350          * assumes packets must have BDs.
2351          */
2352         wmb();
2353
2354         fp->tx_db.data.prod += nbd;
2355         barrier();
2356
2357         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2358
2359         mmiowb();
2360
2361         fp->tx_bd_prod += nbd;
2362
2363         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2364                 netif_tx_stop_queue(txq);
2365
2366                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2367                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2368                  * fp->bd_tx_cons */
2369                 smp_mb();
2370
2371                 fp->eth_q_stats.driver_xoff++;
2372                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2373                         netif_tx_wake_queue(txq);
2374         }
2375         fp->tx_pkt++;
2376
2377         return NETDEV_TX_OK;
2378 }
2379
2380 /* called with rtnl_lock */
2381 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2382 {
2383         struct sockaddr *addr = p;
2384         struct bnx2x *bp = netdev_priv(dev);
2385
2386         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2387                 return -EINVAL;
2388
2389         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2390         if (netif_running(dev))
2391                 bnx2x_set_eth_mac(bp, 1);
2392
2393         return 0;
2394 }
2395
2396
2397 static int bnx2x_setup_irqs(struct bnx2x *bp)
2398 {
2399         int rc = 0;
2400         if (bp->flags & USING_MSIX_FLAG) {
2401                 rc = bnx2x_req_msix_irqs(bp);
2402                 if (rc)
2403                         return rc;
2404         } else {
2405                 bnx2x_ack_int(bp);
2406                 rc = bnx2x_req_irq(bp);
2407                 if (rc) {
2408                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2409                         return rc;
2410                 }
2411                 if (bp->flags & USING_MSI_FLAG) {
2412                         bp->dev->irq = bp->pdev->irq;
2413                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2414                                bp->pdev->irq);
2415                 }
2416         }
2417
2418         return 0;
2419 }
2420
2421 void bnx2x_free_mem_bp(struct bnx2x *bp)
2422 {
2423         kfree(bp->fp);
2424         kfree(bp->msix_table);
2425         kfree(bp->ilt);
2426 }
2427
2428 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2429 {
2430         struct bnx2x_fastpath *fp;
2431         struct msix_entry *tbl;
2432         struct bnx2x_ilt *ilt;
2433
2434         /* fp array */
2435         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2436         if (!fp)
2437                 goto alloc_err;
2438         bp->fp = fp;
2439
2440         /* msix table */
2441         tbl = kzalloc((FP_SB_COUNT(bp->l2_cid_count) + 1) * sizeof(*tbl),
2442                                   GFP_KERNEL);
2443         if (!tbl)
2444                 goto alloc_err;
2445         bp->msix_table = tbl;
2446
2447         /* ilt */
2448         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2449         if (!ilt)
2450                 goto alloc_err;
2451         bp->ilt = ilt;
2452
2453         return 0;
2454 alloc_err:
2455         bnx2x_free_mem_bp(bp);
2456         return -ENOMEM;
2457
2458 }
2459
2460 /* called with rtnl_lock */
2461 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2462 {
2463         struct bnx2x *bp = netdev_priv(dev);
2464         int rc = 0;
2465
2466         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2467                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2468                 return -EAGAIN;
2469         }
2470
2471         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2472             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2473                 return -EINVAL;
2474
2475         /* This does not race with packet allocation
2476          * because the actual alloc size is
2477          * only updated as part of load
2478          */
2479         dev->mtu = new_mtu;
2480
2481         if (netif_running(dev)) {
2482                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2483                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2484         }
2485
2486         return rc;
2487 }
2488
2489 void bnx2x_tx_timeout(struct net_device *dev)
2490 {
2491         struct bnx2x *bp = netdev_priv(dev);
2492
2493 #ifdef BNX2X_STOP_ON_ERROR
2494         if (!bp->panic)
2495                 bnx2x_panic();
2496 #endif
2497         /* This allows the netif to be shutdown gracefully before resetting */
2498         schedule_delayed_work(&bp->reset_task, 0);
2499 }
2500
2501 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2502 {
2503         struct net_device *dev = pci_get_drvdata(pdev);
2504         struct bnx2x *bp;
2505
2506         if (!dev) {
2507                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2508                 return -ENODEV;
2509         }
2510         bp = netdev_priv(dev);
2511
2512         rtnl_lock();
2513
2514         pci_save_state(pdev);
2515
2516         if (!netif_running(dev)) {
2517                 rtnl_unlock();
2518                 return 0;
2519         }
2520
2521         netif_device_detach(dev);
2522
2523         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2524
2525         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2526
2527         rtnl_unlock();
2528
2529         return 0;
2530 }
2531
2532 int bnx2x_resume(struct pci_dev *pdev)
2533 {
2534         struct net_device *dev = pci_get_drvdata(pdev);
2535         struct bnx2x *bp;
2536         int rc;
2537
2538         if (!dev) {
2539                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2540                 return -ENODEV;
2541         }
2542         bp = netdev_priv(dev);
2543
2544         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2545                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2546                 return -EAGAIN;
2547         }
2548
2549         rtnl_lock();
2550
2551         pci_restore_state(pdev);
2552
2553         if (!netif_running(dev)) {
2554                 rtnl_unlock();
2555                 return 0;
2556         }
2557
2558         bnx2x_set_power_state(bp, PCI_D0);
2559         netif_device_attach(dev);
2560
2561         /* Since the chip was reset, clear the FW sequence number */
2562         bp->fw_seq = 0;
2563         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2564
2565         rtnl_unlock();
2566
2567         return rc;
2568 }