drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <linux/if_vlan.h>
  20 #include <linux/ip.h>
  21 #include <net/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #include "bnx2x_init.h"
  27
  28 static int bnx2x_setup_irqs(struct bnx2x *bp);
  29
  30 /* free skb in the packet ring at pos idx
  31  * return idx of last bd freed
  32  */
  33 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  34                              u16 idx)
  35 {
  36         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  37         struct eth_tx_start_bd *tx_start_bd;
  38         struct eth_tx_bd *tx_data_bd;
  39         struct sk_buff *skb = tx_buf->skb;
  40         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  41         int nbd;
  42
  43         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  44         prefetch(&skb->end);
  45
  46         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  47            idx, tx_buf, skb);
  48
  49         /* unmap first bd */
  50         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  51         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  52         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  53                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  54
  55         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  56 #ifdef BNX2X_STOP_ON_ERROR
  57         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  58                 BNX2X_ERR("BAD nbd!\n");
  59                 bnx2x_panic();
  60         }
  61 #endif
  62         new_cons = nbd + tx_buf->first_bd;
  63
  64         /* Get the next bd */
  65         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  66
  67         /* Skip a parse bd... */
  68         --nbd;
  69         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  70
  71         /* ...and the TSO split header bd since they have no mapping */
  72         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  73                 --nbd;
  74                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  75         }
  76
  77         /* now free frags */
  78         while (nbd > 0) {
  79
  80                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  81                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  82                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  83                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  84                 if (--nbd)
  85                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  86         }
  87
  88         /* release skb */
  89         WARN_ON(!skb);
  90         dev_kfree_skb(skb);
  91         tx_buf->first_bd = 0;
  92         tx_buf->skb = NULL;
  93
  94         return new_cons;
  95 }
  96
  97 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
  98 {
  99         struct bnx2x *bp = fp->bp;
 100         struct netdev_queue *txq;
 101         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 102
 103 #ifdef BNX2X_STOP_ON_ERROR
 104         if (unlikely(bp->panic))
 105                 return -1;
 106 #endif
 107
 108         txq = netdev_get_tx_queue(bp->dev, fp->index);
 109         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 110         sw_cons = fp->tx_pkt_cons;
 111
 112         while (sw_cons != hw_cons) {
 113                 u16 pkt_cons;
 114
 115                 pkt_cons = TX_BD(sw_cons);
 116
 117                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 118                                       " pkt_cons %u\n",
 119                    fp->index, hw_cons, sw_cons, pkt_cons);
 120
 121                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 122                 sw_cons++;
 123         }
 124
 125         fp->tx_pkt_cons = sw_cons;
 126         fp->tx_bd_cons = bd_cons;
 127
 128         /* Need to make the tx_bd_cons update visible to start_xmit()
 129          * before checking for netif_tx_queue_stopped().  Without the
 130          * memory barrier, there is a small possibility that
 131          * start_xmit() will miss it and cause the queue to be stopped
 132          * forever.
 133          */
 134         smp_mb();
 135
 136         if (unlikely(netif_tx_queue_stopped(txq))) {
 137                 /* Taking tx_lock() is needed to prevent reenabling the queue
 138                  * while it's empty. This could have happen if rx_action() gets
 139                  * suspended in bnx2x_tx_int() after the condition before
 140                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 141                  *
 142                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 143                  * sends some packets consuming the whole queue again->
 144                  * stops the queue
 145                  */
 146
 147                 __netif_tx_lock(txq, smp_processor_id());
 148
 149                 if ((netif_tx_queue_stopped(txq)) &&
 150                     (bp->state == BNX2X_STATE_OPEN) &&
 151                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 152                         netif_tx_wake_queue(txq);
 153
 154                 __netif_tx_unlock(txq);
 155         }
 156         return 0;
 157 }
 158
 159 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 160                                              u16 idx)
 161 {
 162         u16 last_max = fp->last_max_sge;
 163
 164         if (SUB_S16(idx, last_max) > 0)
 165                 fp->last_max_sge = idx;
 166 }
 167
 168 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 169                                   struct eth_fast_path_rx_cqe *fp_cqe)
 170 {
 171         struct bnx2x *bp = fp->bp;
 172         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 173                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 174                       SGE_PAGE_SHIFT;
 175         u16 last_max, last_elem, first_elem;
 176         u16 delta = 0;
 177         u16 i;
 178
 179         if (!sge_len)
 180                 return;
 181
 182         /* First mark all used pages */
 183         for (i = 0; i < sge_len; i++)
 184                 SGE_MASK_CLEAR_BIT(fp,
 185                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 186
 187         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 188            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 189
 190         /* Here we assume that the last SGE index is the biggest */
 191         prefetch((void *)(fp->sge_mask));
 192         bnx2x_update_last_max_sge(fp,
 193                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 194
 195         last_max = RX_SGE(fp->last_max_sge);
 196         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 197         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 198
 199         /* If ring is not full */
 200         if (last_elem + 1 != first_elem)
 201                 last_elem++;
 202
 203         /* Now update the prod */
 204         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 205                 if (likely(fp->sge_mask[i]))
 206                         break;
 207
 208                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 209                 delta += RX_SGE_MASK_ELEM_SZ;
 210         }
 211
 212         if (delta > 0) {
 213                 fp->rx_sge_prod += delta;
 214                 /* clear page-end entries */
 215                 bnx2x_clear_sge_mask_next_elems(fp);
 216         }
 217
 218         DP(NETIF_MSG_RX_STATUS,
 219            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 220            fp->last_max_sge, fp->rx_sge_prod);
 221 }
 222
 223 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 224                             struct sk_buff *skb, u16 cons, u16 prod)
 225 {
 226         struct bnx2x *bp = fp->bp;
 227         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 228         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 229         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 230         dma_addr_t mapping;
 231
 232         /* move empty skb from pool to prod and map it */
 233         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 234         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 235                                  bp->rx_buf_size, DMA_FROM_DEVICE);
 236         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 237
 238         /* move partial skb from cons to pool (don't unmap yet) */
 239         fp->tpa_pool[queue] = *cons_rx_buf;
 240
 241         /* mark bin state as start - print error if current state != stop */
 242         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 243                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 244
 245         fp->tpa_state[queue] = BNX2X_TPA_START;
 246
 247         /* point prod_bd to new skb */
 248         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 249         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 250
 251 #ifdef BNX2X_STOP_ON_ERROR
 252         fp->tpa_queue_used |= (1 << queue);
 253 #ifdef _ASM_GENERIC_INT_L64_H
 254         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 255 #else
 256         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 257 #endif
 258            fp->tpa_queue_used);
 259 #endif
 260 }
 261
 262 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 263                                struct sk_buff *skb,
 264                                struct eth_fast_path_rx_cqe *fp_cqe,
 265                                u16 cqe_idx)
 266 {
 267         struct sw_rx_page *rx_pg, old_rx_pg;
 268         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 269         u32 i, frag_len, frag_size, pages;
 270         int err;
 271         int j;
 272
 273         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 274         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 275
 276         /* This is needed in order to enable forwarding support */
 277         if (frag_size)
 278                 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
 279                                                max(frag_size, (u32)len_on_bd));
 280
 281 #ifdef BNX2X_STOP_ON_ERROR
 282         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 283                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 284                           pages, cqe_idx);
 285                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 286                           fp_cqe->pkt_len, len_on_bd);
 287                 bnx2x_panic();
 288                 return -EINVAL;
 289         }
 290 #endif
 291
 292         /* Run through the SGL and compose the fragmented skb */
 293         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 294                 u16 sge_idx =
 295                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
 296
 297                 /* FW gives the indices of the SGE as if the ring is an array
 298                    (meaning that "next" element will consume 2 indices) */
 299                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 300                 rx_pg = &fp->rx_page_ring[sge_idx];
 301                 old_rx_pg = *rx_pg;
 302
 303                 /* If we fail to allocate a substitute page, we simply stop
 304                    where we are and drop the whole packet */
 305                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 306                 if (unlikely(err)) {
 307                         fp->eth_q_stats.rx_skb_alloc_failed++;
 308                         return err;
 309                 }
 310
 311                 /* Unmap the page as we r going to pass it to the stack */
 312                 dma_unmap_page(&bp->pdev->dev,
 313                                dma_unmap_addr(&old_rx_pg, mapping),
 314                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 315
 316                 /* Add one frag and update the appropriate fields in the skb */
 317                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 318
 319                 skb->data_len += frag_len;
 320                 skb->truesize += frag_len;
 321                 skb->len += frag_len;
 322
 323                 frag_size -= frag_len;
 324         }
 325
 326         return 0;
 327 }
 328
 329 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 330                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 331                            u16 cqe_idx)
 332 {
 333         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 334         struct sk_buff *skb = rx_buf->skb;
 335         /* alloc new skb */
 336         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 337
 338         /* Unmap skb in the pool anyway, as we are going to change
 339            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 340            fails. */
 341         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 342                          bp->rx_buf_size, DMA_FROM_DEVICE);
 343
 344         if (likely(new_skb)) {
 345                 /* fix ip xsum and give it to the stack */
 346                 /* (no need to map the new skb) */
 347
 348                 prefetch(skb);
 349                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 350
 351 #ifdef BNX2X_STOP_ON_ERROR
 352                 if (pad + len > bp->rx_buf_size) {
 353                         BNX2X_ERR("skb_put is about to fail...  "
 354                                   "pad %d  len %d  rx_buf_size %d\n",
 355                                   pad, len, bp->rx_buf_size);
 356                         bnx2x_panic();
 357                         return;
 358                 }
 359 #endif
 360
 361                 skb_reserve(skb, pad);
 362                 skb_put(skb, len);
 363
 364                 skb->protocol = eth_type_trans(skb, bp->dev);
 365                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 366
 367                 {
 368                         struct iphdr *iph;
 369
 370                         iph = (struct iphdr *)skb->data;
 371                         iph->check = 0;
 372                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 373                 }
 374
 375                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 376                                          &cqe->fast_path_cqe, cqe_idx)) {
 377                         if ((le16_to_cpu(cqe->fast_path_cqe.
 378                             pars_flags.flags) & PARSING_FLAGS_VLAN))
 379                                 __vlan_hwaccel_put_tag(skb,
 380                                                  le16_to_cpu(cqe->fast_path_cqe.
 381                                                              vlan_tag));
 382                         napi_gro_receive(&fp->napi, skb);
 383                 } else {
 384                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 385                            " - dropping packet!\n");
 386                         dev_kfree_skb(skb);
 387                 }
 388
 389
 390                 /* put new skb in bin */
 391                 fp->tpa_pool[queue].skb = new_skb;
 392
 393         } else {
 394                 /* else drop the packet and keep the buffer in the bin */
 395                 DP(NETIF_MSG_RX_STATUS,
 396                    "Failed to allocate new skb - dropping packet!\n");
 397                 fp->eth_q_stats.rx_skb_alloc_failed++;
 398         }
 399
 400         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 401 }
 402
 403 /* Set Toeplitz hash value in the skb using the value from the
 404  * CQE (calculated by HW).
 405  */
 406 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 407                                         struct sk_buff *skb)
 408 {
 409         /* Set Toeplitz hash from CQE */
 410         if ((bp->dev->features & NETIF_F_RXHASH) &&
 411             (cqe->fast_path_cqe.status_flags &
 412              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 413                 skb->rxhash =
 414                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 415 }
 416
 417 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 418 {
 419         struct bnx2x *bp = fp->bp;
 420         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 421         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 422         int rx_pkt = 0;
 423
 424 #ifdef BNX2X_STOP_ON_ERROR
 425         if (unlikely(bp->panic))
 426                 return 0;
 427 #endif
 428
 429         /* CQ "next element" is of the size of the regular element,
 430            that's why it's ok here */
 431         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 432         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 433                 hw_comp_cons++;
 434
 435         bd_cons = fp->rx_bd_cons;
 436         bd_prod = fp->rx_bd_prod;
 437         bd_prod_fw = bd_prod;
 438         sw_comp_cons = fp->rx_comp_cons;
 439         sw_comp_prod = fp->rx_comp_prod;
 440
 441         /* Memory barrier necessary as speculative reads of the rx
 442          * buffer can be ahead of the index in the status block
 443          */
 444         rmb();
 445
 446         DP(NETIF_MSG_RX_STATUS,
 447            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 448            fp->index, hw_comp_cons, sw_comp_cons);
 449
 450         while (sw_comp_cons != hw_comp_cons) {
 451                 struct sw_rx_bd *rx_buf = NULL;
 452                 struct sk_buff *skb;
 453                 union eth_rx_cqe *cqe;
 454                 u8 cqe_fp_flags;
 455                 u16 len, pad;
 456
 457                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 458                 bd_prod = RX_BD(bd_prod);
 459                 bd_cons = RX_BD(bd_cons);
 460
 461                 /* Prefetch the page containing the BD descriptor
 462                    at producer's index. It will be needed when new skb is
 463                    allocated */
 464                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 465                                              (&fp->rx_desc_ring[bd_prod])) -
 466                                   PAGE_SIZE + 1));
 467
 468                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 469                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 470
 471                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 472                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 473                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 474                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 475                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 476                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 477
 478                 /* is this a slowpath msg? */
 479                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 480                         bnx2x_sp_event(fp, cqe);
 481                         goto next_cqe;
 482
 483                 /* this is an rx packet */
 484                 } else {
 485                         rx_buf = &fp->rx_buf_ring[bd_cons];
 486                         skb = rx_buf->skb;
 487                         prefetch(skb);
 488                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 489                         pad = cqe->fast_path_cqe.placement_offset;
 490
 491                         /* - If CQE is marked both TPA_START and TPA_END it is
 492                          *   a non-TPA CQE.
 493                          * - FP CQE will always have either TPA_START or/and
 494                          *   TPA_STOP flags set.
 495                          */
 496                         if ((!fp->disable_tpa) &&
 497                             (TPA_TYPE(cqe_fp_flags) !=
 498                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 499                                 u16 queue = cqe->fast_path_cqe.queue_index;
 500
 501                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 502                                         DP(NETIF_MSG_RX_STATUS,
 503                                            "calling tpa_start on queue %d\n",
 504                                            queue);
 505
 506                                         bnx2x_tpa_start(fp, queue, skb,
 507                                                         bd_cons, bd_prod);
 508
 509                                         /* Set Toeplitz hash for an LRO skb */
 510                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 511
 512                                         goto next_rx;
 513                                 } else { /* TPA_STOP */
 514                                         DP(NETIF_MSG_RX_STATUS,
 515                                            "calling tpa_stop on queue %d\n",
 516                                            queue);
 517
 518                                         if (!BNX2X_RX_SUM_FIX(cqe))
 519                                                 BNX2X_ERR("STOP on none TCP "
 520                                                           "data\n");
 521
 522                                         /* This is a size of the linear data
 523                                            on this skb */
 524                                         len = le16_to_cpu(cqe->fast_path_cqe.
 525                                                                 len_on_bd);
 526                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 527                                                     len, cqe, comp_ring_cons);
 528 #ifdef BNX2X_STOP_ON_ERROR
 529                                         if (bp->panic)
 530                                                 return 0;
 531 #endif
 532
 533                                         bnx2x_update_sge_prod(fp,
 534                                                         &cqe->fast_path_cqe);
 535                                         goto next_cqe;
 536                                 }
 537                         }
 538
 539                         dma_sync_single_for_device(&bp->pdev->dev,
 540                                         dma_unmap_addr(rx_buf, mapping),
 541                                                    pad + RX_COPY_THRESH,
 542                                                    DMA_FROM_DEVICE);
 543                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 544
 545                         /* is this an error packet? */
 546                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 547                                 DP(NETIF_MSG_RX_ERR,
 548                                    "ERROR  flags %x  rx packet %u\n",
 549                                    cqe_fp_flags, sw_comp_cons);
 550                                 fp->eth_q_stats.rx_err_discard_pkt++;
 551                                 goto reuse_rx;
 552                         }
 553
 554                         /* Since we don't have a jumbo ring
 555                          * copy small packets if mtu > 1500
 556                          */
 557                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 558                             (len <= RX_COPY_THRESH)) {
 559                                 struct sk_buff *new_skb;
 560
 561                                 new_skb = netdev_alloc_skb(bp->dev,
 562                                                            len + pad);
 563                                 if (new_skb == NULL) {
 564                                         DP(NETIF_MSG_RX_ERR,
 565                                            "ERROR  packet dropped "
 566                                            "because of alloc failure\n");
 567                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 568                                         goto reuse_rx;
 569                                 }
 570
 571                                 /* aligned copy */
 572                                 skb_copy_from_linear_data_offset(skb, pad,
 573                                                     new_skb->data + pad, len);
 574                                 skb_reserve(new_skb, pad);
 575                                 skb_put(new_skb, len);
 576
 577                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 578
 579                                 skb = new_skb;
 580
 581                         } else
 582                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 583                                 dma_unmap_single(&bp->pdev->dev,
 584                                         dma_unmap_addr(rx_buf, mapping),
 585                                                  bp->rx_buf_size,
 586                                                  DMA_FROM_DEVICE);
 587                                 skb_reserve(skb, pad);
 588                                 skb_put(skb, len);
 589
 590                         } else {
 591                                 DP(NETIF_MSG_RX_ERR,
 592                                    "ERROR  packet dropped because "
 593                                    "of alloc failure\n");
 594                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 595 reuse_rx:
 596                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 597                                 goto next_rx;
 598                         }
 599
 600                         skb->protocol = eth_type_trans(skb, bp->dev);
 601
 602                         /* Set Toeplitz hash for a none-LRO skb */
 603                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 604
 605                         skb_checksum_none_assert(skb);
 606
 607                         if (bp->rx_csum) {
 608                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 609                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 610                                 else
 611                                         fp->eth_q_stats.hw_csum_err++;
 612                         }
 613                 }
 614
 615                 skb_record_rx_queue(skb, fp->index);
 616
 617                 if (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 618                      PARSING_FLAGS_VLAN)
 619                         __vlan_hwaccel_put_tag(skb,
 620                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag));
 621                 napi_gro_receive(&fp->napi, skb);
 622
 623
 624 next_rx:
 625                 rx_buf->skb = NULL;
 626
 627                 bd_cons = NEXT_RX_IDX(bd_cons);
 628                 bd_prod = NEXT_RX_IDX(bd_prod);
 629                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 630                 rx_pkt++;
 631 next_cqe:
 632                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 633                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 634
 635                 if (rx_pkt == budget)
 636                         break;
 637         } /* while */
 638
 639         fp->rx_bd_cons = bd_cons;
 640         fp->rx_bd_prod = bd_prod_fw;
 641         fp->rx_comp_cons = sw_comp_cons;
 642         fp->rx_comp_prod = sw_comp_prod;
 643
 644         /* Update producers */
 645         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 646                              fp->rx_sge_prod);
 647
 648         fp->rx_pkt += rx_pkt;
 649         fp->rx_calls++;
 650
 651         return rx_pkt;
 652 }
 653
 654 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 655 {
 656         struct bnx2x_fastpath *fp = fp_cookie;
 657         struct bnx2x *bp = fp->bp;
 658
 659         /* Return here if interrupt is disabled */
 660         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 661                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 662                 return IRQ_HANDLED;
 663         }
 664
 665         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 666                          "[fp %d fw_sd %d igusb %d]\n",
 667            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 668         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 669
 670 #ifdef BNX2X_STOP_ON_ERROR
 671         if (unlikely(bp->panic))
 672                 return IRQ_HANDLED;
 673 #endif
 674
 675         /* Handle Rx and Tx according to MSI-X vector */
 676         prefetch(fp->rx_cons_sb);
 677         prefetch(fp->tx_cons_sb);
 678         prefetch(&fp->sb_running_index[SM_RX_ID]);
 679         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 680
 681         return IRQ_HANDLED;
 682 }
 683
 684 /* HW Lock for shared dual port PHYs */
 685 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 686 {
 687         mutex_lock(&bp->port.phy_mutex);
 688
 689         if (bp->port.need_hw_lock)
 690                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 691 }
 692
 693 void bnx2x_release_phy_lock(struct bnx2x *bp)
 694 {
 695         if (bp->port.need_hw_lock)
 696                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 697
 698         mutex_unlock(&bp->port.phy_mutex);
 699 }
 700
 701 void bnx2x_link_report(struct bnx2x *bp)
 702 {
 703         if (bp->flags & MF_FUNC_DIS) {
 704                 netif_carrier_off(bp->dev);
 705                 netdev_err(bp->dev, "NIC Link is Down\n");
 706                 return;
 707         }
 708
 709         if (bp->link_vars.link_up) {
 710                 u16 line_speed;
 711
 712                 if (bp->state == BNX2X_STATE_OPEN)
 713                         netif_carrier_on(bp->dev);
 714                 netdev_info(bp->dev, "NIC Link is Up, ");
 715
 716                 line_speed = bp->link_vars.line_speed;
 717                 if (IS_MF(bp)) {
 718                         u16 vn_max_rate;
 719
 720                         vn_max_rate =
 721                                 ((bp->mf_config[BP_VN(bp)] &
 722                                   FUNC_MF_CFG_MAX_BW_MASK) >>
 723                                                 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
 724                         if (vn_max_rate < line_speed)
 725                                 line_speed = vn_max_rate;
 726                 }
 727                 pr_cont("%d Mbps ", line_speed);
 728
 729                 if (bp->link_vars.duplex == DUPLEX_FULL)
 730                         pr_cont("full duplex");
 731                 else
 732                         pr_cont("half duplex");
 733
 734                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 735                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 736                                 pr_cont(", receive ");
 737                                 if (bp->link_vars.flow_ctrl &
 738                                     BNX2X_FLOW_CTRL_TX)
 739                                         pr_cont("& transmit ");
 740                         } else {
 741                                 pr_cont(", transmit ");
 742                         }
 743                         pr_cont("flow control ON");
 744                 }
 745                 pr_cont("\n");
 746
 747         } else { /* link_down */
 748                 netif_carrier_off(bp->dev);
 749                 netdev_err(bp->dev, "NIC Link is Down\n");
 750         }
 751 }
 752
 753 /* Returns the number of actually allocated BDs */
 754 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
 755                                       int rx_ring_size)
 756 {
 757         struct bnx2x *bp = fp->bp;
 758         u16 ring_prod, cqe_ring_prod;
 759         int i;
 760
 761         fp->rx_comp_cons = 0;
 762         cqe_ring_prod = ring_prod = 0;
 763         for (i = 0; i < rx_ring_size; i++) {
 764                 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 765                         BNX2X_ERR("was only able to allocate "
 766                                   "%d rx skbs on queue[%d]\n", i, fp->index);
 767                         fp->eth_q_stats.rx_skb_alloc_failed++;
 768                         break;
 769                 }
 770                 ring_prod = NEXT_RX_IDX(ring_prod);
 771                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 772                 WARN_ON(ring_prod <= i);
 773         }
 774
 775         fp->rx_bd_prod = ring_prod;
 776         /* Limit the CQE producer by the CQE ring size */
 777         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 778                                cqe_ring_prod);
 779         fp->rx_pkt = fp->rx_calls = 0;
 780
 781         return i;
 782 }
 783
 784 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath *fp)
 785 {
 786         struct bnx2x *bp = fp->bp;
 787         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
 788                                               MAX_RX_AVAIL/bp->num_queues;
 789
 790         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
 791
 792         bnx2x_alloc_rx_bds(fp, rx_ring_size);
 793
 794         /* Warning!
 795          * this will generate an interrupt (to the TSTORM)
 796          * must only be done after chip is initialized
 797          */
 798         bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
 799                              fp->rx_sge_prod);
 800 }
 801
 802 void bnx2x_init_rx_rings(struct bnx2x *bp)
 803 {
 804         int func = BP_FUNC(bp);
 805         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 806                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 807         u16 ring_prod;
 808         int i, j;
 809
 810         bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
 811                 IP_HEADER_ALIGNMENT_PADDING;
 812
 813         DP(NETIF_MSG_IFUP,
 814            "mtu %d  rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
 815
 816         for_each_queue(bp, j) {
 817                 struct bnx2x_fastpath *fp = &bp->fp[j];
 818
 819                 if (!fp->disable_tpa) {
 820                         for (i = 0; i < max_agg_queues; i++) {
 821                                 fp->tpa_pool[i].skb =
 822                                    netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 823                                 if (!fp->tpa_pool[i].skb) {
 824                                         BNX2X_ERR("Failed to allocate TPA "
 825                                                   "skb pool for queue[%d] - "
 826                                                   "disabling TPA on this "
 827                                                   "queue!\n", j);
 828                                         bnx2x_free_tpa_pool(bp, fp, i);
 829                                         fp->disable_tpa = 1;
 830                                         break;
 831                                 }
 832                                 dma_unmap_addr_set((struct sw_rx_bd *)
 833                                                         &bp->fp->tpa_pool[i],
 834                                                    mapping, 0);
 835                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 836                         }
 837
 838                         /* "next page" elements initialization */
 839                         bnx2x_set_next_page_sgl(fp);
 840
 841                         /* set SGEs bit mask */
 842                         bnx2x_init_sge_ring_bit_mask(fp);
 843
 844                         /* Allocate SGEs and initialize the ring elements */
 845                         for (i = 0, ring_prod = 0;
 846                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 847
 848                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 849                                         BNX2X_ERR("was only able to allocate "
 850                                                   "%d rx sges\n", i);
 851                                         BNX2X_ERR("disabling TPA for"
 852                                                   " queue[%d]\n", j);
 853                                         /* Cleanup already allocated elements */
 854                                         bnx2x_free_rx_sge_range(bp,
 855                                                                 fp, ring_prod);
 856                                         bnx2x_free_tpa_pool(bp,
 857                                                             fp, max_agg_queues);
 858                                         fp->disable_tpa = 1;
 859                                         ring_prod = 0;
 860                                         break;
 861                                 }
 862                                 ring_prod = NEXT_SGE_IDX(ring_prod);
 863                         }
 864
 865                         fp->rx_sge_prod = ring_prod;
 866                 }
 867         }
 868
 869         for_each_queue(bp, j) {
 870                 struct bnx2x_fastpath *fp = &bp->fp[j];
 871
 872                 fp->rx_bd_cons = 0;
 873
 874                 bnx2x_set_next_page_rx_bd(fp);
 875
 876                 /* CQ ring */
 877                 bnx2x_set_next_page_rx_cq(fp);
 878
 879                 /* Allocate BDs and initialize BD ring */
 880                 bnx2x_alloc_rx_bd_ring(fp);
 881
 882                 if (j != 0)
 883                         continue;
 884
 885                 if (!CHIP_IS_E2(bp)) {
 886                         REG_WR(bp, BAR_USTRORM_INTMEM +
 887                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 888                                U64_LO(fp->rx_comp_mapping));
 889                         REG_WR(bp, BAR_USTRORM_INTMEM +
 890                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 891                                U64_HI(fp->rx_comp_mapping));
 892                 }
 893         }
 894 }
 895
 896 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 897 {
 898         int i;
 899
 900         for_each_queue(bp, i) {
 901                 struct bnx2x_fastpath *fp = &bp->fp[i];
 902
 903                 u16 bd_cons = fp->tx_bd_cons;
 904                 u16 sw_prod = fp->tx_pkt_prod;
 905                 u16 sw_cons = fp->tx_pkt_cons;
 906
 907                 while (sw_cons != sw_prod) {
 908                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 909                         sw_cons++;
 910                 }
 911         }
 912 }
 913
 914 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 915 {
 916         int i, j;
 917
 918         for_each_queue(bp, j) {
 919                 struct bnx2x_fastpath *fp = &bp->fp[j];
 920
 921                 for (i = 0; i < NUM_RX_BD; i++) {
 922                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 923                         struct sk_buff *skb = rx_buf->skb;
 924
 925                         if (skb == NULL)
 926                                 continue;
 927
 928                         dma_unmap_single(&bp->pdev->dev,
 929                                          dma_unmap_addr(rx_buf, mapping),
 930                                          bp->rx_buf_size, DMA_FROM_DEVICE);
 931
 932                         rx_buf->skb = NULL;
 933                         dev_kfree_skb(skb);
 934                 }
 935                 if (!fp->disable_tpa)
 936                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 937                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 938                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 939         }
 940 }
 941
 942 void bnx2x_free_skbs(struct bnx2x *bp)
 943 {
 944         bnx2x_free_tx_skbs(bp);
 945         bnx2x_free_rx_skbs(bp);
 946 }
 947
 948 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
 949 {
 950         int i, offset = 1;
 951
 952         free_irq(bp->msix_table[0].vector, bp->dev);
 953         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
 954            bp->msix_table[0].vector);
 955
 956 #ifdef BCM_CNIC
 957         offset++;
 958 #endif
 959         for_each_queue(bp, i) {
 960                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
 961                    "state %x\n", i, bp->msix_table[i + offset].vector,
 962                    bnx2x_fp(bp, i, state));
 963
 964                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
 965         }
 966 }
 967
 968 void bnx2x_free_irq(struct bnx2x *bp)
 969 {
 970         if (bp->flags & USING_MSIX_FLAG)
 971                 bnx2x_free_msix_irqs(bp);
 972         else if (bp->flags & USING_MSI_FLAG)
 973                 free_irq(bp->pdev->irq, bp->dev);
 974         else
 975                 free_irq(bp->pdev->irq, bp->dev);
 976 }
 977
 978 int bnx2x_enable_msix(struct bnx2x *bp)
 979 {
 980         int msix_vec = 0, i, rc, req_cnt;
 981
 982         bp->msix_table[msix_vec].entry = msix_vec;
 983         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
 984            bp->msix_table[0].entry);
 985         msix_vec++;
 986
 987 #ifdef BCM_CNIC
 988         bp->msix_table[msix_vec].entry = msix_vec;
 989         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
 990            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
 991         msix_vec++;
 992 #endif
 993         for_each_queue(bp, i) {
 994                 bp->msix_table[msix_vec].entry = msix_vec;
 995                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
 996                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
 997                 msix_vec++;
 998         }
 999
1000         req_cnt = BNX2X_NUM_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1001
1002         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1003
1004         /*
1005          * reconfigure number of tx/rx queues according to available
1006          * MSI-X vectors
1007          */
1008         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1009                 /* how less vectors we will have? */
1010                 int diff = req_cnt - rc;
1011
1012                 DP(NETIF_MSG_IFUP,
1013                    "Trying to use less MSI-X vectors: %d\n", rc);
1014
1015                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1016
1017                 if (rc) {
1018                         DP(NETIF_MSG_IFUP,
1019                            "MSI-X is not attainable  rc %d\n", rc);
1020                         return rc;
1021                 }
1022                 /*
1023                  * decrease number of queues by number of unallocated entries
1024                  */
1025                 bp->num_queues -= diff;
1026
1027                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1028                                   bp->num_queues);
1029         } else if (rc) {
1030                 /* fall to INTx if not enough memory */
1031                 if (rc == -ENOMEM)
1032                         bp->flags |= DISABLE_MSI_FLAG;
1033                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1034                 return rc;
1035         }
1036
1037         bp->flags |= USING_MSIX_FLAG;
1038
1039         return 0;
1040 }
1041
1042 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1043 {
1044         int i, rc, offset = 1;
1045
1046         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1047                          bp->dev->name, bp->dev);
1048         if (rc) {
1049                 BNX2X_ERR("request sp irq failed\n");
1050                 return -EBUSY;
1051         }
1052
1053 #ifdef BCM_CNIC
1054         offset++;
1055 #endif
1056         for_each_queue(bp, i) {
1057                 struct bnx2x_fastpath *fp = &bp->fp[i];
1058                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1059                          bp->dev->name, i);
1060
1061                 rc = request_irq(bp->msix_table[offset].vector,
1062                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1063                 if (rc) {
1064                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1065                         bnx2x_free_msix_irqs(bp);
1066                         return -EBUSY;
1067                 }
1068
1069                 offset++;
1070                 fp->state = BNX2X_FP_STATE_IRQ;
1071         }
1072
1073         i = BNX2X_NUM_QUEUES(bp);
1074         offset = 1 + CNIC_CONTEXT_USE;
1075         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1076                " ... fp[%d] %d\n",
1077                bp->msix_table[0].vector,
1078                0, bp->msix_table[offset].vector,
1079                i - 1, bp->msix_table[offset + i - 1].vector);
1080
1081         return 0;
1082 }
1083
1084 int bnx2x_enable_msi(struct bnx2x *bp)
1085 {
1086         int rc;
1087
1088         rc = pci_enable_msi(bp->pdev);
1089         if (rc) {
1090                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1091                 return -1;
1092         }
1093         bp->flags |= USING_MSI_FLAG;
1094
1095         return 0;
1096 }
1097
1098 static int bnx2x_req_irq(struct bnx2x *bp)
1099 {
1100         unsigned long flags;
1101         int rc;
1102
1103         if (bp->flags & USING_MSI_FLAG)
1104                 flags = 0;
1105         else
1106                 flags = IRQF_SHARED;
1107
1108         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1109                          bp->dev->name, bp->dev);
1110         if (!rc)
1111                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1112
1113         return rc;
1114 }
1115
1116 static void bnx2x_napi_enable(struct bnx2x *bp)
1117 {
1118         int i;
1119
1120         for_each_queue(bp, i)
1121                 napi_enable(&bnx2x_fp(bp, i, napi));
1122 }
1123
1124 static void bnx2x_napi_disable(struct bnx2x *bp)
1125 {
1126         int i;
1127
1128         for_each_queue(bp, i)
1129                 napi_disable(&bnx2x_fp(bp, i, napi));
1130 }
1131
1132 void bnx2x_netif_start(struct bnx2x *bp)
1133 {
1134         int intr_sem;
1135
1136         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1137         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1138
1139         if (intr_sem) {
1140                 if (netif_running(bp->dev)) {
1141                         bnx2x_napi_enable(bp);
1142                         bnx2x_int_enable(bp);
1143                         if (bp->state == BNX2X_STATE_OPEN)
1144                                 netif_tx_wake_all_queues(bp->dev);
1145                 }
1146         }
1147 }
1148
1149 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1150 {
1151         bnx2x_int_disable_sync(bp, disable_hw);
1152         bnx2x_napi_disable(bp);
1153         netif_tx_disable(bp->dev);
1154 }
1155
1156 void bnx2x_set_num_queues(struct bnx2x *bp)
1157 {
1158         switch (bp->multi_mode) {
1159         case ETH_RSS_MODE_DISABLED:
1160                 bp->num_queues = 1;
1161                 break;
1162         case ETH_RSS_MODE_REGULAR:
1163                 bp->num_queues = bnx2x_calc_num_queues(bp);
1164                 break;
1165
1166         default:
1167                 bp->num_queues = 1;
1168                 break;
1169         }
1170 }
1171
1172 static void bnx2x_release_firmware(struct bnx2x *bp)
1173 {
1174         kfree(bp->init_ops_offsets);
1175         kfree(bp->init_ops);
1176         kfree(bp->init_data);
1177         release_firmware(bp->firmware);
1178 }
1179
1180 /* must be called with rtnl_lock */
1181 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1182 {
1183         u32 load_code;
1184         int i, rc;
1185
1186         /* Set init arrays */
1187         rc = bnx2x_init_firmware(bp);
1188         if (rc) {
1189                 BNX2X_ERR("Error loading firmware\n");
1190                 return rc;
1191         }
1192
1193 #ifdef BNX2X_STOP_ON_ERROR
1194         if (unlikely(bp->panic))
1195                 return -EPERM;
1196 #endif
1197
1198         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1199
1200         /* must be called before memory allocation and HW init */
1201         bnx2x_ilt_set_info(bp);
1202
1203         if (bnx2x_alloc_mem(bp))
1204                 return -ENOMEM;
1205
1206         netif_set_real_num_tx_queues(bp->dev, bp->num_queues);
1207         rc = netif_set_real_num_rx_queues(bp->dev, bp->num_queues);
1208         if (rc) {
1209                 BNX2X_ERR("Unable to update real_num_rx_queues\n");
1210                 goto load_error0;
1211         }
1212
1213         for_each_queue(bp, i)
1214                 bnx2x_fp(bp, i, disable_tpa) =
1215                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1216
1217         bnx2x_napi_enable(bp);
1218
1219         /* Send LOAD_REQUEST command to MCP
1220            Returns the type of LOAD command:
1221            if it is the first port to be initialized
1222            common blocks should be initialized, otherwise - not
1223         */
1224         if (!BP_NOMCP(bp)) {
1225                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1226                 if (!load_code) {
1227                         BNX2X_ERR("MCP response failure, aborting\n");
1228                         rc = -EBUSY;
1229                         goto load_error1;
1230                 }
1231                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1232                         rc = -EBUSY; /* other port in diagnostic mode */
1233                         goto load_error1;
1234                 }
1235
1236         } else {
1237                 int path = BP_PATH(bp);
1238                 int port = BP_PORT(bp);
1239
1240                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1241                    path, load_count[path][0], load_count[path][1],
1242                    load_count[path][2]);
1243                 load_count[path][0]++;
1244                 load_count[path][1 + port]++;
1245                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1246                    path, load_count[path][0], load_count[path][1],
1247                    load_count[path][2]);
1248                 if (load_count[path][0] == 1)
1249                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1250                 else if (load_count[path][1 + port] == 1)
1251                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1252                 else
1253                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1254         }
1255
1256         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1257             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1258             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1259                 bp->port.pmf = 1;
1260         else
1261                 bp->port.pmf = 0;
1262         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1263
1264         /* Initialize HW */
1265         rc = bnx2x_init_hw(bp, load_code);
1266         if (rc) {
1267                 BNX2X_ERR("HW init failed, aborting\n");
1268                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1269                 goto load_error2;
1270         }
1271
1272         /* Connect to IRQs */
1273         rc = bnx2x_setup_irqs(bp);
1274         if (rc) {
1275                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1276                 goto load_error2;
1277         }
1278
1279         /* Setup NIC internals and enable interrupts */
1280         bnx2x_nic_init(bp, load_code);
1281
1282         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1283             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1284             (bp->common.shmem2_base))
1285                 SHMEM2_WR(bp, dcc_support,
1286                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1287                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1288
1289         /* Send LOAD_DONE command to MCP */
1290         if (!BP_NOMCP(bp)) {
1291                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1292                 if (!load_code) {
1293                         BNX2X_ERR("MCP response failure, aborting\n");
1294                         rc = -EBUSY;
1295                         goto load_error3;
1296                 }
1297         }
1298
1299         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1300
1301         rc = bnx2x_func_start(bp);
1302         if (rc) {
1303                 BNX2X_ERR("Function start failed!\n");
1304 #ifndef BNX2X_STOP_ON_ERROR
1305                 goto load_error3;
1306 #else
1307                 bp->panic = 1;
1308                 return -EBUSY;
1309 #endif
1310         }
1311
1312         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1313         if (rc) {
1314                 BNX2X_ERR("Setup leading failed!\n");
1315 #ifndef BNX2X_STOP_ON_ERROR
1316                 goto load_error3;
1317 #else
1318                 bp->panic = 1;
1319                 return -EBUSY;
1320 #endif
1321         }
1322
1323         if (!CHIP_IS_E1(bp) &&
1324             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1325                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1326                 bp->flags |= MF_FUNC_DIS;
1327         }
1328
1329 #ifdef BCM_CNIC
1330         /* Enable Timer scan */
1331         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1332 #endif
1333
1334         for_each_nondefault_queue(bp, i) {
1335                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1336                 if (rc)
1337 #ifdef BCM_CNIC
1338                         goto load_error4;
1339 #else
1340                         goto load_error3;
1341 #endif
1342         }
1343
1344         /* Now when Clients are configured we are ready to work */
1345         bp->state = BNX2X_STATE_OPEN;
1346
1347         bnx2x_set_eth_mac(bp, 1);
1348
1349         if (bp->port.pmf)
1350                 bnx2x_initial_phy_init(bp, load_mode);
1351
1352         /* Start fast path */
1353         switch (load_mode) {
1354         case LOAD_NORMAL:
1355                 /* Tx queue should be only reenabled */
1356                 netif_tx_wake_all_queues(bp->dev);
1357                 /* Initialize the receive filter. */
1358                 bnx2x_set_rx_mode(bp->dev);
1359                 break;
1360
1361         case LOAD_OPEN:
1362                 netif_tx_start_all_queues(bp->dev);
1363                 smp_mb__after_clear_bit();
1364                 /* Initialize the receive filter. */
1365                 bnx2x_set_rx_mode(bp->dev);
1366                 break;
1367
1368         case LOAD_DIAG:
1369                 /* Initialize the receive filter. */
1370                 bnx2x_set_rx_mode(bp->dev);
1371                 bp->state = BNX2X_STATE_DIAG;
1372                 break;
1373
1374         default:
1375                 break;
1376         }
1377
1378         if (!bp->port.pmf)
1379                 bnx2x__link_status_update(bp);
1380
1381         /* start the timer */
1382         mod_timer(&bp->timer, jiffies + bp->current_interval);
1383
1384 #ifdef BCM_CNIC
1385         bnx2x_setup_cnic_irq_info(bp);
1386         if (bp->state == BNX2X_STATE_OPEN)
1387                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1388 #endif
1389         bnx2x_inc_load_cnt(bp);
1390
1391         bnx2x_release_firmware(bp);
1392
1393         return 0;
1394
1395 #ifdef BCM_CNIC
1396 load_error4:
1397         /* Disable Timer scan */
1398         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1399 #endif
1400 load_error3:
1401         bnx2x_int_disable_sync(bp, 1);
1402
1403         /* Free SKBs, SGEs, TPA pool and driver internals */
1404         bnx2x_free_skbs(bp);
1405         for_each_queue(bp, i)
1406                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1407
1408         /* Release IRQs */
1409         bnx2x_free_irq(bp);
1410 load_error2:
1411         if (!BP_NOMCP(bp)) {
1412                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1413                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1414         }
1415
1416         bp->port.pmf = 0;
1417 load_error1:
1418         bnx2x_napi_disable(bp);
1419 load_error0:
1420         bnx2x_free_mem(bp);
1421
1422         bnx2x_release_firmware(bp);
1423
1424         return rc;
1425 }
1426
1427 /* must be called with rtnl_lock */
1428 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1429 {
1430         int i;
1431
1432         if (bp->state == BNX2X_STATE_CLOSED) {
1433                 /* Interface has been removed - nothing to recover */
1434                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1435                 bp->is_leader = 0;
1436                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1437                 smp_wmb();
1438
1439                 return -EINVAL;
1440         }
1441
1442 #ifdef BCM_CNIC
1443         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1444 #endif
1445         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1446
1447         /* Set "drop all" */
1448         bp->rx_mode = BNX2X_RX_MODE_NONE;
1449         bnx2x_set_storm_rx_mode(bp);
1450
1451         /* Stop Tx */
1452         bnx2x_tx_disable(bp);
1453
1454         del_timer_sync(&bp->timer);
1455
1456         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1457                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1458
1459         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1460
1461         /* Cleanup the chip if needed */
1462         if (unload_mode != UNLOAD_RECOVERY)
1463                 bnx2x_chip_cleanup(bp, unload_mode);
1464         else {
1465                 /* Disable HW interrupts, NAPI and Tx */
1466                 bnx2x_netif_stop(bp, 1);
1467
1468                 /* Release IRQs */
1469                 bnx2x_free_irq(bp);
1470         }
1471
1472         bp->port.pmf = 0;
1473
1474         /* Free SKBs, SGEs, TPA pool and driver internals */
1475         bnx2x_free_skbs(bp);
1476         for_each_queue(bp, i)
1477                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1478
1479         bnx2x_free_mem(bp);
1480
1481         bp->state = BNX2X_STATE_CLOSED;
1482
1483         /* The last driver must disable a "close the gate" if there is no
1484          * parity attention or "process kill" pending.
1485          */
1486         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1487             bnx2x_reset_is_done(bp))
1488                 bnx2x_disable_close_the_gate(bp);
1489
1490         /* Reset MCP mail box sequence if there is on going recovery */
1491         if (unload_mode == UNLOAD_RECOVERY)
1492                 bp->fw_seq = 0;
1493
1494         return 0;
1495 }
1496
1497 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1498 {
1499         u16 pmcsr;
1500
1501         /* If there is no power capability, silently succeed */
1502         if (!bp->pm_cap) {
1503                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1504                 return 0;
1505         }
1506
1507         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1508
1509         switch (state) {
1510         case PCI_D0:
1511                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1512                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1513                                        PCI_PM_CTRL_PME_STATUS));
1514
1515                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1516                         /* delay required during transition out of D3hot */
1517                         msleep(20);
1518                 break;
1519
1520         case PCI_D3hot:
1521                 /* If there are other clients above don't
1522                    shut down the power */
1523                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1524                         return 0;
1525                 /* Don't shut down the power for emulation and FPGA */
1526                 if (CHIP_REV_IS_SLOW(bp))
1527                         return 0;
1528
1529                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1530                 pmcsr |= 3;
1531
1532                 if (bp->wol)
1533                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1534
1535                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1536                                       pmcsr);
1537
1538                 /* No more memory access after this point until
1539                 * device is brought back to D0.
1540                 */
1541                 break;
1542
1543         default:
1544                 return -EINVAL;
1545         }
1546         return 0;
1547 }
1548
1549 /*
1550  * net_device service functions
1551  */
1552 int bnx2x_poll(struct napi_struct *napi, int budget)
1553 {
1554         int work_done = 0;
1555         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1556                                                  napi);
1557         struct bnx2x *bp = fp->bp;
1558
1559         while (1) {
1560 #ifdef BNX2X_STOP_ON_ERROR
1561                 if (unlikely(bp->panic)) {
1562                         napi_complete(napi);
1563                         return 0;
1564                 }
1565 #endif
1566
1567                 if (bnx2x_has_tx_work(fp))
1568                         bnx2x_tx_int(fp);
1569
1570                 if (bnx2x_has_rx_work(fp)) {
1571                         work_done += bnx2x_rx_int(fp, budget - work_done);
1572
1573                         /* must not complete if we consumed full budget */
1574                         if (work_done >= budget)
1575                                 break;
1576                 }
1577
1578                 /* Fall out from the NAPI loop if needed */
1579                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1580                         bnx2x_update_fpsb_idx(fp);
1581                         /* bnx2x_has_rx_work() reads the status block,
1582                          * thus we need to ensure that status block indices
1583                          * have been actually read (bnx2x_update_fpsb_idx)
1584                          * prior to this check (bnx2x_has_rx_work) so that
1585                          * we won't write the "newer" value of the status block
1586                          * to IGU (if there was a DMA right after
1587                          * bnx2x_has_rx_work and if there is no rmb, the memory
1588                          * reading (bnx2x_update_fpsb_idx) may be postponed
1589                          * to right before bnx2x_ack_sb). In this case there
1590                          * will never be another interrupt until there is
1591                          * another update of the status block, while there
1592                          * is still unhandled work.
1593                          */
1594                         rmb();
1595
1596                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1597                                 napi_complete(napi);
1598                                 /* Re-enable interrupts */
1599                                 DP(NETIF_MSG_HW,
1600                                    "Update index to %d\n", fp->fp_hc_idx);
1601                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1602                                              le16_to_cpu(fp->fp_hc_idx),
1603                                              IGU_INT_ENABLE, 1);
1604                                 break;
1605                         }
1606                 }
1607         }
1608
1609         return work_done;
1610 }
1611
1612 /* we split the first BD into headers and data BDs
1613  * to ease the pain of our fellow microcode engineers
1614  * we use one mapping for both BDs
1615  * So far this has only been observed to happen
1616  * in Other Operating Systems(TM)
1617  */
1618 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1619                                    struct bnx2x_fastpath *fp,
1620                                    struct sw_tx_bd *tx_buf,
1621                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1622                                    u16 bd_prod, int nbd)
1623 {
1624         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1625         struct eth_tx_bd *d_tx_bd;
1626         dma_addr_t mapping;
1627         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1628
1629         /* first fix first BD */
1630         h_tx_bd->nbd = cpu_to_le16(nbd);
1631         h_tx_bd->nbytes = cpu_to_le16(hlen);
1632
1633         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1634            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1635            h_tx_bd->addr_lo, h_tx_bd->nbd);
1636
1637         /* now get a new data BD
1638          * (after the pbd) and fill it */
1639         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1640         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1641
1642         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1643                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1644
1645         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1646         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1647         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1648
1649         /* this marks the BD as one that has no individual mapping */
1650         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1651
1652         DP(NETIF_MSG_TX_QUEUED,
1653            "TSO split data size is %d (%x:%x)\n",
1654            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1655
1656         /* update tx_bd */
1657         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1658
1659         return bd_prod;
1660 }
1661
1662 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1663 {
1664         if (fix > 0)
1665                 csum = (u16) ~csum_fold(csum_sub(csum,
1666                                 csum_partial(t_header - fix, fix, 0)));
1667
1668         else if (fix < 0)
1669                 csum = (u16) ~csum_fold(csum_add(csum,
1670                                 csum_partial(t_header, -fix, 0)));
1671
1672         return swab16(csum);
1673 }
1674
1675 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1676 {
1677         u32 rc;
1678
1679         if (skb->ip_summed != CHECKSUM_PARTIAL)
1680                 rc = XMIT_PLAIN;
1681
1682         else {
1683                 if (skb->protocol == htons(ETH_P_IPV6)) {
1684                         rc = XMIT_CSUM_V6;
1685                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1686                                 rc |= XMIT_CSUM_TCP;
1687
1688                 } else {
1689                         rc = XMIT_CSUM_V4;
1690                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1691                                 rc |= XMIT_CSUM_TCP;
1692                 }
1693         }
1694
1695         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1696                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1697
1698         else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1699                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1700
1701         return rc;
1702 }
1703
1704 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1705 /* check if packet requires linearization (packet is too fragmented)
1706    no need to check fragmentation if page size > 8K (there will be no
1707    violation to FW restrictions) */
1708 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1709                              u32 xmit_type)
1710 {
1711         int to_copy = 0;
1712         int hlen = 0;
1713         int first_bd_sz = 0;
1714
1715         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1716         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1717
1718                 if (xmit_type & XMIT_GSO) {
1719                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1720                         /* Check if LSO packet needs to be copied:
1721                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1722                         int wnd_size = MAX_FETCH_BD - 3;
1723                         /* Number of windows to check */
1724                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1725                         int wnd_idx = 0;
1726                         int frag_idx = 0;
1727                         u32 wnd_sum = 0;
1728
1729                         /* Headers length */
1730                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1731                                 tcp_hdrlen(skb);
1732
1733                         /* Amount of data (w/o headers) on linear part of SKB*/
1734                         first_bd_sz = skb_headlen(skb) - hlen;
1735
1736                         wnd_sum  = first_bd_sz;
1737
1738                         /* Calculate the first sum - it's special */
1739                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1740                                 wnd_sum +=
1741                                         skb_shinfo(skb)->frags[frag_idx].size;
1742
1743                         /* If there was data on linear skb data - check it */
1744                         if (first_bd_sz > 0) {
1745                                 if (unlikely(wnd_sum < lso_mss)) {
1746                                         to_copy = 1;
1747                                         goto exit_lbl;
1748                                 }
1749
1750                                 wnd_sum -= first_bd_sz;
1751                         }
1752
1753                         /* Others are easier: run through the frag list and
1754                            check all windows */
1755                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1756                                 wnd_sum +=
1757                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1758
1759                                 if (unlikely(wnd_sum < lso_mss)) {
1760                                         to_copy = 1;
1761                                         break;
1762                                 }
1763                                 wnd_sum -=
1764                                         skb_shinfo(skb)->frags[wnd_idx].size;
1765                         }
1766                 } else {
1767                         /* in non-LSO too fragmented packet should always
1768                            be linearized */
1769                         to_copy = 1;
1770                 }
1771         }
1772
1773 exit_lbl:
1774         if (unlikely(to_copy))
1775                 DP(NETIF_MSG_TX_QUEUED,
1776                    "Linearization IS REQUIRED for %s packet. "
1777                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1778                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1779                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1780
1781         return to_copy;
1782 }
1783 #endif
1784
1785 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb,
1786                                      struct eth_tx_parse_bd_e2 *pbd,
1787                                      u32 xmit_type)
1788 {
1789         pbd->parsing_data |= cpu_to_le16(skb_shinfo(skb)->gso_size) <<
1790                 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT;
1791         if ((xmit_type & XMIT_GSO_V6) &&
1792             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
1793                 pbd->parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
1794 }
1795
1796 /**
1797  * Update PBD in GSO case.
1798  *
1799  * @param skb
1800  * @param tx_start_bd
1801  * @param pbd
1802  * @param xmit_type
1803  */
1804 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
1805                                      struct eth_tx_parse_bd_e1x *pbd,
1806                                      u32 xmit_type)
1807 {
1808         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1809         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1810         pbd->tcp_flags = pbd_tcp_flags(skb);
1811
1812         if (xmit_type & XMIT_GSO_V4) {
1813                 pbd->ip_id = swab16(ip_hdr(skb)->id);
1814                 pbd->tcp_pseudo_csum =
1815                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1816                                                   ip_hdr(skb)->daddr,
1817                                                   0, IPPROTO_TCP, 0));
1818
1819         } else
1820                 pbd->tcp_pseudo_csum =
1821                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1822                                                 &ipv6_hdr(skb)->daddr,
1823                                                 0, IPPROTO_TCP, 0));
1824
1825         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
1826 }
1827
1828 /**
1829  *
1830  * @param skb
1831  * @param tx_start_bd
1832  * @param pbd_e2
1833  * @param xmit_type
1834  *
1835  * @return header len
1836  */
1837 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
1838         struct eth_tx_parse_bd_e2 *pbd,
1839         u32 xmit_type)
1840 {
1841         pbd->parsing_data |= cpu_to_le16(tcp_hdrlen(skb)/4) <<
1842                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT;
1843
1844         pbd->parsing_data |= cpu_to_le16(((unsigned char *)tcp_hdr(skb) -
1845                                           skb->data) / 2) <<
1846                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT;
1847
1848         return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
1849 }
1850
1851 /**
1852  *
1853  * @param skb
1854  * @param tx_start_bd
1855  * @param pbd
1856  * @param xmit_type
1857  *
1858  * @return Header length
1859  */
1860 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
1861         struct eth_tx_parse_bd_e1x *pbd,
1862         u32 xmit_type)
1863 {
1864         u8 hlen = (skb_network_header(skb) - skb->data) / 2;
1865
1866         /* for now NS flag is not used in Linux */
1867         pbd->global_data =
1868                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1869                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
1870
1871         pbd->ip_hlen_w = (skb_transport_header(skb) -
1872                         skb_network_header(skb)) / 2;
1873
1874         hlen += pbd->ip_hlen_w + tcp_hdrlen(skb) / 2;
1875
1876         pbd->total_hlen_w = cpu_to_le16(hlen);
1877         hlen = hlen*2;
1878
1879         if (xmit_type & XMIT_CSUM_TCP) {
1880                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1881
1882         } else {
1883                 s8 fix = SKB_CS_OFF(skb); /* signed! */
1884
1885                 DP(NETIF_MSG_TX_QUEUED,
1886                    "hlen %d  fix %d  csum before fix %x\n",
1887                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
1888
1889                 /* HW bug: fixup the CSUM */
1890                 pbd->tcp_pseudo_csum =
1891                         bnx2x_csum_fix(skb_transport_header(skb),
1892                                        SKB_CS(skb), fix);
1893
1894                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1895                    pbd->tcp_pseudo_csum);
1896         }
1897
1898         return hlen;
1899 }
1900
1901 /* called with netif_tx_lock
1902  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1903  * netif_wake_queue()
1904  */
1905 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1906 {
1907         struct bnx2x *bp = netdev_priv(dev);
1908         struct bnx2x_fastpath *fp;
1909         struct netdev_queue *txq;
1910         struct sw_tx_bd *tx_buf;
1911         struct eth_tx_start_bd *tx_start_bd;
1912         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1913         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
1914         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
1915         u16 pkt_prod, bd_prod;
1916         int nbd, fp_index;
1917         dma_addr_t mapping;
1918         u32 xmit_type = bnx2x_xmit_type(bp, skb);
1919         int i;
1920         u8 hlen = 0;
1921         __le16 pkt_size = 0;
1922         struct ethhdr *eth;
1923         u8 mac_type = UNICAST_ADDRESS;
1924
1925 #ifdef BNX2X_STOP_ON_ERROR
1926         if (unlikely(bp->panic))
1927                 return NETDEV_TX_BUSY;
1928 #endif
1929
1930         fp_index = skb_get_queue_mapping(skb);
1931         txq = netdev_get_tx_queue(dev, fp_index);
1932
1933         fp = &bp->fp[fp_index];
1934
1935         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1936                 fp->eth_q_stats.driver_xoff++;
1937                 netif_tx_stop_queue(txq);
1938                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1939                 return NETDEV_TX_BUSY;
1940         }
1941
1942         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
1943                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
1944            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1945            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1946
1947         eth = (struct ethhdr *)skb->data;
1948
1949         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1950         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1951                 if (is_broadcast_ether_addr(eth->h_dest))
1952                         mac_type = BROADCAST_ADDRESS;
1953                 else
1954                         mac_type = MULTICAST_ADDRESS;
1955         }
1956
1957 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1958         /* First, check if we need to linearize the skb (due to FW
1959            restrictions). No need to check fragmentation if page size > 8K
1960            (there will be no violation to FW restrictions) */
1961         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1962                 /* Statistics of linearization */
1963                 bp->lin_cnt++;
1964                 if (skb_linearize(skb) != 0) {
1965                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1966                            "silently dropping this SKB\n");
1967                         dev_kfree_skb_any(skb);
1968                         return NETDEV_TX_OK;
1969                 }
1970         }
1971 #endif
1972
1973         /*
1974         Please read carefully. First we use one BD which we mark as start,
1975         then we have a parsing info BD (used for TSO or xsum),
1976         and only then we have the rest of the TSO BDs.
1977         (don't forget to mark the last one as last,
1978         and to unmap only AFTER you write to the BD ...)
1979         And above all, all pdb sizes are in words - NOT DWORDS!
1980         */
1981
1982         pkt_prod = fp->tx_pkt_prod++;
1983         bd_prod = TX_BD(fp->tx_bd_prod);
1984
1985         /* get a tx_buf and first BD */
1986         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1987         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1988
1989         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1990         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
1991                  mac_type);
1992
1993         /* header nbd */
1994         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
1995
1996         /* remember the first BD of the packet */
1997         tx_buf->first_bd = fp->tx_bd_prod;
1998         tx_buf->skb = skb;
1999         tx_buf->flags = 0;
2000
2001         DP(NETIF_MSG_TX_QUEUED,
2002            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2003            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2004
2005         if (vlan_tx_tag_present(skb)) {
2006                 tx_start_bd->vlan_or_ethertype =
2007                     cpu_to_le16(vlan_tx_tag_get(skb));
2008                 tx_start_bd->bd_flags.as_bitfield |=
2009                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2010         } else
2011                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2012
2013         /* turn on parsing and get a BD */
2014         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2015
2016         if (xmit_type & XMIT_CSUM) {
2017                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2018
2019                 if (xmit_type & XMIT_CSUM_V4)
2020                         tx_start_bd->bd_flags.as_bitfield |=
2021                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2022                 else
2023                         tx_start_bd->bd_flags.as_bitfield |=
2024                                                 ETH_TX_BD_FLAGS_IPV6;
2025
2026                 if (!(xmit_type & XMIT_CSUM_TCP))
2027                         tx_start_bd->bd_flags.as_bitfield |=
2028                                                 ETH_TX_BD_FLAGS_IS_UDP;
2029         }
2030
2031         if (CHIP_IS_E2(bp)) {
2032                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2033                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2034                 /* Set PBD in checksum offload case */
2035                 if (xmit_type & XMIT_CSUM)
2036                         hlen = bnx2x_set_pbd_csum_e2(bp,
2037                                                      skb, pbd_e2, xmit_type);
2038         } else {
2039                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2040                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2041                 /* Set PBD in checksum offload case */
2042                 if (xmit_type & XMIT_CSUM)
2043                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2044
2045         }
2046
2047         /* Map skb linear data for DMA */
2048         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2049                                  skb_headlen(skb), DMA_TO_DEVICE);
2050
2051         /* Setup the data pointer of the first BD of the packet */
2052         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2053         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2054         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2055         tx_start_bd->nbd = cpu_to_le16(nbd);
2056         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2057         pkt_size = tx_start_bd->nbytes;
2058
2059         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2060            "  nbytes %d  flags %x  vlan %x\n",
2061            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2062            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2063            tx_start_bd->bd_flags.as_bitfield,
2064            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2065
2066         if (xmit_type & XMIT_GSO) {
2067
2068                 DP(NETIF_MSG_TX_QUEUED,
2069                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2070                    skb->len, hlen, skb_headlen(skb),
2071                    skb_shinfo(skb)->gso_size);
2072
2073                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2074
2075                 if (unlikely(skb_headlen(skb) > hlen))
2076                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2077                                                  hlen, bd_prod, ++nbd);
2078                 if (CHIP_IS_E2(bp))
2079                         bnx2x_set_pbd_gso_e2(skb, pbd_e2, xmit_type);
2080                 else
2081                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2082         }
2083         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2084
2085         /* Handle fragmented skb */
2086         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2087                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2088
2089                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2090                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2091                 if (total_pkt_bd == NULL)
2092                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2093
2094                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2095                                        frag->page_offset,
2096                                        frag->size, DMA_TO_DEVICE);
2097
2098                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2099                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2100                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2101                 le16_add_cpu(&pkt_size, frag->size);
2102
2103                 DP(NETIF_MSG_TX_QUEUED,
2104                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2105                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2106                    le16_to_cpu(tx_data_bd->nbytes));
2107         }
2108
2109         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2110
2111         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2112
2113         /* now send a tx doorbell, counting the next BD
2114          * if the packet contains or ends with it
2115          */
2116         if (TX_BD_POFF(bd_prod) < nbd)
2117                 nbd++;
2118
2119         if (total_pkt_bd != NULL)
2120                 total_pkt_bd->total_pkt_bytes = pkt_size;
2121
2122         if (pbd_e1x)
2123                 DP(NETIF_MSG_TX_QUEUED,
2124                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2125                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2126                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2127                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2128                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2129                     le16_to_cpu(pbd_e1x->total_hlen_w));
2130         if (pbd_e2)
2131                 DP(NETIF_MSG_TX_QUEUED,
2132                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2133                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2134                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2135                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2136                    pbd_e2->parsing_data);
2137         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2138
2139         /*
2140          * Make sure that the BD data is updated before updating the producer
2141          * since FW might read the BD right after the producer is updated.
2142          * This is only applicable for weak-ordered memory model archs such
2143          * as IA-64. The following barrier is also mandatory since FW will
2144          * assumes packets must have BDs.
2145          */
2146         wmb();
2147
2148         fp->tx_db.data.prod += nbd;
2149         barrier();
2150
2151         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2152
2153         mmiowb();
2154
2155         fp->tx_bd_prod += nbd;
2156
2157         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2158                 netif_tx_stop_queue(txq);
2159
2160                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2161                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2162                  * fp->bd_tx_cons */
2163                 smp_mb();
2164
2165                 fp->eth_q_stats.driver_xoff++;
2166                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2167                         netif_tx_wake_queue(txq);
2168         }
2169         fp->tx_pkt++;
2170
2171         return NETDEV_TX_OK;
2172 }
2173
2174 /* called with rtnl_lock */
2175 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2176 {
2177         struct sockaddr *addr = p;
2178         struct bnx2x *bp = netdev_priv(dev);
2179
2180         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2181                 return -EINVAL;
2182
2183         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2184         if (netif_running(dev))
2185                 bnx2x_set_eth_mac(bp, 1);
2186
2187         return 0;
2188 }
2189
2190
2191 static int bnx2x_setup_irqs(struct bnx2x *bp)
2192 {
2193         int rc = 0;
2194         if (bp->flags & USING_MSIX_FLAG) {
2195                 rc = bnx2x_req_msix_irqs(bp);
2196                 if (rc)
2197                         return rc;
2198         } else {
2199                 bnx2x_ack_int(bp);
2200                 rc = bnx2x_req_irq(bp);
2201                 if (rc) {
2202                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2203                         return rc;
2204                 }
2205                 if (bp->flags & USING_MSI_FLAG) {
2206                         bp->dev->irq = bp->pdev->irq;
2207                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2208                                bp->pdev->irq);
2209                 }
2210         }
2211
2212         return 0;
2213 }
2214
2215 void bnx2x_free_mem_bp(struct bnx2x *bp)
2216 {
2217         kfree(bp->fp);
2218         kfree(bp->msix_table);
2219         kfree(bp->ilt);
2220 }
2221
2222 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2223 {
2224         struct bnx2x_fastpath *fp;
2225         struct msix_entry *tbl;
2226         struct bnx2x_ilt *ilt;
2227
2228         /* fp array */
2229         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2230         if (!fp)
2231                 goto alloc_err;
2232         bp->fp = fp;
2233
2234         /* msix table */
2235         tbl = kzalloc((bp->l2_cid_count + 1) * sizeof(*tbl),
2236                                   GFP_KERNEL);
2237         if (!tbl)
2238                 goto alloc_err;
2239         bp->msix_table = tbl;
2240
2241         /* ilt */
2242         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2243         if (!ilt)
2244                 goto alloc_err;
2245         bp->ilt = ilt;
2246
2247         return 0;
2248 alloc_err:
2249         bnx2x_free_mem_bp(bp);
2250         return -ENOMEM;
2251
2252 }
2253
2254 /* called with rtnl_lock */
2255 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2256 {
2257         struct bnx2x *bp = netdev_priv(dev);
2258         int rc = 0;
2259
2260         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2261                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2262                 return -EAGAIN;
2263         }
2264
2265         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2266             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2267                 return -EINVAL;
2268
2269         /* This does not race with packet allocation
2270          * because the actual alloc size is
2271          * only updated as part of load
2272          */
2273         dev->mtu = new_mtu;
2274
2275         if (netif_running(dev)) {
2276                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2277                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2278         }
2279
2280         return rc;
2281 }
2282
2283 void bnx2x_tx_timeout(struct net_device *dev)
2284 {
2285         struct bnx2x *bp = netdev_priv(dev);
2286
2287 #ifdef BNX2X_STOP_ON_ERROR
2288         if (!bp->panic)
2289                 bnx2x_panic();
2290 #endif
2291         /* This allows the netif to be shutdown gracefully before resetting */
2292         schedule_delayed_work(&bp->reset_task, 0);
2293 }
2294
2295 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2296 {
2297         struct net_device *dev = pci_get_drvdata(pdev);
2298         struct bnx2x *bp;
2299
2300         if (!dev) {
2301                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2302                 return -ENODEV;
2303         }
2304         bp = netdev_priv(dev);
2305
2306         rtnl_lock();
2307
2308         pci_save_state(pdev);
2309
2310         if (!netif_running(dev)) {
2311                 rtnl_unlock();
2312                 return 0;
2313         }
2314
2315         netif_device_detach(dev);
2316
2317         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2318
2319         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2320
2321         rtnl_unlock();
2322
2323         return 0;
2324 }
2325
2326 int bnx2x_resume(struct pci_dev *pdev)
2327 {
2328         struct net_device *dev = pci_get_drvdata(pdev);
2329         struct bnx2x *bp;
2330         int rc;
2331
2332         if (!dev) {
2333                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2334                 return -ENODEV;
2335         }
2336         bp = netdev_priv(dev);
2337
2338         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2339                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2340                 return -EAGAIN;
2341         }
2342
2343         rtnl_lock();
2344
2345         pci_restore_state(pdev);
2346
2347         if (!netif_running(dev)) {
2348                 rtnl_unlock();
2349                 return 0;
2350         }
2351
2352         bnx2x_set_power_state(bp, PCI_D0);
2353         netif_device_attach(dev);
2354
2355         /* Since the chip was reset, clear the FW sequence number */
2356         bp->fw_seq = 0;
2357         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2358
2359         rtnl_unlock();
2360
2361         return rc;
2362 }