drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18
  19 #include <linux/etherdevice.h>
  20 #include <linux/ip.h>
  21 #include <linux/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #ifdef BCM_VLAN
  27 #include <linux/if_vlan.h>
  28 #endif
  29
  30 static int bnx2x_poll(struct napi_struct *napi, int budget);
  31
  32 /* free skb in the packet ring at pos idx
  33  * return idx of last bd freed
  34  */
  35 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  36                              u16 idx)
  37 {
  38         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  39         struct eth_tx_start_bd *tx_start_bd;
  40         struct eth_tx_bd *tx_data_bd;
  41         struct sk_buff *skb = tx_buf->skb;
  42         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  43         int nbd;
  44
  45         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  46         prefetch(&skb->end);
  47
  48         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  49            idx, tx_buf, skb);
  50
  51         /* unmap first bd */
  52         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  53         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  54         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  55                          BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
  56
  57         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  58 #ifdef BNX2X_STOP_ON_ERROR
  59         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  60                 BNX2X_ERR("BAD nbd!\n");
  61                 bnx2x_panic();
  62         }
  63 #endif
  64         new_cons = nbd + tx_buf->first_bd;
  65
  66         /* Get the next bd */
  67         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  68
  69         /* Skip a parse bd... */
  70         --nbd;
  71         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  72
  73         /* ...and the TSO split header bd since they have no mapping */
  74         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  75                 --nbd;
  76                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  77         }
  78
  79         /* now free frags */
  80         while (nbd > 0) {
  81
  82                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  83                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  84                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  85                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  86                 if (--nbd)
  87                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  88         }
  89
  90         /* release skb */
  91         WARN_ON(!skb);
  92         dev_kfree_skb(skb);
  93         tx_buf->first_bd = 0;
  94         tx_buf->skb = NULL;
  95
  96         return new_cons;
  97 }
  98
  99 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
 100 {
 101         struct bnx2x *bp = fp->bp;
 102         struct netdev_queue *txq;
 103         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 104
 105 #ifdef BNX2X_STOP_ON_ERROR
 106         if (unlikely(bp->panic))
 107                 return -1;
 108 #endif
 109
 110         txq = netdev_get_tx_queue(bp->dev, fp->index);
 111         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 112         sw_cons = fp->tx_pkt_cons;
 113
 114         while (sw_cons != hw_cons) {
 115                 u16 pkt_cons;
 116
 117                 pkt_cons = TX_BD(sw_cons);
 118
 119                 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
 120
 121                 DP(NETIF_MSG_TX_DONE, "hw_cons %u  sw_cons %u  pkt_cons %u\n",
 122                    hw_cons, sw_cons, pkt_cons);
 123
 124 /*              if (NEXT_TX_IDX(sw_cons) != hw_cons) {
 125                         rmb();
 126                         prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
 127                 }
 128 */
 129                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 130                 sw_cons++;
 131         }
 132
 133         fp->tx_pkt_cons = sw_cons;
 134         fp->tx_bd_cons = bd_cons;
 135
 136         /* Need to make the tx_bd_cons update visible to start_xmit()
 137          * before checking for netif_tx_queue_stopped().  Without the
 138          * memory barrier, there is a small possibility that
 139          * start_xmit() will miss it and cause the queue to be stopped
 140          * forever.
 141          */
 142         smp_mb();
 143
 144         /* TBD need a thresh? */
 145         if (unlikely(netif_tx_queue_stopped(txq))) {
 146                 /* Taking tx_lock() is needed to prevent reenabling the queue
 147                  * while it's empty. This could have happen if rx_action() gets
 148                  * suspended in bnx2x_tx_int() after the condition before
 149                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 150                  *
 151                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 152                  * sends some packets consuming the whole queue again->
 153                  * stops the queue
 154                  */
 155
 156                 __netif_tx_lock(txq, smp_processor_id());
 157
 158                 if ((netif_tx_queue_stopped(txq)) &&
 159                     (bp->state == BNX2X_STATE_OPEN) &&
 160                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 161                         netif_tx_wake_queue(txq);
 162
 163                 __netif_tx_unlock(txq);
 164         }
 165         return 0;
 166 }
 167
 168 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 169                                              u16 idx)
 170 {
 171         u16 last_max = fp->last_max_sge;
 172
 173         if (SUB_S16(idx, last_max) > 0)
 174                 fp->last_max_sge = idx;
 175 }
 176
 177 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 178                                   struct eth_fast_path_rx_cqe *fp_cqe)
 179 {
 180         struct bnx2x *bp = fp->bp;
 181         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 182                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 183                       SGE_PAGE_SHIFT;
 184         u16 last_max, last_elem, first_elem;
 185         u16 delta = 0;
 186         u16 i;
 187
 188         if (!sge_len)
 189                 return;
 190
 191         /* First mark all used pages */
 192         for (i = 0; i < sge_len; i++)
 193                 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
 194
 195         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 196            sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
 197
 198         /* Here we assume that the last SGE index is the biggest */
 199         prefetch((void *)(fp->sge_mask));
 200         bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
 201
 202         last_max = RX_SGE(fp->last_max_sge);
 203         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 204         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 205
 206         /* If ring is not full */
 207         if (last_elem + 1 != first_elem)
 208                 last_elem++;
 209
 210         /* Now update the prod */
 211         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 212                 if (likely(fp->sge_mask[i]))
 213                         break;
 214
 215                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 216                 delta += RX_SGE_MASK_ELEM_SZ;
 217         }
 218
 219         if (delta > 0) {
 220                 fp->rx_sge_prod += delta;
 221                 /* clear page-end entries */
 222                 bnx2x_clear_sge_mask_next_elems(fp);
 223         }
 224
 225         DP(NETIF_MSG_RX_STATUS,
 226            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 227            fp->last_max_sge, fp->rx_sge_prod);
 228 }
 229
 230 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 231                             struct sk_buff *skb, u16 cons, u16 prod)
 232 {
 233         struct bnx2x *bp = fp->bp;
 234         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 235         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 236         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 237         dma_addr_t mapping;
 238
 239         /* move empty skb from pool to prod and map it */
 240         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 241         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 242                                  bp->rx_buf_size, DMA_FROM_DEVICE);
 243         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 244
 245         /* move partial skb from cons to pool (don't unmap yet) */
 246         fp->tpa_pool[queue] = *cons_rx_buf;
 247
 248         /* mark bin state as start - print error if current state != stop */
 249         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 250                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 251
 252         fp->tpa_state[queue] = BNX2X_TPA_START;
 253
 254         /* point prod_bd to new skb */
 255         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 256         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 257
 258 #ifdef BNX2X_STOP_ON_ERROR
 259         fp->tpa_queue_used |= (1 << queue);
 260 #ifdef _ASM_GENERIC_INT_L64_H
 261         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 262 #else
 263         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 264 #endif
 265            fp->tpa_queue_used);
 266 #endif
 267 }
 268
 269 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 270                                struct sk_buff *skb,
 271                                struct eth_fast_path_rx_cqe *fp_cqe,
 272                                u16 cqe_idx)
 273 {
 274         struct sw_rx_page *rx_pg, old_rx_pg;
 275         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 276         u32 i, frag_len, frag_size, pages;
 277         int err;
 278         int j;
 279
 280         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 281         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 282
 283         /* This is needed in order to enable forwarding support */
 284         if (frag_size)
 285                 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
 286                                                max(frag_size, (u32)len_on_bd));
 287
 288 #ifdef BNX2X_STOP_ON_ERROR
 289         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 290                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 291                           pages, cqe_idx);
 292                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 293                           fp_cqe->pkt_len, len_on_bd);
 294                 bnx2x_panic();
 295                 return -EINVAL;
 296         }
 297 #endif
 298
 299         /* Run through the SGL and compose the fragmented skb */
 300         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 301                 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
 302
 303                 /* FW gives the indices of the SGE as if the ring is an array
 304                    (meaning that "next" element will consume 2 indices) */
 305                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 306                 rx_pg = &fp->rx_page_ring[sge_idx];
 307                 old_rx_pg = *rx_pg;
 308
 309                 /* If we fail to allocate a substitute page, we simply stop
 310                    where we are and drop the whole packet */
 311                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 312                 if (unlikely(err)) {
 313                         fp->eth_q_stats.rx_skb_alloc_failed++;
 314                         return err;
 315                 }
 316
 317                 /* Unmap the page as we r going to pass it to the stack */
 318                 dma_unmap_page(&bp->pdev->dev,
 319                                dma_unmap_addr(&old_rx_pg, mapping),
 320                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 321
 322                 /* Add one frag and update the appropriate fields in the skb */
 323                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 324
 325                 skb->data_len += frag_len;
 326                 skb->truesize += frag_len;
 327                 skb->len += frag_len;
 328
 329                 frag_size -= frag_len;
 330         }
 331
 332         return 0;
 333 }
 334
 335 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 336                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 337                            u16 cqe_idx)
 338 {
 339         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 340         struct sk_buff *skb = rx_buf->skb;
 341         /* alloc new skb */
 342         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 343
 344         /* Unmap skb in the pool anyway, as we are going to change
 345            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 346            fails. */
 347         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 348                          bp->rx_buf_size, DMA_FROM_DEVICE);
 349
 350         if (likely(new_skb)) {
 351                 /* fix ip xsum and give it to the stack */
 352                 /* (no need to map the new skb) */
 353 #ifdef BCM_VLAN
 354                 int is_vlan_cqe =
 355                         (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 356                          PARSING_FLAGS_VLAN);
 357                 int is_not_hwaccel_vlan_cqe =
 358                         (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
 359 #endif
 360
 361                 prefetch(skb);
 362                 prefetch(((char *)(skb)) + 128);
 363
 364 #ifdef BNX2X_STOP_ON_ERROR
 365                 if (pad + len > bp->rx_buf_size) {
 366                         BNX2X_ERR("skb_put is about to fail...  "
 367                                   "pad %d  len %d  rx_buf_size %d\n",
 368                                   pad, len, bp->rx_buf_size);
 369                         bnx2x_panic();
 370                         return;
 371                 }
 372 #endif
 373
 374                 skb_reserve(skb, pad);
 375                 skb_put(skb, len);
 376
 377                 skb->protocol = eth_type_trans(skb, bp->dev);
 378                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 379
 380                 {
 381                         struct iphdr *iph;
 382
 383                         iph = (struct iphdr *)skb->data;
 384 #ifdef BCM_VLAN
 385                         /* If there is no Rx VLAN offloading -
 386                            take VLAN tag into an account */
 387                         if (unlikely(is_not_hwaccel_vlan_cqe))
 388                                 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
 389 #endif
 390                         iph->check = 0;
 391                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 392                 }
 393
 394                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 395                                          &cqe->fast_path_cqe, cqe_idx)) {
 396 #ifdef BCM_VLAN
 397                         if ((bp->vlgrp != NULL) && is_vlan_cqe &&
 398                             (!is_not_hwaccel_vlan_cqe))
 399                                 vlan_gro_receive(&fp->napi, bp->vlgrp,
 400                                                  le16_to_cpu(cqe->fast_path_cqe.
 401                                                              vlan_tag), skb);
 402                         else
 403 #endif
 404                                 napi_gro_receive(&fp->napi, skb);
 405                 } else {
 406                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 407                            " - dropping packet!\n");
 408                         dev_kfree_skb(skb);
 409                 }
 410
 411
 412                 /* put new skb in bin */
 413                 fp->tpa_pool[queue].skb = new_skb;
 414
 415         } else {
 416                 /* else drop the packet and keep the buffer in the bin */
 417                 DP(NETIF_MSG_RX_STATUS,
 418                    "Failed to allocate new skb - dropping packet!\n");
 419                 fp->eth_q_stats.rx_skb_alloc_failed++;
 420         }
 421
 422         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 423 }
 424
 425 /* Set Toeplitz hash value in the skb using the value from the
 426  * CQE (calculated by HW).
 427  */
 428 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 429                                         struct sk_buff *skb)
 430 {
 431         /* Set Toeplitz hash from CQE */
 432         if ((bp->dev->features & NETIF_F_RXHASH) &&
 433             (cqe->fast_path_cqe.status_flags &
 434              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 435                 skb->rxhash =
 436                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 437 }
 438
 439 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 440 {
 441         struct bnx2x *bp = fp->bp;
 442         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 443         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 444         int rx_pkt = 0;
 445
 446 #ifdef BNX2X_STOP_ON_ERROR
 447         if (unlikely(bp->panic))
 448                 return 0;
 449 #endif
 450
 451         /* CQ "next element" is of the size of the regular element,
 452            that's why it's ok here */
 453         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 454         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 455                 hw_comp_cons++;
 456
 457         bd_cons = fp->rx_bd_cons;
 458         bd_prod = fp->rx_bd_prod;
 459         bd_prod_fw = bd_prod;
 460         sw_comp_cons = fp->rx_comp_cons;
 461         sw_comp_prod = fp->rx_comp_prod;
 462
 463         /* Memory barrier necessary as speculative reads of the rx
 464          * buffer can be ahead of the index in the status block
 465          */
 466         rmb();
 467
 468         DP(NETIF_MSG_RX_STATUS,
 469            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 470            fp->index, hw_comp_cons, sw_comp_cons);
 471
 472         while (sw_comp_cons != hw_comp_cons) {
 473                 struct sw_rx_bd *rx_buf = NULL;
 474                 struct sk_buff *skb;
 475                 union eth_rx_cqe *cqe;
 476                 u8 cqe_fp_flags;
 477                 u16 len, pad;
 478
 479                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 480                 bd_prod = RX_BD(bd_prod);
 481                 bd_cons = RX_BD(bd_cons);
 482
 483                 /* Prefetch the page containing the BD descriptor
 484                    at producer's index. It will be needed when new skb is
 485                    allocated */
 486                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 487                                              (&fp->rx_desc_ring[bd_prod])) -
 488                                   PAGE_SIZE + 1));
 489
 490                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 491                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 492
 493                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 494                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 495                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 496                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 497                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 498                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 499
 500                 /* is this a slowpath msg? */
 501                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 502                         bnx2x_sp_event(fp, cqe);
 503                         goto next_cqe;
 504
 505                 /* this is an rx packet */
 506                 } else {
 507                         rx_buf = &fp->rx_buf_ring[bd_cons];
 508                         skb = rx_buf->skb;
 509                         prefetch(skb);
 510                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 511                         pad = cqe->fast_path_cqe.placement_offset;
 512
 513                         /* If CQE is marked both TPA_START and TPA_END
 514                            it is a non-TPA CQE */
 515                         if ((!fp->disable_tpa) &&
 516                             (TPA_TYPE(cqe_fp_flags) !=
 517                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 518                                 u16 queue = cqe->fast_path_cqe.queue_index;
 519
 520                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 521                                         DP(NETIF_MSG_RX_STATUS,
 522                                            "calling tpa_start on queue %d\n",
 523                                            queue);
 524
 525                                         bnx2x_tpa_start(fp, queue, skb,
 526                                                         bd_cons, bd_prod);
 527
 528                                         /* Set Toeplitz hash for an LRO skb */
 529                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 530
 531                                         goto next_rx;
 532                                 }
 533
 534                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
 535                                         DP(NETIF_MSG_RX_STATUS,
 536                                            "calling tpa_stop on queue %d\n",
 537                                            queue);
 538
 539                                         if (!BNX2X_RX_SUM_FIX(cqe))
 540                                                 BNX2X_ERR("STOP on none TCP "
 541                                                           "data\n");
 542
 543                                         /* This is a size of the linear data
 544                                            on this skb */
 545                                         len = le16_to_cpu(cqe->fast_path_cqe.
 546                                                                 len_on_bd);
 547                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 548                                                     len, cqe, comp_ring_cons);
 549 #ifdef BNX2X_STOP_ON_ERROR
 550                                         if (bp->panic)
 551                                                 return 0;
 552 #endif
 553
 554                                         bnx2x_update_sge_prod(fp,
 555                                                         &cqe->fast_path_cqe);
 556                                         goto next_cqe;
 557                                 }
 558                         }
 559
 560                         dma_sync_single_for_device(&bp->pdev->dev,
 561                                         dma_unmap_addr(rx_buf, mapping),
 562                                                    pad + RX_COPY_THRESH,
 563                                                    DMA_FROM_DEVICE);
 564                         prefetch(((char *)(skb)) + 128);
 565
 566                         /* is this an error packet? */
 567                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 568                                 DP(NETIF_MSG_RX_ERR,
 569                                    "ERROR  flags %x  rx packet %u\n",
 570                                    cqe_fp_flags, sw_comp_cons);
 571                                 fp->eth_q_stats.rx_err_discard_pkt++;
 572                                 goto reuse_rx;
 573                         }
 574
 575                         /* Since we don't have a jumbo ring
 576                          * copy small packets if mtu > 1500
 577                          */
 578                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 579                             (len <= RX_COPY_THRESH)) {
 580                                 struct sk_buff *new_skb;
 581
 582                                 new_skb = netdev_alloc_skb(bp->dev,
 583                                                            len + pad);
 584                                 if (new_skb == NULL) {
 585                                         DP(NETIF_MSG_RX_ERR,
 586                                            "ERROR  packet dropped "
 587                                            "because of alloc failure\n");
 588                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 589                                         goto reuse_rx;
 590                                 }
 591
 592                                 /* aligned copy */
 593                                 skb_copy_from_linear_data_offset(skb, pad,
 594                                                     new_skb->data + pad, len);
 595                                 skb_reserve(new_skb, pad);
 596                                 skb_put(new_skb, len);
 597
 598                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
 599
 600                                 skb = new_skb;
 601
 602                         } else
 603                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 604                                 dma_unmap_single(&bp->pdev->dev,
 605                                         dma_unmap_addr(rx_buf, mapping),
 606                                                  bp->rx_buf_size,
 607                                                  DMA_FROM_DEVICE);
 608                                 skb_reserve(skb, pad);
 609                                 skb_put(skb, len);
 610
 611                         } else {
 612                                 DP(NETIF_MSG_RX_ERR,
 613                                    "ERROR  packet dropped because "
 614                                    "of alloc failure\n");
 615                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 616 reuse_rx:
 617                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
 618                                 goto next_rx;
 619                         }
 620
 621                         skb->protocol = eth_type_trans(skb, bp->dev);
 622
 623                         /* Set Toeplitz hash for a none-LRO skb */
 624                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 625
 626                         skb_checksum_none_assert(skb);
 627                         if (bp->rx_csum) {
 628                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 629                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 630                                 else
 631                                         fp->eth_q_stats.hw_csum_err++;
 632                         }
 633                 }
 634
 635                 skb_record_rx_queue(skb, fp->index);
 636
 637 #ifdef BCM_VLAN
 638                 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
 639                     (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 640                      PARSING_FLAGS_VLAN))
 641                         vlan_gro_receive(&fp->napi, bp->vlgrp,
 642                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
 643                 else
 644 #endif
 645                         napi_gro_receive(&fp->napi, skb);
 646
 647
 648 next_rx:
 649                 rx_buf->skb = NULL;
 650
 651                 bd_cons = NEXT_RX_IDX(bd_cons);
 652                 bd_prod = NEXT_RX_IDX(bd_prod);
 653                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 654                 rx_pkt++;
 655 next_cqe:
 656                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 657                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 658
 659                 if (rx_pkt == budget)
 660                         break;
 661         } /* while */
 662
 663         fp->rx_bd_cons = bd_cons;
 664         fp->rx_bd_prod = bd_prod_fw;
 665         fp->rx_comp_cons = sw_comp_cons;
 666         fp->rx_comp_prod = sw_comp_prod;
 667
 668         /* Update producers */
 669         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 670                              fp->rx_sge_prod);
 671
 672         fp->rx_pkt += rx_pkt;
 673         fp->rx_calls++;
 674
 675         return rx_pkt;
 676 }
 677
 678 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 679 {
 680         struct bnx2x_fastpath *fp = fp_cookie;
 681         struct bnx2x *bp = fp->bp;
 682
 683         /* Return here if interrupt is disabled */
 684         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 685                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 686                 return IRQ_HANDLED;
 687         }
 688
 689         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
 690            fp->index, fp->sb_id);
 691         bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 692
 693 #ifdef BNX2X_STOP_ON_ERROR
 694         if (unlikely(bp->panic))
 695                 return IRQ_HANDLED;
 696 #endif
 697
 698         /* Handle Rx and Tx according to MSI-X vector */
 699         prefetch(fp->rx_cons_sb);
 700         prefetch(fp->tx_cons_sb);
 701         prefetch(&fp->status_blk->u_status_block.status_block_index);
 702         prefetch(&fp->status_blk->c_status_block.status_block_index);
 703         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 704
 705         return IRQ_HANDLED;
 706 }
 707
 708
 709 /* HW Lock for shared dual port PHYs */
 710 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 711 {
 712         mutex_lock(&bp->port.phy_mutex);
 713
 714         if (bp->port.need_hw_lock)
 715                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 716 }
 717
 718 void bnx2x_release_phy_lock(struct bnx2x *bp)
 719 {
 720         if (bp->port.need_hw_lock)
 721                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 722
 723         mutex_unlock(&bp->port.phy_mutex);
 724 }
 725
 726 void bnx2x_link_report(struct bnx2x *bp)
 727 {
 728         if (bp->flags & MF_FUNC_DIS) {
 729                 netif_carrier_off(bp->dev);
 730                 netdev_err(bp->dev, "NIC Link is Down\n");
 731                 return;
 732         }
 733
 734         if (bp->link_vars.link_up) {
 735                 u16 line_speed;
 736
 737                 if (bp->state == BNX2X_STATE_OPEN)
 738                         netif_carrier_on(bp->dev);
 739                 netdev_info(bp->dev, "NIC Link is Up, ");
 740
 741                 line_speed = bp->link_vars.line_speed;
 742                 if (IS_E1HMF(bp)) {
 743                         u16 vn_max_rate;
 744
 745                         vn_max_rate =
 746                                 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
 747                                  FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
 748                         if (vn_max_rate < line_speed)
 749                                 line_speed = vn_max_rate;
 750                 }
 751                 pr_cont("%d Mbps ", line_speed);
 752
 753                 if (bp->link_vars.duplex == DUPLEX_FULL)
 754                         pr_cont("full duplex");
 755                 else
 756                         pr_cont("half duplex");
 757
 758                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 759                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 760                                 pr_cont(", receive ");
 761                                 if (bp->link_vars.flow_ctrl &
 762                                     BNX2X_FLOW_CTRL_TX)
 763                                         pr_cont("& transmit ");
 764                         } else {
 765                                 pr_cont(", transmit ");
 766                         }
 767                         pr_cont("flow control ON");
 768                 }
 769                 pr_cont("\n");
 770
 771         } else { /* link_down */
 772                 netif_carrier_off(bp->dev);
 773                 netdev_err(bp->dev, "NIC Link is Down\n");
 774         }
 775 }
 776
 777 void bnx2x_init_rx_rings(struct bnx2x *bp)
 778 {
 779         int func = BP_FUNC(bp);
 780         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 781                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 782         u16 ring_prod, cqe_ring_prod;
 783         int i, j;
 784         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
 785                                               MAX_RX_AVAIL/bp->num_queues;
 786
 787         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
 788
 789         bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
 790         DP(NETIF_MSG_IFUP,
 791            "mtu %d  rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
 792
 793         if (bp->flags & TPA_ENABLE_FLAG) {
 794
 795                 for_each_queue(bp, j) {
 796                         struct bnx2x_fastpath *fp = &bp->fp[j];
 797
 798                         for (i = 0; i < max_agg_queues; i++) {
 799                                 fp->tpa_pool[i].skb =
 800                                    netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 801                                 if (!fp->tpa_pool[i].skb) {
 802                                         BNX2X_ERR("Failed to allocate TPA "
 803                                                   "skb pool for queue[%d] - "
 804                                                   "disabling TPA on this "
 805                                                   "queue!\n", j);
 806                                         bnx2x_free_tpa_pool(bp, fp, i);
 807                                         fp->disable_tpa = 1;
 808                                         break;
 809                                 }
 810                                 dma_unmap_addr_set((struct sw_rx_bd *)
 811                                                         &bp->fp->tpa_pool[i],
 812                                                    mapping, 0);
 813                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 814                         }
 815                 }
 816         }
 817
 818         for_each_queue(bp, j) {
 819                 struct bnx2x_fastpath *fp = &bp->fp[j];
 820
 821                 fp->rx_bd_cons = 0;
 822                 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
 823                 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
 824
 825                 /* "next page" elements initialization */
 826                 /* SGE ring */
 827                 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
 828                         struct eth_rx_sge *sge;
 829
 830                         sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
 831                         sge->addr_hi =
 832                                 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
 833                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
 834                         sge->addr_lo =
 835                                 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
 836                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
 837                 }
 838
 839                 bnx2x_init_sge_ring_bit_mask(fp);
 840
 841                 /* RX BD ring */
 842                 for (i = 1; i <= NUM_RX_RINGS; i++) {
 843                         struct eth_rx_bd *rx_bd;
 844
 845                         rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
 846                         rx_bd->addr_hi =
 847                                 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
 848                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
 849                         rx_bd->addr_lo =
 850                                 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
 851                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
 852                 }
 853
 854                 /* CQ ring */
 855                 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
 856                         struct eth_rx_cqe_next_page *nextpg;
 857
 858                         nextpg = (struct eth_rx_cqe_next_page *)
 859                                 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
 860                         nextpg->addr_hi =
 861                                 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
 862                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
 863                         nextpg->addr_lo =
 864                                 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
 865                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
 866                 }
 867
 868                 /* Allocate SGEs and initialize the ring elements */
 869                 for (i = 0, ring_prod = 0;
 870                      i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 871
 872                         if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 873                                 BNX2X_ERR("was only able to allocate "
 874                                           "%d rx sges\n", i);
 875                                 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
 876                                 /* Cleanup already allocated elements */
 877                                 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
 878                                 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
 879                                 fp->disable_tpa = 1;
 880                                 ring_prod = 0;
 881                                 break;
 882                         }
 883                         ring_prod = NEXT_SGE_IDX(ring_prod);
 884                 }
 885                 fp->rx_sge_prod = ring_prod;
 886
 887                 /* Allocate BDs and initialize BD ring */
 888                 fp->rx_comp_cons = 0;
 889                 cqe_ring_prod = ring_prod = 0;
 890                 for (i = 0; i < rx_ring_size; i++) {
 891                         if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 892                                 BNX2X_ERR("was only able to allocate "
 893                                           "%d rx skbs on queue[%d]\n", i, j);
 894                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 895                                 break;
 896                         }
 897                         ring_prod = NEXT_RX_IDX(ring_prod);
 898                         cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 899                         WARN_ON(ring_prod <= i);
 900                 }
 901
 902                 fp->rx_bd_prod = ring_prod;
 903                 /* must not have more available CQEs than BDs */
 904                 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 905                                          cqe_ring_prod);
 906                 fp->rx_pkt = fp->rx_calls = 0;
 907
 908                 /* Warning!
 909                  * this will generate an interrupt (to the TSTORM)
 910                  * must only be done after chip is initialized
 911                  */
 912                 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
 913                                      fp->rx_sge_prod);
 914                 if (j != 0)
 915                         continue;
 916
 917                 REG_WR(bp, BAR_USTRORM_INTMEM +
 918                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 919                        U64_LO(fp->rx_comp_mapping));
 920                 REG_WR(bp, BAR_USTRORM_INTMEM +
 921                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 922                        U64_HI(fp->rx_comp_mapping));
 923         }
 924 }
 925 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 926 {
 927         int i;
 928
 929         for_each_queue(bp, i) {
 930                 struct bnx2x_fastpath *fp = &bp->fp[i];
 931
 932                 u16 bd_cons = fp->tx_bd_cons;
 933                 u16 sw_prod = fp->tx_pkt_prod;
 934                 u16 sw_cons = fp->tx_pkt_cons;
 935
 936                 while (sw_cons != sw_prod) {
 937                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 938                         sw_cons++;
 939                 }
 940         }
 941 }
 942
 943 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 944 {
 945         int i, j;
 946
 947         for_each_queue(bp, j) {
 948                 struct bnx2x_fastpath *fp = &bp->fp[j];
 949
 950                 for (i = 0; i < NUM_RX_BD; i++) {
 951                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 952                         struct sk_buff *skb = rx_buf->skb;
 953
 954                         if (skb == NULL)
 955                                 continue;
 956
 957                         dma_unmap_single(&bp->pdev->dev,
 958                                          dma_unmap_addr(rx_buf, mapping),
 959                                          bp->rx_buf_size, DMA_FROM_DEVICE);
 960
 961                         rx_buf->skb = NULL;
 962                         dev_kfree_skb(skb);
 963                 }
 964                 if (!fp->disable_tpa)
 965                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 966                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 967                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 968         }
 969 }
 970
 971 void bnx2x_free_skbs(struct bnx2x *bp)
 972 {
 973         bnx2x_free_tx_skbs(bp);
 974         bnx2x_free_rx_skbs(bp);
 975 }
 976
 977 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
 978 {
 979         int i, offset = 1;
 980
 981         free_irq(bp->msix_table[0].vector, bp->dev);
 982         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
 983            bp->msix_table[0].vector);
 984
 985 #ifdef BCM_CNIC
 986         offset++;
 987 #endif
 988         for_each_queue(bp, i) {
 989                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
 990                    "state %x\n", i, bp->msix_table[i + offset].vector,
 991                    bnx2x_fp(bp, i, state));
 992
 993                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
 994         }
 995 }
 996
 997 void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
 998 {
 999         if (bp->flags & USING_MSIX_FLAG) {
1000                 if (!disable_only)
1001                         bnx2x_free_msix_irqs(bp);
1002                 pci_disable_msix(bp->pdev);
1003                 bp->flags &= ~USING_MSIX_FLAG;
1004
1005         } else if (bp->flags & USING_MSI_FLAG) {
1006                 if (!disable_only)
1007                         free_irq(bp->pdev->irq, bp->dev);
1008                 pci_disable_msi(bp->pdev);
1009                 bp->flags &= ~USING_MSI_FLAG;
1010
1011         } else if (!disable_only)
1012                 free_irq(bp->pdev->irq, bp->dev);
1013 }
1014
1015 static int bnx2x_enable_msix(struct bnx2x *bp)
1016 {
1017         int i, rc, offset = 1;
1018         int igu_vec = 0;
1019
1020         bp->msix_table[0].entry = igu_vec;
1021         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1022
1023 #ifdef BCM_CNIC
1024         igu_vec = BP_L_ID(bp) + offset;
1025         bp->msix_table[1].entry = igu_vec;
1026         DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1027         offset++;
1028 #endif
1029         for_each_queue(bp, i) {
1030                 igu_vec = BP_L_ID(bp) + offset + i;
1031                 bp->msix_table[i + offset].entry = igu_vec;
1032                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1033                    "(fastpath #%u)\n", i + offset, igu_vec, i);
1034         }
1035
1036         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1037                              BNX2X_NUM_QUEUES(bp) + offset);
1038
1039         /*
1040          * reconfigure number of tx/rx queues according to available
1041          * MSI-X vectors
1042          */
1043         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1044                 /* vectors available for FP */
1045                 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1046
1047                 DP(NETIF_MSG_IFUP,
1048                    "Trying to use less MSI-X vectors: %d\n", rc);
1049
1050                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1051
1052                 if (rc) {
1053                         DP(NETIF_MSG_IFUP,
1054                            "MSI-X is not attainable  rc %d\n", rc);
1055                         return rc;
1056                 }
1057
1058                 bp->num_queues = min(bp->num_queues, fp_vec);
1059
1060                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1061                                   bp->num_queues);
1062         } else if (rc) {
1063                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1064                 return rc;
1065         }
1066
1067         bp->flags |= USING_MSIX_FLAG;
1068
1069         return 0;
1070 }
1071
1072 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1073 {
1074         int i, rc, offset = 1;
1075
1076         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1077                          bp->dev->name, bp->dev);
1078         if (rc) {
1079                 BNX2X_ERR("request sp irq failed\n");
1080                 return -EBUSY;
1081         }
1082
1083 #ifdef BCM_CNIC
1084         offset++;
1085 #endif
1086         for_each_queue(bp, i) {
1087                 struct bnx2x_fastpath *fp = &bp->fp[i];
1088                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1089                          bp->dev->name, i);
1090
1091                 rc = request_irq(bp->msix_table[i + offset].vector,
1092                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1093                 if (rc) {
1094                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1095                         bnx2x_free_msix_irqs(bp);
1096                         return -EBUSY;
1097                 }
1098
1099                 fp->state = BNX2X_FP_STATE_IRQ;
1100         }
1101
1102         i = BNX2X_NUM_QUEUES(bp);
1103         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1104                " ... fp[%d] %d\n",
1105                bp->msix_table[0].vector,
1106                0, bp->msix_table[offset].vector,
1107                i - 1, bp->msix_table[offset + i - 1].vector);
1108
1109         return 0;
1110 }
1111
1112 static int bnx2x_enable_msi(struct bnx2x *bp)
1113 {
1114         int rc;
1115
1116         rc = pci_enable_msi(bp->pdev);
1117         if (rc) {
1118                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1119                 return -1;
1120         }
1121         bp->flags |= USING_MSI_FLAG;
1122
1123         return 0;
1124 }
1125
1126 static int bnx2x_req_irq(struct bnx2x *bp)
1127 {
1128         unsigned long flags;
1129         int rc;
1130
1131         if (bp->flags & USING_MSI_FLAG)
1132                 flags = 0;
1133         else
1134                 flags = IRQF_SHARED;
1135
1136         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1137                          bp->dev->name, bp->dev);
1138         if (!rc)
1139                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1140
1141         return rc;
1142 }
1143
1144 static void bnx2x_napi_enable(struct bnx2x *bp)
1145 {
1146         int i;
1147
1148         for_each_queue(bp, i)
1149                 napi_enable(&bnx2x_fp(bp, i, napi));
1150 }
1151
1152 static void bnx2x_napi_disable(struct bnx2x *bp)
1153 {
1154         int i;
1155
1156         for_each_queue(bp, i)
1157                 napi_disable(&bnx2x_fp(bp, i, napi));
1158 }
1159
1160 void bnx2x_netif_start(struct bnx2x *bp)
1161 {
1162         int intr_sem;
1163
1164         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1165         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1166
1167         if (intr_sem) {
1168                 if (netif_running(bp->dev)) {
1169                         bnx2x_napi_enable(bp);
1170                         bnx2x_int_enable(bp);
1171                         if (bp->state == BNX2X_STATE_OPEN)
1172                                 netif_tx_wake_all_queues(bp->dev);
1173                 }
1174         }
1175 }
1176
1177 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1178 {
1179         bnx2x_int_disable_sync(bp, disable_hw);
1180         bnx2x_napi_disable(bp);
1181         netif_tx_disable(bp->dev);
1182 }
1183 static int bnx2x_set_num_queues(struct bnx2x *bp)
1184 {
1185         int rc = 0;
1186
1187         switch (bp->int_mode) {
1188         case INT_MODE_MSI:
1189                 bnx2x_enable_msi(bp);
1190                 /* falling through... */
1191         case INT_MODE_INTx:
1192                 bp->num_queues = 1;
1193                 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1194                 break;
1195         default:
1196                 /* Set number of queues according to bp->multi_mode value */
1197                 bnx2x_set_num_queues_msix(bp);
1198
1199                 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1200                    bp->num_queues);
1201
1202                 /* if we can't use MSI-X we only need one fp,
1203                  * so try to enable MSI-X with the requested number of fp's
1204                  * and fallback to MSI or legacy INTx with one fp
1205                  */
1206                 rc = bnx2x_enable_msix(bp);
1207                 if (rc) {
1208                         /* failed to enable MSI-X */
1209                         bp->num_queues = 1;
1210
1211                         /* Fall to INTx if failed to enable MSI-X due to lack of
1212                          * memory (in bnx2x_set_num_queues()) */
1213                         if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1214                                 bnx2x_enable_msi(bp);
1215                 }
1216
1217                 break;
1218         }
1219         netif_set_real_num_tx_queues(bp->dev, bp->num_queues);
1220         return netif_set_real_num_rx_queues(bp->dev, bp->num_queues);
1221 }
1222
1223 static void bnx2x_release_firmware(struct bnx2x *bp)
1224 {
1225         kfree(bp->init_ops_offsets);
1226         kfree(bp->init_ops);
1227         kfree(bp->init_data);
1228         release_firmware(bp->firmware);
1229 }
1230
1231 /* must be called with rtnl_lock */
1232 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1233 {
1234         u32 load_code;
1235         int i, rc;
1236
1237         /* Set init arrays */
1238         rc = bnx2x_init_firmware(bp);
1239         if (rc) {
1240                 BNX2X_ERR("Error loading firmware\n");
1241                 return rc;
1242         }
1243
1244 #ifdef BNX2X_STOP_ON_ERROR
1245         if (unlikely(bp->panic))
1246                 return -EPERM;
1247 #endif
1248
1249         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1250
1251         rc = bnx2x_set_num_queues(bp);
1252         if (rc)
1253                 return rc;
1254
1255         if (bnx2x_alloc_mem(bp)) {
1256                 bnx2x_free_irq(bp, true);
1257                 return -ENOMEM;
1258         }
1259
1260         for_each_queue(bp, i)
1261                 bnx2x_fp(bp, i, disable_tpa) =
1262                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1263
1264         for_each_queue(bp, i)
1265                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1266                                bnx2x_poll, 128);
1267
1268         bnx2x_napi_enable(bp);
1269
1270         if (bp->flags & USING_MSIX_FLAG) {
1271                 rc = bnx2x_req_msix_irqs(bp);
1272                 if (rc) {
1273                         bnx2x_free_irq(bp, true);
1274                         goto load_error1;
1275                 }
1276         } else {
1277                 bnx2x_ack_int(bp);
1278                 rc = bnx2x_req_irq(bp);
1279                 if (rc) {
1280                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1281                         bnx2x_free_irq(bp, true);
1282                         goto load_error1;
1283                 }
1284                 if (bp->flags & USING_MSI_FLAG) {
1285                         bp->dev->irq = bp->pdev->irq;
1286                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
1287                                     bp->pdev->irq);
1288                 }
1289         }
1290
1291         /* Send LOAD_REQUEST command to MCP
1292            Returns the type of LOAD command:
1293            if it is the first port to be initialized
1294            common blocks should be initialized, otherwise - not
1295         */
1296         if (!BP_NOMCP(bp)) {
1297                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1298                 if (!load_code) {
1299                         BNX2X_ERR("MCP response failure, aborting\n");
1300                         rc = -EBUSY;
1301                         goto load_error2;
1302                 }
1303                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1304                         rc = -EBUSY; /* other port in diagnostic mode */
1305                         goto load_error2;
1306                 }
1307
1308         } else {
1309                 int port = BP_PORT(bp);
1310
1311                 DP(NETIF_MSG_IFUP, "NO MCP - load counts      %d, %d, %d\n",
1312                    load_count[0], load_count[1], load_count[2]);
1313                 load_count[0]++;
1314                 load_count[1 + port]++;
1315                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts  %d, %d, %d\n",
1316                    load_count[0], load_count[1], load_count[2]);
1317                 if (load_count[0] == 1)
1318                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1319                 else if (load_count[1 + port] == 1)
1320                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1321                 else
1322                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1323         }
1324
1325         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1326             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1327                 bp->port.pmf = 1;
1328         else
1329                 bp->port.pmf = 0;
1330         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1331
1332         /* Initialize HW */
1333         rc = bnx2x_init_hw(bp, load_code);
1334         if (rc) {
1335                 BNX2X_ERR("HW init failed, aborting\n");
1336                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1337                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1338                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1339                 goto load_error2;
1340         }
1341
1342         /* Setup NIC internals and enable interrupts */
1343         bnx2x_nic_init(bp, load_code);
1344
1345         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1346             (bp->common.shmem2_base))
1347                 SHMEM2_WR(bp, dcc_support,
1348                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1349                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1350
1351         /* Send LOAD_DONE command to MCP */
1352         if (!BP_NOMCP(bp)) {
1353                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1354                 if (!load_code) {
1355                         BNX2X_ERR("MCP response failure, aborting\n");
1356                         rc = -EBUSY;
1357                         goto load_error3;
1358                 }
1359         }
1360
1361         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1362
1363         rc = bnx2x_setup_leading(bp);
1364         if (rc) {
1365                 BNX2X_ERR("Setup leading failed!\n");
1366 #ifndef BNX2X_STOP_ON_ERROR
1367                 goto load_error3;
1368 #else
1369                 bp->panic = 1;
1370                 return -EBUSY;
1371 #endif
1372         }
1373
1374         if (CHIP_IS_E1H(bp))
1375                 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1376                         DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1377                         bp->flags |= MF_FUNC_DIS;
1378                 }
1379
1380         if (bp->state == BNX2X_STATE_OPEN) {
1381 #ifdef BCM_CNIC
1382                 /* Enable Timer scan */
1383                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1384 #endif
1385                 for_each_nondefault_queue(bp, i) {
1386                         rc = bnx2x_setup_multi(bp, i);
1387                         if (rc)
1388 #ifdef BCM_CNIC
1389                                 goto load_error4;
1390 #else
1391                                 goto load_error3;
1392 #endif
1393                 }
1394
1395                 if (CHIP_IS_E1(bp))
1396                         bnx2x_set_eth_mac_addr_e1(bp, 1);
1397                 else
1398                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
1399 #ifdef BCM_CNIC
1400                 /* Set iSCSI L2 MAC */
1401                 mutex_lock(&bp->cnic_mutex);
1402                 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1403                         bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1404                         bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1405                         bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1406                                       CNIC_SB_ID(bp));
1407                 }
1408                 mutex_unlock(&bp->cnic_mutex);
1409 #endif
1410         }
1411
1412         if (bp->port.pmf)
1413                 bnx2x_initial_phy_init(bp, load_mode);
1414
1415         /* Start fast path */
1416         switch (load_mode) {
1417         case LOAD_NORMAL:
1418                 if (bp->state == BNX2X_STATE_OPEN) {
1419                         /* Tx queue should be only reenabled */
1420                         netif_tx_wake_all_queues(bp->dev);
1421                 }
1422                 /* Initialize the receive filter. */
1423                 bnx2x_set_rx_mode(bp->dev);
1424                 break;
1425
1426         case LOAD_OPEN:
1427                 netif_tx_start_all_queues(bp->dev);
1428                 if (bp->state != BNX2X_STATE_OPEN)
1429                         netif_tx_disable(bp->dev);
1430                 /* Initialize the receive filter. */
1431                 bnx2x_set_rx_mode(bp->dev);
1432                 break;
1433
1434         case LOAD_DIAG:
1435                 /* Initialize the receive filter. */
1436                 bnx2x_set_rx_mode(bp->dev);
1437                 bp->state = BNX2X_STATE_DIAG;
1438                 break;
1439
1440         default:
1441                 break;
1442         }
1443
1444         if (!bp->port.pmf)
1445                 bnx2x__link_status_update(bp);
1446
1447         /* start the timer */
1448         mod_timer(&bp->timer, jiffies + bp->current_interval);
1449
1450 #ifdef BCM_CNIC
1451         bnx2x_setup_cnic_irq_info(bp);
1452         if (bp->state == BNX2X_STATE_OPEN)
1453                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1454 #endif
1455         bnx2x_inc_load_cnt(bp);
1456
1457         bnx2x_release_firmware(bp);
1458
1459         return 0;
1460
1461 #ifdef BCM_CNIC
1462 load_error4:
1463         /* Disable Timer scan */
1464         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1465 #endif
1466 load_error3:
1467         bnx2x_int_disable_sync(bp, 1);
1468         if (!BP_NOMCP(bp)) {
1469                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1470                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1471         }
1472         bp->port.pmf = 0;
1473         /* Free SKBs, SGEs, TPA pool and driver internals */
1474         bnx2x_free_skbs(bp);
1475         for_each_queue(bp, i)
1476                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1477 load_error2:
1478         /* Release IRQs */
1479         bnx2x_free_irq(bp, false);
1480 load_error1:
1481         bnx2x_napi_disable(bp);
1482         for_each_queue(bp, i)
1483                 netif_napi_del(&bnx2x_fp(bp, i, napi));
1484         bnx2x_free_mem(bp);
1485
1486         bnx2x_release_firmware(bp);
1487
1488         return rc;
1489 }
1490
1491 /* must be called with rtnl_lock */
1492 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1493 {
1494         int i;
1495
1496         if (bp->state == BNX2X_STATE_CLOSED) {
1497                 /* Interface has been removed - nothing to recover */
1498                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1499                 bp->is_leader = 0;
1500                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1501                 smp_wmb();
1502
1503                 return -EINVAL;
1504         }
1505
1506 #ifdef BCM_CNIC
1507         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1508 #endif
1509         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1510
1511         /* Set "drop all" */
1512         bp->rx_mode = BNX2X_RX_MODE_NONE;
1513         bnx2x_set_storm_rx_mode(bp);
1514
1515         /* Disable HW interrupts, NAPI and Tx */
1516         bnx2x_netif_stop(bp, 1);
1517         netif_carrier_off(bp->dev);
1518
1519         del_timer_sync(&bp->timer);
1520         SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1521                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1522         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1523
1524         /* Release IRQs */
1525         bnx2x_free_irq(bp, false);
1526
1527         /* Cleanup the chip if needed */
1528         if (unload_mode != UNLOAD_RECOVERY)
1529                 bnx2x_chip_cleanup(bp, unload_mode);
1530
1531         bp->port.pmf = 0;
1532
1533         /* Free SKBs, SGEs, TPA pool and driver internals */
1534         bnx2x_free_skbs(bp);
1535         for_each_queue(bp, i)
1536                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1537         for_each_queue(bp, i)
1538                 netif_napi_del(&bnx2x_fp(bp, i, napi));
1539         bnx2x_free_mem(bp);
1540
1541         bp->state = BNX2X_STATE_CLOSED;
1542
1543         /* The last driver must disable a "close the gate" if there is no
1544          * parity attention or "process kill" pending.
1545          */
1546         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1547             bnx2x_reset_is_done(bp))
1548                 bnx2x_disable_close_the_gate(bp);
1549
1550         /* Reset MCP mail box sequence if there is on going recovery */
1551         if (unload_mode == UNLOAD_RECOVERY)
1552                 bp->fw_seq = 0;
1553
1554         return 0;
1555 }
1556 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1557 {
1558         u16 pmcsr;
1559
1560         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1561
1562         switch (state) {
1563         case PCI_D0:
1564                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1565                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1566                                        PCI_PM_CTRL_PME_STATUS));
1567
1568                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1569                         /* delay required during transition out of D3hot */
1570                         msleep(20);
1571                 break;
1572
1573         case PCI_D3hot:
1574                 /* If there are other clients above don't
1575                    shut down the power */
1576                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1577                         return 0;
1578                 /* Don't shut down the power for emulation and FPGA */
1579                 if (CHIP_REV_IS_SLOW(bp))
1580                         return 0;
1581
1582                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1583                 pmcsr |= 3;
1584
1585                 if (bp->wol)
1586                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1587
1588                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1589                                       pmcsr);
1590
1591                 /* No more memory access after this point until
1592                 * device is brought back to D0.
1593                 */
1594                 break;
1595
1596         default:
1597                 return -EINVAL;
1598         }
1599         return 0;
1600 }
1601
1602
1603
1604 /*
1605  * net_device service functions
1606  */
1607
1608 static int bnx2x_poll(struct napi_struct *napi, int budget)
1609 {
1610         int work_done = 0;
1611         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1612                                                  napi);
1613         struct bnx2x *bp = fp->bp;
1614
1615         while (1) {
1616 #ifdef BNX2X_STOP_ON_ERROR
1617                 if (unlikely(bp->panic)) {
1618                         napi_complete(napi);
1619                         return 0;
1620                 }
1621 #endif
1622
1623                 if (bnx2x_has_tx_work(fp))
1624                         bnx2x_tx_int(fp);
1625
1626                 if (bnx2x_has_rx_work(fp)) {
1627                         work_done += bnx2x_rx_int(fp, budget - work_done);
1628
1629                         /* must not complete if we consumed full budget */
1630                         if (work_done >= budget)
1631                                 break;
1632                 }
1633
1634                 /* Fall out from the NAPI loop if needed */
1635                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1636                         bnx2x_update_fpsb_idx(fp);
1637                 /* bnx2x_has_rx_work() reads the status block, thus we need
1638                  * to ensure that status block indices have been actually read
1639                  * (bnx2x_update_fpsb_idx) prior to this check
1640                  * (bnx2x_has_rx_work) so that we won't write the "newer"
1641                  * value of the status block to IGU (if there was a DMA right
1642                  * after bnx2x_has_rx_work and if there is no rmb, the memory
1643                  * reading (bnx2x_update_fpsb_idx) may be postponed to right
1644                  * before bnx2x_ack_sb). In this case there will never be
1645                  * another interrupt until there is another update of the
1646                  * status block, while there is still unhandled work.
1647                  */
1648                         rmb();
1649
1650                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1651                                 napi_complete(napi);
1652                                 /* Re-enable interrupts */
1653                                 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1654                                              le16_to_cpu(fp->fp_c_idx),
1655                                              IGU_INT_NOP, 1);
1656                                 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1657                                              le16_to_cpu(fp->fp_u_idx),
1658                                              IGU_INT_ENABLE, 1);
1659                                 break;
1660                         }
1661                 }
1662         }
1663
1664         return work_done;
1665 }
1666
1667
1668 /* we split the first BD into headers and data BDs
1669  * to ease the pain of our fellow microcode engineers
1670  * we use one mapping for both BDs
1671  * So far this has only been observed to happen
1672  * in Other Operating Systems(TM)
1673  */
1674 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1675                                    struct bnx2x_fastpath *fp,
1676                                    struct sw_tx_bd *tx_buf,
1677                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1678                                    u16 bd_prod, int nbd)
1679 {
1680         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1681         struct eth_tx_bd *d_tx_bd;
1682         dma_addr_t mapping;
1683         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1684
1685         /* first fix first BD */
1686         h_tx_bd->nbd = cpu_to_le16(nbd);
1687         h_tx_bd->nbytes = cpu_to_le16(hlen);
1688
1689         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1690            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1691            h_tx_bd->addr_lo, h_tx_bd->nbd);
1692
1693         /* now get a new data BD
1694          * (after the pbd) and fill it */
1695         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1696         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1697
1698         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1699                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1700
1701         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1702         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1703         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1704
1705         /* this marks the BD as one that has no individual mapping */
1706         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1707
1708         DP(NETIF_MSG_TX_QUEUED,
1709            "TSO split data size is %d (%x:%x)\n",
1710            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1711
1712         /* update tx_bd */
1713         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1714
1715         return bd_prod;
1716 }
1717
1718 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1719 {
1720         if (fix > 0)
1721                 csum = (u16) ~csum_fold(csum_sub(csum,
1722                                 csum_partial(t_header - fix, fix, 0)));
1723
1724         else if (fix < 0)
1725                 csum = (u16) ~csum_fold(csum_add(csum,
1726                                 csum_partial(t_header, -fix, 0)));
1727
1728         return swab16(csum);
1729 }
1730
1731 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1732 {
1733         u32 rc;
1734
1735         if (skb->ip_summed != CHECKSUM_PARTIAL)
1736                 rc = XMIT_PLAIN;
1737
1738         else {
1739                 if (skb->protocol == htons(ETH_P_IPV6)) {
1740                         rc = XMIT_CSUM_V6;
1741                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1742                                 rc |= XMIT_CSUM_TCP;
1743
1744                 } else {
1745                         rc = XMIT_CSUM_V4;
1746                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1747                                 rc |= XMIT_CSUM_TCP;
1748                 }
1749         }
1750
1751         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1752                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1753
1754         else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1755                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1756
1757         return rc;
1758 }
1759
1760 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1761 /* check if packet requires linearization (packet is too fragmented)
1762    no need to check fragmentation if page size > 8K (there will be no
1763    violation to FW restrictions) */
1764 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1765                              u32 xmit_type)
1766 {
1767         int to_copy = 0;
1768         int hlen = 0;
1769         int first_bd_sz = 0;
1770
1771         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1772         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1773
1774                 if (xmit_type & XMIT_GSO) {
1775                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1776                         /* Check if LSO packet needs to be copied:
1777                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1778                         int wnd_size = MAX_FETCH_BD - 3;
1779                         /* Number of windows to check */
1780                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1781                         int wnd_idx = 0;
1782                         int frag_idx = 0;
1783                         u32 wnd_sum = 0;
1784
1785                         /* Headers length */
1786                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1787                                 tcp_hdrlen(skb);
1788
1789                         /* Amount of data (w/o headers) on linear part of SKB*/
1790                         first_bd_sz = skb_headlen(skb) - hlen;
1791
1792                         wnd_sum  = first_bd_sz;
1793
1794                         /* Calculate the first sum - it's special */
1795                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1796                                 wnd_sum +=
1797                                         skb_shinfo(skb)->frags[frag_idx].size;
1798
1799                         /* If there was data on linear skb data - check it */
1800                         if (first_bd_sz > 0) {
1801                                 if (unlikely(wnd_sum < lso_mss)) {
1802                                         to_copy = 1;
1803                                         goto exit_lbl;
1804                                 }
1805
1806                                 wnd_sum -= first_bd_sz;
1807                         }
1808
1809                         /* Others are easier: run through the frag list and
1810                            check all windows */
1811                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1812                                 wnd_sum +=
1813                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1814
1815                                 if (unlikely(wnd_sum < lso_mss)) {
1816                                         to_copy = 1;
1817                                         break;
1818                                 }
1819                                 wnd_sum -=
1820                                         skb_shinfo(skb)->frags[wnd_idx].size;
1821                         }
1822                 } else {
1823                         /* in non-LSO too fragmented packet should always
1824                            be linearized */
1825                         to_copy = 1;
1826                 }
1827         }
1828
1829 exit_lbl:
1830         if (unlikely(to_copy))
1831                 DP(NETIF_MSG_TX_QUEUED,
1832                    "Linearization IS REQUIRED for %s packet. "
1833                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1834                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1835                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1836
1837         return to_copy;
1838 }
1839 #endif
1840
1841 /* called with netif_tx_lock
1842  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1843  * netif_wake_queue()
1844  */
1845 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1846 {
1847         struct bnx2x *bp = netdev_priv(dev);
1848         struct bnx2x_fastpath *fp;
1849         struct netdev_queue *txq;
1850         struct sw_tx_bd *tx_buf;
1851         struct eth_tx_start_bd *tx_start_bd;
1852         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1853         struct eth_tx_parse_bd *pbd = NULL;
1854         u16 pkt_prod, bd_prod;
1855         int nbd, fp_index;
1856         dma_addr_t mapping;
1857         u32 xmit_type = bnx2x_xmit_type(bp, skb);
1858         int i;
1859         u8 hlen = 0;
1860         __le16 pkt_size = 0;
1861         struct ethhdr *eth;
1862         u8 mac_type = UNICAST_ADDRESS;
1863
1864 #ifdef BNX2X_STOP_ON_ERROR
1865         if (unlikely(bp->panic))
1866                 return NETDEV_TX_BUSY;
1867 #endif
1868
1869         fp_index = skb_get_queue_mapping(skb);
1870         txq = netdev_get_tx_queue(dev, fp_index);
1871
1872         fp = &bp->fp[fp_index];
1873
1874         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1875                 fp->eth_q_stats.driver_xoff++;
1876                 netif_tx_stop_queue(txq);
1877                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1878                 return NETDEV_TX_BUSY;
1879         }
1880
1881         DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x  protocol %x  protocol(%x,%x)"
1882            "  gso type %x  xmit_type %x\n",
1883            skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1884            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1885
1886         eth = (struct ethhdr *)skb->data;
1887
1888         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1889         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1890                 if (is_broadcast_ether_addr(eth->h_dest))
1891                         mac_type = BROADCAST_ADDRESS;
1892                 else
1893                         mac_type = MULTICAST_ADDRESS;
1894         }
1895
1896 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1897         /* First, check if we need to linearize the skb (due to FW
1898            restrictions). No need to check fragmentation if page size > 8K
1899            (there will be no violation to FW restrictions) */
1900         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1901                 /* Statistics of linearization */
1902                 bp->lin_cnt++;
1903                 if (skb_linearize(skb) != 0) {
1904                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1905                            "silently dropping this SKB\n");
1906                         dev_kfree_skb_any(skb);
1907                         return NETDEV_TX_OK;
1908                 }
1909         }
1910 #endif
1911
1912         /*
1913         Please read carefully. First we use one BD which we mark as start,
1914         then we have a parsing info BD (used for TSO or xsum),
1915         and only then we have the rest of the TSO BDs.
1916         (don't forget to mark the last one as last,
1917         and to unmap only AFTER you write to the BD ...)
1918         And above all, all pdb sizes are in words - NOT DWORDS!
1919         */
1920
1921         pkt_prod = fp->tx_pkt_prod++;
1922         bd_prod = TX_BD(fp->tx_bd_prod);
1923
1924         /* get a tx_buf and first BD */
1925         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1926         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1927
1928         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1929         tx_start_bd->general_data =  (mac_type <<
1930                                         ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1931         /* header nbd */
1932         tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1933
1934         /* remember the first BD of the packet */
1935         tx_buf->first_bd = fp->tx_bd_prod;
1936         tx_buf->skb = skb;
1937         tx_buf->flags = 0;
1938
1939         DP(NETIF_MSG_TX_QUEUED,
1940            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
1941            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1942
1943 #ifdef BCM_VLAN
1944         if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1945             (bp->flags & HW_VLAN_TX_FLAG)) {
1946                 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1947                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1948         } else
1949 #endif
1950                 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1951
1952         /* turn on parsing and get a BD */
1953         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1954         pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1955
1956         memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1957
1958         if (xmit_type & XMIT_CSUM) {
1959                 hlen = (skb_network_header(skb) - skb->data) / 2;
1960
1961                 /* for now NS flag is not used in Linux */
1962                 pbd->global_data =
1963                         (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1964                                  ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1965
1966                 pbd->ip_hlen = (skb_transport_header(skb) -
1967                                 skb_network_header(skb)) / 2;
1968
1969                 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1970
1971                 pbd->total_hlen = cpu_to_le16(hlen);
1972                 hlen = hlen*2;
1973
1974                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1975
1976                 if (xmit_type & XMIT_CSUM_V4)
1977                         tx_start_bd->bd_flags.as_bitfield |=
1978                                                 ETH_TX_BD_FLAGS_IP_CSUM;
1979                 else
1980                         tx_start_bd->bd_flags.as_bitfield |=
1981                                                 ETH_TX_BD_FLAGS_IPV6;
1982
1983                 if (xmit_type & XMIT_CSUM_TCP) {
1984                         pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1985
1986                 } else {
1987                         s8 fix = SKB_CS_OFF(skb); /* signed! */
1988
1989                         pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1990
1991                         DP(NETIF_MSG_TX_QUEUED,
1992                            "hlen %d  fix %d  csum before fix %x\n",
1993                            le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1994
1995                         /* HW bug: fixup the CSUM */
1996                         pbd->tcp_pseudo_csum =
1997                                 bnx2x_csum_fix(skb_transport_header(skb),
1998                                                SKB_CS(skb), fix);
1999
2000                         DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
2001                            pbd->tcp_pseudo_csum);
2002                 }
2003         }
2004
2005         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2006                                  skb_headlen(skb), DMA_TO_DEVICE);
2007
2008         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2009         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2010         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2011         tx_start_bd->nbd = cpu_to_le16(nbd);
2012         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2013         pkt_size = tx_start_bd->nbytes;
2014
2015         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2016            "  nbytes %d  flags %x  vlan %x\n",
2017            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2018            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2019            tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
2020
2021         if (xmit_type & XMIT_GSO) {
2022
2023                 DP(NETIF_MSG_TX_QUEUED,
2024                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2025                    skb->len, hlen, skb_headlen(skb),
2026                    skb_shinfo(skb)->gso_size);
2027
2028                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2029
2030                 if (unlikely(skb_headlen(skb) > hlen))
2031                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2032                                                  hlen, bd_prod, ++nbd);
2033
2034                 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2035                 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2036                 pbd->tcp_flags = pbd_tcp_flags(skb);
2037
2038                 if (xmit_type & XMIT_GSO_V4) {
2039                         pbd->ip_id = swab16(ip_hdr(skb)->id);
2040                         pbd->tcp_pseudo_csum =
2041                                 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2042                                                           ip_hdr(skb)->daddr,
2043                                                           0, IPPROTO_TCP, 0));
2044
2045                 } else
2046                         pbd->tcp_pseudo_csum =
2047                                 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2048                                                         &ipv6_hdr(skb)->daddr,
2049                                                         0, IPPROTO_TCP, 0));
2050
2051                 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2052         }
2053         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2054
2055         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2056                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2057
2058                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2059                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2060                 if (total_pkt_bd == NULL)
2061                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2062
2063                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2064                                        frag->page_offset,
2065                                        frag->size, DMA_TO_DEVICE);
2066
2067                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2068                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2069                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2070                 le16_add_cpu(&pkt_size, frag->size);
2071
2072                 DP(NETIF_MSG_TX_QUEUED,
2073                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2074                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2075                    le16_to_cpu(tx_data_bd->nbytes));
2076         }
2077
2078         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2079
2080         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2081
2082         /* now send a tx doorbell, counting the next BD
2083          * if the packet contains or ends with it
2084          */
2085         if (TX_BD_POFF(bd_prod) < nbd)
2086                 nbd++;
2087
2088         if (total_pkt_bd != NULL)
2089                 total_pkt_bd->total_pkt_bytes = pkt_size;
2090
2091         if (pbd)
2092                 DP(NETIF_MSG_TX_QUEUED,
2093                    "PBD @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2094                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2095                    pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2096                    pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2097                    pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2098
2099         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2100
2101         /*
2102          * Make sure that the BD data is updated before updating the producer
2103          * since FW might read the BD right after the producer is updated.
2104          * This is only applicable for weak-ordered memory model archs such
2105          * as IA-64. The following barrier is also mandatory since FW will
2106          * assumes packets must have BDs.
2107          */
2108         wmb();
2109
2110         fp->tx_db.data.prod += nbd;
2111         barrier();
2112         DOORBELL(bp, fp->index, fp->tx_db.raw);
2113
2114         mmiowb();
2115
2116         fp->tx_bd_prod += nbd;
2117
2118         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2119                 netif_tx_stop_queue(txq);
2120
2121                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2122                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2123                  * fp->bd_tx_cons */
2124                 smp_mb();
2125
2126                 fp->eth_q_stats.driver_xoff++;
2127                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2128                         netif_tx_wake_queue(txq);
2129         }
2130         fp->tx_pkt++;
2131
2132         return NETDEV_TX_OK;
2133 }
2134 /* called with rtnl_lock */
2135 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2136 {
2137         struct sockaddr *addr = p;
2138         struct bnx2x *bp = netdev_priv(dev);
2139
2140         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2141                 return -EINVAL;
2142
2143         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2144         if (netif_running(dev)) {
2145                 if (CHIP_IS_E1(bp))
2146                         bnx2x_set_eth_mac_addr_e1(bp, 1);
2147                 else
2148                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
2149         }
2150
2151         return 0;
2152 }
2153
2154 /* called with rtnl_lock */
2155 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2156 {
2157         struct bnx2x *bp = netdev_priv(dev);
2158         int rc = 0;
2159
2160         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2161                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2162                 return -EAGAIN;
2163         }
2164
2165         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2166             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2167                 return -EINVAL;
2168
2169         /* This does not race with packet allocation
2170          * because the actual alloc size is
2171          * only updated as part of load
2172          */
2173         dev->mtu = new_mtu;
2174
2175         if (netif_running(dev)) {
2176                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2177                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2178         }
2179
2180         return rc;
2181 }
2182
2183 void bnx2x_tx_timeout(struct net_device *dev)
2184 {
2185         struct bnx2x *bp = netdev_priv(dev);
2186
2187 #ifdef BNX2X_STOP_ON_ERROR
2188         if (!bp->panic)
2189                 bnx2x_panic();
2190 #endif
2191         /* This allows the netif to be shutdown gracefully before resetting */
2192         schedule_delayed_work(&bp->reset_task, 0);
2193 }
2194
2195 #ifdef BCM_VLAN
2196 /* called with rtnl_lock */
2197 void bnx2x_vlan_rx_register(struct net_device *dev,
2198                                    struct vlan_group *vlgrp)
2199 {
2200         struct bnx2x *bp = netdev_priv(dev);
2201
2202         bp->vlgrp = vlgrp;
2203
2204         /* Set flags according to the required capabilities */
2205         bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2206
2207         if (dev->features & NETIF_F_HW_VLAN_TX)
2208                 bp->flags |= HW_VLAN_TX_FLAG;
2209
2210         if (dev->features & NETIF_F_HW_VLAN_RX)
2211                 bp->flags |= HW_VLAN_RX_FLAG;
2212
2213         if (netif_running(dev))
2214                 bnx2x_set_client_config(bp);
2215 }
2216
2217 #endif
2218 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2219 {
2220         struct net_device *dev = pci_get_drvdata(pdev);
2221         struct bnx2x *bp;
2222
2223         if (!dev) {
2224                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2225                 return -ENODEV;
2226         }
2227         bp = netdev_priv(dev);
2228
2229         rtnl_lock();
2230
2231         pci_save_state(pdev);
2232
2233         if (!netif_running(dev)) {
2234                 rtnl_unlock();
2235                 return 0;
2236         }
2237
2238         netif_device_detach(dev);
2239
2240         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2241
2242         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2243
2244         rtnl_unlock();
2245
2246         return 0;
2247 }
2248
2249 int bnx2x_resume(struct pci_dev *pdev)
2250 {
2251         struct net_device *dev = pci_get_drvdata(pdev);
2252         struct bnx2x *bp;
2253         int rc;
2254
2255         if (!dev) {
2256                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2257                 return -ENODEV;
2258         }
2259         bp = netdev_priv(dev);
2260
2261         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2262                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2263                 return -EAGAIN;
2264         }
2265
2266         rtnl_lock();
2267
2268         pci_restore_state(pdev);
2269
2270         if (!netif_running(dev)) {
2271                 rtnl_unlock();
2272                 return 0;
2273         }
2274
2275         bnx2x_set_power_state(bp, PCI_D0);
2276         netif_device_attach(dev);
2277
2278         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2279
2280         rtnl_unlock();
2281
2282         return rc;
2283 }