drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18
  19 #include <linux/etherdevice.h>
  20 #include <linux/ip.h>
  21 #include <linux/ipv6.h>
  22 #include <net/ip6_checksum.h>
  23 #include <linux/firmware.h>
  24 #include "bnx2x_cmn.h"
  25
  26 #ifdef BCM_VLAN
  27 #include <linux/if_vlan.h>
  28 #endif
  29
  30 static int bnx2x_poll(struct napi_struct *napi, int budget);
  31
  32 /* free skb in the packet ring at pos idx
  33  * return idx of last bd freed
  34  */
  35 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  36                              u16 idx)
  37 {
  38         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  39         struct eth_tx_start_bd *tx_start_bd;
  40         struct eth_tx_bd *tx_data_bd;
  41         struct sk_buff *skb = tx_buf->skb;
  42         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  43         int nbd;
  44
  45         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  46         prefetch(&skb->end);
  47
  48         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  49            idx, tx_buf, skb);
  50
  51         /* unmap first bd */
  52         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  53         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  54         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  55                          BD_UNMAP_LEN(tx_start_bd), PCI_DMA_TODEVICE);
  56
  57         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  58 #ifdef BNX2X_STOP_ON_ERROR
  59         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  60                 BNX2X_ERR("BAD nbd!\n");
  61                 bnx2x_panic();
  62         }
  63 #endif
  64         new_cons = nbd + tx_buf->first_bd;
  65
  66         /* Get the next bd */
  67         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  68
  69         /* Skip a parse bd... */
  70         --nbd;
  71         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  72
  73         /* ...and the TSO split header bd since they have no mapping */
  74         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  75                 --nbd;
  76                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  77         }
  78
  79         /* now free frags */
  80         while (nbd > 0) {
  81
  82                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  83                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  84                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  85                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  86                 if (--nbd)
  87                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  88         }
  89
  90         /* release skb */
  91         WARN_ON(!skb);
  92         dev_kfree_skb(skb);
  93         tx_buf->first_bd = 0;
  94         tx_buf->skb = NULL;
  95
  96         return new_cons;
  97 }
  98
  99 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
 100 {
 101         struct bnx2x *bp = fp->bp;
 102         struct netdev_queue *txq;
 103         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 104
 105 #ifdef BNX2X_STOP_ON_ERROR
 106         if (unlikely(bp->panic))
 107                 return -1;
 108 #endif
 109
 110         txq = netdev_get_tx_queue(bp->dev, fp->index);
 111         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 112         sw_cons = fp->tx_pkt_cons;
 113
 114         while (sw_cons != hw_cons) {
 115                 u16 pkt_cons;
 116
 117                 pkt_cons = TX_BD(sw_cons);
 118
 119                 /* prefetch(bp->tx_buf_ring[pkt_cons].skb); */
 120
 121                 DP(NETIF_MSG_TX_DONE, "hw_cons %u  sw_cons %u  pkt_cons %u\n",
 122                    hw_cons, sw_cons, pkt_cons);
 123
 124 /*              if (NEXT_TX_IDX(sw_cons) != hw_cons) {
 125                         rmb();
 126                         prefetch(fp->tx_buf_ring[NEXT_TX_IDX(sw_cons)].skb);
 127                 }
 128 */
 129                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 130                 sw_cons++;
 131         }
 132
 133         fp->tx_pkt_cons = sw_cons;
 134         fp->tx_bd_cons = bd_cons;
 135
 136         /* Need to make the tx_bd_cons update visible to start_xmit()
 137          * before checking for netif_tx_queue_stopped().  Without the
 138          * memory barrier, there is a small possibility that
 139          * start_xmit() will miss it and cause the queue to be stopped
 140          * forever.
 141          */
 142         smp_mb();
 143
 144         /* TBD need a thresh? */
 145         if (unlikely(netif_tx_queue_stopped(txq))) {
 146                 /* Taking tx_lock() is needed to prevent reenabling the queue
 147                  * while it's empty. This could have happen if rx_action() gets
 148                  * suspended in bnx2x_tx_int() after the condition before
 149                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 150                  *
 151                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 152                  * sends some packets consuming the whole queue again->
 153                  * stops the queue
 154                  */
 155
 156                 __netif_tx_lock(txq, smp_processor_id());
 157
 158                 if ((netif_tx_queue_stopped(txq)) &&
 159                     (bp->state == BNX2X_STATE_OPEN) &&
 160                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 161                         netif_tx_wake_queue(txq);
 162
 163                 __netif_tx_unlock(txq);
 164         }
 165         return 0;
 166 }
 167
 168 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 169                                              u16 idx)
 170 {
 171         u16 last_max = fp->last_max_sge;
 172
 173         if (SUB_S16(idx, last_max) > 0)
 174                 fp->last_max_sge = idx;
 175 }
 176
 177 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 178                                   struct eth_fast_path_rx_cqe *fp_cqe)
 179 {
 180         struct bnx2x *bp = fp->bp;
 181         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 182                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 183                       SGE_PAGE_SHIFT;
 184         u16 last_max, last_elem, first_elem;
 185         u16 delta = 0;
 186         u16 i;
 187
 188         if (!sge_len)
 189                 return;
 190
 191         /* First mark all used pages */
 192         for (i = 0; i < sge_len; i++)
 193                 SGE_MASK_CLEAR_BIT(fp, RX_SGE(le16_to_cpu(fp_cqe->sgl[i])));
 194
 195         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 196            sge_len - 1, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
 197
 198         /* Here we assume that the last SGE index is the biggest */
 199         prefetch((void *)(fp->sge_mask));
 200         bnx2x_update_last_max_sge(fp, le16_to_cpu(fp_cqe->sgl[sge_len - 1]));
 201
 202         last_max = RX_SGE(fp->last_max_sge);
 203         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 204         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 205
 206         /* If ring is not full */
 207         if (last_elem + 1 != first_elem)
 208                 last_elem++;
 209
 210         /* Now update the prod */
 211         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 212                 if (likely(fp->sge_mask[i]))
 213                         break;
 214
 215                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 216                 delta += RX_SGE_MASK_ELEM_SZ;
 217         }
 218
 219         if (delta > 0) {
 220                 fp->rx_sge_prod += delta;
 221                 /* clear page-end entries */
 222                 bnx2x_clear_sge_mask_next_elems(fp);
 223         }
 224
 225         DP(NETIF_MSG_RX_STATUS,
 226            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 227            fp->last_max_sge, fp->rx_sge_prod);
 228 }
 229
 230 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 231                             struct sk_buff *skb, u16 cons, u16 prod)
 232 {
 233         struct bnx2x *bp = fp->bp;
 234         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 235         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 236         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 237         dma_addr_t mapping;
 238
 239         /* move empty skb from pool to prod and map it */
 240         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 241         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 242                                  bp->rx_buf_size, DMA_FROM_DEVICE);
 243         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 244
 245         /* move partial skb from cons to pool (don't unmap yet) */
 246         fp->tpa_pool[queue] = *cons_rx_buf;
 247
 248         /* mark bin state as start - print error if current state != stop */
 249         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 250                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 251
 252         fp->tpa_state[queue] = BNX2X_TPA_START;
 253
 254         /* point prod_bd to new skb */
 255         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 256         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 257
 258 #ifdef BNX2X_STOP_ON_ERROR
 259         fp->tpa_queue_used |= (1 << queue);
 260 #ifdef _ASM_GENERIC_INT_L64_H
 261         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 262 #else
 263         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 264 #endif
 265            fp->tpa_queue_used);
 266 #endif
 267 }
 268
 269 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 270                                struct sk_buff *skb,
 271                                struct eth_fast_path_rx_cqe *fp_cqe,
 272                                u16 cqe_idx)
 273 {
 274         struct sw_rx_page *rx_pg, old_rx_pg;
 275         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 276         u32 i, frag_len, frag_size, pages;
 277         int err;
 278         int j;
 279
 280         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 281         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 282
 283         /* This is needed in order to enable forwarding support */
 284         if (frag_size)
 285                 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
 286                                                max(frag_size, (u32)len_on_bd));
 287
 288 #ifdef BNX2X_STOP_ON_ERROR
 289         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 290                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 291                           pages, cqe_idx);
 292                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 293                           fp_cqe->pkt_len, len_on_bd);
 294                 bnx2x_panic();
 295                 return -EINVAL;
 296         }
 297 #endif
 298
 299         /* Run through the SGL and compose the fragmented skb */
 300         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 301                 u16 sge_idx = RX_SGE(le16_to_cpu(fp_cqe->sgl[j]));
 302
 303                 /* FW gives the indices of the SGE as if the ring is an array
 304                    (meaning that "next" element will consume 2 indices) */
 305                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 306                 rx_pg = &fp->rx_page_ring[sge_idx];
 307                 old_rx_pg = *rx_pg;
 308
 309                 /* If we fail to allocate a substitute page, we simply stop
 310                    where we are and drop the whole packet */
 311                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 312                 if (unlikely(err)) {
 313                         fp->eth_q_stats.rx_skb_alloc_failed++;
 314                         return err;
 315                 }
 316
 317                 /* Unmap the page as we r going to pass it to the stack */
 318                 dma_unmap_page(&bp->pdev->dev,
 319                                dma_unmap_addr(&old_rx_pg, mapping),
 320                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 321
 322                 /* Add one frag and update the appropriate fields in the skb */
 323                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 324
 325                 skb->data_len += frag_len;
 326                 skb->truesize += frag_len;
 327                 skb->len += frag_len;
 328
 329                 frag_size -= frag_len;
 330         }
 331
 332         return 0;
 333 }
 334
 335 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 336                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 337                            u16 cqe_idx)
 338 {
 339         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 340         struct sk_buff *skb = rx_buf->skb;
 341         /* alloc new skb */
 342         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 343
 344         /* Unmap skb in the pool anyway, as we are going to change
 345            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 346            fails. */
 347         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 348                          bp->rx_buf_size, DMA_FROM_DEVICE);
 349
 350         if (likely(new_skb)) {
 351                 /* fix ip xsum and give it to the stack */
 352                 /* (no need to map the new skb) */
 353 #ifdef BCM_VLAN
 354                 int is_vlan_cqe =
 355                         (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 356                          PARSING_FLAGS_VLAN);
 357                 int is_not_hwaccel_vlan_cqe =
 358                         (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
 359 #endif
 360
 361                 prefetch(skb);
 362                 prefetch(((char *)(skb)) + 128);
 363
 364 #ifdef BNX2X_STOP_ON_ERROR
 365                 if (pad + len > bp->rx_buf_size) {
 366                         BNX2X_ERR("skb_put is about to fail...  "
 367                                   "pad %d  len %d  rx_buf_size %d\n",
 368                                   pad, len, bp->rx_buf_size);
 369                         bnx2x_panic();
 370                         return;
 371                 }
 372 #endif
 373
 374                 skb_reserve(skb, pad);
 375                 skb_put(skb, len);
 376
 377                 skb->protocol = eth_type_trans(skb, bp->dev);
 378                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 379
 380                 {
 381                         struct iphdr *iph;
 382
 383                         iph = (struct iphdr *)skb->data;
 384 #ifdef BCM_VLAN
 385                         /* If there is no Rx VLAN offloading -
 386                            take VLAN tag into an account */
 387                         if (unlikely(is_not_hwaccel_vlan_cqe))
 388                                 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
 389 #endif
 390                         iph->check = 0;
 391                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 392                 }
 393
 394                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 395                                          &cqe->fast_path_cqe, cqe_idx)) {
 396 #ifdef BCM_VLAN
 397                         if ((bp->vlgrp != NULL) && is_vlan_cqe &&
 398                             (!is_not_hwaccel_vlan_cqe))
 399                                 vlan_gro_receive(&fp->napi, bp->vlgrp,
 400                                                  le16_to_cpu(cqe->fast_path_cqe.
 401                                                              vlan_tag), skb);
 402                         else
 403 #endif
 404                                 napi_gro_receive(&fp->napi, skb);
 405                 } else {
 406                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 407                            " - dropping packet!\n");
 408                         dev_kfree_skb(skb);
 409                 }
 410
 411
 412                 /* put new skb in bin */
 413                 fp->tpa_pool[queue].skb = new_skb;
 414
 415         } else {
 416                 /* else drop the packet and keep the buffer in the bin */
 417                 DP(NETIF_MSG_RX_STATUS,
 418                    "Failed to allocate new skb - dropping packet!\n");
 419                 fp->eth_q_stats.rx_skb_alloc_failed++;
 420         }
 421
 422         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 423 }
 424
 425 /* Set Toeplitz hash value in the skb using the value from the
 426  * CQE (calculated by HW).
 427  */
 428 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 429                                         struct sk_buff *skb)
 430 {
 431         /* Set Toeplitz hash from CQE */
 432         if ((bp->dev->features & NETIF_F_RXHASH) &&
 433             (cqe->fast_path_cqe.status_flags &
 434              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 435                 skb->rxhash =
 436                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 437 }
 438
 439 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 440 {
 441         struct bnx2x *bp = fp->bp;
 442         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 443         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 444         int rx_pkt = 0;
 445
 446 #ifdef BNX2X_STOP_ON_ERROR
 447         if (unlikely(bp->panic))
 448                 return 0;
 449 #endif
 450
 451         /* CQ "next element" is of the size of the regular element,
 452            that's why it's ok here */
 453         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 454         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 455                 hw_comp_cons++;
 456
 457         bd_cons = fp->rx_bd_cons;
 458         bd_prod = fp->rx_bd_prod;
 459         bd_prod_fw = bd_prod;
 460         sw_comp_cons = fp->rx_comp_cons;
 461         sw_comp_prod = fp->rx_comp_prod;
 462
 463         /* Memory barrier necessary as speculative reads of the rx
 464          * buffer can be ahead of the index in the status block
 465          */
 466         rmb();
 467
 468         DP(NETIF_MSG_RX_STATUS,
 469            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 470            fp->index, hw_comp_cons, sw_comp_cons);
 471
 472         while (sw_comp_cons != hw_comp_cons) {
 473                 struct sw_rx_bd *rx_buf = NULL;
 474                 struct sk_buff *skb;
 475                 union eth_rx_cqe *cqe;
 476                 u8 cqe_fp_flags;
 477                 u16 len, pad;
 478
 479                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 480                 bd_prod = RX_BD(bd_prod);
 481                 bd_cons = RX_BD(bd_cons);
 482
 483                 /* Prefetch the page containing the BD descriptor
 484                    at producer's index. It will be needed when new skb is
 485                    allocated */
 486                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 487                                              (&fp->rx_desc_ring[bd_prod])) -
 488                                   PAGE_SIZE + 1));
 489
 490                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 491                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 492
 493                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 494                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 495                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 496                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 497                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 498                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 499
 500                 /* is this a slowpath msg? */
 501                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 502                         bnx2x_sp_event(fp, cqe);
 503                         goto next_cqe;
 504
 505                 /* this is an rx packet */
 506                 } else {
 507                         rx_buf = &fp->rx_buf_ring[bd_cons];
 508                         skb = rx_buf->skb;
 509                         prefetch(skb);
 510                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 511                         pad = cqe->fast_path_cqe.placement_offset;
 512
 513                         /* If CQE is marked both TPA_START and TPA_END
 514                            it is a non-TPA CQE */
 515                         if ((!fp->disable_tpa) &&
 516                             (TPA_TYPE(cqe_fp_flags) !=
 517                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 518                                 u16 queue = cqe->fast_path_cqe.queue_index;
 519
 520                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 521                                         DP(NETIF_MSG_RX_STATUS,
 522                                            "calling tpa_start on queue %d\n",
 523                                            queue);
 524
 525                                         bnx2x_tpa_start(fp, queue, skb,
 526                                                         bd_cons, bd_prod);
 527
 528                                         /* Set Toeplitz hash for an LRO skb */
 529                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 530
 531                                         goto next_rx;
 532                                 }
 533
 534                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_END) {
 535                                         DP(NETIF_MSG_RX_STATUS,
 536                                            "calling tpa_stop on queue %d\n",
 537                                            queue);
 538
 539                                         if (!BNX2X_RX_SUM_FIX(cqe))
 540                                                 BNX2X_ERR("STOP on none TCP "
 541                                                           "data\n");
 542
 543                                         /* This is a size of the linear data
 544                                            on this skb */
 545                                         len = le16_to_cpu(cqe->fast_path_cqe.
 546                                                                 len_on_bd);
 547                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 548                                                     len, cqe, comp_ring_cons);
 549 #ifdef BNX2X_STOP_ON_ERROR
 550                                         if (bp->panic)
 551                                                 return 0;
 552 #endif
 553
 554                                         bnx2x_update_sge_prod(fp,
 555                                                         &cqe->fast_path_cqe);
 556                                         goto next_cqe;
 557                                 }
 558                         }
 559
 560                         dma_sync_single_for_device(&bp->pdev->dev,
 561                                         dma_unmap_addr(rx_buf, mapping),
 562                                                    pad + RX_COPY_THRESH,
 563                                                    DMA_FROM_DEVICE);
 564                         prefetch(((char *)(skb)) + 128);
 565
 566                         /* is this an error packet? */
 567                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 568                                 DP(NETIF_MSG_RX_ERR,
 569                                    "ERROR  flags %x  rx packet %u\n",
 570                                    cqe_fp_flags, sw_comp_cons);
 571                                 fp->eth_q_stats.rx_err_discard_pkt++;
 572                                 goto reuse_rx;
 573                         }
 574
 575                         /* Since we don't have a jumbo ring
 576                          * copy small packets if mtu > 1500
 577                          */
 578                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 579                             (len <= RX_COPY_THRESH)) {
 580                                 struct sk_buff *new_skb;
 581
 582                                 new_skb = netdev_alloc_skb(bp->dev,
 583                                                            len + pad);
 584                                 if (new_skb == NULL) {
 585                                         DP(NETIF_MSG_RX_ERR,
 586                                            "ERROR  packet dropped "
 587                                            "because of alloc failure\n");
 588                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 589                                         goto reuse_rx;
 590                                 }
 591
 592                                 /* aligned copy */
 593                                 skb_copy_from_linear_data_offset(skb, pad,
 594                                                     new_skb->data + pad, len);
 595                                 skb_reserve(new_skb, pad);
 596                                 skb_put(new_skb, len);
 597
 598                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
 599
 600                                 skb = new_skb;
 601
 602                         } else
 603                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 604                                 dma_unmap_single(&bp->pdev->dev,
 605                                         dma_unmap_addr(rx_buf, mapping),
 606                                                  bp->rx_buf_size,
 607                                                  DMA_FROM_DEVICE);
 608                                 skb_reserve(skb, pad);
 609                                 skb_put(skb, len);
 610
 611                         } else {
 612                                 DP(NETIF_MSG_RX_ERR,
 613                                    "ERROR  packet dropped because "
 614                                    "of alloc failure\n");
 615                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 616 reuse_rx:
 617                                 bnx2x_reuse_rx_skb(fp, skb, bd_cons, bd_prod);
 618                                 goto next_rx;
 619                         }
 620
 621                         skb->protocol = eth_type_trans(skb, bp->dev);
 622
 623                         /* Set Toeplitz hash for a none-LRO skb */
 624                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 625
 626                         skb_checksum_none_assert(skb);
 627                         if (bp->rx_csum) {
 628                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 629                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 630                                 else
 631                                         fp->eth_q_stats.hw_csum_err++;
 632                         }
 633                 }
 634
 635                 skb_record_rx_queue(skb, fp->index);
 636
 637 #ifdef BCM_VLAN
 638                 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
 639                     (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 640                      PARSING_FLAGS_VLAN))
 641                         vlan_gro_receive(&fp->napi, bp->vlgrp,
 642                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
 643                 else
 644 #endif
 645                         napi_gro_receive(&fp->napi, skb);
 646
 647
 648 next_rx:
 649                 rx_buf->skb = NULL;
 650
 651                 bd_cons = NEXT_RX_IDX(bd_cons);
 652                 bd_prod = NEXT_RX_IDX(bd_prod);
 653                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 654                 rx_pkt++;
 655 next_cqe:
 656                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 657                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 658
 659                 if (rx_pkt == budget)
 660                         break;
 661         } /* while */
 662
 663         fp->rx_bd_cons = bd_cons;
 664         fp->rx_bd_prod = bd_prod_fw;
 665         fp->rx_comp_cons = sw_comp_cons;
 666         fp->rx_comp_prod = sw_comp_prod;
 667
 668         /* Update producers */
 669         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 670                              fp->rx_sge_prod);
 671
 672         fp->rx_pkt += rx_pkt;
 673         fp->rx_calls++;
 674
 675         return rx_pkt;
 676 }
 677
 678 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 679 {
 680         struct bnx2x_fastpath *fp = fp_cookie;
 681         struct bnx2x *bp = fp->bp;
 682
 683         /* Return here if interrupt is disabled */
 684         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 685                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 686                 return IRQ_HANDLED;
 687         }
 688
 689         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB [%d:%d]\n",
 690            fp->index, fp->sb_id);
 691         bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 692
 693 #ifdef BNX2X_STOP_ON_ERROR
 694         if (unlikely(bp->panic))
 695                 return IRQ_HANDLED;
 696 #endif
 697
 698         /* Handle Rx and Tx according to MSI-X vector */
 699         prefetch(fp->rx_cons_sb);
 700         prefetch(fp->tx_cons_sb);
 701         prefetch(&fp->status_blk->u_status_block.status_block_index);
 702         prefetch(&fp->status_blk->c_status_block.status_block_index);
 703         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 704
 705         return IRQ_HANDLED;
 706 }
 707
 708
 709 /* HW Lock for shared dual port PHYs */
 710 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 711 {
 712         mutex_lock(&bp->port.phy_mutex);
 713
 714         if (bp->port.need_hw_lock)
 715                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 716 }
 717
 718 void bnx2x_release_phy_lock(struct bnx2x *bp)
 719 {
 720         if (bp->port.need_hw_lock)
 721                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 722
 723         mutex_unlock(&bp->port.phy_mutex);
 724 }
 725
 726 void bnx2x_link_report(struct bnx2x *bp)
 727 {
 728         if (bp->flags & MF_FUNC_DIS) {
 729                 netif_carrier_off(bp->dev);
 730                 netdev_err(bp->dev, "NIC Link is Down\n");
 731                 return;
 732         }
 733
 734         if (bp->link_vars.link_up) {
 735                 u16 line_speed;
 736
 737                 if (bp->state == BNX2X_STATE_OPEN)
 738                         netif_carrier_on(bp->dev);
 739                 netdev_info(bp->dev, "NIC Link is Up, ");
 740
 741                 line_speed = bp->link_vars.line_speed;
 742                 if (IS_E1HMF(bp)) {
 743                         u16 vn_max_rate;
 744
 745                         vn_max_rate =
 746                                 ((bp->mf_config & FUNC_MF_CFG_MAX_BW_MASK) >>
 747                                  FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
 748                         if (vn_max_rate < line_speed)
 749                                 line_speed = vn_max_rate;
 750                 }
 751                 pr_cont("%d Mbps ", line_speed);
 752
 753                 if (bp->link_vars.duplex == DUPLEX_FULL)
 754                         pr_cont("full duplex");
 755                 else
 756                         pr_cont("half duplex");
 757
 758                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 759                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 760                                 pr_cont(", receive ");
 761                                 if (bp->link_vars.flow_ctrl &
 762                                     BNX2X_FLOW_CTRL_TX)
 763                                         pr_cont("& transmit ");
 764                         } else {
 765                                 pr_cont(", transmit ");
 766                         }
 767                         pr_cont("flow control ON");
 768                 }
 769                 pr_cont("\n");
 770
 771         } else { /* link_down */
 772                 netif_carrier_off(bp->dev);
 773                 netdev_err(bp->dev, "NIC Link is Down\n");
 774         }
 775 }
 776
 777 void bnx2x_init_rx_rings(struct bnx2x *bp)
 778 {
 779         int func = BP_FUNC(bp);
 780         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 781                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 782         u16 ring_prod, cqe_ring_prod;
 783         int i, j;
 784
 785         bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN;
 786         DP(NETIF_MSG_IFUP,
 787            "mtu %d  rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
 788
 789         if (bp->flags & TPA_ENABLE_FLAG) {
 790
 791                 for_each_queue(bp, j) {
 792                         struct bnx2x_fastpath *fp = &bp->fp[j];
 793
 794                         for (i = 0; i < max_agg_queues; i++) {
 795                                 fp->tpa_pool[i].skb =
 796                                    netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 797                                 if (!fp->tpa_pool[i].skb) {
 798                                         BNX2X_ERR("Failed to allocate TPA "
 799                                                   "skb pool for queue[%d] - "
 800                                                   "disabling TPA on this "
 801                                                   "queue!\n", j);
 802                                         bnx2x_free_tpa_pool(bp, fp, i);
 803                                         fp->disable_tpa = 1;
 804                                         break;
 805                                 }
 806                                 dma_unmap_addr_set((struct sw_rx_bd *)
 807                                                         &bp->fp->tpa_pool[i],
 808                                                    mapping, 0);
 809                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 810                         }
 811                 }
 812         }
 813
 814         for_each_queue(bp, j) {
 815                 struct bnx2x_fastpath *fp = &bp->fp[j];
 816
 817                 fp->rx_bd_cons = 0;
 818                 fp->rx_cons_sb = BNX2X_RX_SB_INDEX;
 819                 fp->rx_bd_cons_sb = BNX2X_RX_SB_BD_INDEX;
 820
 821                 /* "next page" elements initialization */
 822                 /* SGE ring */
 823                 for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
 824                         struct eth_rx_sge *sge;
 825
 826                         sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
 827                         sge->addr_hi =
 828                                 cpu_to_le32(U64_HI(fp->rx_sge_mapping +
 829                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
 830                         sge->addr_lo =
 831                                 cpu_to_le32(U64_LO(fp->rx_sge_mapping +
 832                                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
 833                 }
 834
 835                 bnx2x_init_sge_ring_bit_mask(fp);
 836
 837                 /* RX BD ring */
 838                 for (i = 1; i <= NUM_RX_RINGS; i++) {
 839                         struct eth_rx_bd *rx_bd;
 840
 841                         rx_bd = &fp->rx_desc_ring[RX_DESC_CNT * i - 2];
 842                         rx_bd->addr_hi =
 843                                 cpu_to_le32(U64_HI(fp->rx_desc_mapping +
 844                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
 845                         rx_bd->addr_lo =
 846                                 cpu_to_le32(U64_LO(fp->rx_desc_mapping +
 847                                             BCM_PAGE_SIZE*(i % NUM_RX_RINGS)));
 848                 }
 849
 850                 /* CQ ring */
 851                 for (i = 1; i <= NUM_RCQ_RINGS; i++) {
 852                         struct eth_rx_cqe_next_page *nextpg;
 853
 854                         nextpg = (struct eth_rx_cqe_next_page *)
 855                                 &fp->rx_comp_ring[RCQ_DESC_CNT * i - 1];
 856                         nextpg->addr_hi =
 857                                 cpu_to_le32(U64_HI(fp->rx_comp_mapping +
 858                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
 859                         nextpg->addr_lo =
 860                                 cpu_to_le32(U64_LO(fp->rx_comp_mapping +
 861                                            BCM_PAGE_SIZE*(i % NUM_RCQ_RINGS)));
 862                 }
 863
 864                 /* Allocate SGEs and initialize the ring elements */
 865                 for (i = 0, ring_prod = 0;
 866                      i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 867
 868                         if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 869                                 BNX2X_ERR("was only able to allocate "
 870                                           "%d rx sges\n", i);
 871                                 BNX2X_ERR("disabling TPA for queue[%d]\n", j);
 872                                 /* Cleanup already allocated elements */
 873                                 bnx2x_free_rx_sge_range(bp, fp, ring_prod);
 874                                 bnx2x_free_tpa_pool(bp, fp, max_agg_queues);
 875                                 fp->disable_tpa = 1;
 876                                 ring_prod = 0;
 877                                 break;
 878                         }
 879                         ring_prod = NEXT_SGE_IDX(ring_prod);
 880                 }
 881                 fp->rx_sge_prod = ring_prod;
 882
 883                 /* Allocate BDs and initialize BD ring */
 884                 fp->rx_comp_cons = 0;
 885                 cqe_ring_prod = ring_prod = 0;
 886                 for (i = 0; i < bp->rx_ring_size; i++) {
 887                         if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 888                                 BNX2X_ERR("was only able to allocate "
 889                                           "%d rx skbs on queue[%d]\n", i, j);
 890                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 891                                 break;
 892                         }
 893                         ring_prod = NEXT_RX_IDX(ring_prod);
 894                         cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 895                         WARN_ON(ring_prod <= i);
 896                 }
 897
 898                 fp->rx_bd_prod = ring_prod;
 899                 /* must not have more available CQEs than BDs */
 900                 fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 901                                          cqe_ring_prod);
 902                 fp->rx_pkt = fp->rx_calls = 0;
 903
 904                 /* Warning!
 905                  * this will generate an interrupt (to the TSTORM)
 906                  * must only be done after chip is initialized
 907                  */
 908                 bnx2x_update_rx_prod(bp, fp, ring_prod, fp->rx_comp_prod,
 909                                      fp->rx_sge_prod);
 910                 if (j != 0)
 911                         continue;
 912
 913                 REG_WR(bp, BAR_USTRORM_INTMEM +
 914                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 915                        U64_LO(fp->rx_comp_mapping));
 916                 REG_WR(bp, BAR_USTRORM_INTMEM +
 917                        USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 918                        U64_HI(fp->rx_comp_mapping));
 919         }
 920 }
 921 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 922 {
 923         int i;
 924
 925         for_each_queue(bp, i) {
 926                 struct bnx2x_fastpath *fp = &bp->fp[i];
 927
 928                 u16 bd_cons = fp->tx_bd_cons;
 929                 u16 sw_prod = fp->tx_pkt_prod;
 930                 u16 sw_cons = fp->tx_pkt_cons;
 931
 932                 while (sw_cons != sw_prod) {
 933                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 934                         sw_cons++;
 935                 }
 936         }
 937 }
 938
 939 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 940 {
 941         int i, j;
 942
 943         for_each_queue(bp, j) {
 944                 struct bnx2x_fastpath *fp = &bp->fp[j];
 945
 946                 for (i = 0; i < NUM_RX_BD; i++) {
 947                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 948                         struct sk_buff *skb = rx_buf->skb;
 949
 950                         if (skb == NULL)
 951                                 continue;
 952
 953                         dma_unmap_single(&bp->pdev->dev,
 954                                          dma_unmap_addr(rx_buf, mapping),
 955                                          bp->rx_buf_size, DMA_FROM_DEVICE);
 956
 957                         rx_buf->skb = NULL;
 958                         dev_kfree_skb(skb);
 959                 }
 960                 if (!fp->disable_tpa)
 961                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 962                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 963                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 964         }
 965 }
 966
 967 void bnx2x_free_skbs(struct bnx2x *bp)
 968 {
 969         bnx2x_free_tx_skbs(bp);
 970         bnx2x_free_rx_skbs(bp);
 971 }
 972
 973 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
 974 {
 975         int i, offset = 1;
 976
 977         free_irq(bp->msix_table[0].vector, bp->dev);
 978         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
 979            bp->msix_table[0].vector);
 980
 981 #ifdef BCM_CNIC
 982         offset++;
 983 #endif
 984         for_each_queue(bp, i) {
 985                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
 986                    "state %x\n", i, bp->msix_table[i + offset].vector,
 987                    bnx2x_fp(bp, i, state));
 988
 989                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
 990         }
 991 }
 992
 993 void bnx2x_free_irq(struct bnx2x *bp, bool disable_only)
 994 {
 995         if (bp->flags & USING_MSIX_FLAG) {
 996                 if (!disable_only)
 997                         bnx2x_free_msix_irqs(bp);
 998                 pci_disable_msix(bp->pdev);
 999                 bp->flags &= ~USING_MSIX_FLAG;
1000
1001         } else if (bp->flags & USING_MSI_FLAG) {
1002                 if (!disable_only)
1003                         free_irq(bp->pdev->irq, bp->dev);
1004                 pci_disable_msi(bp->pdev);
1005                 bp->flags &= ~USING_MSI_FLAG;
1006
1007         } else if (!disable_only)
1008                 free_irq(bp->pdev->irq, bp->dev);
1009 }
1010
1011 static int bnx2x_enable_msix(struct bnx2x *bp)
1012 {
1013         int i, rc, offset = 1;
1014         int igu_vec = 0;
1015
1016         bp->msix_table[0].entry = igu_vec;
1017         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n", igu_vec);
1018
1019 #ifdef BCM_CNIC
1020         igu_vec = BP_L_ID(bp) + offset;
1021         bp->msix_table[1].entry = igu_vec;
1022         DP(NETIF_MSG_IFUP, "msix_table[1].entry = %d (CNIC)\n", igu_vec);
1023         offset++;
1024 #endif
1025         for_each_queue(bp, i) {
1026                 igu_vec = BP_L_ID(bp) + offset + i;
1027                 bp->msix_table[i + offset].entry = igu_vec;
1028                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1029                    "(fastpath #%u)\n", i + offset, igu_vec, i);
1030         }
1031
1032         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0],
1033                              BNX2X_NUM_QUEUES(bp) + offset);
1034
1035         /*
1036          * reconfigure number of tx/rx queues according to available
1037          * MSI-X vectors
1038          */
1039         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1040                 /* vectors available for FP */
1041                 int fp_vec = rc - BNX2X_MSIX_VEC_FP_START;
1042
1043                 DP(NETIF_MSG_IFUP,
1044                    "Trying to use less MSI-X vectors: %d\n", rc);
1045
1046                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1047
1048                 if (rc) {
1049                         DP(NETIF_MSG_IFUP,
1050                            "MSI-X is not attainable  rc %d\n", rc);
1051                         return rc;
1052                 }
1053
1054                 bp->num_queues = min(bp->num_queues, fp_vec);
1055
1056                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1057                                   bp->num_queues);
1058         } else if (rc) {
1059                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1060                 return rc;
1061         }
1062
1063         bp->flags |= USING_MSIX_FLAG;
1064
1065         return 0;
1066 }
1067
1068 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1069 {
1070         int i, rc, offset = 1;
1071
1072         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1073                          bp->dev->name, bp->dev);
1074         if (rc) {
1075                 BNX2X_ERR("request sp irq failed\n");
1076                 return -EBUSY;
1077         }
1078
1079 #ifdef BCM_CNIC
1080         offset++;
1081 #endif
1082         for_each_queue(bp, i) {
1083                 struct bnx2x_fastpath *fp = &bp->fp[i];
1084                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1085                          bp->dev->name, i);
1086
1087                 rc = request_irq(bp->msix_table[i + offset].vector,
1088                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1089                 if (rc) {
1090                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1091                         bnx2x_free_msix_irqs(bp);
1092                         return -EBUSY;
1093                 }
1094
1095                 fp->state = BNX2X_FP_STATE_IRQ;
1096         }
1097
1098         i = BNX2X_NUM_QUEUES(bp);
1099         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1100                " ... fp[%d] %d\n",
1101                bp->msix_table[0].vector,
1102                0, bp->msix_table[offset].vector,
1103                i - 1, bp->msix_table[offset + i - 1].vector);
1104
1105         return 0;
1106 }
1107
1108 static int bnx2x_enable_msi(struct bnx2x *bp)
1109 {
1110         int rc;
1111
1112         rc = pci_enable_msi(bp->pdev);
1113         if (rc) {
1114                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1115                 return -1;
1116         }
1117         bp->flags |= USING_MSI_FLAG;
1118
1119         return 0;
1120 }
1121
1122 static int bnx2x_req_irq(struct bnx2x *bp)
1123 {
1124         unsigned long flags;
1125         int rc;
1126
1127         if (bp->flags & USING_MSI_FLAG)
1128                 flags = 0;
1129         else
1130                 flags = IRQF_SHARED;
1131
1132         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1133                          bp->dev->name, bp->dev);
1134         if (!rc)
1135                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1136
1137         return rc;
1138 }
1139
1140 static void bnx2x_napi_enable(struct bnx2x *bp)
1141 {
1142         int i;
1143
1144         for_each_queue(bp, i)
1145                 napi_enable(&bnx2x_fp(bp, i, napi));
1146 }
1147
1148 static void bnx2x_napi_disable(struct bnx2x *bp)
1149 {
1150         int i;
1151
1152         for_each_queue(bp, i)
1153                 napi_disable(&bnx2x_fp(bp, i, napi));
1154 }
1155
1156 void bnx2x_netif_start(struct bnx2x *bp)
1157 {
1158         int intr_sem;
1159
1160         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1161         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1162
1163         if (intr_sem) {
1164                 if (netif_running(bp->dev)) {
1165                         bnx2x_napi_enable(bp);
1166                         bnx2x_int_enable(bp);
1167                         if (bp->state == BNX2X_STATE_OPEN)
1168                                 netif_tx_wake_all_queues(bp->dev);
1169                 }
1170         }
1171 }
1172
1173 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1174 {
1175         bnx2x_int_disable_sync(bp, disable_hw);
1176         bnx2x_napi_disable(bp);
1177         netif_tx_disable(bp->dev);
1178 }
1179 static int bnx2x_set_num_queues(struct bnx2x *bp)
1180 {
1181         int rc = 0;
1182
1183         switch (bp->int_mode) {
1184         case INT_MODE_INTx:
1185         case INT_MODE_MSI:
1186                 bp->num_queues = 1;
1187                 DP(NETIF_MSG_IFUP, "set number of queues to 1\n");
1188                 break;
1189         default:
1190                 /* Set number of queues according to bp->multi_mode value */
1191                 bnx2x_set_num_queues_msix(bp);
1192
1193                 DP(NETIF_MSG_IFUP, "set number of queues to %d\n",
1194                    bp->num_queues);
1195
1196                 /* if we can't use MSI-X we only need one fp,
1197                  * so try to enable MSI-X with the requested number of fp's
1198                  * and fallback to MSI or legacy INTx with one fp
1199                  */
1200                 rc = bnx2x_enable_msix(bp);
1201                 if (rc)
1202                         /* failed to enable MSI-X */
1203                         bp->num_queues = 1;
1204                 break;
1205         }
1206         bp->dev->real_num_tx_queues = bp->num_queues;
1207         return rc;
1208 }
1209
1210 static void bnx2x_release_firmware(struct bnx2x *bp)
1211 {
1212         kfree(bp->init_ops_offsets);
1213         kfree(bp->init_ops);
1214         kfree(bp->init_data);
1215         release_firmware(bp->firmware);
1216 }
1217
1218 /* must be called with rtnl_lock */
1219 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1220 {
1221         u32 load_code;
1222         int i, rc;
1223
1224         /* Set init arrays */
1225         rc = bnx2x_init_firmware(bp);
1226         if (rc) {
1227                 BNX2X_ERR("Error loading firmware\n");
1228                 return rc;
1229         }
1230
1231 #ifdef BNX2X_STOP_ON_ERROR
1232         if (unlikely(bp->panic))
1233                 return -EPERM;
1234 #endif
1235
1236         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1237
1238         rc = bnx2x_set_num_queues(bp);
1239
1240         if (bnx2x_alloc_mem(bp)) {
1241                 bnx2x_free_irq(bp, true);
1242                 return -ENOMEM;
1243         }
1244
1245         for_each_queue(bp, i)
1246                 bnx2x_fp(bp, i, disable_tpa) =
1247                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1248
1249         for_each_queue(bp, i)
1250                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
1251                                bnx2x_poll, 128);
1252
1253         bnx2x_napi_enable(bp);
1254
1255         if (bp->flags & USING_MSIX_FLAG) {
1256                 rc = bnx2x_req_msix_irqs(bp);
1257                 if (rc) {
1258                         bnx2x_free_irq(bp, true);
1259                         goto load_error1;
1260                 }
1261         } else {
1262                 /* Fall to INTx if failed to enable MSI-X due to lack of
1263                    memory (in bnx2x_set_num_queues()) */
1264                 if ((rc != -ENOMEM) && (bp->int_mode != INT_MODE_INTx))
1265                         bnx2x_enable_msi(bp);
1266                 bnx2x_ack_int(bp);
1267                 rc = bnx2x_req_irq(bp);
1268                 if (rc) {
1269                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1270                         bnx2x_free_irq(bp, true);
1271                         goto load_error1;
1272                 }
1273                 if (bp->flags & USING_MSI_FLAG) {
1274                         bp->dev->irq = bp->pdev->irq;
1275                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
1276                                     bp->pdev->irq);
1277                 }
1278         }
1279
1280         /* Send LOAD_REQUEST command to MCP
1281            Returns the type of LOAD command:
1282            if it is the first port to be initialized
1283            common blocks should be initialized, otherwise - not
1284         */
1285         if (!BP_NOMCP(bp)) {
1286                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1287                 if (!load_code) {
1288                         BNX2X_ERR("MCP response failure, aborting\n");
1289                         rc = -EBUSY;
1290                         goto load_error2;
1291                 }
1292                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1293                         rc = -EBUSY; /* other port in diagnostic mode */
1294                         goto load_error2;
1295                 }
1296
1297         } else {
1298                 int port = BP_PORT(bp);
1299
1300                 DP(NETIF_MSG_IFUP, "NO MCP - load counts      %d, %d, %d\n",
1301                    load_count[0], load_count[1], load_count[2]);
1302                 load_count[0]++;
1303                 load_count[1 + port]++;
1304                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts  %d, %d, %d\n",
1305                    load_count[0], load_count[1], load_count[2]);
1306                 if (load_count[0] == 1)
1307                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1308                 else if (load_count[1 + port] == 1)
1309                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1310                 else
1311                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1312         }
1313
1314         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1315             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1316                 bp->port.pmf = 1;
1317         else
1318                 bp->port.pmf = 0;
1319         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1320
1321         /* Initialize HW */
1322         rc = bnx2x_init_hw(bp, load_code);
1323         if (rc) {
1324                 BNX2X_ERR("HW init failed, aborting\n");
1325                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1326                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1327                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1328                 goto load_error2;
1329         }
1330
1331         /* Setup NIC internals and enable interrupts */
1332         bnx2x_nic_init(bp, load_code);
1333
1334         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) &&
1335             (bp->common.shmem2_base))
1336                 SHMEM2_WR(bp, dcc_support,
1337                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1338                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1339
1340         /* Send LOAD_DONE command to MCP */
1341         if (!BP_NOMCP(bp)) {
1342                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1343                 if (!load_code) {
1344                         BNX2X_ERR("MCP response failure, aborting\n");
1345                         rc = -EBUSY;
1346                         goto load_error3;
1347                 }
1348         }
1349
1350         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1351
1352         rc = bnx2x_setup_leading(bp);
1353         if (rc) {
1354                 BNX2X_ERR("Setup leading failed!\n");
1355 #ifndef BNX2X_STOP_ON_ERROR
1356                 goto load_error3;
1357 #else
1358                 bp->panic = 1;
1359                 return -EBUSY;
1360 #endif
1361         }
1362
1363         if (CHIP_IS_E1H(bp))
1364                 if (bp->mf_config & FUNC_MF_CFG_FUNC_DISABLED) {
1365                         DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1366                         bp->flags |= MF_FUNC_DIS;
1367                 }
1368
1369         if (bp->state == BNX2X_STATE_OPEN) {
1370 #ifdef BCM_CNIC
1371                 /* Enable Timer scan */
1372                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1373 #endif
1374                 for_each_nondefault_queue(bp, i) {
1375                         rc = bnx2x_setup_multi(bp, i);
1376                         if (rc)
1377 #ifdef BCM_CNIC
1378                                 goto load_error4;
1379 #else
1380                                 goto load_error3;
1381 #endif
1382                 }
1383
1384                 if (CHIP_IS_E1(bp))
1385                         bnx2x_set_eth_mac_addr_e1(bp, 1);
1386                 else
1387                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
1388 #ifdef BCM_CNIC
1389                 /* Set iSCSI L2 MAC */
1390                 mutex_lock(&bp->cnic_mutex);
1391                 if (bp->cnic_eth_dev.drv_state & CNIC_DRV_STATE_REGD) {
1392                         bnx2x_set_iscsi_eth_mac_addr(bp, 1);
1393                         bp->cnic_flags |= BNX2X_CNIC_FLAG_MAC_SET;
1394                         bnx2x_init_sb(bp, bp->cnic_sb, bp->cnic_sb_mapping,
1395                                       CNIC_SB_ID(bp));
1396                 }
1397                 mutex_unlock(&bp->cnic_mutex);
1398 #endif
1399         }
1400
1401         if (bp->port.pmf)
1402                 bnx2x_initial_phy_init(bp, load_mode);
1403
1404         /* Start fast path */
1405         switch (load_mode) {
1406         case LOAD_NORMAL:
1407                 if (bp->state == BNX2X_STATE_OPEN) {
1408                         /* Tx queue should be only reenabled */
1409                         netif_tx_wake_all_queues(bp->dev);
1410                 }
1411                 /* Initialize the receive filter. */
1412                 bnx2x_set_rx_mode(bp->dev);
1413                 break;
1414
1415         case LOAD_OPEN:
1416                 netif_tx_start_all_queues(bp->dev);
1417                 if (bp->state != BNX2X_STATE_OPEN)
1418                         netif_tx_disable(bp->dev);
1419                 /* Initialize the receive filter. */
1420                 bnx2x_set_rx_mode(bp->dev);
1421                 break;
1422
1423         case LOAD_DIAG:
1424                 /* Initialize the receive filter. */
1425                 bnx2x_set_rx_mode(bp->dev);
1426                 bp->state = BNX2X_STATE_DIAG;
1427                 break;
1428
1429         default:
1430                 break;
1431         }
1432
1433         if (!bp->port.pmf)
1434                 bnx2x__link_status_update(bp);
1435
1436         /* start the timer */
1437         mod_timer(&bp->timer, jiffies + bp->current_interval);
1438
1439 #ifdef BCM_CNIC
1440         bnx2x_setup_cnic_irq_info(bp);
1441         if (bp->state == BNX2X_STATE_OPEN)
1442                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1443 #endif
1444         bnx2x_inc_load_cnt(bp);
1445
1446         bnx2x_release_firmware(bp);
1447
1448         return 0;
1449
1450 #ifdef BCM_CNIC
1451 load_error4:
1452         /* Disable Timer scan */
1453         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1454 #endif
1455 load_error3:
1456         bnx2x_int_disable_sync(bp, 1);
1457         if (!BP_NOMCP(bp)) {
1458                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1459                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1460         }
1461         bp->port.pmf = 0;
1462         /* Free SKBs, SGEs, TPA pool and driver internals */
1463         bnx2x_free_skbs(bp);
1464         for_each_queue(bp, i)
1465                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1466 load_error2:
1467         /* Release IRQs */
1468         bnx2x_free_irq(bp, false);
1469 load_error1:
1470         bnx2x_napi_disable(bp);
1471         for_each_queue(bp, i)
1472                 netif_napi_del(&bnx2x_fp(bp, i, napi));
1473         bnx2x_free_mem(bp);
1474
1475         bnx2x_release_firmware(bp);
1476
1477         return rc;
1478 }
1479
1480 /* must be called with rtnl_lock */
1481 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1482 {
1483         int i;
1484
1485         if (bp->state == BNX2X_STATE_CLOSED) {
1486                 /* Interface has been removed - nothing to recover */
1487                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1488                 bp->is_leader = 0;
1489                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1490                 smp_wmb();
1491
1492                 return -EINVAL;
1493         }
1494
1495 #ifdef BCM_CNIC
1496         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1497 #endif
1498         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1499
1500         /* Set "drop all" */
1501         bp->rx_mode = BNX2X_RX_MODE_NONE;
1502         bnx2x_set_storm_rx_mode(bp);
1503
1504         /* Disable HW interrupts, NAPI and Tx */
1505         bnx2x_netif_stop(bp, 1);
1506         netif_carrier_off(bp->dev);
1507
1508         del_timer_sync(&bp->timer);
1509         SHMEM_WR(bp, func_mb[BP_FUNC(bp)].drv_pulse_mb,
1510                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1511         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1512
1513         /* Release IRQs */
1514         bnx2x_free_irq(bp, false);
1515
1516         /* Cleanup the chip if needed */
1517         if (unload_mode != UNLOAD_RECOVERY)
1518                 bnx2x_chip_cleanup(bp, unload_mode);
1519
1520         bp->port.pmf = 0;
1521
1522         /* Free SKBs, SGEs, TPA pool and driver internals */
1523         bnx2x_free_skbs(bp);
1524         for_each_queue(bp, i)
1525                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1526         for_each_queue(bp, i)
1527                 netif_napi_del(&bnx2x_fp(bp, i, napi));
1528         bnx2x_free_mem(bp);
1529
1530         bp->state = BNX2X_STATE_CLOSED;
1531
1532         /* The last driver must disable a "close the gate" if there is no
1533          * parity attention or "process kill" pending.
1534          */
1535         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1536             bnx2x_reset_is_done(bp))
1537                 bnx2x_disable_close_the_gate(bp);
1538
1539         /* Reset MCP mail box sequence if there is on going recovery */
1540         if (unload_mode == UNLOAD_RECOVERY)
1541                 bp->fw_seq = 0;
1542
1543         return 0;
1544 }
1545 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1546 {
1547         u16 pmcsr;
1548
1549         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1550
1551         switch (state) {
1552         case PCI_D0:
1553                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1554                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1555                                        PCI_PM_CTRL_PME_STATUS));
1556
1557                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1558                         /* delay required during transition out of D3hot */
1559                         msleep(20);
1560                 break;
1561
1562         case PCI_D3hot:
1563                 /* If there are other clients above don't
1564                    shut down the power */
1565                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1566                         return 0;
1567                 /* Don't shut down the power for emulation and FPGA */
1568                 if (CHIP_REV_IS_SLOW(bp))
1569                         return 0;
1570
1571                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1572                 pmcsr |= 3;
1573
1574                 if (bp->wol)
1575                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1576
1577                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1578                                       pmcsr);
1579
1580                 /* No more memory access after this point until
1581                 * device is brought back to D0.
1582                 */
1583                 break;
1584
1585         default:
1586                 return -EINVAL;
1587         }
1588         return 0;
1589 }
1590
1591
1592
1593 /*
1594  * net_device service functions
1595  */
1596
1597 static int bnx2x_poll(struct napi_struct *napi, int budget)
1598 {
1599         int work_done = 0;
1600         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1601                                                  napi);
1602         struct bnx2x *bp = fp->bp;
1603
1604         while (1) {
1605 #ifdef BNX2X_STOP_ON_ERROR
1606                 if (unlikely(bp->panic)) {
1607                         napi_complete(napi);
1608                         return 0;
1609                 }
1610 #endif
1611
1612                 if (bnx2x_has_tx_work(fp))
1613                         bnx2x_tx_int(fp);
1614
1615                 if (bnx2x_has_rx_work(fp)) {
1616                         work_done += bnx2x_rx_int(fp, budget - work_done);
1617
1618                         /* must not complete if we consumed full budget */
1619                         if (work_done >= budget)
1620                                 break;
1621                 }
1622
1623                 /* Fall out from the NAPI loop if needed */
1624                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1625                         bnx2x_update_fpsb_idx(fp);
1626                 /* bnx2x_has_rx_work() reads the status block, thus we need
1627                  * to ensure that status block indices have been actually read
1628                  * (bnx2x_update_fpsb_idx) prior to this check
1629                  * (bnx2x_has_rx_work) so that we won't write the "newer"
1630                  * value of the status block to IGU (if there was a DMA right
1631                  * after bnx2x_has_rx_work and if there is no rmb, the memory
1632                  * reading (bnx2x_update_fpsb_idx) may be postponed to right
1633                  * before bnx2x_ack_sb). In this case there will never be
1634                  * another interrupt until there is another update of the
1635                  * status block, while there is still unhandled work.
1636                  */
1637                         rmb();
1638
1639                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1640                                 napi_complete(napi);
1641                                 /* Re-enable interrupts */
1642                                 bnx2x_ack_sb(bp, fp->sb_id, CSTORM_ID,
1643                                              le16_to_cpu(fp->fp_c_idx),
1644                                              IGU_INT_NOP, 1);
1645                                 bnx2x_ack_sb(bp, fp->sb_id, USTORM_ID,
1646                                              le16_to_cpu(fp->fp_u_idx),
1647                                              IGU_INT_ENABLE, 1);
1648                                 break;
1649                         }
1650                 }
1651         }
1652
1653         return work_done;
1654 }
1655
1656
1657 /* we split the first BD into headers and data BDs
1658  * to ease the pain of our fellow microcode engineers
1659  * we use one mapping for both BDs
1660  * So far this has only been observed to happen
1661  * in Other Operating Systems(TM)
1662  */
1663 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1664                                    struct bnx2x_fastpath *fp,
1665                                    struct sw_tx_bd *tx_buf,
1666                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1667                                    u16 bd_prod, int nbd)
1668 {
1669         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1670         struct eth_tx_bd *d_tx_bd;
1671         dma_addr_t mapping;
1672         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1673
1674         /* first fix first BD */
1675         h_tx_bd->nbd = cpu_to_le16(nbd);
1676         h_tx_bd->nbytes = cpu_to_le16(hlen);
1677
1678         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1679            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1680            h_tx_bd->addr_lo, h_tx_bd->nbd);
1681
1682         /* now get a new data BD
1683          * (after the pbd) and fill it */
1684         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1685         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1686
1687         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1688                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1689
1690         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1691         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1692         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1693
1694         /* this marks the BD as one that has no individual mapping */
1695         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1696
1697         DP(NETIF_MSG_TX_QUEUED,
1698            "TSO split data size is %d (%x:%x)\n",
1699            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1700
1701         /* update tx_bd */
1702         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1703
1704         return bd_prod;
1705 }
1706
1707 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1708 {
1709         if (fix > 0)
1710                 csum = (u16) ~csum_fold(csum_sub(csum,
1711                                 csum_partial(t_header - fix, fix, 0)));
1712
1713         else if (fix < 0)
1714                 csum = (u16) ~csum_fold(csum_add(csum,
1715                                 csum_partial(t_header, -fix, 0)));
1716
1717         return swab16(csum);
1718 }
1719
1720 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1721 {
1722         u32 rc;
1723
1724         if (skb->ip_summed != CHECKSUM_PARTIAL)
1725                 rc = XMIT_PLAIN;
1726
1727         else {
1728                 if (skb->protocol == htons(ETH_P_IPV6)) {
1729                         rc = XMIT_CSUM_V6;
1730                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1731                                 rc |= XMIT_CSUM_TCP;
1732
1733                 } else {
1734                         rc = XMIT_CSUM_V4;
1735                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1736                                 rc |= XMIT_CSUM_TCP;
1737                 }
1738         }
1739
1740         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1741                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1742
1743         else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1744                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1745
1746         return rc;
1747 }
1748
1749 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1750 /* check if packet requires linearization (packet is too fragmented)
1751    no need to check fragmentation if page size > 8K (there will be no
1752    violation to FW restrictions) */
1753 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1754                              u32 xmit_type)
1755 {
1756         int to_copy = 0;
1757         int hlen = 0;
1758         int first_bd_sz = 0;
1759
1760         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1761         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1762
1763                 if (xmit_type & XMIT_GSO) {
1764                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1765                         /* Check if LSO packet needs to be copied:
1766                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1767                         int wnd_size = MAX_FETCH_BD - 3;
1768                         /* Number of windows to check */
1769                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1770                         int wnd_idx = 0;
1771                         int frag_idx = 0;
1772                         u32 wnd_sum = 0;
1773
1774                         /* Headers length */
1775                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1776                                 tcp_hdrlen(skb);
1777
1778                         /* Amount of data (w/o headers) on linear part of SKB*/
1779                         first_bd_sz = skb_headlen(skb) - hlen;
1780
1781                         wnd_sum  = first_bd_sz;
1782
1783                         /* Calculate the first sum - it's special */
1784                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1785                                 wnd_sum +=
1786                                         skb_shinfo(skb)->frags[frag_idx].size;
1787
1788                         /* If there was data on linear skb data - check it */
1789                         if (first_bd_sz > 0) {
1790                                 if (unlikely(wnd_sum < lso_mss)) {
1791                                         to_copy = 1;
1792                                         goto exit_lbl;
1793                                 }
1794
1795                                 wnd_sum -= first_bd_sz;
1796                         }
1797
1798                         /* Others are easier: run through the frag list and
1799                            check all windows */
1800                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1801                                 wnd_sum +=
1802                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1803
1804                                 if (unlikely(wnd_sum < lso_mss)) {
1805                                         to_copy = 1;
1806                                         break;
1807                                 }
1808                                 wnd_sum -=
1809                                         skb_shinfo(skb)->frags[wnd_idx].size;
1810                         }
1811                 } else {
1812                         /* in non-LSO too fragmented packet should always
1813                            be linearized */
1814                         to_copy = 1;
1815                 }
1816         }
1817
1818 exit_lbl:
1819         if (unlikely(to_copy))
1820                 DP(NETIF_MSG_TX_QUEUED,
1821                    "Linearization IS REQUIRED for %s packet. "
1822                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1823                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1824                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1825
1826         return to_copy;
1827 }
1828 #endif
1829
1830 /* called with netif_tx_lock
1831  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1832  * netif_wake_queue()
1833  */
1834 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1835 {
1836         struct bnx2x *bp = netdev_priv(dev);
1837         struct bnx2x_fastpath *fp;
1838         struct netdev_queue *txq;
1839         struct sw_tx_bd *tx_buf;
1840         struct eth_tx_start_bd *tx_start_bd;
1841         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1842         struct eth_tx_parse_bd *pbd = NULL;
1843         u16 pkt_prod, bd_prod;
1844         int nbd, fp_index;
1845         dma_addr_t mapping;
1846         u32 xmit_type = bnx2x_xmit_type(bp, skb);
1847         int i;
1848         u8 hlen = 0;
1849         __le16 pkt_size = 0;
1850         struct ethhdr *eth;
1851         u8 mac_type = UNICAST_ADDRESS;
1852
1853 #ifdef BNX2X_STOP_ON_ERROR
1854         if (unlikely(bp->panic))
1855                 return NETDEV_TX_BUSY;
1856 #endif
1857
1858         fp_index = skb_get_queue_mapping(skb);
1859         txq = netdev_get_tx_queue(dev, fp_index);
1860
1861         fp = &bp->fp[fp_index];
1862
1863         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1864                 fp->eth_q_stats.driver_xoff++;
1865                 netif_tx_stop_queue(txq);
1866                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1867                 return NETDEV_TX_BUSY;
1868         }
1869
1870         DP(NETIF_MSG_TX_QUEUED, "SKB: summed %x  protocol %x  protocol(%x,%x)"
1871            "  gso type %x  xmit_type %x\n",
1872            skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1873            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1874
1875         eth = (struct ethhdr *)skb->data;
1876
1877         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1878         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1879                 if (is_broadcast_ether_addr(eth->h_dest))
1880                         mac_type = BROADCAST_ADDRESS;
1881                 else
1882                         mac_type = MULTICAST_ADDRESS;
1883         }
1884
1885 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1886         /* First, check if we need to linearize the skb (due to FW
1887            restrictions). No need to check fragmentation if page size > 8K
1888            (there will be no violation to FW restrictions) */
1889         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1890                 /* Statistics of linearization */
1891                 bp->lin_cnt++;
1892                 if (skb_linearize(skb) != 0) {
1893                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1894                            "silently dropping this SKB\n");
1895                         dev_kfree_skb_any(skb);
1896                         return NETDEV_TX_OK;
1897                 }
1898         }
1899 #endif
1900
1901         /*
1902         Please read carefully. First we use one BD which we mark as start,
1903         then we have a parsing info BD (used for TSO or xsum),
1904         and only then we have the rest of the TSO BDs.
1905         (don't forget to mark the last one as last,
1906         and to unmap only AFTER you write to the BD ...)
1907         And above all, all pdb sizes are in words - NOT DWORDS!
1908         */
1909
1910         pkt_prod = fp->tx_pkt_prod++;
1911         bd_prod = TX_BD(fp->tx_bd_prod);
1912
1913         /* get a tx_buf and first BD */
1914         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
1915         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
1916
1917         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
1918         tx_start_bd->general_data =  (mac_type <<
1919                                         ETH_TX_START_BD_ETH_ADDR_TYPE_SHIFT);
1920         /* header nbd */
1921         tx_start_bd->general_data |= (1 << ETH_TX_START_BD_HDR_NBDS_SHIFT);
1922
1923         /* remember the first BD of the packet */
1924         tx_buf->first_bd = fp->tx_bd_prod;
1925         tx_buf->skb = skb;
1926         tx_buf->flags = 0;
1927
1928         DP(NETIF_MSG_TX_QUEUED,
1929            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
1930            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
1931
1932 #ifdef BCM_VLAN
1933         if ((bp->vlgrp != NULL) && vlan_tx_tag_present(skb) &&
1934             (bp->flags & HW_VLAN_TX_FLAG)) {
1935                 tx_start_bd->vlan = cpu_to_le16(vlan_tx_tag_get(skb));
1936                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_VLAN_TAG;
1937         } else
1938 #endif
1939                 tx_start_bd->vlan = cpu_to_le16(pkt_prod);
1940
1941         /* turn on parsing and get a BD */
1942         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1943         pbd = &fp->tx_desc_ring[bd_prod].parse_bd;
1944
1945         memset(pbd, 0, sizeof(struct eth_tx_parse_bd));
1946
1947         if (xmit_type & XMIT_CSUM) {
1948                 hlen = (skb_network_header(skb) - skb->data) / 2;
1949
1950                 /* for now NS flag is not used in Linux */
1951                 pbd->global_data =
1952                         (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1953                                  ETH_TX_PARSE_BD_LLC_SNAP_EN_SHIFT));
1954
1955                 pbd->ip_hlen = (skb_transport_header(skb) -
1956                                 skb_network_header(skb)) / 2;
1957
1958                 hlen += pbd->ip_hlen + tcp_hdrlen(skb) / 2;
1959
1960                 pbd->total_hlen = cpu_to_le16(hlen);
1961                 hlen = hlen*2;
1962
1963                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
1964
1965                 if (xmit_type & XMIT_CSUM_V4)
1966                         tx_start_bd->bd_flags.as_bitfield |=
1967                                                 ETH_TX_BD_FLAGS_IP_CSUM;
1968                 else
1969                         tx_start_bd->bd_flags.as_bitfield |=
1970                                                 ETH_TX_BD_FLAGS_IPV6;
1971
1972                 if (xmit_type & XMIT_CSUM_TCP) {
1973                         pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1974
1975                 } else {
1976                         s8 fix = SKB_CS_OFF(skb); /* signed! */
1977
1978                         pbd->global_data |= ETH_TX_PARSE_BD_UDP_CS_FLG;
1979
1980                         DP(NETIF_MSG_TX_QUEUED,
1981                            "hlen %d  fix %d  csum before fix %x\n",
1982                            le16_to_cpu(pbd->total_hlen), fix, SKB_CS(skb));
1983
1984                         /* HW bug: fixup the CSUM */
1985                         pbd->tcp_pseudo_csum =
1986                                 bnx2x_csum_fix(skb_transport_header(skb),
1987                                                SKB_CS(skb), fix);
1988
1989                         DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1990                            pbd->tcp_pseudo_csum);
1991                 }
1992         }
1993
1994         mapping = dma_map_single(&bp->pdev->dev, skb->data,
1995                                  skb_headlen(skb), DMA_TO_DEVICE);
1996
1997         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1998         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1999         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2000         tx_start_bd->nbd = cpu_to_le16(nbd);
2001         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2002         pkt_size = tx_start_bd->nbytes;
2003
2004         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2005            "  nbytes %d  flags %x  vlan %x\n",
2006            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2007            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2008            tx_start_bd->bd_flags.as_bitfield, le16_to_cpu(tx_start_bd->vlan));
2009
2010         if (xmit_type & XMIT_GSO) {
2011
2012                 DP(NETIF_MSG_TX_QUEUED,
2013                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2014                    skb->len, hlen, skb_headlen(skb),
2015                    skb_shinfo(skb)->gso_size);
2016
2017                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2018
2019                 if (unlikely(skb_headlen(skb) > hlen))
2020                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2021                                                  hlen, bd_prod, ++nbd);
2022
2023                 pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2024                 pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
2025                 pbd->tcp_flags = pbd_tcp_flags(skb);
2026
2027                 if (xmit_type & XMIT_GSO_V4) {
2028                         pbd->ip_id = swab16(ip_hdr(skb)->id);
2029                         pbd->tcp_pseudo_csum =
2030                                 swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
2031                                                           ip_hdr(skb)->daddr,
2032                                                           0, IPPROTO_TCP, 0));
2033
2034                 } else
2035                         pbd->tcp_pseudo_csum =
2036                                 swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2037                                                         &ipv6_hdr(skb)->daddr,
2038                                                         0, IPPROTO_TCP, 0));
2039
2040                 pbd->global_data |= ETH_TX_PARSE_BD_PSEUDO_CS_WITHOUT_LEN;
2041         }
2042         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2043
2044         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2045                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2046
2047                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2048                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2049                 if (total_pkt_bd == NULL)
2050                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2051
2052                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2053                                        frag->page_offset,
2054                                        frag->size, DMA_TO_DEVICE);
2055
2056                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2057                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2058                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2059                 le16_add_cpu(&pkt_size, frag->size);
2060
2061                 DP(NETIF_MSG_TX_QUEUED,
2062                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2063                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2064                    le16_to_cpu(tx_data_bd->nbytes));
2065         }
2066
2067         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2068
2069         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2070
2071         /* now send a tx doorbell, counting the next BD
2072          * if the packet contains or ends with it
2073          */
2074         if (TX_BD_POFF(bd_prod) < nbd)
2075                 nbd++;
2076
2077         if (total_pkt_bd != NULL)
2078                 total_pkt_bd->total_pkt_bytes = pkt_size;
2079
2080         if (pbd)
2081                 DP(NETIF_MSG_TX_QUEUED,
2082                    "PBD @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2083                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2084                    pbd, pbd->global_data, pbd->ip_hlen, pbd->ip_id,
2085                    pbd->lso_mss, pbd->tcp_flags, pbd->tcp_pseudo_csum,
2086                    pbd->tcp_send_seq, le16_to_cpu(pbd->total_hlen));
2087
2088         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2089
2090         /*
2091          * Make sure that the BD data is updated before updating the producer
2092          * since FW might read the BD right after the producer is updated.
2093          * This is only applicable for weak-ordered memory model archs such
2094          * as IA-64. The following barrier is also mandatory since FW will
2095          * assumes packets must have BDs.
2096          */
2097         wmb();
2098
2099         fp->tx_db.data.prod += nbd;
2100         barrier();
2101         DOORBELL(bp, fp->index, fp->tx_db.raw);
2102
2103         mmiowb();
2104
2105         fp->tx_bd_prod += nbd;
2106
2107         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2108                 netif_tx_stop_queue(txq);
2109
2110                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2111                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2112                  * fp->bd_tx_cons */
2113                 smp_mb();
2114
2115                 fp->eth_q_stats.driver_xoff++;
2116                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2117                         netif_tx_wake_queue(txq);
2118         }
2119         fp->tx_pkt++;
2120
2121         return NETDEV_TX_OK;
2122 }
2123 /* called with rtnl_lock */
2124 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2125 {
2126         struct sockaddr *addr = p;
2127         struct bnx2x *bp = netdev_priv(dev);
2128
2129         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2130                 return -EINVAL;
2131
2132         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2133         if (netif_running(dev)) {
2134                 if (CHIP_IS_E1(bp))
2135                         bnx2x_set_eth_mac_addr_e1(bp, 1);
2136                 else
2137                         bnx2x_set_eth_mac_addr_e1h(bp, 1);
2138         }
2139
2140         return 0;
2141 }
2142
2143 /* called with rtnl_lock */
2144 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2145 {
2146         struct bnx2x *bp = netdev_priv(dev);
2147         int rc = 0;
2148
2149         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2150                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2151                 return -EAGAIN;
2152         }
2153
2154         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2155             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2156                 return -EINVAL;
2157
2158         /* This does not race with packet allocation
2159          * because the actual alloc size is
2160          * only updated as part of load
2161          */
2162         dev->mtu = new_mtu;
2163
2164         if (netif_running(dev)) {
2165                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2166                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2167         }
2168
2169         return rc;
2170 }
2171
2172 void bnx2x_tx_timeout(struct net_device *dev)
2173 {
2174         struct bnx2x *bp = netdev_priv(dev);
2175
2176 #ifdef BNX2X_STOP_ON_ERROR
2177         if (!bp->panic)
2178                 bnx2x_panic();
2179 #endif
2180         /* This allows the netif to be shutdown gracefully before resetting */
2181         schedule_delayed_work(&bp->reset_task, 0);
2182 }
2183
2184 #ifdef BCM_VLAN
2185 /* called with rtnl_lock */
2186 void bnx2x_vlan_rx_register(struct net_device *dev,
2187                                    struct vlan_group *vlgrp)
2188 {
2189         struct bnx2x *bp = netdev_priv(dev);
2190
2191         bp->vlgrp = vlgrp;
2192
2193         /* Set flags according to the required capabilities */
2194         bp->flags &= ~(HW_VLAN_RX_FLAG | HW_VLAN_TX_FLAG);
2195
2196         if (dev->features & NETIF_F_HW_VLAN_TX)
2197                 bp->flags |= HW_VLAN_TX_FLAG;
2198
2199         if (dev->features & NETIF_F_HW_VLAN_RX)
2200                 bp->flags |= HW_VLAN_RX_FLAG;
2201
2202         if (netif_running(dev))
2203                 bnx2x_set_client_config(bp);
2204 }
2205
2206 #endif
2207 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2208 {
2209         struct net_device *dev = pci_get_drvdata(pdev);
2210         struct bnx2x *bp;
2211
2212         if (!dev) {
2213                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2214                 return -ENODEV;
2215         }
2216         bp = netdev_priv(dev);
2217
2218         rtnl_lock();
2219
2220         pci_save_state(pdev);
2221
2222         if (!netif_running(dev)) {
2223                 rtnl_unlock();
2224                 return 0;
2225         }
2226
2227         netif_device_detach(dev);
2228
2229         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2230
2231         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2232
2233         rtnl_unlock();
2234
2235         return 0;
2236 }
2237
2238 int bnx2x_resume(struct pci_dev *pdev)
2239 {
2240         struct net_device *dev = pci_get_drvdata(pdev);
2241         struct bnx2x *bp;
2242         int rc;
2243
2244         if (!dev) {
2245                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2246                 return -ENODEV;
2247         }
2248         bp = netdev_priv(dev);
2249
2250         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2251                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2252                 return -EAGAIN;
2253         }
2254
2255         rtnl_lock();
2256
2257         pci_restore_state(pdev);
2258
2259         if (!netif_running(dev)) {
2260                 rtnl_unlock();
2261                 return 0;
2262         }
2263
2264         bnx2x_set_power_state(bp, PCI_D0);
2265         netif_device_attach(dev);
2266
2267         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2268
2269         rtnl_unlock();
2270
2271         return rc;
2272 }