drivers/net/bnx2x/bnx2x_cmn.c

   1 /* bnx2x_cmn.c: Broadcom Everest network driver.
   2  *
   3  * Copyright (c) 2007-2010 Broadcom Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  *
   9  * Maintained by: Eilon Greenstein <eilong@broadcom.com>
  10  * Written by: Eliezer Tamir
  11  * Based on code from Michael Chan's bnx2 driver
  12  * UDP CSUM errata workaround by Arik Gendelman
  13  * Slowpath and fastpath rework by Vladislav Zolotarov
  14  * Statistics and Link management by Yitchak Gertner
  15  *
  16  */
  17
  18 #include <linux/etherdevice.h>
  19 #include <linux/ip.h>
  20 #include <net/ipv6.h>
  21 #include <net/ip6_checksum.h>
  22 #include <linux/firmware.h>
  23 #include "bnx2x_cmn.h"
  24
  25 #ifdef BCM_VLAN
  26 #include <linux/if_vlan.h>
  27 #endif
  28
  29 #include "bnx2x_init.h"
  30
  31
  32 /* free skb in the packet ring at pos idx
  33  * return idx of last bd freed
  34  */
  35 static u16 bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fastpath *fp,
  36                              u16 idx)
  37 {
  38         struct sw_tx_bd *tx_buf = &fp->tx_buf_ring[idx];
  39         struct eth_tx_start_bd *tx_start_bd;
  40         struct eth_tx_bd *tx_data_bd;
  41         struct sk_buff *skb = tx_buf->skb;
  42         u16 bd_idx = TX_BD(tx_buf->first_bd), new_cons;
  43         int nbd;
  44
  45         /* prefetch skb end pointer to speedup dev_kfree_skb() */
  46         prefetch(&skb->end);
  47
  48         DP(BNX2X_MSG_OFF, "pkt_idx %d  buff @(%p)->skb %p\n",
  49            idx, tx_buf, skb);
  50
  51         /* unmap first bd */
  52         DP(BNX2X_MSG_OFF, "free bd_idx %d\n", bd_idx);
  53         tx_start_bd = &fp->tx_desc_ring[bd_idx].start_bd;
  54         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
  55                          BD_UNMAP_LEN(tx_start_bd), DMA_TO_DEVICE);
  56
  57         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
  58 #ifdef BNX2X_STOP_ON_ERROR
  59         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
  60                 BNX2X_ERR("BAD nbd!\n");
  61                 bnx2x_panic();
  62         }
  63 #endif
  64         new_cons = nbd + tx_buf->first_bd;
  65
  66         /* Get the next bd */
  67         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  68
  69         /* Skip a parse bd... */
  70         --nbd;
  71         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  72
  73         /* ...and the TSO split header bd since they have no mapping */
  74         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
  75                 --nbd;
  76                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  77         }
  78
  79         /* now free frags */
  80         while (nbd > 0) {
  81
  82                 DP(BNX2X_MSG_OFF, "free frag bd_idx %d\n", bd_idx);
  83                 tx_data_bd = &fp->tx_desc_ring[bd_idx].reg_bd;
  84                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
  85                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
  86                 if (--nbd)
  87                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
  88         }
  89
  90         /* release skb */
  91         WARN_ON(!skb);
  92         dev_kfree_skb(skb);
  93         tx_buf->first_bd = 0;
  94         tx_buf->skb = NULL;
  95
  96         return new_cons;
  97 }
  98
  99 int bnx2x_tx_int(struct bnx2x_fastpath *fp)
 100 {
 101         struct bnx2x *bp = fp->bp;
 102         struct netdev_queue *txq;
 103         u16 hw_cons, sw_cons, bd_cons = fp->tx_bd_cons;
 104
 105 #ifdef BNX2X_STOP_ON_ERROR
 106         if (unlikely(bp->panic))
 107                 return -1;
 108 #endif
 109
 110         txq = netdev_get_tx_queue(bp->dev, fp->index);
 111         hw_cons = le16_to_cpu(*fp->tx_cons_sb);
 112         sw_cons = fp->tx_pkt_cons;
 113
 114         while (sw_cons != hw_cons) {
 115                 u16 pkt_cons;
 116
 117                 pkt_cons = TX_BD(sw_cons);
 118
 119                 DP(NETIF_MSG_TX_DONE, "queue[%d]: hw_cons %u  sw_cons %u "
 120                                       " pkt_cons %u\n",
 121                    fp->index, hw_cons, sw_cons, pkt_cons);
 122
 123                 bd_cons = bnx2x_free_tx_pkt(bp, fp, pkt_cons);
 124                 sw_cons++;
 125         }
 126
 127         fp->tx_pkt_cons = sw_cons;
 128         fp->tx_bd_cons = bd_cons;
 129
 130         /* Need to make the tx_bd_cons update visible to start_xmit()
 131          * before checking for netif_tx_queue_stopped().  Without the
 132          * memory barrier, there is a small possibility that
 133          * start_xmit() will miss it and cause the queue to be stopped
 134          * forever.
 135          */
 136         smp_mb();
 137
 138         if (unlikely(netif_tx_queue_stopped(txq))) {
 139                 /* Taking tx_lock() is needed to prevent reenabling the queue
 140                  * while it's empty. This could have happen if rx_action() gets
 141                  * suspended in bnx2x_tx_int() after the condition before
 142                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
 143                  *
 144                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
 145                  * sends some packets consuming the whole queue again->
 146                  * stops the queue
 147                  */
 148
 149                 __netif_tx_lock(txq, smp_processor_id());
 150
 151                 if ((netif_tx_queue_stopped(txq)) &&
 152                     (bp->state == BNX2X_STATE_OPEN) &&
 153                     (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3))
 154                         netif_tx_wake_queue(txq);
 155
 156                 __netif_tx_unlock(txq);
 157         }
 158         return 0;
 159 }
 160
 161 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
 162                                              u16 idx)
 163 {
 164         u16 last_max = fp->last_max_sge;
 165
 166         if (SUB_S16(idx, last_max) > 0)
 167                 fp->last_max_sge = idx;
 168 }
 169
 170 static void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
 171                                   struct eth_fast_path_rx_cqe *fp_cqe)
 172 {
 173         struct bnx2x *bp = fp->bp;
 174         u16 sge_len = SGE_PAGE_ALIGN(le16_to_cpu(fp_cqe->pkt_len) -
 175                                      le16_to_cpu(fp_cqe->len_on_bd)) >>
 176                       SGE_PAGE_SHIFT;
 177         u16 last_max, last_elem, first_elem;
 178         u16 delta = 0;
 179         u16 i;
 180
 181         if (!sge_len)
 182                 return;
 183
 184         /* First mark all used pages */
 185         for (i = 0; i < sge_len; i++)
 186                 SGE_MASK_CLEAR_BIT(fp,
 187                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[i])));
 188
 189         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
 190            sge_len - 1, le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 191
 192         /* Here we assume that the last SGE index is the biggest */
 193         prefetch((void *)(fp->sge_mask));
 194         bnx2x_update_last_max_sge(fp,
 195                 le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[sge_len - 1]));
 196
 197         last_max = RX_SGE(fp->last_max_sge);
 198         last_elem = last_max >> RX_SGE_MASK_ELEM_SHIFT;
 199         first_elem = RX_SGE(fp->rx_sge_prod) >> RX_SGE_MASK_ELEM_SHIFT;
 200
 201         /* If ring is not full */
 202         if (last_elem + 1 != first_elem)
 203                 last_elem++;
 204
 205         /* Now update the prod */
 206         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
 207                 if (likely(fp->sge_mask[i]))
 208                         break;
 209
 210                 fp->sge_mask[i] = RX_SGE_MASK_ELEM_ONE_MASK;
 211                 delta += RX_SGE_MASK_ELEM_SZ;
 212         }
 213
 214         if (delta > 0) {
 215                 fp->rx_sge_prod += delta;
 216                 /* clear page-end entries */
 217                 bnx2x_clear_sge_mask_next_elems(fp);
 218         }
 219
 220         DP(NETIF_MSG_RX_STATUS,
 221            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
 222            fp->last_max_sge, fp->rx_sge_prod);
 223 }
 224
 225 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, u16 queue,
 226                             struct sk_buff *skb, u16 cons, u16 prod)
 227 {
 228         struct bnx2x *bp = fp->bp;
 229         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
 230         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
 231         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
 232         dma_addr_t mapping;
 233
 234         /* move empty skb from pool to prod and map it */
 235         prod_rx_buf->skb = fp->tpa_pool[queue].skb;
 236         mapping = dma_map_single(&bp->pdev->dev, fp->tpa_pool[queue].skb->data,
 237                                  bp->rx_buf_size, DMA_FROM_DEVICE);
 238         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
 239
 240         /* move partial skb from cons to pool (don't unmap yet) */
 241         fp->tpa_pool[queue] = *cons_rx_buf;
 242
 243         /* mark bin state as start - print error if current state != stop */
 244         if (fp->tpa_state[queue] != BNX2X_TPA_STOP)
 245                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
 246
 247         fp->tpa_state[queue] = BNX2X_TPA_START;
 248
 249         /* point prod_bd to new skb */
 250         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
 251         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
 252
 253 #ifdef BNX2X_STOP_ON_ERROR
 254         fp->tpa_queue_used |= (1 << queue);
 255 #ifdef _ASM_GENERIC_INT_L64_H
 256         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%lx\n",
 257 #else
 258         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
 259 #endif
 260            fp->tpa_queue_used);
 261 #endif
 262 }
 263
 264 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 265                                struct sk_buff *skb,
 266                                struct eth_fast_path_rx_cqe *fp_cqe,
 267                                u16 cqe_idx)
 268 {
 269         struct sw_rx_page *rx_pg, old_rx_pg;
 270         u16 len_on_bd = le16_to_cpu(fp_cqe->len_on_bd);
 271         u32 i, frag_len, frag_size, pages;
 272         int err;
 273         int j;
 274
 275         frag_size = le16_to_cpu(fp_cqe->pkt_len) - len_on_bd;
 276         pages = SGE_PAGE_ALIGN(frag_size) >> SGE_PAGE_SHIFT;
 277
 278         /* This is needed in order to enable forwarding support */
 279         if (frag_size)
 280                 skb_shinfo(skb)->gso_size = min((u32)SGE_PAGE_SIZE,
 281                                                max(frag_size, (u32)len_on_bd));
 282
 283 #ifdef BNX2X_STOP_ON_ERROR
 284         if (pages > min_t(u32, 8, MAX_SKB_FRAGS)*SGE_PAGE_SIZE*PAGES_PER_SGE) {
 285                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
 286                           pages, cqe_idx);
 287                 BNX2X_ERR("fp_cqe->pkt_len = %d  fp_cqe->len_on_bd = %d\n",
 288                           fp_cqe->pkt_len, len_on_bd);
 289                 bnx2x_panic();
 290                 return -EINVAL;
 291         }
 292 #endif
 293
 294         /* Run through the SGL and compose the fragmented skb */
 295         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
 296                 u16 sge_idx =
 297                         RX_SGE(le16_to_cpu(fp_cqe->sgl_or_raw_data.sgl[j]));
 298
 299                 /* FW gives the indices of the SGE as if the ring is an array
 300                    (meaning that "next" element will consume 2 indices) */
 301                 frag_len = min(frag_size, (u32)(SGE_PAGE_SIZE*PAGES_PER_SGE));
 302                 rx_pg = &fp->rx_page_ring[sge_idx];
 303                 old_rx_pg = *rx_pg;
 304
 305                 /* If we fail to allocate a substitute page, we simply stop
 306                    where we are and drop the whole packet */
 307                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx);
 308                 if (unlikely(err)) {
 309                         fp->eth_q_stats.rx_skb_alloc_failed++;
 310                         return err;
 311                 }
 312
 313                 /* Unmap the page as we r going to pass it to the stack */
 314                 dma_unmap_page(&bp->pdev->dev,
 315                                dma_unmap_addr(&old_rx_pg, mapping),
 316                                SGE_PAGE_SIZE*PAGES_PER_SGE, DMA_FROM_DEVICE);
 317
 318                 /* Add one frag and update the appropriate fields in the skb */
 319                 skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
 320
 321                 skb->data_len += frag_len;
 322                 skb->truesize += frag_len;
 323                 skb->len += frag_len;
 324
 325                 frag_size -= frag_len;
 326         }
 327
 328         return 0;
 329 }
 330
 331 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 332                            u16 queue, int pad, int len, union eth_rx_cqe *cqe,
 333                            u16 cqe_idx)
 334 {
 335         struct sw_rx_bd *rx_buf = &fp->tpa_pool[queue];
 336         struct sk_buff *skb = rx_buf->skb;
 337         /* alloc new skb */
 338         struct sk_buff *new_skb = netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 339
 340         /* Unmap skb in the pool anyway, as we are going to change
 341            pool entry status to BNX2X_TPA_STOP even if new skb allocation
 342            fails. */
 343         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
 344                          bp->rx_buf_size, DMA_FROM_DEVICE);
 345
 346         if (likely(new_skb)) {
 347                 /* fix ip xsum and give it to the stack */
 348                 /* (no need to map the new skb) */
 349 #ifdef BCM_VLAN
 350                 int is_vlan_cqe =
 351                         (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 352                          PARSING_FLAGS_VLAN);
 353                 int is_not_hwaccel_vlan_cqe =
 354                         (is_vlan_cqe && (!(bp->flags & HW_VLAN_RX_FLAG)));
 355 #endif
 356
 357                 prefetch(skb);
 358                 prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 359
 360 #ifdef BNX2X_STOP_ON_ERROR
 361                 if (pad + len > bp->rx_buf_size) {
 362                         BNX2X_ERR("skb_put is about to fail...  "
 363                                   "pad %d  len %d  rx_buf_size %d\n",
 364                                   pad, len, bp->rx_buf_size);
 365                         bnx2x_panic();
 366                         return;
 367                 }
 368 #endif
 369
 370                 skb_reserve(skb, pad);
 371                 skb_put(skb, len);
 372
 373                 skb->protocol = eth_type_trans(skb, bp->dev);
 374                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 375
 376                 {
 377                         struct iphdr *iph;
 378
 379                         iph = (struct iphdr *)skb->data;
 380 #ifdef BCM_VLAN
 381                         /* If there is no Rx VLAN offloading -
 382                            take VLAN tag into an account */
 383                         if (unlikely(is_not_hwaccel_vlan_cqe))
 384                                 iph = (struct iphdr *)((u8 *)iph + VLAN_HLEN);
 385 #endif
 386                         iph->check = 0;
 387                         iph->check = ip_fast_csum((u8 *)iph, iph->ihl);
 388                 }
 389
 390                 if (!bnx2x_fill_frag_skb(bp, fp, skb,
 391                                          &cqe->fast_path_cqe, cqe_idx)) {
 392 #ifdef BCM_VLAN
 393                         if ((bp->vlgrp != NULL) &&
 394                                 (le16_to_cpu(cqe->fast_path_cqe.
 395                                 pars_flags.flags) & PARSING_FLAGS_VLAN))
 396                                 vlan_gro_receive(&fp->napi, bp->vlgrp,
 397                                                  le16_to_cpu(cqe->fast_path_cqe.
 398                                                              vlan_tag), skb);
 399                         else
 400 #endif
 401                                 napi_gro_receive(&fp->napi, skb);
 402                 } else {
 403                         DP(NETIF_MSG_RX_STATUS, "Failed to allocate new pages"
 404                            " - dropping packet!\n");
 405                         dev_kfree_skb(skb);
 406                 }
 407
 408
 409                 /* put new skb in bin */
 410                 fp->tpa_pool[queue].skb = new_skb;
 411
 412         } else {
 413                 /* else drop the packet and keep the buffer in the bin */
 414                 DP(NETIF_MSG_RX_STATUS,
 415                    "Failed to allocate new skb - dropping packet!\n");
 416                 fp->eth_q_stats.rx_skb_alloc_failed++;
 417         }
 418
 419         fp->tpa_state[queue] = BNX2X_TPA_STOP;
 420 }
 421
 422 /* Set Toeplitz hash value in the skb using the value from the
 423  * CQE (calculated by HW).
 424  */
 425 static inline void bnx2x_set_skb_rxhash(struct bnx2x *bp, union eth_rx_cqe *cqe,
 426                                         struct sk_buff *skb)
 427 {
 428         /* Set Toeplitz hash from CQE */
 429         if ((bp->dev->features & NETIF_F_RXHASH) &&
 430             (cqe->fast_path_cqe.status_flags &
 431              ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG))
 432                 skb->rxhash =
 433                 le32_to_cpu(cqe->fast_path_cqe.rss_hash_result);
 434 }
 435
 436 int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
 437 {
 438         struct bnx2x *bp = fp->bp;
 439         u16 bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
 440         u16 hw_comp_cons, sw_comp_cons, sw_comp_prod;
 441         int rx_pkt = 0;
 442
 443 #ifdef BNX2X_STOP_ON_ERROR
 444         if (unlikely(bp->panic))
 445                 return 0;
 446 #endif
 447
 448         /* CQ "next element" is of the size of the regular element,
 449            that's why it's ok here */
 450         hw_comp_cons = le16_to_cpu(*fp->rx_cons_sb);
 451         if ((hw_comp_cons & MAX_RCQ_DESC_CNT) == MAX_RCQ_DESC_CNT)
 452                 hw_comp_cons++;
 453
 454         bd_cons = fp->rx_bd_cons;
 455         bd_prod = fp->rx_bd_prod;
 456         bd_prod_fw = bd_prod;
 457         sw_comp_cons = fp->rx_comp_cons;
 458         sw_comp_prod = fp->rx_comp_prod;
 459
 460         /* Memory barrier necessary as speculative reads of the rx
 461          * buffer can be ahead of the index in the status block
 462          */
 463         rmb();
 464
 465         DP(NETIF_MSG_RX_STATUS,
 466            "queue[%d]:  hw_comp_cons %u  sw_comp_cons %u\n",
 467            fp->index, hw_comp_cons, sw_comp_cons);
 468
 469         while (sw_comp_cons != hw_comp_cons) {
 470                 struct sw_rx_bd *rx_buf = NULL;
 471                 struct sk_buff *skb;
 472                 union eth_rx_cqe *cqe;
 473                 u8 cqe_fp_flags;
 474                 u16 len, pad;
 475
 476                 comp_ring_cons = RCQ_BD(sw_comp_cons);
 477                 bd_prod = RX_BD(bd_prod);
 478                 bd_cons = RX_BD(bd_cons);
 479
 480                 /* Prefetch the page containing the BD descriptor
 481                    at producer's index. It will be needed when new skb is
 482                    allocated */
 483                 prefetch((void *)(PAGE_ALIGN((unsigned long)
 484                                              (&fp->rx_desc_ring[bd_prod])) -
 485                                   PAGE_SIZE + 1));
 486
 487                 cqe = &fp->rx_comp_ring[comp_ring_cons];
 488                 cqe_fp_flags = cqe->fast_path_cqe.type_error_flags;
 489
 490                 DP(NETIF_MSG_RX_STATUS, "CQE type %x  err %x  status %x"
 491                    "  queue %x  vlan %x  len %u\n", CQE_TYPE(cqe_fp_flags),
 492                    cqe_fp_flags, cqe->fast_path_cqe.status_flags,
 493                    le32_to_cpu(cqe->fast_path_cqe.rss_hash_result),
 494                    le16_to_cpu(cqe->fast_path_cqe.vlan_tag),
 495                    le16_to_cpu(cqe->fast_path_cqe.pkt_len));
 496
 497                 /* is this a slowpath msg? */
 498                 if (unlikely(CQE_TYPE(cqe_fp_flags))) {
 499                         bnx2x_sp_event(fp, cqe);
 500                         goto next_cqe;
 501
 502                 /* this is an rx packet */
 503                 } else {
 504                         rx_buf = &fp->rx_buf_ring[bd_cons];
 505                         skb = rx_buf->skb;
 506                         prefetch(skb);
 507                         len = le16_to_cpu(cqe->fast_path_cqe.pkt_len);
 508                         pad = cqe->fast_path_cqe.placement_offset;
 509
 510                         /* - If CQE is marked both TPA_START and TPA_END it is
 511                          *   a non-TPA CQE.
 512                          * - FP CQE will always have either TPA_START or/and
 513                          *   TPA_STOP flags set.
 514                          */
 515                         if ((!fp->disable_tpa) &&
 516                             (TPA_TYPE(cqe_fp_flags) !=
 517                                         (TPA_TYPE_START | TPA_TYPE_END))) {
 518                                 u16 queue = cqe->fast_path_cqe.queue_index;
 519
 520                                 if (TPA_TYPE(cqe_fp_flags) == TPA_TYPE_START) {
 521                                         DP(NETIF_MSG_RX_STATUS,
 522                                            "calling tpa_start on queue %d\n",
 523                                            queue);
 524
 525                                         bnx2x_tpa_start(fp, queue, skb,
 526                                                         bd_cons, bd_prod);
 527
 528                                         /* Set Toeplitz hash for an LRO skb */
 529                                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 530
 531                                         goto next_rx;
 532                                 } else { /* TPA_STOP */
 533                                         DP(NETIF_MSG_RX_STATUS,
 534                                            "calling tpa_stop on queue %d\n",
 535                                            queue);
 536
 537                                         if (!BNX2X_RX_SUM_FIX(cqe))
 538                                                 BNX2X_ERR("STOP on none TCP "
 539                                                           "data\n");
 540
 541                                         /* This is a size of the linear data
 542                                            on this skb */
 543                                         len = le16_to_cpu(cqe->fast_path_cqe.
 544                                                                 len_on_bd);
 545                                         bnx2x_tpa_stop(bp, fp, queue, pad,
 546                                                     len, cqe, comp_ring_cons);
 547 #ifdef BNX2X_STOP_ON_ERROR
 548                                         if (bp->panic)
 549                                                 return 0;
 550 #endif
 551
 552                                         bnx2x_update_sge_prod(fp,
 553                                                         &cqe->fast_path_cqe);
 554                                         goto next_cqe;
 555                                 }
 556                         }
 557
 558                         dma_sync_single_for_device(&bp->pdev->dev,
 559                                         dma_unmap_addr(rx_buf, mapping),
 560                                                    pad + RX_COPY_THRESH,
 561                                                    DMA_FROM_DEVICE);
 562                         prefetch(((char *)(skb)) + L1_CACHE_BYTES);
 563
 564                         /* is this an error packet? */
 565                         if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
 566                                 DP(NETIF_MSG_RX_ERR,
 567                                    "ERROR  flags %x  rx packet %u\n",
 568                                    cqe_fp_flags, sw_comp_cons);
 569                                 fp->eth_q_stats.rx_err_discard_pkt++;
 570                                 goto reuse_rx;
 571                         }
 572
 573                         /* Since we don't have a jumbo ring
 574                          * copy small packets if mtu > 1500
 575                          */
 576                         if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
 577                             (len <= RX_COPY_THRESH)) {
 578                                 struct sk_buff *new_skb;
 579
 580                                 new_skb = netdev_alloc_skb(bp->dev,
 581                                                            len + pad);
 582                                 if (new_skb == NULL) {
 583                                         DP(NETIF_MSG_RX_ERR,
 584                                            "ERROR  packet dropped "
 585                                            "because of alloc failure\n");
 586                                         fp->eth_q_stats.rx_skb_alloc_failed++;
 587                                         goto reuse_rx;
 588                                 }
 589
 590                                 /* aligned copy */
 591                                 skb_copy_from_linear_data_offset(skb, pad,
 592                                                     new_skb->data + pad, len);
 593                                 skb_reserve(new_skb, pad);
 594                                 skb_put(new_skb, len);
 595
 596                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 597
 598                                 skb = new_skb;
 599
 600                         } else
 601                         if (likely(bnx2x_alloc_rx_skb(bp, fp, bd_prod) == 0)) {
 602                                 dma_unmap_single(&bp->pdev->dev,
 603                                         dma_unmap_addr(rx_buf, mapping),
 604                                                  bp->rx_buf_size,
 605                                                  DMA_FROM_DEVICE);
 606                                 skb_reserve(skb, pad);
 607                                 skb_put(skb, len);
 608
 609                         } else {
 610                                 DP(NETIF_MSG_RX_ERR,
 611                                    "ERROR  packet dropped because "
 612                                    "of alloc failure\n");
 613                                 fp->eth_q_stats.rx_skb_alloc_failed++;
 614 reuse_rx:
 615                                 bnx2x_reuse_rx_skb(fp, bd_cons, bd_prod);
 616                                 goto next_rx;
 617                         }
 618
 619                         skb->protocol = eth_type_trans(skb, bp->dev);
 620
 621                         /* Set Toeplitz hash for a none-LRO skb */
 622                         bnx2x_set_skb_rxhash(bp, cqe, skb);
 623
 624                         skb_checksum_none_assert(skb);
 625
 626                         if (bp->rx_csum) {
 627                                 if (likely(BNX2X_RX_CSUM_OK(cqe)))
 628                                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 629                                 else
 630                                         fp->eth_q_stats.hw_csum_err++;
 631                         }
 632                 }
 633
 634                 skb_record_rx_queue(skb, fp->index);
 635
 636 #ifdef BCM_VLAN
 637                 if ((bp->vlgrp != NULL) && (bp->flags & HW_VLAN_RX_FLAG) &&
 638                     (le16_to_cpu(cqe->fast_path_cqe.pars_flags.flags) &
 639                      PARSING_FLAGS_VLAN))
 640                         vlan_gro_receive(&fp->napi, bp->vlgrp,
 641                                 le16_to_cpu(cqe->fast_path_cqe.vlan_tag), skb);
 642                 else
 643 #endif
 644                         napi_gro_receive(&fp->napi, skb);
 645
 646
 647 next_rx:
 648                 rx_buf->skb = NULL;
 649
 650                 bd_cons = NEXT_RX_IDX(bd_cons);
 651                 bd_prod = NEXT_RX_IDX(bd_prod);
 652                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
 653                 rx_pkt++;
 654 next_cqe:
 655                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
 656                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
 657
 658                 if (rx_pkt == budget)
 659                         break;
 660         } /* while */
 661
 662         fp->rx_bd_cons = bd_cons;
 663         fp->rx_bd_prod = bd_prod_fw;
 664         fp->rx_comp_cons = sw_comp_cons;
 665         fp->rx_comp_prod = sw_comp_prod;
 666
 667         /* Update producers */
 668         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
 669                              fp->rx_sge_prod);
 670
 671         fp->rx_pkt += rx_pkt;
 672         fp->rx_calls++;
 673
 674         return rx_pkt;
 675 }
 676
 677 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
 678 {
 679         struct bnx2x_fastpath *fp = fp_cookie;
 680         struct bnx2x *bp = fp->bp;
 681
 682         /* Return here if interrupt is disabled */
 683         if (unlikely(atomic_read(&bp->intr_sem) != 0)) {
 684                 DP(NETIF_MSG_INTR, "called but intr_sem not 0, returning\n");
 685                 return IRQ_HANDLED;
 686         }
 687
 688         DP(BNX2X_MSG_FP, "got an MSI-X interrupt on IDX:SB "
 689                          "[fp %d fw_sd %d igusb %d]\n",
 690            fp->index, fp->fw_sb_id, fp->igu_sb_id);
 691         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
 692
 693 #ifdef BNX2X_STOP_ON_ERROR
 694         if (unlikely(bp->panic))
 695                 return IRQ_HANDLED;
 696 #endif
 697
 698         /* Handle Rx and Tx according to MSI-X vector */
 699         prefetch(fp->rx_cons_sb);
 700         prefetch(fp->tx_cons_sb);
 701         prefetch(&fp->sb_running_index[SM_RX_ID]);
 702         napi_schedule(&bnx2x_fp(bp, fp->index, napi));
 703
 704         return IRQ_HANDLED;
 705 }
 706
 707 /* HW Lock for shared dual port PHYs */
 708 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
 709 {
 710         mutex_lock(&bp->port.phy_mutex);
 711
 712         if (bp->port.need_hw_lock)
 713                 bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 714 }
 715
 716 void bnx2x_release_phy_lock(struct bnx2x *bp)
 717 {
 718         if (bp->port.need_hw_lock)
 719                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
 720
 721         mutex_unlock(&bp->port.phy_mutex);
 722 }
 723
 724 void bnx2x_link_report(struct bnx2x *bp)
 725 {
 726         if (bp->flags & MF_FUNC_DIS) {
 727                 netif_carrier_off(bp->dev);
 728                 netdev_err(bp->dev, "NIC Link is Down\n");
 729                 return;
 730         }
 731
 732         if (bp->link_vars.link_up) {
 733                 u16 line_speed;
 734
 735                 if (bp->state == BNX2X_STATE_OPEN)
 736                         netif_carrier_on(bp->dev);
 737                 netdev_info(bp->dev, "NIC Link is Up, ");
 738
 739                 line_speed = bp->link_vars.line_speed;
 740                 if (IS_MF(bp)) {
 741                         u16 vn_max_rate;
 742
 743                         vn_max_rate =
 744                                 ((bp->mf_config[BP_VN(bp)] &
 745                                   FUNC_MF_CFG_MAX_BW_MASK) >>
 746                                                 FUNC_MF_CFG_MAX_BW_SHIFT) * 100;
 747                         if (vn_max_rate < line_speed)
 748                                 line_speed = vn_max_rate;
 749                 }
 750                 pr_cont("%d Mbps ", line_speed);
 751
 752                 if (bp->link_vars.duplex == DUPLEX_FULL)
 753                         pr_cont("full duplex");
 754                 else
 755                         pr_cont("half duplex");
 756
 757                 if (bp->link_vars.flow_ctrl != BNX2X_FLOW_CTRL_NONE) {
 758                         if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX) {
 759                                 pr_cont(", receive ");
 760                                 if (bp->link_vars.flow_ctrl &
 761                                     BNX2X_FLOW_CTRL_TX)
 762                                         pr_cont("& transmit ");
 763                         } else {
 764                                 pr_cont(", transmit ");
 765                         }
 766                         pr_cont("flow control ON");
 767                 }
 768                 pr_cont("\n");
 769
 770         } else { /* link_down */
 771                 netif_carrier_off(bp->dev);
 772                 netdev_err(bp->dev, "NIC Link is Down\n");
 773         }
 774 }
 775
 776 /* Returns the number of actually allocated BDs */
 777 static inline int bnx2x_alloc_rx_bds(struct bnx2x_fastpath *fp,
 778                                       int rx_ring_size)
 779 {
 780         struct bnx2x *bp = fp->bp;
 781         u16 ring_prod, cqe_ring_prod;
 782         int i;
 783
 784         fp->rx_comp_cons = 0;
 785         cqe_ring_prod = ring_prod = 0;
 786         for (i = 0; i < rx_ring_size; i++) {
 787                 if (bnx2x_alloc_rx_skb(bp, fp, ring_prod) < 0) {
 788                         BNX2X_ERR("was only able to allocate "
 789                                   "%d rx skbs on queue[%d]\n", i, fp->index);
 790                         fp->eth_q_stats.rx_skb_alloc_failed++;
 791                         break;
 792                 }
 793                 ring_prod = NEXT_RX_IDX(ring_prod);
 794                 cqe_ring_prod = NEXT_RCQ_IDX(cqe_ring_prod);
 795                 WARN_ON(ring_prod <= i);
 796         }
 797
 798         fp->rx_bd_prod = ring_prod;
 799         /* Limit the CQE producer by the CQE ring size */
 800         fp->rx_comp_prod = min_t(u16, NUM_RCQ_RINGS*RCQ_DESC_CNT,
 801                                cqe_ring_prod);
 802         fp->rx_pkt = fp->rx_calls = 0;
 803
 804         return i;
 805 }
 806
 807 static inline void bnx2x_alloc_rx_bd_ring(struct bnx2x_fastpath *fp)
 808 {
 809         struct bnx2x *bp = fp->bp;
 810         int rx_ring_size = bp->rx_ring_size ? bp->rx_ring_size :
 811                                               MAX_RX_AVAIL/bp->num_queues;
 812
 813         rx_ring_size = max_t(int, MIN_RX_AVAIL, rx_ring_size);
 814
 815         bnx2x_alloc_rx_bds(fp, rx_ring_size);
 816
 817         /* Warning!
 818          * this will generate an interrupt (to the TSTORM)
 819          * must only be done after chip is initialized
 820          */
 821         bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
 822                              fp->rx_sge_prod);
 823 }
 824
 825 void bnx2x_init_rx_rings(struct bnx2x *bp)
 826 {
 827         int func = BP_FUNC(bp);
 828         int max_agg_queues = CHIP_IS_E1(bp) ? ETH_MAX_AGGREGATION_QUEUES_E1 :
 829                                               ETH_MAX_AGGREGATION_QUEUES_E1H;
 830         u16 ring_prod;
 831         int i, j;
 832
 833         bp->rx_buf_size = bp->dev->mtu + ETH_OVREHEAD + BNX2X_RX_ALIGN +
 834                 IP_HEADER_ALIGNMENT_PADDING;
 835
 836         DP(NETIF_MSG_IFUP,
 837            "mtu %d  rx_buf_size %d\n", bp->dev->mtu, bp->rx_buf_size);
 838
 839         for_each_queue(bp, j) {
 840                 struct bnx2x_fastpath *fp = &bp->fp[j];
 841
 842                 if (!fp->disable_tpa) {
 843                         for (i = 0; i < max_agg_queues; i++) {
 844                                 fp->tpa_pool[i].skb =
 845                                    netdev_alloc_skb(bp->dev, bp->rx_buf_size);
 846                                 if (!fp->tpa_pool[i].skb) {
 847                                         BNX2X_ERR("Failed to allocate TPA "
 848                                                   "skb pool for queue[%d] - "
 849                                                   "disabling TPA on this "
 850                                                   "queue!\n", j);
 851                                         bnx2x_free_tpa_pool(bp, fp, i);
 852                                         fp->disable_tpa = 1;
 853                                         break;
 854                                 }
 855                                 dma_unmap_addr_set((struct sw_rx_bd *)
 856                                                         &bp->fp->tpa_pool[i],
 857                                                    mapping, 0);
 858                                 fp->tpa_state[i] = BNX2X_TPA_STOP;
 859                         }
 860
 861                         /* "next page" elements initialization */
 862                         bnx2x_set_next_page_sgl(fp);
 863
 864                         /* set SGEs bit mask */
 865                         bnx2x_init_sge_ring_bit_mask(fp);
 866
 867                         /* Allocate SGEs and initialize the ring elements */
 868                         for (i = 0, ring_prod = 0;
 869                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
 870
 871                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod) < 0) {
 872                                         BNX2X_ERR("was only able to allocate "
 873                                                   "%d rx sges\n", i);
 874                                         BNX2X_ERR("disabling TPA for"
 875                                                   " queue[%d]\n", j);
 876                                         /* Cleanup already allocated elements */
 877                                         bnx2x_free_rx_sge_range(bp,
 878                                                                 fp, ring_prod);
 879                                         bnx2x_free_tpa_pool(bp,
 880                                                             fp, max_agg_queues);
 881                                         fp->disable_tpa = 1;
 882                                         ring_prod = 0;
 883                                         break;
 884                                 }
 885                                 ring_prod = NEXT_SGE_IDX(ring_prod);
 886                         }
 887
 888                         fp->rx_sge_prod = ring_prod;
 889                 }
 890         }
 891
 892         for_each_queue(bp, j) {
 893                 struct bnx2x_fastpath *fp = &bp->fp[j];
 894
 895                 fp->rx_bd_cons = 0;
 896
 897                 bnx2x_set_next_page_rx_bd(fp);
 898
 899                 /* CQ ring */
 900                 bnx2x_set_next_page_rx_cq(fp);
 901
 902                 /* Allocate BDs and initialize BD ring */
 903                 bnx2x_alloc_rx_bd_ring(fp);
 904
 905                 if (j != 0)
 906                         continue;
 907
 908                 if (!CHIP_IS_E2(bp)) {
 909                         REG_WR(bp, BAR_USTRORM_INTMEM +
 910                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
 911                                U64_LO(fp->rx_comp_mapping));
 912                         REG_WR(bp, BAR_USTRORM_INTMEM +
 913                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
 914                                U64_HI(fp->rx_comp_mapping));
 915                 }
 916         }
 917 }
 918
 919 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
 920 {
 921         int i;
 922
 923         for_each_queue(bp, i) {
 924                 struct bnx2x_fastpath *fp = &bp->fp[i];
 925
 926                 u16 bd_cons = fp->tx_bd_cons;
 927                 u16 sw_prod = fp->tx_pkt_prod;
 928                 u16 sw_cons = fp->tx_pkt_cons;
 929
 930                 while (sw_cons != sw_prod) {
 931                         bd_cons = bnx2x_free_tx_pkt(bp, fp, TX_BD(sw_cons));
 932                         sw_cons++;
 933                 }
 934         }
 935 }
 936
 937 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
 938 {
 939         int i, j;
 940
 941         for_each_queue(bp, j) {
 942                 struct bnx2x_fastpath *fp = &bp->fp[j];
 943
 944                 for (i = 0; i < NUM_RX_BD; i++) {
 945                         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
 946                         struct sk_buff *skb = rx_buf->skb;
 947
 948                         if (skb == NULL)
 949                                 continue;
 950
 951                         dma_unmap_single(&bp->pdev->dev,
 952                                          dma_unmap_addr(rx_buf, mapping),
 953                                          bp->rx_buf_size, DMA_FROM_DEVICE);
 954
 955                         rx_buf->skb = NULL;
 956                         dev_kfree_skb(skb);
 957                 }
 958                 if (!fp->disable_tpa)
 959                         bnx2x_free_tpa_pool(bp, fp, CHIP_IS_E1(bp) ?
 960                                             ETH_MAX_AGGREGATION_QUEUES_E1 :
 961                                             ETH_MAX_AGGREGATION_QUEUES_E1H);
 962         }
 963 }
 964
 965 void bnx2x_free_skbs(struct bnx2x *bp)
 966 {
 967         bnx2x_free_tx_skbs(bp);
 968         bnx2x_free_rx_skbs(bp);
 969 }
 970
 971 static void bnx2x_free_msix_irqs(struct bnx2x *bp)
 972 {
 973         int i, offset = 1;
 974
 975         free_irq(bp->msix_table[0].vector, bp->dev);
 976         DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
 977            bp->msix_table[0].vector);
 978
 979 #ifdef BCM_CNIC
 980         offset++;
 981 #endif
 982         for_each_queue(bp, i) {
 983                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq  "
 984                    "state %x\n", i, bp->msix_table[i + offset].vector,
 985                    bnx2x_fp(bp, i, state));
 986
 987                 free_irq(bp->msix_table[i + offset].vector, &bp->fp[i]);
 988         }
 989 }
 990
 991 void bnx2x_free_irq(struct bnx2x *bp)
 992 {
 993         if (bp->flags & USING_MSIX_FLAG)
 994                 bnx2x_free_msix_irqs(bp);
 995         else if (bp->flags & USING_MSI_FLAG)
 996                 free_irq(bp->pdev->irq, bp->dev);
 997         else
 998                 free_irq(bp->pdev->irq, bp->dev);
 999 }
1000
1001 int bnx2x_enable_msix(struct bnx2x *bp)
1002 {
1003         int msix_vec = 0, i, rc, req_cnt;
1004
1005         bp->msix_table[msix_vec].entry = msix_vec;
1006         DP(NETIF_MSG_IFUP, "msix_table[0].entry = %d (slowpath)\n",
1007            bp->msix_table[0].entry);
1008         msix_vec++;
1009
1010 #ifdef BCM_CNIC
1011         bp->msix_table[msix_vec].entry = msix_vec;
1012         DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d (CNIC)\n",
1013            bp->msix_table[msix_vec].entry, bp->msix_table[msix_vec].entry);
1014         msix_vec++;
1015 #endif
1016         for_each_queue(bp, i) {
1017                 bp->msix_table[msix_vec].entry = msix_vec;
1018                 DP(NETIF_MSG_IFUP, "msix_table[%d].entry = %d "
1019                    "(fastpath #%u)\n", msix_vec, msix_vec, i);
1020                 msix_vec++;
1021         }
1022
1023         req_cnt = BNX2X_NUM_QUEUES(bp) + CNIC_CONTEXT_USE + 1;
1024
1025         rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], req_cnt);
1026
1027         /*
1028          * reconfigure number of tx/rx queues according to available
1029          * MSI-X vectors
1030          */
1031         if (rc >= BNX2X_MIN_MSIX_VEC_CNT) {
1032                 /* how less vectors we will have? */
1033                 int diff = req_cnt - rc;
1034
1035                 DP(NETIF_MSG_IFUP,
1036                    "Trying to use less MSI-X vectors: %d\n", rc);
1037
1038                 rc = pci_enable_msix(bp->pdev, &bp->msix_table[0], rc);
1039
1040                 if (rc) {
1041                         DP(NETIF_MSG_IFUP,
1042                            "MSI-X is not attainable  rc %d\n", rc);
1043                         return rc;
1044                 }
1045                 /*
1046                  * decrease number of queues by number of unallocated entries
1047                  */
1048                 bp->num_queues -= diff;
1049
1050                 DP(NETIF_MSG_IFUP, "New queue configuration set: %d\n",
1051                                   bp->num_queues);
1052         } else if (rc) {
1053                 /* fall to INTx if not enough memory */
1054                 if (rc == -ENOMEM)
1055                         bp->flags |= DISABLE_MSI_FLAG;
1056                 DP(NETIF_MSG_IFUP, "MSI-X is not attainable  rc %d\n", rc);
1057                 return rc;
1058         }
1059
1060         bp->flags |= USING_MSIX_FLAG;
1061
1062         return 0;
1063 }
1064
1065 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1066 {
1067         int i, rc, offset = 1;
1068
1069         rc = request_irq(bp->msix_table[0].vector, bnx2x_msix_sp_int, 0,
1070                          bp->dev->name, bp->dev);
1071         if (rc) {
1072                 BNX2X_ERR("request sp irq failed\n");
1073                 return -EBUSY;
1074         }
1075
1076 #ifdef BCM_CNIC
1077         offset++;
1078 #endif
1079         for_each_queue(bp, i) {
1080                 struct bnx2x_fastpath *fp = &bp->fp[i];
1081                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1082                          bp->dev->name, i);
1083
1084                 rc = request_irq(bp->msix_table[offset].vector,
1085                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1086                 if (rc) {
1087                         BNX2X_ERR("request fp #%d irq failed  rc %d\n", i, rc);
1088                         bnx2x_free_msix_irqs(bp);
1089                         return -EBUSY;
1090                 }
1091
1092                 offset++;
1093                 fp->state = BNX2X_FP_STATE_IRQ;
1094         }
1095
1096         i = BNX2X_NUM_QUEUES(bp);
1097         offset = 1 + CNIC_CONTEXT_USE;
1098         netdev_info(bp->dev, "using MSI-X  IRQs: sp %d  fp[%d] %d"
1099                " ... fp[%d] %d\n",
1100                bp->msix_table[0].vector,
1101                0, bp->msix_table[offset].vector,
1102                i - 1, bp->msix_table[offset + i - 1].vector);
1103
1104         return 0;
1105 }
1106
1107 int bnx2x_enable_msi(struct bnx2x *bp)
1108 {
1109         int rc;
1110
1111         rc = pci_enable_msi(bp->pdev);
1112         if (rc) {
1113                 DP(NETIF_MSG_IFUP, "MSI is not attainable\n");
1114                 return -1;
1115         }
1116         bp->flags |= USING_MSI_FLAG;
1117
1118         return 0;
1119 }
1120
1121 static int bnx2x_req_irq(struct bnx2x *bp)
1122 {
1123         unsigned long flags;
1124         int rc;
1125
1126         if (bp->flags & USING_MSI_FLAG)
1127                 flags = 0;
1128         else
1129                 flags = IRQF_SHARED;
1130
1131         rc = request_irq(bp->pdev->irq, bnx2x_interrupt, flags,
1132                          bp->dev->name, bp->dev);
1133         if (!rc)
1134                 bnx2x_fp(bp, 0, state) = BNX2X_FP_STATE_IRQ;
1135
1136         return rc;
1137 }
1138
1139 static void bnx2x_napi_enable(struct bnx2x *bp)
1140 {
1141         int i;
1142
1143         for_each_queue(bp, i)
1144                 napi_enable(&bnx2x_fp(bp, i, napi));
1145 }
1146
1147 static void bnx2x_napi_disable(struct bnx2x *bp)
1148 {
1149         int i;
1150
1151         for_each_queue(bp, i)
1152                 napi_disable(&bnx2x_fp(bp, i, napi));
1153 }
1154
1155 void bnx2x_netif_start(struct bnx2x *bp)
1156 {
1157         int intr_sem;
1158
1159         intr_sem = atomic_dec_and_test(&bp->intr_sem);
1160         smp_wmb(); /* Ensure that bp->intr_sem update is SMP-safe */
1161
1162         if (intr_sem) {
1163                 if (netif_running(bp->dev)) {
1164                         bnx2x_napi_enable(bp);
1165                         bnx2x_int_enable(bp);
1166                         if (bp->state == BNX2X_STATE_OPEN)
1167                                 netif_tx_wake_all_queues(bp->dev);
1168                 }
1169         }
1170 }
1171
1172 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1173 {
1174         bnx2x_int_disable_sync(bp, disable_hw);
1175         bnx2x_napi_disable(bp);
1176         netif_tx_disable(bp->dev);
1177 }
1178
1179 void bnx2x_set_num_queues(struct bnx2x *bp)
1180 {
1181         switch (bp->multi_mode) {
1182         case ETH_RSS_MODE_DISABLED:
1183                 bp->num_queues = 1;
1184                 break;
1185         case ETH_RSS_MODE_REGULAR:
1186                 bp->num_queues = bnx2x_calc_num_queues(bp);
1187                 break;
1188
1189         default:
1190                 bp->num_queues = 1;
1191                 break;
1192         }
1193 }
1194
1195 static void bnx2x_release_firmware(struct bnx2x *bp)
1196 {
1197         kfree(bp->init_ops_offsets);
1198         kfree(bp->init_ops);
1199         kfree(bp->init_data);
1200         release_firmware(bp->firmware);
1201 }
1202
1203 /* must be called with rtnl_lock */
1204 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
1205 {
1206         u32 load_code;
1207         int i, rc;
1208
1209         /* Set init arrays */
1210         rc = bnx2x_init_firmware(bp);
1211         if (rc) {
1212                 BNX2X_ERR("Error loading firmware\n");
1213                 return rc;
1214         }
1215
1216 #ifdef BNX2X_STOP_ON_ERROR
1217         if (unlikely(bp->panic))
1218                 return -EPERM;
1219 #endif
1220
1221         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
1222
1223         /* must be called before memory allocation and HW init */
1224         bnx2x_ilt_set_info(bp);
1225
1226         if (bnx2x_alloc_mem(bp))
1227                 return -ENOMEM;
1228
1229         netif_set_real_num_tx_queues(bp->dev, bp->num_queues);
1230         rc = netif_set_real_num_rx_queues(bp->dev, bp->num_queues);
1231         if (rc) {
1232                 BNX2X_ERR("Unable to update real_num_rx_queues\n");
1233                 goto load_error0;
1234         }
1235
1236         for_each_queue(bp, i)
1237                 bnx2x_fp(bp, i, disable_tpa) =
1238                                         ((bp->flags & TPA_ENABLE_FLAG) == 0);
1239
1240         bnx2x_napi_enable(bp);
1241
1242         /* Send LOAD_REQUEST command to MCP
1243            Returns the type of LOAD command:
1244            if it is the first port to be initialized
1245            common blocks should be initialized, otherwise - not
1246         */
1247         if (!BP_NOMCP(bp)) {
1248                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, 0);
1249                 if (!load_code) {
1250                         BNX2X_ERR("MCP response failure, aborting\n");
1251                         rc = -EBUSY;
1252                         goto load_error1;
1253                 }
1254                 if (load_code == FW_MSG_CODE_DRV_LOAD_REFUSED) {
1255                         rc = -EBUSY; /* other port in diagnostic mode */
1256                         goto load_error1;
1257                 }
1258
1259         } else {
1260                 int path = BP_PATH(bp);
1261                 int port = BP_PORT(bp);
1262
1263                 DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
1264                    path, load_count[path][0], load_count[path][1],
1265                    load_count[path][2]);
1266                 load_count[path][0]++;
1267                 load_count[path][1 + port]++;
1268                 DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
1269                    path, load_count[path][0], load_count[path][1],
1270                    load_count[path][2]);
1271                 if (load_count[path][0] == 1)
1272                         load_code = FW_MSG_CODE_DRV_LOAD_COMMON;
1273                 else if (load_count[path][1 + port] == 1)
1274                         load_code = FW_MSG_CODE_DRV_LOAD_PORT;
1275                 else
1276                         load_code = FW_MSG_CODE_DRV_LOAD_FUNCTION;
1277         }
1278
1279         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1280             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
1281             (load_code == FW_MSG_CODE_DRV_LOAD_PORT))
1282                 bp->port.pmf = 1;
1283         else
1284                 bp->port.pmf = 0;
1285         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
1286
1287         /* Initialize HW */
1288         rc = bnx2x_init_hw(bp, load_code);
1289         if (rc) {
1290                 BNX2X_ERR("HW init failed, aborting\n");
1291                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1292                 goto load_error2;
1293         }
1294
1295         /* Connect to IRQs */
1296         rc = bnx2x_setup_irqs(bp);
1297         if (rc) {
1298                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1299                 goto load_error2;
1300         }
1301
1302         /* Setup NIC internals and enable interrupts */
1303         bnx2x_nic_init(bp, load_code);
1304
1305         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
1306             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
1307             (bp->common.shmem2_base))
1308                 SHMEM2_WR(bp, dcc_support,
1309                           (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
1310                            SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
1311
1312         /* Send LOAD_DONE command to MCP */
1313         if (!BP_NOMCP(bp)) {
1314                 load_code = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
1315                 if (!load_code) {
1316                         BNX2X_ERR("MCP response failure, aborting\n");
1317                         rc = -EBUSY;
1318                         goto load_error3;
1319                 }
1320         }
1321
1322         bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
1323
1324         rc = bnx2x_func_start(bp);
1325         if (rc) {
1326                 BNX2X_ERR("Function start failed!\n");
1327 #ifndef BNX2X_STOP_ON_ERROR
1328                 goto load_error3;
1329 #else
1330                 bp->panic = 1;
1331                 return -EBUSY;
1332 #endif
1333         }
1334
1335         rc = bnx2x_setup_client(bp, &bp->fp[0], 1 /* Leading */);
1336         if (rc) {
1337                 BNX2X_ERR("Setup leading failed!\n");
1338 #ifndef BNX2X_STOP_ON_ERROR
1339                 goto load_error3;
1340 #else
1341                 bp->panic = 1;
1342                 return -EBUSY;
1343 #endif
1344         }
1345
1346         if (!CHIP_IS_E1(bp) &&
1347             (bp->mf_config[BP_VN(bp)] & FUNC_MF_CFG_FUNC_DISABLED)) {
1348                 DP(NETIF_MSG_IFUP, "mf_cfg function disabled\n");
1349                 bp->flags |= MF_FUNC_DIS;
1350         }
1351
1352 #ifdef BCM_CNIC
1353         /* Enable Timer scan */
1354         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 1);
1355 #endif
1356
1357         for_each_nondefault_queue(bp, i) {
1358                 rc = bnx2x_setup_client(bp, &bp->fp[i], 0);
1359                 if (rc)
1360 #ifdef BCM_CNIC
1361                         goto load_error4;
1362 #else
1363                         goto load_error3;
1364 #endif
1365         }
1366
1367         /* Now when Clients are configured we are ready to work */
1368         bp->state = BNX2X_STATE_OPEN;
1369
1370         bnx2x_set_eth_mac(bp, 1);
1371
1372         if (bp->port.pmf)
1373                 bnx2x_initial_phy_init(bp, load_mode);
1374
1375         /* Start fast path */
1376         switch (load_mode) {
1377         case LOAD_NORMAL:
1378                 /* Tx queue should be only reenabled */
1379                 netif_tx_wake_all_queues(bp->dev);
1380                 /* Initialize the receive filter. */
1381                 bnx2x_set_rx_mode(bp->dev);
1382                 break;
1383
1384         case LOAD_OPEN:
1385                 netif_tx_start_all_queues(bp->dev);
1386                 smp_mb__after_clear_bit();
1387                 /* Initialize the receive filter. */
1388                 bnx2x_set_rx_mode(bp->dev);
1389                 break;
1390
1391         case LOAD_DIAG:
1392                 /* Initialize the receive filter. */
1393                 bnx2x_set_rx_mode(bp->dev);
1394                 bp->state = BNX2X_STATE_DIAG;
1395                 break;
1396
1397         default:
1398                 break;
1399         }
1400
1401         if (!bp->port.pmf)
1402                 bnx2x__link_status_update(bp);
1403
1404         /* start the timer */
1405         mod_timer(&bp->timer, jiffies + bp->current_interval);
1406
1407 #ifdef BCM_CNIC
1408         bnx2x_setup_cnic_irq_info(bp);
1409         if (bp->state == BNX2X_STATE_OPEN)
1410                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
1411 #endif
1412         bnx2x_inc_load_cnt(bp);
1413
1414         bnx2x_release_firmware(bp);
1415
1416         return 0;
1417
1418 #ifdef BCM_CNIC
1419 load_error4:
1420         /* Disable Timer scan */
1421         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + BP_PORT(bp)*4, 0);
1422 #endif
1423 load_error3:
1424         bnx2x_int_disable_sync(bp, 1);
1425
1426         /* Free SKBs, SGEs, TPA pool and driver internals */
1427         bnx2x_free_skbs(bp);
1428         for_each_queue(bp, i)
1429                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1430
1431         /* Release IRQs */
1432         bnx2x_free_irq(bp);
1433 load_error2:
1434         if (!BP_NOMCP(bp)) {
1435                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
1436                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
1437         }
1438
1439         bp->port.pmf = 0;
1440 load_error1:
1441         bnx2x_napi_disable(bp);
1442 load_error0:
1443         bnx2x_free_mem(bp);
1444
1445         bnx2x_release_firmware(bp);
1446
1447         return rc;
1448 }
1449
1450 /* must be called with rtnl_lock */
1451 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode)
1452 {
1453         int i;
1454
1455         if (bp->state == BNX2X_STATE_CLOSED) {
1456                 /* Interface has been removed - nothing to recover */
1457                 bp->recovery_state = BNX2X_RECOVERY_DONE;
1458                 bp->is_leader = 0;
1459                 bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_RESERVED_08);
1460                 smp_wmb();
1461
1462                 return -EINVAL;
1463         }
1464
1465 #ifdef BCM_CNIC
1466         bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
1467 #endif
1468         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
1469
1470         /* Set "drop all" */
1471         bp->rx_mode = BNX2X_RX_MODE_NONE;
1472         bnx2x_set_storm_rx_mode(bp);
1473
1474         /* Stop Tx */
1475         bnx2x_tx_disable(bp);
1476
1477         del_timer_sync(&bp->timer);
1478
1479         SHMEM_WR(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb,
1480                  (DRV_PULSE_ALWAYS_ALIVE | bp->fw_drv_pulse_wr_seq));
1481
1482         bnx2x_stats_handle(bp, STATS_EVENT_STOP);
1483
1484         /* Cleanup the chip if needed */
1485         if (unload_mode != UNLOAD_RECOVERY)
1486                 bnx2x_chip_cleanup(bp, unload_mode);
1487         else {
1488                 /* Disable HW interrupts, NAPI and Tx */
1489                 bnx2x_netif_stop(bp, 1);
1490
1491                 /* Release IRQs */
1492                 bnx2x_free_irq(bp);
1493         }
1494
1495         bp->port.pmf = 0;
1496
1497         /* Free SKBs, SGEs, TPA pool and driver internals */
1498         bnx2x_free_skbs(bp);
1499         for_each_queue(bp, i)
1500                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
1501
1502         bnx2x_free_mem(bp);
1503
1504         bp->state = BNX2X_STATE_CLOSED;
1505
1506         /* The last driver must disable a "close the gate" if there is no
1507          * parity attention or "process kill" pending.
1508          */
1509         if ((!bnx2x_dec_load_cnt(bp)) && (!bnx2x_chk_parity_attn(bp)) &&
1510             bnx2x_reset_is_done(bp))
1511                 bnx2x_disable_close_the_gate(bp);
1512
1513         /* Reset MCP mail box sequence if there is on going recovery */
1514         if (unload_mode == UNLOAD_RECOVERY)
1515                 bp->fw_seq = 0;
1516
1517         return 0;
1518 }
1519
1520 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
1521 {
1522         u16 pmcsr;
1523
1524         /* If there is no power capability, silently succeed */
1525         if (!bp->pm_cap) {
1526                 DP(NETIF_MSG_HW, "No power capability. Breaking.\n");
1527                 return 0;
1528         }
1529
1530         pci_read_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL, &pmcsr);
1531
1532         switch (state) {
1533         case PCI_D0:
1534                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1535                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
1536                                        PCI_PM_CTRL_PME_STATUS));
1537
1538                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
1539                         /* delay required during transition out of D3hot */
1540                         msleep(20);
1541                 break;
1542
1543         case PCI_D3hot:
1544                 /* If there are other clients above don't
1545                    shut down the power */
1546                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
1547                         return 0;
1548                 /* Don't shut down the power for emulation and FPGA */
1549                 if (CHIP_REV_IS_SLOW(bp))
1550                         return 0;
1551
1552                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1553                 pmcsr |= 3;
1554
1555                 if (bp->wol)
1556                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1557
1558                 pci_write_config_word(bp->pdev, bp->pm_cap + PCI_PM_CTRL,
1559                                       pmcsr);
1560
1561                 /* No more memory access after this point until
1562                 * device is brought back to D0.
1563                 */
1564                 break;
1565
1566         default:
1567                 return -EINVAL;
1568         }
1569         return 0;
1570 }
1571
1572 /*
1573  * net_device service functions
1574  */
1575 int bnx2x_poll(struct napi_struct *napi, int budget)
1576 {
1577         int work_done = 0;
1578         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
1579                                                  napi);
1580         struct bnx2x *bp = fp->bp;
1581
1582         while (1) {
1583 #ifdef BNX2X_STOP_ON_ERROR
1584                 if (unlikely(bp->panic)) {
1585                         napi_complete(napi);
1586                         return 0;
1587                 }
1588 #endif
1589
1590                 if (bnx2x_has_tx_work(fp))
1591                         bnx2x_tx_int(fp);
1592
1593                 if (bnx2x_has_rx_work(fp)) {
1594                         work_done += bnx2x_rx_int(fp, budget - work_done);
1595
1596                         /* must not complete if we consumed full budget */
1597                         if (work_done >= budget)
1598                                 break;
1599                 }
1600
1601                 /* Fall out from the NAPI loop if needed */
1602                 if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1603                         bnx2x_update_fpsb_idx(fp);
1604                         /* bnx2x_has_rx_work() reads the status block,
1605                          * thus we need to ensure that status block indices
1606                          * have been actually read (bnx2x_update_fpsb_idx)
1607                          * prior to this check (bnx2x_has_rx_work) so that
1608                          * we won't write the "newer" value of the status block
1609                          * to IGU (if there was a DMA right after
1610                          * bnx2x_has_rx_work and if there is no rmb, the memory
1611                          * reading (bnx2x_update_fpsb_idx) may be postponed
1612                          * to right before bnx2x_ack_sb). In this case there
1613                          * will never be another interrupt until there is
1614                          * another update of the status block, while there
1615                          * is still unhandled work.
1616                          */
1617                         rmb();
1618
1619                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
1620                                 napi_complete(napi);
1621                                 /* Re-enable interrupts */
1622                                 DP(NETIF_MSG_HW,
1623                                    "Update index to %d\n", fp->fp_hc_idx);
1624                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
1625                                              le16_to_cpu(fp->fp_hc_idx),
1626                                              IGU_INT_ENABLE, 1);
1627                                 break;
1628                         }
1629                 }
1630         }
1631
1632         return work_done;
1633 }
1634
1635 /* we split the first BD into headers and data BDs
1636  * to ease the pain of our fellow microcode engineers
1637  * we use one mapping for both BDs
1638  * So far this has only been observed to happen
1639  * in Other Operating Systems(TM)
1640  */
1641 static noinline u16 bnx2x_tx_split(struct bnx2x *bp,
1642                                    struct bnx2x_fastpath *fp,
1643                                    struct sw_tx_bd *tx_buf,
1644                                    struct eth_tx_start_bd **tx_bd, u16 hlen,
1645                                    u16 bd_prod, int nbd)
1646 {
1647         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
1648         struct eth_tx_bd *d_tx_bd;
1649         dma_addr_t mapping;
1650         int old_len = le16_to_cpu(h_tx_bd->nbytes);
1651
1652         /* first fix first BD */
1653         h_tx_bd->nbd = cpu_to_le16(nbd);
1654         h_tx_bd->nbytes = cpu_to_le16(hlen);
1655
1656         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d "
1657            "(%x:%x) nbd %d\n", h_tx_bd->nbytes, h_tx_bd->addr_hi,
1658            h_tx_bd->addr_lo, h_tx_bd->nbd);
1659
1660         /* now get a new data BD
1661          * (after the pbd) and fill it */
1662         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
1663         d_tx_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
1664
1665         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
1666                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
1667
1668         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
1669         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
1670         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
1671
1672         /* this marks the BD as one that has no individual mapping */
1673         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
1674
1675         DP(NETIF_MSG_TX_QUEUED,
1676            "TSO split data size is %d (%x:%x)\n",
1677            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
1678
1679         /* update tx_bd */
1680         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
1681
1682         return bd_prod;
1683 }
1684
1685 static inline u16 bnx2x_csum_fix(unsigned char *t_header, u16 csum, s8 fix)
1686 {
1687         if (fix > 0)
1688                 csum = (u16) ~csum_fold(csum_sub(csum,
1689                                 csum_partial(t_header - fix, fix, 0)));
1690
1691         else if (fix < 0)
1692                 csum = (u16) ~csum_fold(csum_add(csum,
1693                                 csum_partial(t_header, -fix, 0)));
1694
1695         return swab16(csum);
1696 }
1697
1698 static inline u32 bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
1699 {
1700         u32 rc;
1701
1702         if (skb->ip_summed != CHECKSUM_PARTIAL)
1703                 rc = XMIT_PLAIN;
1704
1705         else {
1706                 if (skb->protocol == htons(ETH_P_IPV6)) {
1707                         rc = XMIT_CSUM_V6;
1708                         if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
1709                                 rc |= XMIT_CSUM_TCP;
1710
1711                 } else {
1712                         rc = XMIT_CSUM_V4;
1713                         if (ip_hdr(skb)->protocol == IPPROTO_TCP)
1714                                 rc |= XMIT_CSUM_TCP;
1715                 }
1716         }
1717
1718         if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4)
1719                 rc |= (XMIT_GSO_V4 | XMIT_CSUM_V4 | XMIT_CSUM_TCP);
1720
1721         else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6)
1722                 rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP | XMIT_CSUM_V6);
1723
1724         return rc;
1725 }
1726
1727 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1728 /* check if packet requires linearization (packet is too fragmented)
1729    no need to check fragmentation if page size > 8K (there will be no
1730    violation to FW restrictions) */
1731 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
1732                              u32 xmit_type)
1733 {
1734         int to_copy = 0;
1735         int hlen = 0;
1736         int first_bd_sz = 0;
1737
1738         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
1739         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
1740
1741                 if (xmit_type & XMIT_GSO) {
1742                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
1743                         /* Check if LSO packet needs to be copied:
1744                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
1745                         int wnd_size = MAX_FETCH_BD - 3;
1746                         /* Number of windows to check */
1747                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
1748                         int wnd_idx = 0;
1749                         int frag_idx = 0;
1750                         u32 wnd_sum = 0;
1751
1752                         /* Headers length */
1753                         hlen = (int)(skb_transport_header(skb) - skb->data) +
1754                                 tcp_hdrlen(skb);
1755
1756                         /* Amount of data (w/o headers) on linear part of SKB*/
1757                         first_bd_sz = skb_headlen(skb) - hlen;
1758
1759                         wnd_sum  = first_bd_sz;
1760
1761                         /* Calculate the first sum - it's special */
1762                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
1763                                 wnd_sum +=
1764                                         skb_shinfo(skb)->frags[frag_idx].size;
1765
1766                         /* If there was data on linear skb data - check it */
1767                         if (first_bd_sz > 0) {
1768                                 if (unlikely(wnd_sum < lso_mss)) {
1769                                         to_copy = 1;
1770                                         goto exit_lbl;
1771                                 }
1772
1773                                 wnd_sum -= first_bd_sz;
1774                         }
1775
1776                         /* Others are easier: run through the frag list and
1777                            check all windows */
1778                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
1779                                 wnd_sum +=
1780                           skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1].size;
1781
1782                                 if (unlikely(wnd_sum < lso_mss)) {
1783                                         to_copy = 1;
1784                                         break;
1785                                 }
1786                                 wnd_sum -=
1787                                         skb_shinfo(skb)->frags[wnd_idx].size;
1788                         }
1789                 } else {
1790                         /* in non-LSO too fragmented packet should always
1791                            be linearized */
1792                         to_copy = 1;
1793                 }
1794         }
1795
1796 exit_lbl:
1797         if (unlikely(to_copy))
1798                 DP(NETIF_MSG_TX_QUEUED,
1799                    "Linearization IS REQUIRED for %s packet. "
1800                    "num_frags %d  hlen %d  first_bd_sz %d\n",
1801                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
1802                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
1803
1804         return to_copy;
1805 }
1806 #endif
1807
1808 static inline void bnx2x_set_pbd_gso_e2(struct sk_buff *skb,
1809                                      struct eth_tx_parse_bd_e2 *pbd,
1810                                      u32 xmit_type)
1811 {
1812         pbd->parsing_data |= cpu_to_le16(skb_shinfo(skb)->gso_size) <<
1813                 ETH_TX_PARSE_BD_E2_LSO_MSS_SHIFT;
1814         if ((xmit_type & XMIT_GSO_V6) &&
1815             (ipv6_hdr(skb)->nexthdr == NEXTHDR_IPV6))
1816                 pbd->parsing_data |= ETH_TX_PARSE_BD_E2_IPV6_WITH_EXT_HDR;
1817 }
1818
1819 /**
1820  * Update PBD in GSO case.
1821  *
1822  * @param skb
1823  * @param tx_start_bd
1824  * @param pbd
1825  * @param xmit_type
1826  */
1827 static inline void bnx2x_set_pbd_gso(struct sk_buff *skb,
1828                                      struct eth_tx_parse_bd_e1x *pbd,
1829                                      u32 xmit_type)
1830 {
1831         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
1832         pbd->tcp_send_seq = swab32(tcp_hdr(skb)->seq);
1833         pbd->tcp_flags = pbd_tcp_flags(skb);
1834
1835         if (xmit_type & XMIT_GSO_V4) {
1836                 pbd->ip_id = swab16(ip_hdr(skb)->id);
1837                 pbd->tcp_pseudo_csum =
1838                         swab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1839                                                   ip_hdr(skb)->daddr,
1840                                                   0, IPPROTO_TCP, 0));
1841
1842         } else
1843                 pbd->tcp_pseudo_csum =
1844                         swab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1845                                                 &ipv6_hdr(skb)->daddr,
1846                                                 0, IPPROTO_TCP, 0));
1847
1848         pbd->global_data |= ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN;
1849 }
1850
1851 /**
1852  *
1853  * @param skb
1854  * @param tx_start_bd
1855  * @param pbd_e2
1856  * @param xmit_type
1857  *
1858  * @return header len
1859  */
1860 static inline  u8 bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
1861         struct eth_tx_parse_bd_e2 *pbd,
1862         u32 xmit_type)
1863 {
1864         pbd->parsing_data |= cpu_to_le16(tcp_hdrlen(skb)/4) <<
1865                 ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT;
1866
1867         pbd->parsing_data |= cpu_to_le16(((unsigned char *)tcp_hdr(skb) -
1868                                           skb->data) / 2) <<
1869                 ETH_TX_PARSE_BD_E2_TCP_HDR_START_OFFSET_W_SHIFT;
1870
1871         return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
1872 }
1873
1874 /**
1875  *
1876  * @param skb
1877  * @param tx_start_bd
1878  * @param pbd
1879  * @param xmit_type
1880  *
1881  * @return Header length
1882  */
1883 static inline u8 bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
1884         struct eth_tx_parse_bd_e1x *pbd,
1885         u32 xmit_type)
1886 {
1887         u8 hlen = (skb_network_header(skb) - skb->data) / 2;
1888
1889         /* for now NS flag is not used in Linux */
1890         pbd->global_data =
1891                 (hlen | ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
1892                          ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
1893
1894         pbd->ip_hlen_w = (skb_transport_header(skb) -
1895                         skb_network_header(skb)) / 2;
1896
1897         hlen += pbd->ip_hlen_w + tcp_hdrlen(skb) / 2;
1898
1899         pbd->total_hlen_w = cpu_to_le16(hlen);
1900         hlen = hlen*2;
1901
1902         if (xmit_type & XMIT_CSUM_TCP) {
1903                 pbd->tcp_pseudo_csum = swab16(tcp_hdr(skb)->check);
1904
1905         } else {
1906                 s8 fix = SKB_CS_OFF(skb); /* signed! */
1907
1908                 DP(NETIF_MSG_TX_QUEUED,
1909                    "hlen %d  fix %d  csum before fix %x\n",
1910                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
1911
1912                 /* HW bug: fixup the CSUM */
1913                 pbd->tcp_pseudo_csum =
1914                         bnx2x_csum_fix(skb_transport_header(skb),
1915                                        SKB_CS(skb), fix);
1916
1917                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
1918                    pbd->tcp_pseudo_csum);
1919         }
1920
1921         return hlen;
1922 }
1923
1924 /* called with netif_tx_lock
1925  * bnx2x_tx_int() runs without netif_tx_lock unless it needs to call
1926  * netif_wake_queue()
1927  */
1928 netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev)
1929 {
1930         struct bnx2x *bp = netdev_priv(dev);
1931         struct bnx2x_fastpath *fp;
1932         struct netdev_queue *txq;
1933         struct sw_tx_bd *tx_buf;
1934         struct eth_tx_start_bd *tx_start_bd;
1935         struct eth_tx_bd *tx_data_bd, *total_pkt_bd = NULL;
1936         struct eth_tx_parse_bd_e1x *pbd_e1x = NULL;
1937         struct eth_tx_parse_bd_e2 *pbd_e2 = NULL;
1938         u16 pkt_prod, bd_prod;
1939         int nbd, fp_index;
1940         dma_addr_t mapping;
1941         u32 xmit_type = bnx2x_xmit_type(bp, skb);
1942         int i;
1943         u8 hlen = 0;
1944         __le16 pkt_size = 0;
1945         struct ethhdr *eth;
1946         u8 mac_type = UNICAST_ADDRESS;
1947
1948 #ifdef BNX2X_STOP_ON_ERROR
1949         if (unlikely(bp->panic))
1950                 return NETDEV_TX_BUSY;
1951 #endif
1952
1953         fp_index = skb_get_queue_mapping(skb);
1954         txq = netdev_get_tx_queue(dev, fp_index);
1955
1956         fp = &bp->fp[fp_index];
1957
1958         if (unlikely(bnx2x_tx_avail(fp) < (skb_shinfo(skb)->nr_frags + 3))) {
1959                 fp->eth_q_stats.driver_xoff++;
1960                 netif_tx_stop_queue(txq);
1961                 BNX2X_ERR("BUG! Tx ring full when queue awake!\n");
1962                 return NETDEV_TX_BUSY;
1963         }
1964
1965         DP(NETIF_MSG_TX_QUEUED, "queue[%d]: SKB: summed %x  protocol %x  "
1966                                 "protocol(%x,%x) gso type %x  xmit_type %x\n",
1967            fp_index, skb->ip_summed, skb->protocol, ipv6_hdr(skb)->nexthdr,
1968            ip_hdr(skb)->protocol, skb_shinfo(skb)->gso_type, xmit_type);
1969
1970         eth = (struct ethhdr *)skb->data;
1971
1972         /* set flag according to packet type (UNICAST_ADDRESS is default)*/
1973         if (unlikely(is_multicast_ether_addr(eth->h_dest))) {
1974                 if (is_broadcast_ether_addr(eth->h_dest))
1975                         mac_type = BROADCAST_ADDRESS;
1976                 else
1977                         mac_type = MULTICAST_ADDRESS;
1978         }
1979
1980 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
1981         /* First, check if we need to linearize the skb (due to FW
1982            restrictions). No need to check fragmentation if page size > 8K
1983            (there will be no violation to FW restrictions) */
1984         if (bnx2x_pkt_req_lin(bp, skb, xmit_type)) {
1985                 /* Statistics of linearization */
1986                 bp->lin_cnt++;
1987                 if (skb_linearize(skb) != 0) {
1988                         DP(NETIF_MSG_TX_QUEUED, "SKB linearization failed - "
1989                            "silently dropping this SKB\n");
1990                         dev_kfree_skb_any(skb);
1991                         return NETDEV_TX_OK;
1992                 }
1993         }
1994 #endif
1995
1996         /*
1997         Please read carefully. First we use one BD which we mark as start,
1998         then we have a parsing info BD (used for TSO or xsum),
1999         and only then we have the rest of the TSO BDs.
2000         (don't forget to mark the last one as last,
2001         and to unmap only AFTER you write to the BD ...)
2002         And above all, all pdb sizes are in words - NOT DWORDS!
2003         */
2004
2005         pkt_prod = fp->tx_pkt_prod++;
2006         bd_prod = TX_BD(fp->tx_bd_prod);
2007
2008         /* get a tx_buf and first BD */
2009         tx_buf = &fp->tx_buf_ring[TX_BD(pkt_prod)];
2010         tx_start_bd = &fp->tx_desc_ring[bd_prod].start_bd;
2011
2012         tx_start_bd->bd_flags.as_bitfield = ETH_TX_BD_FLAGS_START_BD;
2013         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_ETH_ADDR_TYPE,
2014                  mac_type);
2015
2016         /* header nbd */
2017         SET_FLAG(tx_start_bd->general_data, ETH_TX_START_BD_HDR_NBDS, 1);
2018
2019         /* remember the first BD of the packet */
2020         tx_buf->first_bd = fp->tx_bd_prod;
2021         tx_buf->skb = skb;
2022         tx_buf->flags = 0;
2023
2024         DP(NETIF_MSG_TX_QUEUED,
2025            "sending pkt %u @%p  next_idx %u  bd %u @%p\n",
2026            pkt_prod, tx_buf, fp->tx_pkt_prod, bd_prod, tx_start_bd);
2027
2028 #ifdef BCM_VLAN
2029         if (vlan_tx_tag_present(skb)) {
2030                 tx_start_bd->vlan_or_ethertype =
2031                     cpu_to_le16(vlan_tx_tag_get(skb));
2032                 tx_start_bd->bd_flags.as_bitfield |=
2033                     (X_ETH_OUTBAND_VLAN << ETH_TX_BD_FLAGS_VLAN_MODE_SHIFT);
2034         } else
2035 #endif
2036                 tx_start_bd->vlan_or_ethertype = cpu_to_le16(pkt_prod);
2037
2038         /* turn on parsing and get a BD */
2039         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2040
2041         if (xmit_type & XMIT_CSUM) {
2042                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
2043
2044                 if (xmit_type & XMIT_CSUM_V4)
2045                         tx_start_bd->bd_flags.as_bitfield |=
2046                                                 ETH_TX_BD_FLAGS_IP_CSUM;
2047                 else
2048                         tx_start_bd->bd_flags.as_bitfield |=
2049                                                 ETH_TX_BD_FLAGS_IPV6;
2050
2051                 if (!(xmit_type & XMIT_CSUM_TCP))
2052                         tx_start_bd->bd_flags.as_bitfield |=
2053                                                 ETH_TX_BD_FLAGS_IS_UDP;
2054         }
2055
2056         if (CHIP_IS_E2(bp)) {
2057                 pbd_e2 = &fp->tx_desc_ring[bd_prod].parse_bd_e2;
2058                 memset(pbd_e2, 0, sizeof(struct eth_tx_parse_bd_e2));
2059                 /* Set PBD in checksum offload case */
2060                 if (xmit_type & XMIT_CSUM)
2061                         hlen = bnx2x_set_pbd_csum_e2(bp,
2062                                                      skb, pbd_e2, xmit_type);
2063         } else {
2064                 pbd_e1x = &fp->tx_desc_ring[bd_prod].parse_bd_e1x;
2065                 memset(pbd_e1x, 0, sizeof(struct eth_tx_parse_bd_e1x));
2066                 /* Set PBD in checksum offload case */
2067                 if (xmit_type & XMIT_CSUM)
2068                         hlen = bnx2x_set_pbd_csum(bp, skb, pbd_e1x, xmit_type);
2069
2070         }
2071
2072         /* Map skb linear data for DMA */
2073         mapping = dma_map_single(&bp->pdev->dev, skb->data,
2074                                  skb_headlen(skb), DMA_TO_DEVICE);
2075
2076         /* Setup the data pointer of the first BD of the packet */
2077         tx_start_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2078         tx_start_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2079         nbd = skb_shinfo(skb)->nr_frags + 2; /* start_bd + pbd + frags */
2080         tx_start_bd->nbd = cpu_to_le16(nbd);
2081         tx_start_bd->nbytes = cpu_to_le16(skb_headlen(skb));
2082         pkt_size = tx_start_bd->nbytes;
2083
2084         DP(NETIF_MSG_TX_QUEUED, "first bd @%p  addr (%x:%x)  nbd %d"
2085            "  nbytes %d  flags %x  vlan %x\n",
2086            tx_start_bd, tx_start_bd->addr_hi, tx_start_bd->addr_lo,
2087            le16_to_cpu(tx_start_bd->nbd), le16_to_cpu(tx_start_bd->nbytes),
2088            tx_start_bd->bd_flags.as_bitfield,
2089            le16_to_cpu(tx_start_bd->vlan_or_ethertype));
2090
2091         if (xmit_type & XMIT_GSO) {
2092
2093                 DP(NETIF_MSG_TX_QUEUED,
2094                    "TSO packet len %d  hlen %d  total len %d  tso size %d\n",
2095                    skb->len, hlen, skb_headlen(skb),
2096                    skb_shinfo(skb)->gso_size);
2097
2098                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_SW_LSO;
2099
2100                 if (unlikely(skb_headlen(skb) > hlen))
2101                         bd_prod = bnx2x_tx_split(bp, fp, tx_buf, &tx_start_bd,
2102                                                  hlen, bd_prod, ++nbd);
2103                 if (CHIP_IS_E2(bp))
2104                         bnx2x_set_pbd_gso_e2(skb, pbd_e2, xmit_type);
2105                 else
2106                         bnx2x_set_pbd_gso(skb, pbd_e1x, xmit_type);
2107         }
2108         tx_data_bd = (struct eth_tx_bd *)tx_start_bd;
2109
2110         /* Handle fragmented skb */
2111         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2112                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2113
2114                 bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2115                 tx_data_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2116                 if (total_pkt_bd == NULL)
2117                         total_pkt_bd = &fp->tx_desc_ring[bd_prod].reg_bd;
2118
2119                 mapping = dma_map_page(&bp->pdev->dev, frag->page,
2120                                        frag->page_offset,
2121                                        frag->size, DMA_TO_DEVICE);
2122
2123                 tx_data_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
2124                 tx_data_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
2125                 tx_data_bd->nbytes = cpu_to_le16(frag->size);
2126                 le16_add_cpu(&pkt_size, frag->size);
2127
2128                 DP(NETIF_MSG_TX_QUEUED,
2129                    "frag %d  bd @%p  addr (%x:%x)  nbytes %d\n",
2130                    i, tx_data_bd, tx_data_bd->addr_hi, tx_data_bd->addr_lo,
2131                    le16_to_cpu(tx_data_bd->nbytes));
2132         }
2133
2134         DP(NETIF_MSG_TX_QUEUED, "last bd @%p\n", tx_data_bd);
2135
2136         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
2137
2138         /* now send a tx doorbell, counting the next BD
2139          * if the packet contains or ends with it
2140          */
2141         if (TX_BD_POFF(bd_prod) < nbd)
2142                 nbd++;
2143
2144         if (total_pkt_bd != NULL)
2145                 total_pkt_bd->total_pkt_bytes = pkt_size;
2146
2147         if (pbd_e1x)
2148                 DP(NETIF_MSG_TX_QUEUED,
2149                    "PBD (E1X) @%p  ip_data %x  ip_hlen %u  ip_id %u  lso_mss %u"
2150                    "  tcp_flags %x  xsum %x  seq %u  hlen %u\n",
2151                    pbd_e1x, pbd_e1x->global_data, pbd_e1x->ip_hlen_w,
2152                    pbd_e1x->ip_id, pbd_e1x->lso_mss, pbd_e1x->tcp_flags,
2153                    pbd_e1x->tcp_pseudo_csum, pbd_e1x->tcp_send_seq,
2154                     le16_to_cpu(pbd_e1x->total_hlen_w));
2155         if (pbd_e2)
2156                 DP(NETIF_MSG_TX_QUEUED,
2157                    "PBD (E2) @%p  dst %x %x %x src %x %x %x parsing_data %x\n",
2158                    pbd_e2, pbd_e2->dst_mac_addr_hi, pbd_e2->dst_mac_addr_mid,
2159                    pbd_e2->dst_mac_addr_lo, pbd_e2->src_mac_addr_hi,
2160                    pbd_e2->src_mac_addr_mid, pbd_e2->src_mac_addr_lo,
2161                    pbd_e2->parsing_data);
2162         DP(NETIF_MSG_TX_QUEUED, "doorbell: nbd %d  bd %u\n", nbd, bd_prod);
2163
2164         /*
2165          * Make sure that the BD data is updated before updating the producer
2166          * since FW might read the BD right after the producer is updated.
2167          * This is only applicable for weak-ordered memory model archs such
2168          * as IA-64. The following barrier is also mandatory since FW will
2169          * assumes packets must have BDs.
2170          */
2171         wmb();
2172
2173         fp->tx_db.data.prod += nbd;
2174         barrier();
2175
2176         DOORBELL(bp, fp->cid, fp->tx_db.raw);
2177
2178         mmiowb();
2179
2180         fp->tx_bd_prod += nbd;
2181
2182         if (unlikely(bnx2x_tx_avail(fp) < MAX_SKB_FRAGS + 3)) {
2183                 netif_tx_stop_queue(txq);
2184
2185                 /* paired memory barrier is in bnx2x_tx_int(), we have to keep
2186                  * ordering of set_bit() in netif_tx_stop_queue() and read of
2187                  * fp->bd_tx_cons */
2188                 smp_mb();
2189
2190                 fp->eth_q_stats.driver_xoff++;
2191                 if (bnx2x_tx_avail(fp) >= MAX_SKB_FRAGS + 3)
2192                         netif_tx_wake_queue(txq);
2193         }
2194         fp->tx_pkt++;
2195
2196         return NETDEV_TX_OK;
2197 }
2198
2199 /* called with rtnl_lock */
2200 int bnx2x_change_mac_addr(struct net_device *dev, void *p)
2201 {
2202         struct sockaddr *addr = p;
2203         struct bnx2x *bp = netdev_priv(dev);
2204
2205         if (!is_valid_ether_addr((u8 *)(addr->sa_data)))
2206                 return -EINVAL;
2207
2208         memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2209         if (netif_running(dev))
2210                 bnx2x_set_eth_mac(bp, 1);
2211
2212         return 0;
2213 }
2214
2215
2216 int bnx2x_setup_irqs(struct bnx2x *bp)
2217 {
2218         int rc = 0;
2219         if (bp->flags & USING_MSIX_FLAG) {
2220                 rc = bnx2x_req_msix_irqs(bp);
2221                 if (rc)
2222                         return rc;
2223         } else {
2224                 bnx2x_ack_int(bp);
2225                 rc = bnx2x_req_irq(bp);
2226                 if (rc) {
2227                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
2228                         return rc;
2229                 }
2230                 if (bp->flags & USING_MSI_FLAG) {
2231                         bp->dev->irq = bp->pdev->irq;
2232                         netdev_info(bp->dev, "using MSI  IRQ %d\n",
2233                                bp->pdev->irq);
2234                 }
2235         }
2236
2237         return 0;
2238 }
2239
2240 void bnx2x_free_mem_bp(struct bnx2x *bp)
2241 {
2242         kfree(bp->fp);
2243         kfree(bp->msix_table);
2244         kfree(bp->ilt);
2245 }
2246
2247 int __devinit bnx2x_alloc_mem_bp(struct bnx2x *bp)
2248 {
2249         struct bnx2x_fastpath *fp;
2250         struct msix_entry *tbl;
2251         struct bnx2x_ilt *ilt;
2252
2253         /* fp array */
2254         fp = kzalloc(L2_FP_COUNT(bp->l2_cid_count)*sizeof(*fp), GFP_KERNEL);
2255         if (!fp)
2256                 goto alloc_err;
2257         bp->fp = fp;
2258
2259         /* msix table */
2260         tbl = kzalloc((bp->l2_cid_count + 1) * sizeof(*tbl),
2261                                   GFP_KERNEL);
2262         if (!tbl)
2263                 goto alloc_err;
2264         bp->msix_table = tbl;
2265
2266         /* ilt */
2267         ilt = kzalloc(sizeof(*ilt), GFP_KERNEL);
2268         if (!ilt)
2269                 goto alloc_err;
2270         bp->ilt = ilt;
2271
2272         return 0;
2273 alloc_err:
2274         bnx2x_free_mem_bp(bp);
2275         return -ENOMEM;
2276
2277 }
2278
2279 /* called with rtnl_lock */
2280 int bnx2x_change_mtu(struct net_device *dev, int new_mtu)
2281 {
2282         struct bnx2x *bp = netdev_priv(dev);
2283         int rc = 0;
2284
2285         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2286                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2287                 return -EAGAIN;
2288         }
2289
2290         if ((new_mtu > ETH_MAX_JUMBO_PACKET_SIZE) ||
2291             ((new_mtu + ETH_HLEN) < ETH_MIN_PACKET_SIZE))
2292                 return -EINVAL;
2293
2294         /* This does not race with packet allocation
2295          * because the actual alloc size is
2296          * only updated as part of load
2297          */
2298         dev->mtu = new_mtu;
2299
2300         if (netif_running(dev)) {
2301                 bnx2x_nic_unload(bp, UNLOAD_NORMAL);
2302                 rc = bnx2x_nic_load(bp, LOAD_NORMAL);
2303         }
2304
2305         return rc;
2306 }
2307
2308 void bnx2x_tx_timeout(struct net_device *dev)
2309 {
2310         struct bnx2x *bp = netdev_priv(dev);
2311
2312 #ifdef BNX2X_STOP_ON_ERROR
2313         if (!bp->panic)
2314                 bnx2x_panic();
2315 #endif
2316         /* This allows the netif to be shutdown gracefully before resetting */
2317         schedule_delayed_work(&bp->reset_task, 0);
2318 }
2319
2320 #ifdef BCM_VLAN
2321 /* called with rtnl_lock */
2322 void bnx2x_vlan_rx_register(struct net_device *dev,
2323                                    struct vlan_group *vlgrp)
2324 {
2325         struct bnx2x *bp = netdev_priv(dev);
2326
2327         bp->vlgrp = vlgrp;
2328 }
2329
2330 #endif
2331
2332 int bnx2x_suspend(struct pci_dev *pdev, pm_message_t state)
2333 {
2334         struct net_device *dev = pci_get_drvdata(pdev);
2335         struct bnx2x *bp;
2336
2337         if (!dev) {
2338                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2339                 return -ENODEV;
2340         }
2341         bp = netdev_priv(dev);
2342
2343         rtnl_lock();
2344
2345         pci_save_state(pdev);
2346
2347         if (!netif_running(dev)) {
2348                 rtnl_unlock();
2349                 return 0;
2350         }
2351
2352         netif_device_detach(dev);
2353
2354         bnx2x_nic_unload(bp, UNLOAD_CLOSE);
2355
2356         bnx2x_set_power_state(bp, pci_choose_state(pdev, state));
2357
2358         rtnl_unlock();
2359
2360         return 0;
2361 }
2362
2363 int bnx2x_resume(struct pci_dev *pdev)
2364 {
2365         struct net_device *dev = pci_get_drvdata(pdev);
2366         struct bnx2x *bp;
2367         int rc;
2368
2369         if (!dev) {
2370                 dev_err(&pdev->dev, "BAD net device from bnx2x_init_one\n");
2371                 return -ENODEV;
2372         }
2373         bp = netdev_priv(dev);
2374
2375         if (bp->recovery_state != BNX2X_RECOVERY_DONE) {
2376                 printk(KERN_ERR "Handling parity error recovery. Try again later\n");
2377                 return -EAGAIN;
2378         }
2379
2380         rtnl_lock();
2381
2382         pci_restore_state(pdev);
2383
2384         if (!netif_running(dev)) {
2385                 rtnl_unlock();
2386                 return 0;
2387         }
2388
2389         bnx2x_set_power_state(bp, PCI_D0);
2390         netif_device_attach(dev);
2391
2392         /* Since the chip was reset, clear the FW sequence number */
2393         bp->fw_seq = 0;
2394         rc = bnx2x_nic_load(bp, LOAD_OPEN);
2395
2396         rtnl_unlock();
2397
2398         return rc;
2399 }