drivers/net/qlge/qlge_main.c

   1 /*
   2  * QLogic qlge NIC HBA Driver
   3  * Copyright (c)  2003-2008 QLogic Corporation
   4  * See LICENSE.qlge for copyright and licensing details.
   5  * Author:     Linux qlge network device driver by
   6  *                      Ron Mercer <ron.mercer@qlogic.com>
   7  */
   8 #include <linux/kernel.h>
   9 #include <linux/init.h>
  10 #include <linux/types.h>
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <linux/pci.h>
  14 #include <linux/dma-mapping.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/sched.h>
  17 #include <linux/slab.h>
  18 #include <linux/dmapool.h>
  19 #include <linux/mempool.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/kthread.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/errno.h>
  24 #include <linux/ioport.h>
  25 #include <linux/in.h>
  26 #include <linux/ip.h>
  27 #include <linux/ipv6.h>
  28 #include <net/ipv6.h>
  29 #include <linux/tcp.h>
  30 #include <linux/udp.h>
  31 #include <linux/if_arp.h>
  32 #include <linux/if_ether.h>
  33 #include <linux/netdevice.h>
  34 #include <linux/etherdevice.h>
  35 #include <linux/ethtool.h>
  36 #include <linux/skbuff.h>
  37 #include <linux/if_vlan.h>
  38 #include <linux/delay.h>
  39 #include <linux/mm.h>
  40 #include <linux/vmalloc.h>
  41 #include <net/ip6_checksum.h>
  42
  43 #include "qlge.h"
  44
  45 char qlge_driver_name[] = DRV_NAME;
  46 const char qlge_driver_version[] = DRV_VERSION;
  47
  48 MODULE_AUTHOR("Ron Mercer <ron.mercer@qlogic.com>");
  49 MODULE_DESCRIPTION(DRV_STRING " ");
  50 MODULE_LICENSE("GPL");
  51 MODULE_VERSION(DRV_VERSION);
  52
  53 static const u32 default_msg =
  54     NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK |
  55 /* NETIF_MSG_TIMER |    */
  56     NETIF_MSG_IFDOWN |
  57     NETIF_MSG_IFUP |
  58     NETIF_MSG_RX_ERR |
  59     NETIF_MSG_TX_ERR |
  60 /*  NETIF_MSG_TX_QUEUED | */
  61 /*  NETIF_MSG_INTR | NETIF_MSG_TX_DONE | NETIF_MSG_RX_STATUS | */
  62 /* NETIF_MSG_PKTDATA | */
  63     NETIF_MSG_HW | NETIF_MSG_WOL | 0;
  64
  65 static int debug = 0x00007fff;  /* defaults above */
  66 module_param(debug, int, 0);
  67 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  68
  69 #define MSIX_IRQ 0
  70 #define MSI_IRQ 1
  71 #define LEG_IRQ 2
  72 static int irq_type = MSIX_IRQ;
  73 module_param(irq_type, int, MSIX_IRQ);
  74 MODULE_PARM_DESC(irq_type, "0 = MSI-X, 1 = MSI, 2 = Legacy.");
  75
  76 static struct pci_device_id qlge_pci_tbl[] __devinitdata = {
  77         {PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, QLGE_DEVICE_ID_8012)},
  78         {PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, QLGE_DEVICE_ID_8000)},
  79         /* required last entry */
  80         {0,}
  81 };
  82
  83 MODULE_DEVICE_TABLE(pci, qlge_pci_tbl);
  84
  85 /* This hardware semaphore causes exclusive access to
  86  * resources shared between the NIC driver, MPI firmware,
  87  * FCOE firmware and the FC driver.
  88  */
  89 static int ql_sem_trylock(struct ql_adapter *qdev, u32 sem_mask)
  90 {
  91         u32 sem_bits = 0;
  92
  93         switch (sem_mask) {
  94         case SEM_XGMAC0_MASK:
  95                 sem_bits = SEM_SET << SEM_XGMAC0_SHIFT;
  96                 break;
  97         case SEM_XGMAC1_MASK:
  98                 sem_bits = SEM_SET << SEM_XGMAC1_SHIFT;
  99                 break;
 100         case SEM_ICB_MASK:
 101                 sem_bits = SEM_SET << SEM_ICB_SHIFT;
 102                 break;
 103         case SEM_MAC_ADDR_MASK:
 104                 sem_bits = SEM_SET << SEM_MAC_ADDR_SHIFT;
 105                 break;
 106         case SEM_FLASH_MASK:
 107                 sem_bits = SEM_SET << SEM_FLASH_SHIFT;
 108                 break;
 109         case SEM_PROBE_MASK:
 110                 sem_bits = SEM_SET << SEM_PROBE_SHIFT;
 111                 break;
 112         case SEM_RT_IDX_MASK:
 113                 sem_bits = SEM_SET << SEM_RT_IDX_SHIFT;
 114                 break;
 115         case SEM_PROC_REG_MASK:
 116                 sem_bits = SEM_SET << SEM_PROC_REG_SHIFT;
 117                 break;
 118         default:
 119                 QPRINTK(qdev, PROBE, ALERT, "Bad Semaphore mask!.\n");
 120                 return -EINVAL;
 121         }
 122
 123         ql_write32(qdev, SEM, sem_bits | sem_mask);
 124         return !(ql_read32(qdev, SEM) & sem_bits);
 125 }
 126
 127 int ql_sem_spinlock(struct ql_adapter *qdev, u32 sem_mask)
 128 {
 129         unsigned int wait_count = 30;
 130         do {
 131                 if (!ql_sem_trylock(qdev, sem_mask))
 132                         return 0;
 133                 udelay(100);
 134         } while (--wait_count);
 135         return -ETIMEDOUT;
 136 }
 137
 138 void ql_sem_unlock(struct ql_adapter *qdev, u32 sem_mask)
 139 {
 140         ql_write32(qdev, SEM, sem_mask);
 141         ql_read32(qdev, SEM);   /* flush */
 142 }
 143
 144 /* This function waits for a specific bit to come ready
 145  * in a given register.  It is used mostly by the initialize
 146  * process, but is also used in kernel thread API such as
 147  * netdev->set_multi, netdev->set_mac_address, netdev->vlan_rx_add_vid.
 148  */
 149 int ql_wait_reg_rdy(struct ql_adapter *qdev, u32 reg, u32 bit, u32 err_bit)
 150 {
 151         u32 temp;
 152         int count = UDELAY_COUNT;
 153
 154         while (count) {
 155                 temp = ql_read32(qdev, reg);
 156
 157                 /* check for errors */
 158                 if (temp & err_bit) {
 159                         QPRINTK(qdev, PROBE, ALERT,
 160                                 "register 0x%.08x access error, value = 0x%.08x!.\n",
 161                                 reg, temp);
 162                         return -EIO;
 163                 } else if (temp & bit)
 164                         return 0;
 165                 udelay(UDELAY_DELAY);
 166                 count--;
 167         }
 168         QPRINTK(qdev, PROBE, ALERT,
 169                 "Timed out waiting for reg %x to come ready.\n", reg);
 170         return -ETIMEDOUT;
 171 }
 172
 173 /* The CFG register is used to download TX and RX control blocks
 174  * to the chip. This function waits for an operation to complete.
 175  */
 176 static int ql_wait_cfg(struct ql_adapter *qdev, u32 bit)
 177 {
 178         int count = UDELAY_COUNT;
 179         u32 temp;
 180
 181         while (count) {
 182                 temp = ql_read32(qdev, CFG);
 183                 if (temp & CFG_LE)
 184                         return -EIO;
 185                 if (!(temp & bit))
 186                         return 0;
 187                 udelay(UDELAY_DELAY);
 188                 count--;
 189         }
 190         return -ETIMEDOUT;
 191 }
 192
 193
 194 /* Used to issue init control blocks to hw. Maps control block,
 195  * sets address, triggers download, waits for completion.
 196  */
 197 int ql_write_cfg(struct ql_adapter *qdev, void *ptr, int size, u32 bit,
 198                  u16 q_id)
 199 {
 200         u64 map;
 201         int status = 0;
 202         int direction;
 203         u32 mask;
 204         u32 value;
 205
 206         direction =
 207             (bit & (CFG_LRQ | CFG_LR | CFG_LCQ)) ? PCI_DMA_TODEVICE :
 208             PCI_DMA_FROMDEVICE;
 209
 210         map = pci_map_single(qdev->pdev, ptr, size, direction);
 211         if (pci_dma_mapping_error(qdev->pdev, map)) {
 212                 QPRINTK(qdev, IFUP, ERR, "Couldn't map DMA area.\n");
 213                 return -ENOMEM;
 214         }
 215
 216         status = ql_sem_spinlock(qdev, SEM_ICB_MASK);
 217         if (status)
 218                 return status;
 219
 220         status = ql_wait_cfg(qdev, bit);
 221         if (status) {
 222                 QPRINTK(qdev, IFUP, ERR,
 223                         "Timed out waiting for CFG to come ready.\n");
 224                 goto exit;
 225         }
 226
 227         ql_write32(qdev, ICB_L, (u32) map);
 228         ql_write32(qdev, ICB_H, (u32) (map >> 32));
 229
 230         mask = CFG_Q_MASK | (bit << 16);
 231         value = bit | (q_id << CFG_Q_SHIFT);
 232         ql_write32(qdev, CFG, (mask | value));
 233
 234         /*
 235          * Wait for the bit to clear after signaling hw.
 236          */
 237         status = ql_wait_cfg(qdev, bit);
 238 exit:
 239         ql_sem_unlock(qdev, SEM_ICB_MASK);      /* does flush too */
 240         pci_unmap_single(qdev->pdev, map, size, direction);
 241         return status;
 242 }
 243
 244 /* Get a specific MAC address from the CAM.  Used for debug and reg dump. */
 245 int ql_get_mac_addr_reg(struct ql_adapter *qdev, u32 type, u16 index,
 246                         u32 *value)
 247 {
 248         u32 offset = 0;
 249         int status;
 250
 251         switch (type) {
 252         case MAC_ADDR_TYPE_MULTI_MAC:
 253         case MAC_ADDR_TYPE_CAM_MAC:
 254                 {
 255                         status =
 256                             ql_wait_reg_rdy(qdev,
 257                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 258                         if (status)
 259                                 goto exit;
 260                         ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
 261                                    (index << MAC_ADDR_IDX_SHIFT) | /* index */
 262                                    MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
 263                         status =
 264                             ql_wait_reg_rdy(qdev,
 265                                 MAC_ADDR_IDX, MAC_ADDR_MR, 0);
 266                         if (status)
 267                                 goto exit;
 268                         *value++ = ql_read32(qdev, MAC_ADDR_DATA);
 269                         status =
 270                             ql_wait_reg_rdy(qdev,
 271                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 272                         if (status)
 273                                 goto exit;
 274                         ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
 275                                    (index << MAC_ADDR_IDX_SHIFT) | /* index */
 276                                    MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
 277                         status =
 278                             ql_wait_reg_rdy(qdev,
 279                                 MAC_ADDR_IDX, MAC_ADDR_MR, 0);
 280                         if (status)
 281                                 goto exit;
 282                         *value++ = ql_read32(qdev, MAC_ADDR_DATA);
 283                         if (type == MAC_ADDR_TYPE_CAM_MAC) {
 284                                 status =
 285                                     ql_wait_reg_rdy(qdev,
 286                                         MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 287                                 if (status)
 288                                         goto exit;
 289                                 ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
 290                                            (index << MAC_ADDR_IDX_SHIFT) | /* index */
 291                                            MAC_ADDR_ADR | MAC_ADDR_RS | type); /* type */
 292                                 status =
 293                                     ql_wait_reg_rdy(qdev, MAC_ADDR_IDX,
 294                                                     MAC_ADDR_MR, 0);
 295                                 if (status)
 296                                         goto exit;
 297                                 *value++ = ql_read32(qdev, MAC_ADDR_DATA);
 298                         }
 299                         break;
 300                 }
 301         case MAC_ADDR_TYPE_VLAN:
 302         case MAC_ADDR_TYPE_MULTI_FLTR:
 303         default:
 304                 QPRINTK(qdev, IFUP, CRIT,
 305                         "Address type %d not yet supported.\n", type);
 306                 status = -EPERM;
 307         }
 308 exit:
 309         return status;
 310 }
 311
 312 /* Set up a MAC, multicast or VLAN address for the
 313  * inbound frame matching.
 314  */
 315 static int ql_set_mac_addr_reg(struct ql_adapter *qdev, u8 *addr, u32 type,
 316                                u16 index)
 317 {
 318         u32 offset = 0;
 319         int status = 0;
 320
 321         switch (type) {
 322         case MAC_ADDR_TYPE_MULTI_MAC:
 323         case MAC_ADDR_TYPE_CAM_MAC:
 324                 {
 325                         u32 cam_output;
 326                         u32 upper = (addr[0] << 8) | addr[1];
 327                         u32 lower =
 328                             (addr[2] << 24) | (addr[3] << 16) | (addr[4] << 8) |
 329                             (addr[5]);
 330
 331                         QPRINTK(qdev, IFUP, DEBUG,
 332                                 "Adding %s address %pM"
 333                                 " at index %d in the CAM.\n",
 334                                 ((type ==
 335                                   MAC_ADDR_TYPE_MULTI_MAC) ? "MULTICAST" :
 336                                  "UNICAST"), addr, index);
 337
 338                         status =
 339                             ql_wait_reg_rdy(qdev,
 340                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 341                         if (status)
 342                                 goto exit;
 343                         ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
 344                                    (index << MAC_ADDR_IDX_SHIFT) | /* index */
 345                                    type);       /* type */
 346                         ql_write32(qdev, MAC_ADDR_DATA, lower);
 347                         status =
 348                             ql_wait_reg_rdy(qdev,
 349                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 350                         if (status)
 351                                 goto exit;
 352                         ql_write32(qdev, MAC_ADDR_IDX, (offset++) | /* offset */
 353                                    (index << MAC_ADDR_IDX_SHIFT) | /* index */
 354                                    type);       /* type */
 355                         ql_write32(qdev, MAC_ADDR_DATA, upper);
 356                         status =
 357                             ql_wait_reg_rdy(qdev,
 358                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 359                         if (status)
 360                                 goto exit;
 361                         ql_write32(qdev, MAC_ADDR_IDX, (offset) |       /* offset */
 362                                    (index << MAC_ADDR_IDX_SHIFT) |      /* index */
 363                                    type);       /* type */
 364                         /* This field should also include the queue id
 365                            and possibly the function id.  Right now we hardcode
 366                            the route field to NIC core.
 367                          */
 368                         if (type == MAC_ADDR_TYPE_CAM_MAC) {
 369                                 cam_output = (CAM_OUT_ROUTE_NIC |
 370                                               (qdev->
 371                                                func << CAM_OUT_FUNC_SHIFT) |
 372                                                 (0 << CAM_OUT_CQ_ID_SHIFT));
 373                                 if (qdev->vlgrp)
 374                                         cam_output |= CAM_OUT_RV;
 375                                 /* route to NIC core */
 376                                 ql_write32(qdev, MAC_ADDR_DATA, cam_output);
 377                         }
 378                         break;
 379                 }
 380         case MAC_ADDR_TYPE_VLAN:
 381                 {
 382                         u32 enable_bit = *((u32 *) &addr[0]);
 383                         /* For VLAN, the addr actually holds a bit that
 384                          * either enables or disables the vlan id we are
 385                          * addressing. It's either MAC_ADDR_E on or off.
 386                          * That's bit-27 we're talking about.
 387                          */
 388                         QPRINTK(qdev, IFUP, INFO, "%s VLAN ID %d %s the CAM.\n",
 389                                 (enable_bit ? "Adding" : "Removing"),
 390                                 index, (enable_bit ? "to" : "from"));
 391
 392                         status =
 393                             ql_wait_reg_rdy(qdev,
 394                                 MAC_ADDR_IDX, MAC_ADDR_MW, 0);
 395                         if (status)
 396                                 goto exit;
 397                         ql_write32(qdev, MAC_ADDR_IDX, offset | /* offset */
 398                                    (index << MAC_ADDR_IDX_SHIFT) |      /* index */
 399                                    type |       /* type */
 400                                    enable_bit); /* enable/disable */
 401                         break;
 402                 }
 403         case MAC_ADDR_TYPE_MULTI_FLTR:
 404         default:
 405                 QPRINTK(qdev, IFUP, CRIT,
 406                         "Address type %d not yet supported.\n", type);
 407                 status = -EPERM;
 408         }
 409 exit:
 410         return status;
 411 }
 412
 413 /* Set or clear MAC address in hardware. We sometimes
 414  * have to clear it to prevent wrong frame routing
 415  * especially in a bonding environment.
 416  */
 417 static int ql_set_mac_addr(struct ql_adapter *qdev, int set)
 418 {
 419         int status;
 420         char zero_mac_addr[ETH_ALEN];
 421         char *addr;
 422
 423         if (set) {
 424                 addr = &qdev->ndev->dev_addr[0];
 425                 QPRINTK(qdev, IFUP, DEBUG,
 426                         "Set Mac addr %02x:%02x:%02x:%02x:%02x:%02x\n",
 427                         addr[0], addr[1], addr[2], addr[3],
 428                         addr[4], addr[5]);
 429         } else {
 430                 memset(zero_mac_addr, 0, ETH_ALEN);
 431                 addr = &zero_mac_addr[0];
 432                 QPRINTK(qdev, IFUP, DEBUG,
 433                                 "Clearing MAC address on %s\n",
 434                                 qdev->ndev->name);
 435         }
 436         status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
 437         if (status)
 438                 return status;
 439         status = ql_set_mac_addr_reg(qdev, (u8 *) addr,
 440                         MAC_ADDR_TYPE_CAM_MAC, qdev->func * MAX_CQ);
 441         ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
 442         if (status)
 443                 QPRINTK(qdev, IFUP, ERR, "Failed to init mac "
 444                         "address.\n");
 445         return status;
 446 }
 447
 448 void ql_link_on(struct ql_adapter *qdev)
 449 {
 450         QPRINTK(qdev, LINK, ERR, "%s: Link is up.\n",
 451                                  qdev->ndev->name);
 452         netif_carrier_on(qdev->ndev);
 453         ql_set_mac_addr(qdev, 1);
 454 }
 455
 456 void ql_link_off(struct ql_adapter *qdev)
 457 {
 458         QPRINTK(qdev, LINK, ERR, "%s: Link is down.\n",
 459                                  qdev->ndev->name);
 460         netif_carrier_off(qdev->ndev);
 461         ql_set_mac_addr(qdev, 0);
 462 }
 463
 464 /* Get a specific frame routing value from the CAM.
 465  * Used for debug and reg dump.
 466  */
 467 int ql_get_routing_reg(struct ql_adapter *qdev, u32 index, u32 *value)
 468 {
 469         int status = 0;
 470
 471         status = ql_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MW, 0);
 472         if (status)
 473                 goto exit;
 474
 475         ql_write32(qdev, RT_IDX,
 476                    RT_IDX_TYPE_NICQ | RT_IDX_RS | (index << RT_IDX_IDX_SHIFT));
 477         status = ql_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MR, 0);
 478         if (status)
 479                 goto exit;
 480         *value = ql_read32(qdev, RT_DATA);
 481 exit:
 482         return status;
 483 }
 484
 485 /* The NIC function for this chip has 16 routing indexes.  Each one can be used
 486  * to route different frame types to various inbound queues.  We send broadcast/
 487  * multicast/error frames to the default queue for slow handling,
 488  * and CAM hit/RSS frames to the fast handling queues.
 489  */
 490 static int ql_set_routing_reg(struct ql_adapter *qdev, u32 index, u32 mask,
 491                               int enable)
 492 {
 493         int status = -EINVAL; /* Return error if no mask match. */
 494         u32 value = 0;
 495
 496         QPRINTK(qdev, IFUP, DEBUG,
 497                 "%s %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s mask %s the routing reg.\n",
 498                 (enable ? "Adding" : "Removing"),
 499                 ((index == RT_IDX_ALL_ERR_SLOT) ? "MAC ERROR/ALL ERROR" : ""),
 500                 ((index == RT_IDX_IP_CSUM_ERR_SLOT) ? "IP CSUM ERROR" : ""),
 501                 ((index ==
 502                   RT_IDX_TCP_UDP_CSUM_ERR_SLOT) ? "TCP/UDP CSUM ERROR" : ""),
 503                 ((index == RT_IDX_BCAST_SLOT) ? "BROADCAST" : ""),
 504                 ((index == RT_IDX_MCAST_MATCH_SLOT) ? "MULTICAST MATCH" : ""),
 505                 ((index == RT_IDX_ALLMULTI_SLOT) ? "ALL MULTICAST MATCH" : ""),
 506                 ((index == RT_IDX_UNUSED6_SLOT) ? "UNUSED6" : ""),
 507                 ((index == RT_IDX_UNUSED7_SLOT) ? "UNUSED7" : ""),
 508                 ((index == RT_IDX_RSS_MATCH_SLOT) ? "RSS ALL/IPV4 MATCH" : ""),
 509                 ((index == RT_IDX_RSS_IPV6_SLOT) ? "RSS IPV6" : ""),
 510                 ((index == RT_IDX_RSS_TCP4_SLOT) ? "RSS TCP4" : ""),
 511                 ((index == RT_IDX_RSS_TCP6_SLOT) ? "RSS TCP6" : ""),
 512                 ((index == RT_IDX_CAM_HIT_SLOT) ? "CAM HIT" : ""),
 513                 ((index == RT_IDX_UNUSED013) ? "UNUSED13" : ""),
 514                 ((index == RT_IDX_UNUSED014) ? "UNUSED14" : ""),
 515                 ((index == RT_IDX_PROMISCUOUS_SLOT) ? "PROMISCUOUS" : ""),
 516                 (enable ? "to" : "from"));
 517
 518         switch (mask) {
 519         case RT_IDX_CAM_HIT:
 520                 {
 521                         value = RT_IDX_DST_CAM_Q |      /* dest */
 522                             RT_IDX_TYPE_NICQ |  /* type */
 523                             (RT_IDX_CAM_HIT_SLOT << RT_IDX_IDX_SHIFT);/* index */
 524                         break;
 525                 }
 526         case RT_IDX_VALID:      /* Promiscuous Mode frames. */
 527                 {
 528                         value = RT_IDX_DST_DFLT_Q |     /* dest */
 529                             RT_IDX_TYPE_NICQ |  /* type */
 530                             (RT_IDX_PROMISCUOUS_SLOT << RT_IDX_IDX_SHIFT);/* index */
 531                         break;
 532                 }
 533         case RT_IDX_ERR:        /* Pass up MAC,IP,TCP/UDP error frames. */
 534                 {
 535                         value = RT_IDX_DST_DFLT_Q |     /* dest */
 536                             RT_IDX_TYPE_NICQ |  /* type */
 537                             (RT_IDX_ALL_ERR_SLOT << RT_IDX_IDX_SHIFT);/* index */
 538                         break;
 539                 }
 540         case RT_IDX_BCAST:      /* Pass up Broadcast frames to default Q. */
 541                 {
 542                         value = RT_IDX_DST_DFLT_Q |     /* dest */
 543                             RT_IDX_TYPE_NICQ |  /* type */
 544                             (RT_IDX_BCAST_SLOT << RT_IDX_IDX_SHIFT);/* index */
 545                         break;
 546                 }
 547         case RT_IDX_MCAST:      /* Pass up All Multicast frames. */
 548                 {
 549                         value = RT_IDX_DST_CAM_Q |      /* dest */
 550                             RT_IDX_TYPE_NICQ |  /* type */
 551                             (RT_IDX_ALLMULTI_SLOT << RT_IDX_IDX_SHIFT);/* index */
 552                         break;
 553                 }
 554         case RT_IDX_MCAST_MATCH:        /* Pass up matched Multicast frames. */
 555                 {
 556                         value = RT_IDX_DST_CAM_Q |      /* dest */
 557                             RT_IDX_TYPE_NICQ |  /* type */
 558                             (RT_IDX_MCAST_MATCH_SLOT << RT_IDX_IDX_SHIFT);/* index */
 559                         break;
 560                 }
 561         case RT_IDX_RSS_MATCH:  /* Pass up matched RSS frames. */
 562                 {
 563                         value = RT_IDX_DST_RSS |        /* dest */
 564                             RT_IDX_TYPE_NICQ |  /* type */
 565                             (RT_IDX_RSS_MATCH_SLOT << RT_IDX_IDX_SHIFT);/* index */
 566                         break;
 567                 }
 568         case 0:         /* Clear the E-bit on an entry. */
 569                 {
 570                         value = RT_IDX_DST_DFLT_Q |     /* dest */
 571                             RT_IDX_TYPE_NICQ |  /* type */
 572                             (index << RT_IDX_IDX_SHIFT);/* index */
 573                         break;
 574                 }
 575         default:
 576                 QPRINTK(qdev, IFUP, ERR, "Mask type %d not yet supported.\n",
 577                         mask);
 578                 status = -EPERM;
 579                 goto exit;
 580         }
 581
 582         if (value) {
 583                 status = ql_wait_reg_rdy(qdev, RT_IDX, RT_IDX_MW, 0);
 584                 if (status)
 585                         goto exit;
 586                 value |= (enable ? RT_IDX_E : 0);
 587                 ql_write32(qdev, RT_IDX, value);
 588                 ql_write32(qdev, RT_DATA, enable ? mask : 0);
 589         }
 590 exit:
 591         return status;
 592 }
 593
 594 static void ql_enable_interrupts(struct ql_adapter *qdev)
 595 {
 596         ql_write32(qdev, INTR_EN, (INTR_EN_EI << 16) | INTR_EN_EI);
 597 }
 598
 599 static void ql_disable_interrupts(struct ql_adapter *qdev)
 600 {
 601         ql_write32(qdev, INTR_EN, (INTR_EN_EI << 16));
 602 }
 603
 604 /* If we're running with multiple MSI-X vectors then we enable on the fly.
 605  * Otherwise, we may have multiple outstanding workers and don't want to
 606  * enable until the last one finishes. In this case, the irq_cnt gets
 607  * incremented everytime we queue a worker and decremented everytime
 608  * a worker finishes.  Once it hits zero we enable the interrupt.
 609  */
 610 u32 ql_enable_completion_interrupt(struct ql_adapter *qdev, u32 intr)
 611 {
 612         u32 var = 0;
 613         unsigned long hw_flags = 0;
 614         struct intr_context *ctx = qdev->intr_context + intr;
 615
 616         if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags) && intr)) {
 617                 /* Always enable if we're MSIX multi interrupts and
 618                  * it's not the default (zeroeth) interrupt.
 619                  */
 620                 ql_write32(qdev, INTR_EN,
 621                            ctx->intr_en_mask);
 622                 var = ql_read32(qdev, STS);
 623                 return var;
 624         }
 625
 626         spin_lock_irqsave(&qdev->hw_lock, hw_flags);
 627         if (atomic_dec_and_test(&ctx->irq_cnt)) {
 628                 ql_write32(qdev, INTR_EN,
 629                            ctx->intr_en_mask);
 630                 var = ql_read32(qdev, STS);
 631         }
 632         spin_unlock_irqrestore(&qdev->hw_lock, hw_flags);
 633         return var;
 634 }
 635
 636 static u32 ql_disable_completion_interrupt(struct ql_adapter *qdev, u32 intr)
 637 {
 638         u32 var = 0;
 639         struct intr_context *ctx;
 640
 641         /* HW disables for us if we're MSIX multi interrupts and
 642          * it's not the default (zeroeth) interrupt.
 643          */
 644         if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags) && intr))
 645                 return 0;
 646
 647         ctx = qdev->intr_context + intr;
 648         spin_lock(&qdev->hw_lock);
 649         if (!atomic_read(&ctx->irq_cnt)) {
 650                 ql_write32(qdev, INTR_EN,
 651                 ctx->intr_dis_mask);
 652                 var = ql_read32(qdev, STS);
 653         }
 654         atomic_inc(&ctx->irq_cnt);
 655         spin_unlock(&qdev->hw_lock);
 656         return var;
 657 }
 658
 659 static void ql_enable_all_completion_interrupts(struct ql_adapter *qdev)
 660 {
 661         int i;
 662         for (i = 0; i < qdev->intr_count; i++) {
 663                 /* The enable call does a atomic_dec_and_test
 664                  * and enables only if the result is zero.
 665                  * So we precharge it here.
 666                  */
 667                 if (unlikely(!test_bit(QL_MSIX_ENABLED, &qdev->flags) ||
 668                         i == 0))
 669                         atomic_set(&qdev->intr_context[i].irq_cnt, 1);
 670                 ql_enable_completion_interrupt(qdev, i);
 671         }
 672
 673 }
 674
 675 static int ql_validate_flash(struct ql_adapter *qdev, u32 size, const char *str)
 676 {
 677         int status, i;
 678         u16 csum = 0;
 679         __le16 *flash = (__le16 *)&qdev->flash;
 680
 681         status = strncmp((char *)&qdev->flash, str, 4);
 682         if (status) {
 683                 QPRINTK(qdev, IFUP, ERR, "Invalid flash signature.\n");
 684                 return  status;
 685         }
 686
 687         for (i = 0; i < size; i++)
 688                 csum += le16_to_cpu(*flash++);
 689
 690         if (csum)
 691                 QPRINTK(qdev, IFUP, ERR,
 692                         "Invalid flash checksum, csum = 0x%.04x.\n", csum);
 693
 694         return csum;
 695 }
 696
 697 static int ql_read_flash_word(struct ql_adapter *qdev, int offset, __le32 *data)
 698 {
 699         int status = 0;
 700         /* wait for reg to come ready */
 701         status = ql_wait_reg_rdy(qdev,
 702                         FLASH_ADDR, FLASH_ADDR_RDY, FLASH_ADDR_ERR);
 703         if (status)
 704                 goto exit;
 705         /* set up for reg read */
 706         ql_write32(qdev, FLASH_ADDR, FLASH_ADDR_R | offset);
 707         /* wait for reg to come ready */
 708         status = ql_wait_reg_rdy(qdev,
 709                         FLASH_ADDR, FLASH_ADDR_RDY, FLASH_ADDR_ERR);
 710         if (status)
 711                 goto exit;
 712          /* This data is stored on flash as an array of
 713          * __le32.  Since ql_read32() returns cpu endian
 714          * we need to swap it back.
 715          */
 716         *data = cpu_to_le32(ql_read32(qdev, FLASH_DATA));
 717 exit:
 718         return status;
 719 }
 720
 721 static int ql_get_8000_flash_params(struct ql_adapter *qdev)
 722 {
 723         u32 i, size;
 724         int status;
 725         __le32 *p = (__le32 *)&qdev->flash;
 726         u32 offset;
 727         u8 mac_addr[6];
 728
 729         /* Get flash offset for function and adjust
 730          * for dword access.
 731          */
 732         if (!qdev->port)
 733                 offset = FUNC0_FLASH_OFFSET / sizeof(u32);
 734         else
 735                 offset = FUNC1_FLASH_OFFSET / sizeof(u32);
 736
 737         if (ql_sem_spinlock(qdev, SEM_FLASH_MASK))
 738                 return -ETIMEDOUT;
 739
 740         size = sizeof(struct flash_params_8000) / sizeof(u32);
 741         for (i = 0; i < size; i++, p++) {
 742                 status = ql_read_flash_word(qdev, i+offset, p);
 743                 if (status) {
 744                         QPRINTK(qdev, IFUP, ERR, "Error reading flash.\n");
 745                         goto exit;
 746                 }
 747         }
 748
 749         status = ql_validate_flash(qdev,
 750                         sizeof(struct flash_params_8000) / sizeof(u16),
 751                         "8000");
 752         if (status) {
 753                 QPRINTK(qdev, IFUP, ERR, "Invalid flash.\n");
 754                 status = -EINVAL;
 755                 goto exit;
 756         }
 757
 758         /* Extract either manufacturer or BOFM modified
 759          * MAC address.
 760          */
 761         if (qdev->flash.flash_params_8000.data_type1 == 2)
 762                 memcpy(mac_addr,
 763                         qdev->flash.flash_params_8000.mac_addr1,
 764                         qdev->ndev->addr_len);
 765         else
 766                 memcpy(mac_addr,
 767                         qdev->flash.flash_params_8000.mac_addr,
 768                         qdev->ndev->addr_len);
 769
 770         if (!is_valid_ether_addr(mac_addr)) {
 771                 QPRINTK(qdev, IFUP, ERR, "Invalid MAC address.\n");
 772                 status = -EINVAL;
 773                 goto exit;
 774         }
 775
 776         memcpy(qdev->ndev->dev_addr,
 777                 mac_addr,
 778                 qdev->ndev->addr_len);
 779
 780 exit:
 781         ql_sem_unlock(qdev, SEM_FLASH_MASK);
 782         return status;
 783 }
 784
 785 static int ql_get_8012_flash_params(struct ql_adapter *qdev)
 786 {
 787         int i;
 788         int status;
 789         __le32 *p = (__le32 *)&qdev->flash;
 790         u32 offset = 0;
 791         u32 size = sizeof(struct flash_params_8012) / sizeof(u32);
 792
 793         /* Second function's parameters follow the first
 794          * function's.
 795          */
 796         if (qdev->port)
 797                 offset = size;
 798
 799         if (ql_sem_spinlock(qdev, SEM_FLASH_MASK))
 800                 return -ETIMEDOUT;
 801
 802         for (i = 0; i < size; i++, p++) {
 803                 status = ql_read_flash_word(qdev, i+offset, p);
 804                 if (status) {
 805                         QPRINTK(qdev, IFUP, ERR, "Error reading flash.\n");
 806                         goto exit;
 807                 }
 808
 809         }
 810
 811         status = ql_validate_flash(qdev,
 812                         sizeof(struct flash_params_8012) / sizeof(u16),
 813                         "8012");
 814         if (status) {
 815                 QPRINTK(qdev, IFUP, ERR, "Invalid flash.\n");
 816                 status = -EINVAL;
 817                 goto exit;
 818         }
 819
 820         if (!is_valid_ether_addr(qdev->flash.flash_params_8012.mac_addr)) {
 821                 status = -EINVAL;
 822                 goto exit;
 823         }
 824
 825         memcpy(qdev->ndev->dev_addr,
 826                 qdev->flash.flash_params_8012.mac_addr,
 827                 qdev->ndev->addr_len);
 828
 829 exit:
 830         ql_sem_unlock(qdev, SEM_FLASH_MASK);
 831         return status;
 832 }
 833
 834 /* xgmac register are located behind the xgmac_addr and xgmac_data
 835  * register pair.  Each read/write requires us to wait for the ready
 836  * bit before reading/writing the data.
 837  */
 838 static int ql_write_xgmac_reg(struct ql_adapter *qdev, u32 reg, u32 data)
 839 {
 840         int status;
 841         /* wait for reg to come ready */
 842         status = ql_wait_reg_rdy(qdev,
 843                         XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
 844         if (status)
 845                 return status;
 846         /* write the data to the data reg */
 847         ql_write32(qdev, XGMAC_DATA, data);
 848         /* trigger the write */
 849         ql_write32(qdev, XGMAC_ADDR, reg);
 850         return status;
 851 }
 852
 853 /* xgmac register are located behind the xgmac_addr and xgmac_data
 854  * register pair.  Each read/write requires us to wait for the ready
 855  * bit before reading/writing the data.
 856  */
 857 int ql_read_xgmac_reg(struct ql_adapter *qdev, u32 reg, u32 *data)
 858 {
 859         int status = 0;
 860         /* wait for reg to come ready */
 861         status = ql_wait_reg_rdy(qdev,
 862                         XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
 863         if (status)
 864                 goto exit;
 865         /* set up for reg read */
 866         ql_write32(qdev, XGMAC_ADDR, reg | XGMAC_ADDR_R);
 867         /* wait for reg to come ready */
 868         status = ql_wait_reg_rdy(qdev,
 869                         XGMAC_ADDR, XGMAC_ADDR_RDY, XGMAC_ADDR_XME);
 870         if (status)
 871                 goto exit;
 872         /* get the data */
 873         *data = ql_read32(qdev, XGMAC_DATA);
 874 exit:
 875         return status;
 876 }
 877
 878 /* This is used for reading the 64-bit statistics regs. */
 879 int ql_read_xgmac_reg64(struct ql_adapter *qdev, u32 reg, u64 *data)
 880 {
 881         int status = 0;
 882         u32 hi = 0;
 883         u32 lo = 0;
 884
 885         status = ql_read_xgmac_reg(qdev, reg, &lo);
 886         if (status)
 887                 goto exit;
 888
 889         status = ql_read_xgmac_reg(qdev, reg + 4, &hi);
 890         if (status)
 891                 goto exit;
 892
 893         *data = (u64) lo | ((u64) hi << 32);
 894
 895 exit:
 896         return status;
 897 }
 898
 899 static int ql_8000_port_initialize(struct ql_adapter *qdev)
 900 {
 901         int status;
 902         /*
 903          * Get MPI firmware version for driver banner
 904          * and ethool info.
 905          */
 906         status = ql_mb_about_fw(qdev);
 907         if (status)
 908                 goto exit;
 909         status = ql_mb_get_fw_state(qdev);
 910         if (status)
 911                 goto exit;
 912         /* Wake up a worker to get/set the TX/RX frame sizes. */
 913         queue_delayed_work(qdev->workqueue, &qdev->mpi_port_cfg_work, 0);
 914 exit:
 915         return status;
 916 }
 917
 918 /* Take the MAC Core out of reset.
 919  * Enable statistics counting.
 920  * Take the transmitter/receiver out of reset.
 921  * This functionality may be done in the MPI firmware at a
 922  * later date.
 923  */
 924 static int ql_8012_port_initialize(struct ql_adapter *qdev)
 925 {
 926         int status = 0;
 927         u32 data;
 928
 929         if (ql_sem_trylock(qdev, qdev->xg_sem_mask)) {
 930                 /* Another function has the semaphore, so
 931                  * wait for the port init bit to come ready.
 932                  */
 933                 QPRINTK(qdev, LINK, INFO,
 934                         "Another function has the semaphore, so wait for the port init bit to come ready.\n");
 935                 status = ql_wait_reg_rdy(qdev, STS, qdev->port_init, 0);
 936                 if (status) {
 937                         QPRINTK(qdev, LINK, CRIT,
 938                                 "Port initialize timed out.\n");
 939                 }
 940                 return status;
 941         }
 942
 943         QPRINTK(qdev, LINK, INFO, "Got xgmac semaphore!.\n");
 944         /* Set the core reset. */
 945         status = ql_read_xgmac_reg(qdev, GLOBAL_CFG, &data);
 946         if (status)
 947                 goto end;
 948         data |= GLOBAL_CFG_RESET;
 949         status = ql_write_xgmac_reg(qdev, GLOBAL_CFG, data);
 950         if (status)
 951                 goto end;
 952
 953         /* Clear the core reset and turn on jumbo for receiver. */
 954         data &= ~GLOBAL_CFG_RESET;      /* Clear core reset. */
 955         data |= GLOBAL_CFG_JUMBO;       /* Turn on jumbo. */
 956         data |= GLOBAL_CFG_TX_STAT_EN;
 957         data |= GLOBAL_CFG_RX_STAT_EN;
 958         status = ql_write_xgmac_reg(qdev, GLOBAL_CFG, data);
 959         if (status)
 960                 goto end;
 961
 962         /* Enable transmitter, and clear it's reset. */
 963         status = ql_read_xgmac_reg(qdev, TX_CFG, &data);
 964         if (status)
 965                 goto end;
 966         data &= ~TX_CFG_RESET;  /* Clear the TX MAC reset. */
 967         data |= TX_CFG_EN;      /* Enable the transmitter. */
 968         status = ql_write_xgmac_reg(qdev, TX_CFG, data);
 969         if (status)
 970                 goto end;
 971
 972         /* Enable receiver and clear it's reset. */
 973         status = ql_read_xgmac_reg(qdev, RX_CFG, &data);
 974         if (status)
 975                 goto end;
 976         data &= ~RX_CFG_RESET;  /* Clear the RX MAC reset. */
 977         data |= RX_CFG_EN;      /* Enable the receiver. */
 978         status = ql_write_xgmac_reg(qdev, RX_CFG, data);
 979         if (status)
 980                 goto end;
 981
 982         /* Turn on jumbo. */
 983         status =
 984             ql_write_xgmac_reg(qdev, MAC_TX_PARAMS, MAC_TX_PARAMS_JUMBO | (0x2580 << 16));
 985         if (status)
 986                 goto end;
 987         status =
 988             ql_write_xgmac_reg(qdev, MAC_RX_PARAMS, 0x2580);
 989         if (status)
 990                 goto end;
 991
 992         /* Signal to the world that the port is enabled.        */
 993         ql_write32(qdev, STS, ((qdev->port_init << 16) | qdev->port_init));
 994 end:
 995         ql_sem_unlock(qdev, qdev->xg_sem_mask);
 996         return status;
 997 }
 998
 999 /* Get the next large buffer. */
1000 static struct bq_desc *ql_get_curr_lbuf(struct rx_ring *rx_ring)
1001 {
1002         struct bq_desc *lbq_desc = &rx_ring->lbq[rx_ring->lbq_curr_idx];
1003         rx_ring->lbq_curr_idx++;
1004         if (rx_ring->lbq_curr_idx == rx_ring->lbq_len)
1005                 rx_ring->lbq_curr_idx = 0;
1006         rx_ring->lbq_free_cnt++;
1007         return lbq_desc;
1008 }
1009
1010 /* Get the next small buffer. */
1011 static struct bq_desc *ql_get_curr_sbuf(struct rx_ring *rx_ring)
1012 {
1013         struct bq_desc *sbq_desc = &rx_ring->sbq[rx_ring->sbq_curr_idx];
1014         rx_ring->sbq_curr_idx++;
1015         if (rx_ring->sbq_curr_idx == rx_ring->sbq_len)
1016                 rx_ring->sbq_curr_idx = 0;
1017         rx_ring->sbq_free_cnt++;
1018         return sbq_desc;
1019 }
1020
1021 /* Update an rx ring index. */
1022 static void ql_update_cq(struct rx_ring *rx_ring)
1023 {
1024         rx_ring->cnsmr_idx++;
1025         rx_ring->curr_entry++;
1026         if (unlikely(rx_ring->cnsmr_idx == rx_ring->cq_len)) {
1027                 rx_ring->cnsmr_idx = 0;
1028                 rx_ring->curr_entry = rx_ring->cq_base;
1029         }
1030 }
1031
1032 static void ql_write_cq_idx(struct rx_ring *rx_ring)
1033 {
1034         ql_write_db_reg(rx_ring->cnsmr_idx, rx_ring->cnsmr_idx_db_reg);
1035 }
1036
1037 /* Process (refill) a large buffer queue. */
1038 static void ql_update_lbq(struct ql_adapter *qdev, struct rx_ring *rx_ring)
1039 {
1040         u32 clean_idx = rx_ring->lbq_clean_idx;
1041         u32 start_idx = clean_idx;
1042         struct bq_desc *lbq_desc;
1043         u64 map;
1044         int i;
1045
1046         while (rx_ring->lbq_free_cnt > 16) {
1047                 for (i = 0; i < 16; i++) {
1048                         QPRINTK(qdev, RX_STATUS, DEBUG,
1049                                 "lbq: try cleaning clean_idx = %d.\n",
1050                                 clean_idx);
1051                         lbq_desc = &rx_ring->lbq[clean_idx];
1052                         if (lbq_desc->p.lbq_page == NULL) {
1053                                 QPRINTK(qdev, RX_STATUS, DEBUG,
1054                                         "lbq: getting new page for index %d.\n",
1055                                         lbq_desc->index);
1056                                 lbq_desc->p.lbq_page = alloc_page(GFP_ATOMIC);
1057                                 if (lbq_desc->p.lbq_page == NULL) {
1058                                         rx_ring->lbq_clean_idx = clean_idx;
1059                                         QPRINTK(qdev, RX_STATUS, ERR,
1060                                                 "Couldn't get a page.\n");
1061                                         return;
1062                                 }
1063                                 map = pci_map_page(qdev->pdev,
1064                                                    lbq_desc->p.lbq_page,
1065                                                    0, PAGE_SIZE,
1066                                                    PCI_DMA_FROMDEVICE);
1067                                 if (pci_dma_mapping_error(qdev->pdev, map)) {
1068                                         rx_ring->lbq_clean_idx = clean_idx;
1069                                         put_page(lbq_desc->p.lbq_page);
1070                                         lbq_desc->p.lbq_page = NULL;
1071                                         QPRINTK(qdev, RX_STATUS, ERR,
1072                                                 "PCI mapping failed.\n");
1073                                         return;
1074                                 }
1075                                 pci_unmap_addr_set(lbq_desc, mapaddr, map);
1076                                 pci_unmap_len_set(lbq_desc, maplen, PAGE_SIZE);
1077                                 *lbq_desc->addr = cpu_to_le64(map);
1078                         }
1079                         clean_idx++;
1080                         if (clean_idx == rx_ring->lbq_len)
1081                                 clean_idx = 0;
1082                 }
1083
1084                 rx_ring->lbq_clean_idx = clean_idx;
1085                 rx_ring->lbq_prod_idx += 16;
1086                 if (rx_ring->lbq_prod_idx == rx_ring->lbq_len)
1087                         rx_ring->lbq_prod_idx = 0;
1088                 rx_ring->lbq_free_cnt -= 16;
1089         }
1090
1091         if (start_idx != clean_idx) {
1092                 QPRINTK(qdev, RX_STATUS, DEBUG,
1093                         "lbq: updating prod idx = %d.\n",
1094                         rx_ring->lbq_prod_idx);
1095                 ql_write_db_reg(rx_ring->lbq_prod_idx,
1096                                 rx_ring->lbq_prod_idx_db_reg);
1097         }
1098 }
1099
1100 /* Process (refill) a small buffer queue. */
1101 static void ql_update_sbq(struct ql_adapter *qdev, struct rx_ring *rx_ring)
1102 {
1103         u32 clean_idx = rx_ring->sbq_clean_idx;
1104         u32 start_idx = clean_idx;
1105         struct bq_desc *sbq_desc;
1106         u64 map;
1107         int i;
1108
1109         while (rx_ring->sbq_free_cnt > 16) {
1110                 for (i = 0; i < 16; i++) {
1111                         sbq_desc = &rx_ring->sbq[clean_idx];
1112                         QPRINTK(qdev, RX_STATUS, DEBUG,
1113                                 "sbq: try cleaning clean_idx = %d.\n",
1114                                 clean_idx);
1115                         if (sbq_desc->p.skb == NULL) {
1116                                 QPRINTK(qdev, RX_STATUS, DEBUG,
1117                                         "sbq: getting new skb for index %d.\n",
1118                                         sbq_desc->index);
1119                                 sbq_desc->p.skb =
1120                                     netdev_alloc_skb(qdev->ndev,
1121                                                      rx_ring->sbq_buf_size);
1122                                 if (sbq_desc->p.skb == NULL) {
1123                                         QPRINTK(qdev, PROBE, ERR,
1124                                                 "Couldn't get an skb.\n");
1125                                         rx_ring->sbq_clean_idx = clean_idx;
1126                                         return;
1127                                 }
1128                                 skb_reserve(sbq_desc->p.skb, QLGE_SB_PAD);
1129                                 map = pci_map_single(qdev->pdev,
1130                                                      sbq_desc->p.skb->data,
1131                                                      rx_ring->sbq_buf_size /
1132                                                      2, PCI_DMA_FROMDEVICE);
1133                                 if (pci_dma_mapping_error(qdev->pdev, map)) {
1134                                         QPRINTK(qdev, IFUP, ERR, "PCI mapping failed.\n");
1135                                         rx_ring->sbq_clean_idx = clean_idx;
1136                                         dev_kfree_skb_any(sbq_desc->p.skb);
1137                                         sbq_desc->p.skb = NULL;
1138                                         return;
1139                                 }
1140                                 pci_unmap_addr_set(sbq_desc, mapaddr, map);
1141                                 pci_unmap_len_set(sbq_desc, maplen,
1142                                                   rx_ring->sbq_buf_size / 2);
1143                                 *sbq_desc->addr = cpu_to_le64(map);
1144                         }
1145
1146                         clean_idx++;
1147                         if (clean_idx == rx_ring->sbq_len)
1148                                 clean_idx = 0;
1149                 }
1150                 rx_ring->sbq_clean_idx = clean_idx;
1151                 rx_ring->sbq_prod_idx += 16;
1152                 if (rx_ring->sbq_prod_idx == rx_ring->sbq_len)
1153                         rx_ring->sbq_prod_idx = 0;
1154                 rx_ring->sbq_free_cnt -= 16;
1155         }
1156
1157         if (start_idx != clean_idx) {
1158                 QPRINTK(qdev, RX_STATUS, DEBUG,
1159                         "sbq: updating prod idx = %d.\n",
1160                         rx_ring->sbq_prod_idx);
1161                 ql_write_db_reg(rx_ring->sbq_prod_idx,
1162                                 rx_ring->sbq_prod_idx_db_reg);
1163         }
1164 }
1165
1166 static void ql_update_buffer_queues(struct ql_adapter *qdev,
1167                                     struct rx_ring *rx_ring)
1168 {
1169         ql_update_sbq(qdev, rx_ring);
1170         ql_update_lbq(qdev, rx_ring);
1171 }
1172
1173 /* Unmaps tx buffers.  Can be called from send() if a pci mapping
1174  * fails at some stage, or from the interrupt when a tx completes.
1175  */
1176 static void ql_unmap_send(struct ql_adapter *qdev,
1177                           struct tx_ring_desc *tx_ring_desc, int mapped)
1178 {
1179         int i;
1180         for (i = 0; i < mapped; i++) {
1181                 if (i == 0 || (i == 7 && mapped > 7)) {
1182                         /*
1183                          * Unmap the skb->data area, or the
1184                          * external sglist (AKA the Outbound
1185                          * Address List (OAL)).
1186                          * If its the zeroeth element, then it's
1187                          * the skb->data area.  If it's the 7th
1188                          * element and there is more than 6 frags,
1189                          * then its an OAL.
1190                          */
1191                         if (i == 7) {
1192                                 QPRINTK(qdev, TX_DONE, DEBUG,
1193                                         "unmapping OAL area.\n");
1194                         }
1195                         pci_unmap_single(qdev->pdev,
1196                                          pci_unmap_addr(&tx_ring_desc->map[i],
1197                                                         mapaddr),
1198                                          pci_unmap_len(&tx_ring_desc->map[i],
1199                                                        maplen),
1200                                          PCI_DMA_TODEVICE);
1201                 } else {
1202                         QPRINTK(qdev, TX_DONE, DEBUG, "unmapping frag %d.\n",
1203                                 i);
1204                         pci_unmap_page(qdev->pdev,
1205                                        pci_unmap_addr(&tx_ring_desc->map[i],
1206                                                       mapaddr),
1207                                        pci_unmap_len(&tx_ring_desc->map[i],
1208                                                      maplen), PCI_DMA_TODEVICE);
1209                 }
1210         }
1211
1212 }
1213
1214 /* Map the buffers for this transmit.  This will return
1215  * NETDEV_TX_BUSY or NETDEV_TX_OK based on success.
1216  */
1217 static int ql_map_send(struct ql_adapter *qdev,
1218                        struct ob_mac_iocb_req *mac_iocb_ptr,
1219                        struct sk_buff *skb, struct tx_ring_desc *tx_ring_desc)
1220 {
1221         int len = skb_headlen(skb);
1222         dma_addr_t map;
1223         int frag_idx, err, map_idx = 0;
1224         struct tx_buf_desc *tbd = mac_iocb_ptr->tbd;
1225         int frag_cnt = skb_shinfo(skb)->nr_frags;
1226
1227         if (frag_cnt) {
1228                 QPRINTK(qdev, TX_QUEUED, DEBUG, "frag_cnt = %d.\n", frag_cnt);
1229         }
1230         /*
1231          * Map the skb buffer first.
1232          */
1233         map = pci_map_single(qdev->pdev, skb->data, len, PCI_DMA_TODEVICE);
1234
1235         err = pci_dma_mapping_error(qdev->pdev, map);
1236         if (err) {
1237                 QPRINTK(qdev, TX_QUEUED, ERR,
1238                         "PCI mapping failed with error: %d\n", err);
1239
1240                 return NETDEV_TX_BUSY;
1241         }
1242
1243         tbd->len = cpu_to_le32(len);
1244         tbd->addr = cpu_to_le64(map);
1245         pci_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr, map);
1246         pci_unmap_len_set(&tx_ring_desc->map[map_idx], maplen, len);
1247         map_idx++;
1248
1249         /*
1250          * This loop fills the remainder of the 8 address descriptors
1251          * in the IOCB.  If there are more than 7 fragments, then the
1252          * eighth address desc will point to an external list (OAL).
1253          * When this happens, the remainder of the frags will be stored
1254          * in this list.
1255          */
1256         for (frag_idx = 0; frag_idx < frag_cnt; frag_idx++, map_idx++) {
1257                 skb_frag_t *frag = &skb_shinfo(skb)->frags[frag_idx];
1258                 tbd++;
1259                 if (frag_idx == 6 && frag_cnt > 7) {
1260                         /* Let's tack on an sglist.
1261                          * Our control block will now
1262                          * look like this:
1263                          * iocb->seg[0] = skb->data
1264                          * iocb->seg[1] = frag[0]
1265                          * iocb->seg[2] = frag[1]
1266                          * iocb->seg[3] = frag[2]
1267                          * iocb->seg[4] = frag[3]
1268                          * iocb->seg[5] = frag[4]
1269                          * iocb->seg[6] = frag[5]
1270                          * iocb->seg[7] = ptr to OAL (external sglist)
1271                          * oal->seg[0] = frag[6]
1272                          * oal->seg[1] = frag[7]
1273                          * oal->seg[2] = frag[8]
1274                          * oal->seg[3] = frag[9]
1275                          * oal->seg[4] = frag[10]
1276                          *      etc...
1277                          */
1278                         /* Tack on the OAL in the eighth segment of IOCB. */
1279                         map = pci_map_single(qdev->pdev, &tx_ring_desc->oal,
1280                                              sizeof(struct oal),
1281                                              PCI_DMA_TODEVICE);
1282                         err = pci_dma_mapping_error(qdev->pdev, map);
1283                         if (err) {
1284                                 QPRINTK(qdev, TX_QUEUED, ERR,
1285                                         "PCI mapping outbound address list with error: %d\n",
1286                                         err);
1287                                 goto map_error;
1288                         }
1289
1290                         tbd->addr = cpu_to_le64(map);
1291                         /*
1292                          * The length is the number of fragments
1293                          * that remain to be mapped times the length
1294                          * of our sglist (OAL).
1295                          */
1296                         tbd->len =
1297                             cpu_to_le32((sizeof(struct tx_buf_desc) *
1298                                          (frag_cnt - frag_idx)) | TX_DESC_C);
1299                         pci_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr,
1300                                            map);
1301                         pci_unmap_len_set(&tx_ring_desc->map[map_idx], maplen,
1302                                           sizeof(struct oal));
1303                         tbd = (struct tx_buf_desc *)&tx_ring_desc->oal;
1304                         map_idx++;
1305                 }
1306
1307                 map =
1308                     pci_map_page(qdev->pdev, frag->page,
1309                                  frag->page_offset, frag->size,
1310                                  PCI_DMA_TODEVICE);
1311
1312                 err = pci_dma_mapping_error(qdev->pdev, map);
1313                 if (err) {
1314                         QPRINTK(qdev, TX_QUEUED, ERR,
1315                                 "PCI mapping frags failed with error: %d.\n",
1316                                 err);
1317                         goto map_error;
1318                 }
1319
1320                 tbd->addr = cpu_to_le64(map);
1321                 tbd->len = cpu_to_le32(frag->size);
1322                 pci_unmap_addr_set(&tx_ring_desc->map[map_idx], mapaddr, map);
1323                 pci_unmap_len_set(&tx_ring_desc->map[map_idx], maplen,
1324                                   frag->size);
1325
1326         }
1327         /* Save the number of segments we've mapped. */
1328         tx_ring_desc->map_cnt = map_idx;
1329         /* Terminate the last segment. */
1330         tbd->len = cpu_to_le32(le32_to_cpu(tbd->len) | TX_DESC_E);
1331         return NETDEV_TX_OK;
1332
1333 map_error:
1334         /*
1335          * If the first frag mapping failed, then i will be zero.
1336          * This causes the unmap of the skb->data area.  Otherwise
1337          * we pass in the number of frags that mapped successfully
1338          * so they can be umapped.
1339          */
1340         ql_unmap_send(qdev, tx_ring_desc, map_idx);
1341         return NETDEV_TX_BUSY;
1342 }
1343
1344 static void ql_realign_skb(struct sk_buff *skb, int len)
1345 {
1346         void *temp_addr = skb->data;
1347
1348         /* Undo the skb_reserve(skb,32) we did before
1349          * giving to hardware, and realign data on
1350          * a 2-byte boundary.
1351          */
1352         skb->data -= QLGE_SB_PAD - NET_IP_ALIGN;
1353         skb->tail -= QLGE_SB_PAD - NET_IP_ALIGN;
1354         skb_copy_to_linear_data(skb, temp_addr,
1355                 (unsigned int)len);
1356 }
1357
1358 /*
1359  * This function builds an skb for the given inbound
1360  * completion.  It will be rewritten for readability in the near
1361  * future, but for not it works well.
1362  */
1363 static struct sk_buff *ql_build_rx_skb(struct ql_adapter *qdev,
1364                                        struct rx_ring *rx_ring,
1365                                        struct ib_mac_iocb_rsp *ib_mac_rsp)
1366 {
1367         struct bq_desc *lbq_desc;
1368         struct bq_desc *sbq_desc;
1369         struct sk_buff *skb = NULL;
1370         u32 length = le32_to_cpu(ib_mac_rsp->data_len);
1371        u32 hdr_len = le32_to_cpu(ib_mac_rsp->hdr_len);
1372
1373         /*
1374          * Handle the header buffer if present.
1375          */
1376         if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HV &&
1377             ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
1378                 QPRINTK(qdev, RX_STATUS, DEBUG, "Header of %d bytes in small buffer.\n", hdr_len);
1379                 /*
1380                  * Headers fit nicely into a small buffer.
1381                  */
1382                 sbq_desc = ql_get_curr_sbuf(rx_ring);
1383                 pci_unmap_single(qdev->pdev,
1384                                 pci_unmap_addr(sbq_desc, mapaddr),
1385                                 pci_unmap_len(sbq_desc, maplen),
1386                                 PCI_DMA_FROMDEVICE);
1387                 skb = sbq_desc->p.skb;
1388                 ql_realign_skb(skb, hdr_len);
1389                 skb_put(skb, hdr_len);
1390                 sbq_desc->p.skb = NULL;
1391         }
1392
1393         /*
1394          * Handle the data buffer(s).
1395          */
1396         if (unlikely(!length)) {        /* Is there data too? */
1397                 QPRINTK(qdev, RX_STATUS, DEBUG,
1398                         "No Data buffer in this packet.\n");
1399                 return skb;
1400         }
1401
1402         if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DS) {
1403                 if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
1404                         QPRINTK(qdev, RX_STATUS, DEBUG,
1405                                 "Headers in small, data of %d bytes in small, combine them.\n", length);
1406                         /*
1407                          * Data is less than small buffer size so it's
1408                          * stuffed in a small buffer.
1409                          * For this case we append the data
1410                          * from the "data" small buffer to the "header" small
1411                          * buffer.
1412                          */
1413                         sbq_desc = ql_get_curr_sbuf(rx_ring);
1414                         pci_dma_sync_single_for_cpu(qdev->pdev,
1415                                                     pci_unmap_addr
1416                                                     (sbq_desc, mapaddr),
1417                                                     pci_unmap_len
1418                                                     (sbq_desc, maplen),
1419                                                     PCI_DMA_FROMDEVICE);
1420                         memcpy(skb_put(skb, length),
1421                                sbq_desc->p.skb->data, length);
1422                         pci_dma_sync_single_for_device(qdev->pdev,
1423                                                        pci_unmap_addr
1424                                                        (sbq_desc,
1425                                                         mapaddr),
1426                                                        pci_unmap_len
1427                                                        (sbq_desc,
1428                                                         maplen),
1429                                                        PCI_DMA_FROMDEVICE);
1430                 } else {
1431                         QPRINTK(qdev, RX_STATUS, DEBUG,
1432                                 "%d bytes in a single small buffer.\n", length);
1433                         sbq_desc = ql_get_curr_sbuf(rx_ring);
1434                         skb = sbq_desc->p.skb;
1435                         ql_realign_skb(skb, length);
1436                         skb_put(skb, length);
1437                         pci_unmap_single(qdev->pdev,
1438                                          pci_unmap_addr(sbq_desc,
1439                                                         mapaddr),
1440                                          pci_unmap_len(sbq_desc,
1441                                                        maplen),
1442                                          PCI_DMA_FROMDEVICE);
1443                         sbq_desc->p.skb = NULL;
1444                 }
1445         } else if (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_DL) {
1446                 if (ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS) {
1447                         QPRINTK(qdev, RX_STATUS, DEBUG,
1448                                 "Header in small, %d bytes in large. Chain large to small!\n", length);
1449                         /*
1450                          * The data is in a single large buffer.  We
1451                          * chain it to the header buffer's skb and let
1452                          * it rip.
1453                          */
1454                         lbq_desc = ql_get_curr_lbuf(rx_ring);
1455                         pci_unmap_page(qdev->pdev,
1456                                        pci_unmap_addr(lbq_desc,
1457                                                       mapaddr),
1458                                        pci_unmap_len(lbq_desc, maplen),
1459                                        PCI_DMA_FROMDEVICE);
1460                         QPRINTK(qdev, RX_STATUS, DEBUG,
1461                                 "Chaining page to skb.\n");
1462                         skb_fill_page_desc(skb, 0, lbq_desc->p.lbq_page,
1463                                            0, length);
1464                         skb->len += length;
1465                         skb->data_len += length;
1466                         skb->truesize += length;
1467                         lbq_desc->p.lbq_page = NULL;
1468                 } else {
1469                         /*
1470                          * The headers and data are in a single large buffer. We
1471                          * copy it to a new skb and let it go. This can happen with
1472                          * jumbo mtu on a non-TCP/UDP frame.
1473                          */
1474                         lbq_desc = ql_get_curr_lbuf(rx_ring);
1475                         skb = netdev_alloc_skb(qdev->ndev, length);
1476                         if (skb == NULL) {
1477                                 QPRINTK(qdev, PROBE, DEBUG,
1478                                         "No skb available, drop the packet.\n");
1479                                 return NULL;
1480                         }
1481                         pci_unmap_page(qdev->pdev,
1482                                        pci_unmap_addr(lbq_desc,
1483                                                       mapaddr),
1484                                        pci_unmap_len(lbq_desc, maplen),
1485                                        PCI_DMA_FROMDEVICE);
1486                         skb_reserve(skb, NET_IP_ALIGN);
1487                         QPRINTK(qdev, RX_STATUS, DEBUG,
1488                                 "%d bytes of headers and data in large. Chain page to new skb and pull tail.\n", length);
1489                         skb_fill_page_desc(skb, 0, lbq_desc->p.lbq_page,
1490                                            0, length);
1491                         skb->len += length;
1492                         skb->data_len += length;
1493                         skb->truesize += length;
1494                         length -= length;
1495                         lbq_desc->p.lbq_page = NULL;
1496                         __pskb_pull_tail(skb,
1497                                 (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) ?
1498                                 VLAN_ETH_HLEN : ETH_HLEN);
1499                 }
1500         } else {
1501                 /*
1502                  * The data is in a chain of large buffers
1503                  * pointed to by a small buffer.  We loop
1504                  * thru and chain them to the our small header
1505                  * buffer's skb.
1506                  * frags:  There are 18 max frags and our small
1507                  *         buffer will hold 32 of them. The thing is,
1508                  *         we'll use 3 max for our 9000 byte jumbo
1509                  *         frames.  If the MTU goes up we could
1510                  *          eventually be in trouble.
1511                  */
1512                 int size, offset, i = 0;
1513                 __le64 *bq, bq_array[8];
1514                 sbq_desc = ql_get_curr_sbuf(rx_ring);
1515                 pci_unmap_single(qdev->pdev,
1516                                  pci_unmap_addr(sbq_desc, mapaddr),
1517                                  pci_unmap_len(sbq_desc, maplen),
1518                                  PCI_DMA_FROMDEVICE);
1519                 if (!(ib_mac_rsp->flags4 & IB_MAC_IOCB_RSP_HS)) {
1520                         /*
1521                          * This is an non TCP/UDP IP frame, so
1522                          * the headers aren't split into a small
1523                          * buffer.  We have to use the small buffer
1524                          * that contains our sg list as our skb to
1525                          * send upstairs. Copy the sg list here to
1526                          * a local buffer and use it to find the
1527                          * pages to chain.
1528                          */
1529                         QPRINTK(qdev, RX_STATUS, DEBUG,
1530                                 "%d bytes of headers & data in chain of large.\n", length);
1531                         skb = sbq_desc->p.skb;
1532                         bq = &bq_array[0];
1533                         memcpy(bq, skb->data, sizeof(bq_array));
1534                         sbq_desc->p.skb = NULL;
1535                         skb_reserve(skb, NET_IP_ALIGN);
1536                 } else {
1537                         QPRINTK(qdev, RX_STATUS, DEBUG,
1538                                 "Headers in small, %d bytes of data in chain of large.\n", length);
1539                         bq = (__le64 *)sbq_desc->p.skb->data;
1540                 }
1541                 while (length > 0) {
1542                         lbq_desc = ql_get_curr_lbuf(rx_ring);
1543                         pci_unmap_page(qdev->pdev,
1544                                        pci_unmap_addr(lbq_desc,
1545                                                       mapaddr),
1546                                        pci_unmap_len(lbq_desc,
1547                                                      maplen),
1548                                        PCI_DMA_FROMDEVICE);
1549                         size = (length < PAGE_SIZE) ? length : PAGE_SIZE;
1550                         offset = 0;
1551
1552                         QPRINTK(qdev, RX_STATUS, DEBUG,
1553                                 "Adding page %d to skb for %d bytes.\n",
1554                                 i, size);
1555                         skb_fill_page_desc(skb, i, lbq_desc->p.lbq_page,
1556                                            offset, size);
1557                         skb->len += size;
1558                         skb->data_len += size;
1559                         skb->truesize += size;
1560                         length -= size;
1561                         lbq_desc->p.lbq_page = NULL;
1562                         bq++;
1563                         i++;
1564                 }
1565                 __pskb_pull_tail(skb, (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) ?
1566                                 VLAN_ETH_HLEN : ETH_HLEN);
1567         }
1568         return skb;
1569 }
1570
1571 /* Process an inbound completion from an rx ring. */
1572 static void ql_process_mac_rx_intr(struct ql_adapter *qdev,
1573                                    struct rx_ring *rx_ring,
1574                                    struct ib_mac_iocb_rsp *ib_mac_rsp)
1575 {
1576         struct net_device *ndev = qdev->ndev;
1577         struct sk_buff *skb = NULL;
1578         u16 vlan_id = (le16_to_cpu(ib_mac_rsp->vlan_id) &
1579                         IB_MAC_IOCB_RSP_VLAN_MASK)
1580
1581         QL_DUMP_IB_MAC_RSP(ib_mac_rsp);
1582
1583         skb = ql_build_rx_skb(qdev, rx_ring, ib_mac_rsp);
1584         if (unlikely(!skb)) {
1585                 QPRINTK(qdev, RX_STATUS, DEBUG,
1586                         "No skb available, drop packet.\n");
1587                 return;
1588         }
1589
1590         /* Frame error, so drop the packet. */
1591         if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_ERR_MASK) {
1592                 QPRINTK(qdev, DRV, ERR, "Receive error, flags2 = 0x%x\n",
1593                                         ib_mac_rsp->flags2);
1594                 dev_kfree_skb_any(skb);
1595                 return;
1596         }
1597
1598         /* The max framesize filter on this chip is set higher than
1599          * MTU since FCoE uses 2k frames.
1600          */
1601         if (skb->len > ndev->mtu + ETH_HLEN) {
1602                 dev_kfree_skb_any(skb);
1603                 return;
1604         }
1605
1606         prefetch(skb->data);
1607         skb->dev = ndev;
1608         if (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) {
1609                 QPRINTK(qdev, RX_STATUS, DEBUG, "%s%s%s Multicast.\n",
1610                         (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
1611                         IB_MAC_IOCB_RSP_M_HASH ? "Hash" : "",
1612                         (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
1613                         IB_MAC_IOCB_RSP_M_REG ? "Registered" : "",
1614                         (ib_mac_rsp->flags1 & IB_MAC_IOCB_RSP_M_MASK) ==
1615                         IB_MAC_IOCB_RSP_M_PROM ? "Promiscuous" : "");
1616         }
1617         if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_P) {
1618                 QPRINTK(qdev, RX_STATUS, DEBUG, "Promiscuous Packet.\n");
1619         }
1620
1621         skb->protocol = eth_type_trans(skb, ndev);
1622         skb->ip_summed = CHECKSUM_NONE;
1623
1624         /* If rx checksum is on, and there are no
1625          * csum or frame errors.
1626          */
1627         if (qdev->rx_csum &&
1628                 !(ib_mac_rsp->flags1 & IB_MAC_CSUM_ERR_MASK)) {
1629                 /* TCP frame. */
1630                 if (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_T) {
1631                         QPRINTK(qdev, RX_STATUS, DEBUG,
1632                                         "TCP checksum done!\n");
1633                         skb->ip_summed = CHECKSUM_UNNECESSARY;
1634                 } else if ((ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_U) &&
1635                                 (ib_mac_rsp->flags3 & IB_MAC_IOCB_RSP_V4)) {
1636                 /* Unfragmented ipv4 UDP frame. */
1637                         struct iphdr *iph = (struct iphdr *) skb->data;
1638                         if (!(iph->frag_off &
1639                                 cpu_to_be16(IP_MF|IP_OFFSET))) {
1640                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1641                                 QPRINTK(qdev, RX_STATUS, DEBUG,
1642                                                 "TCP checksum done!\n");
1643                         }
1644                 }
1645         }
1646
1647         qdev->stats.rx_packets++;
1648         qdev->stats.rx_bytes += skb->len;
1649         skb_record_rx_queue(skb, rx_ring->cq_id);
1650         if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
1651                 if (qdev->vlgrp &&
1652                         (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) &&
1653                         (vlan_id != 0))
1654                         vlan_gro_receive(&rx_ring->napi, qdev->vlgrp,
1655                                 vlan_id, skb);
1656                 else
1657                         napi_gro_receive(&rx_ring->napi, skb);
1658         } else {
1659                 if (qdev->vlgrp &&
1660                         (ib_mac_rsp->flags2 & IB_MAC_IOCB_RSP_V) &&
1661                         (vlan_id != 0))
1662                         vlan_hwaccel_receive_skb(skb, qdev->vlgrp, vlan_id);
1663                 else
1664                         netif_receive_skb(skb);
1665         }
1666 }
1667
1668 /* Process an outbound completion from an rx ring. */
1669 static void ql_process_mac_tx_intr(struct ql_adapter *qdev,
1670                                    struct ob_mac_iocb_rsp *mac_rsp)
1671 {
1672         struct tx_ring *tx_ring;
1673         struct tx_ring_desc *tx_ring_desc;
1674
1675         QL_DUMP_OB_MAC_RSP(mac_rsp);
1676         tx_ring = &qdev->tx_ring[mac_rsp->txq_idx];
1677         tx_ring_desc = &tx_ring->q[mac_rsp->tid];
1678         ql_unmap_send(qdev, tx_ring_desc, tx_ring_desc->map_cnt);
1679         qdev->stats.tx_bytes += (tx_ring_desc->skb)->len;
1680         qdev->stats.tx_packets++;
1681         dev_kfree_skb(tx_ring_desc->skb);
1682         tx_ring_desc->skb = NULL;
1683
1684         if (unlikely(mac_rsp->flags1 & (OB_MAC_IOCB_RSP_E |
1685                                         OB_MAC_IOCB_RSP_S |
1686                                         OB_MAC_IOCB_RSP_L |
1687                                         OB_MAC_IOCB_RSP_P | OB_MAC_IOCB_RSP_B))) {
1688                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_E) {
1689                         QPRINTK(qdev, TX_DONE, WARNING,
1690                                 "Total descriptor length did not match transfer length.\n");
1691                 }
1692                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_S) {
1693                         QPRINTK(qdev, TX_DONE, WARNING,
1694                                 "Frame too short to be legal, not sent.\n");
1695                 }
1696                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_L) {
1697                         QPRINTK(qdev, TX_DONE, WARNING,
1698                                 "Frame too long, but sent anyway.\n");
1699                 }
1700                 if (mac_rsp->flags1 & OB_MAC_IOCB_RSP_B) {
1701                         QPRINTK(qdev, TX_DONE, WARNING,
1702                                 "PCI backplane error. Frame not sent.\n");
1703                 }
1704         }
1705         atomic_inc(&tx_ring->tx_count);
1706 }
1707
1708 /* Fire up a handler to reset the MPI processor. */
1709 void ql_queue_fw_error(struct ql_adapter *qdev)
1710 {
1711         ql_link_off(qdev);
1712         queue_delayed_work(qdev->workqueue, &qdev->mpi_reset_work, 0);
1713 }
1714
1715 void ql_queue_asic_error(struct ql_adapter *qdev)
1716 {
1717         ql_link_off(qdev);
1718         ql_disable_interrupts(qdev);
1719         /* Clear adapter up bit to signal the recovery
1720          * process that it shouldn't kill the reset worker
1721          * thread
1722          */
1723         clear_bit(QL_ADAPTER_UP, &qdev->flags);
1724         queue_delayed_work(qdev->workqueue, &qdev->asic_reset_work, 0);
1725 }
1726
1727 static void ql_process_chip_ae_intr(struct ql_adapter *qdev,
1728                                     struct ib_ae_iocb_rsp *ib_ae_rsp)
1729 {
1730         switch (ib_ae_rsp->event) {
1731         case MGMT_ERR_EVENT:
1732                 QPRINTK(qdev, RX_ERR, ERR,
1733                         "Management Processor Fatal Error.\n");
1734                 ql_queue_fw_error(qdev);
1735                 return;
1736
1737         case CAM_LOOKUP_ERR_EVENT:
1738                 QPRINTK(qdev, LINK, ERR,
1739                         "Multiple CAM hits lookup occurred.\n");
1740                 QPRINTK(qdev, DRV, ERR, "This event shouldn't occur.\n");
1741                 ql_queue_asic_error(qdev);
1742                 return;
1743
1744         case SOFT_ECC_ERROR_EVENT:
1745                 QPRINTK(qdev, RX_ERR, ERR, "Soft ECC error detected.\n");
1746                 ql_queue_asic_error(qdev);
1747                 break;
1748
1749         case PCI_ERR_ANON_BUF_RD:
1750                 QPRINTK(qdev, RX_ERR, ERR,
1751                         "PCI error occurred when reading anonymous buffers from rx_ring %d.\n",
1752                         ib_ae_rsp->q_id);
1753                 ql_queue_asic_error(qdev);
1754                 break;
1755
1756         default:
1757                 QPRINTK(qdev, DRV, ERR, "Unexpected event %d.\n",
1758                         ib_ae_rsp->event);
1759                 ql_queue_asic_error(qdev);
1760                 break;
1761         }
1762 }
1763
1764 static int ql_clean_outbound_rx_ring(struct rx_ring *rx_ring)
1765 {
1766         struct ql_adapter *qdev = rx_ring->qdev;
1767         u32 prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
1768         struct ob_mac_iocb_rsp *net_rsp = NULL;
1769         int count = 0;
1770
1771         struct tx_ring *tx_ring;
1772         /* While there are entries in the completion queue. */
1773         while (prod != rx_ring->cnsmr_idx) {
1774
1775                 QPRINTK(qdev, RX_STATUS, DEBUG,
1776                         "cq_id = %d, prod = %d, cnsmr = %d.\n.", rx_ring->cq_id,
1777                         prod, rx_ring->cnsmr_idx);
1778
1779                 net_rsp = (struct ob_mac_iocb_rsp *)rx_ring->curr_entry;
1780                 rmb();
1781                 switch (net_rsp->opcode) {
1782
1783                 case OPCODE_OB_MAC_TSO_IOCB:
1784                 case OPCODE_OB_MAC_IOCB:
1785                         ql_process_mac_tx_intr(qdev, net_rsp);
1786                         break;
1787                 default:
1788                         QPRINTK(qdev, RX_STATUS, DEBUG,
1789                                 "Hit default case, not handled! dropping the packet, opcode = %x.\n",
1790                                 net_rsp->opcode);
1791                 }
1792                 count++;
1793                 ql_update_cq(rx_ring);
1794                 prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
1795         }
1796         ql_write_cq_idx(rx_ring);
1797         tx_ring = &qdev->tx_ring[net_rsp->txq_idx];
1798         if (__netif_subqueue_stopped(qdev->ndev, tx_ring->wq_id) &&
1799                                         net_rsp != NULL) {
1800                 if (atomic_read(&tx_ring->queue_stopped) &&
1801                     (atomic_read(&tx_ring->tx_count) > (tx_ring->wq_len / 4)))
1802                         /*
1803                          * The queue got stopped because the tx_ring was full.
1804                          * Wake it up, because it's now at least 25% empty.
1805                          */
1806                         netif_wake_subqueue(qdev->ndev, tx_ring->wq_id);
1807         }
1808
1809         return count;
1810 }
1811
1812 static int ql_clean_inbound_rx_ring(struct rx_ring *rx_ring, int budget)
1813 {
1814         struct ql_adapter *qdev = rx_ring->qdev;
1815         u32 prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
1816         struct ql_net_rsp_iocb *net_rsp;
1817         int count = 0;
1818
1819         /* While there are entries in the completion queue. */
1820         while (prod != rx_ring->cnsmr_idx) {
1821
1822                 QPRINTK(qdev, RX_STATUS, DEBUG,
1823                         "cq_id = %d, prod = %d, cnsmr = %d.\n.", rx_ring->cq_id,
1824                         prod, rx_ring->cnsmr_idx);
1825
1826                 net_rsp = rx_ring->curr_entry;
1827                 rmb();
1828                 switch (net_rsp->opcode) {
1829                 case OPCODE_IB_MAC_IOCB:
1830                         ql_process_mac_rx_intr(qdev, rx_ring,
1831                                                (struct ib_mac_iocb_rsp *)
1832                                                net_rsp);
1833                         break;
1834
1835                 case OPCODE_IB_AE_IOCB:
1836                         ql_process_chip_ae_intr(qdev, (struct ib_ae_iocb_rsp *)
1837                                                 net_rsp);
1838                         break;
1839                 default:
1840                         {
1841                                 QPRINTK(qdev, RX_STATUS, DEBUG,
1842                                         "Hit default case, not handled! dropping the packet, opcode = %x.\n",
1843                                         net_rsp->opcode);
1844                         }
1845                 }
1846                 count++;
1847                 ql_update_cq(rx_ring);
1848                 prod = ql_read_sh_reg(rx_ring->prod_idx_sh_reg);
1849                 if (count == budget)
1850                         break;
1851         }
1852         ql_update_buffer_queues(qdev, rx_ring);
1853         ql_write_cq_idx(rx_ring);
1854         return count;
1855 }
1856
1857 static int ql_napi_poll_msix(struct napi_struct *napi, int budget)
1858 {
1859         struct rx_ring *rx_ring = container_of(napi, struct rx_ring, napi);
1860         struct ql_adapter *qdev = rx_ring->qdev;
1861         struct rx_ring *trx_ring;
1862         int i, work_done = 0;
1863         struct intr_context *ctx = &qdev->intr_context[rx_ring->cq_id];
1864
1865         QPRINTK(qdev, RX_STATUS, DEBUG, "Enter, NAPI POLL cq_id = %d.\n",
1866                 rx_ring->cq_id);
1867
1868         /* Service the TX rings first.  They start
1869          * right after the RSS rings. */
1870         for (i = qdev->rss_ring_count; i < qdev->rx_ring_count; i++) {
1871                 trx_ring = &qdev->rx_ring[i];
1872                 /* If this TX completion ring belongs to this vector and
1873                  * it's not empty then service it.
1874                  */
1875                 if ((ctx->irq_mask & (1 << trx_ring->cq_id)) &&
1876                         (ql_read_sh_reg(trx_ring->prod_idx_sh_reg) !=
1877                                         trx_ring->cnsmr_idx)) {
1878                         QPRINTK(qdev, INTR, DEBUG,
1879                                 "%s: Servicing TX completion ring %d.\n",
1880                                 __func__, trx_ring->cq_id);
1881                         ql_clean_outbound_rx_ring(trx_ring);
1882                 }
1883         }
1884
1885         /*
1886          * Now service the RSS ring if it's active.
1887          */
1888         if (ql_read_sh_reg(rx_ring->prod_idx_sh_reg) !=
1889                                         rx_ring->cnsmr_idx) {
1890                 QPRINTK(qdev, INTR, DEBUG,
1891                         "%s: Servicing RX completion ring %d.\n",
1892                         __func__, rx_ring->cq_id);
1893                 work_done = ql_clean_inbound_rx_ring(rx_ring, budget);
1894         }
1895
1896         if (work_done < budget) {
1897                 napi_complete(napi);
1898                 ql_enable_completion_interrupt(qdev, rx_ring->irq);
1899         }
1900         return work_done;
1901 }
1902
1903 static void ql_vlan_rx_register(struct net_device *ndev, struct vlan_group *grp)
1904 {
1905         struct ql_adapter *qdev = netdev_priv(ndev);
1906
1907         qdev->vlgrp = grp;
1908         if (grp) {
1909                 QPRINTK(qdev, IFUP, DEBUG, "Turning on VLAN in NIC_RCV_CFG.\n");
1910                 ql_write32(qdev, NIC_RCV_CFG, NIC_RCV_CFG_VLAN_MASK |
1911                            NIC_RCV_CFG_VLAN_MATCH_AND_NON);
1912         } else {
1913                 QPRINTK(qdev, IFUP, DEBUG,
1914                         "Turning off VLAN in NIC_RCV_CFG.\n");
1915                 ql_write32(qdev, NIC_RCV_CFG, NIC_RCV_CFG_VLAN_MASK);
1916         }
1917 }
1918
1919 static void ql_vlan_rx_add_vid(struct net_device *ndev, u16 vid)
1920 {
1921         struct ql_adapter *qdev = netdev_priv(ndev);
1922         u32 enable_bit = MAC_ADDR_E;
1923         int status;
1924
1925         status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
1926         if (status)
1927                 return;
1928         if (ql_set_mac_addr_reg
1929             (qdev, (u8 *) &enable_bit, MAC_ADDR_TYPE_VLAN, vid)) {
1930                 QPRINTK(qdev, IFUP, ERR, "Failed to init vlan address.\n");
1931         }
1932         ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
1933 }
1934
1935 static void ql_vlan_rx_kill_vid(struct net_device *ndev, u16 vid)
1936 {
1937         struct ql_adapter *qdev = netdev_priv(ndev);
1938         u32 enable_bit = 0;
1939         int status;
1940
1941         status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
1942         if (status)
1943                 return;
1944
1945         if (ql_set_mac_addr_reg
1946             (qdev, (u8 *) &enable_bit, MAC_ADDR_TYPE_VLAN, vid)) {
1947                 QPRINTK(qdev, IFUP, ERR, "Failed to clear vlan address.\n");
1948         }
1949         ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
1950
1951 }
1952
1953 /* MSI-X Multiple Vector Interrupt Handler for inbound completions. */
1954 static irqreturn_t qlge_msix_rx_isr(int irq, void *dev_id)
1955 {
1956         struct rx_ring *rx_ring = dev_id;
1957         napi_schedule(&rx_ring->napi);
1958         return IRQ_HANDLED;
1959 }
1960
1961 /* This handles a fatal error, MPI activity, and the default
1962  * rx_ring in an MSI-X multiple vector environment.
1963  * In MSI/Legacy environment it also process the rest of
1964  * the rx_rings.
1965  */
1966 static irqreturn_t qlge_isr(int irq, void *dev_id)
1967 {
1968         struct rx_ring *rx_ring = dev_id;
1969         struct ql_adapter *qdev = rx_ring->qdev;
1970         struct intr_context *intr_context = &qdev->intr_context[0];
1971         u32 var;
1972         int work_done = 0;
1973
1974         spin_lock(&qdev->hw_lock);
1975         if (atomic_read(&qdev->intr_context[0].irq_cnt)) {
1976                 QPRINTK(qdev, INTR, DEBUG, "Shared Interrupt, Not ours!\n");
1977                 spin_unlock(&qdev->hw_lock);
1978                 return IRQ_NONE;
1979         }
1980         spin_unlock(&qdev->hw_lock);
1981
1982         var = ql_disable_completion_interrupt(qdev, intr_context->intr);
1983
1984         /*
1985          * Check for fatal error.
1986          */
1987         if (var & STS_FE) {
1988                 ql_queue_asic_error(qdev);
1989                 QPRINTK(qdev, INTR, ERR, "Got fatal error, STS = %x.\n", var);
1990                 var = ql_read32(qdev, ERR_STS);
1991                 QPRINTK(qdev, INTR, ERR,
1992                         "Resetting chip. Error Status Register = 0x%x\n", var);
1993                 return IRQ_HANDLED;
1994         }
1995
1996         /*
1997          * Check MPI processor activity.
1998          */
1999         if ((var & STS_PI) &&
2000                 (ql_read32(qdev, INTR_MASK) & INTR_MASK_PI)) {
2001                 /*
2002                  * We've got an async event or mailbox completion.
2003                  * Handle it and clear the source of the interrupt.
2004                  */
2005                 QPRINTK(qdev, INTR, ERR, "Got MPI processor interrupt.\n");
2006                 ql_disable_completion_interrupt(qdev, intr_context->intr);
2007                 ql_write32(qdev, INTR_MASK, (INTR_MASK_PI << 16));
2008                 queue_delayed_work_on(smp_processor_id(),
2009                                 qdev->workqueue, &qdev->mpi_work, 0);
2010                 work_done++;
2011         }
2012
2013         /*
2014          * Get the bit-mask that shows the active queues for this
2015          * pass.  Compare it to the queues that this irq services
2016          * and call napi if there's a match.
2017          */
2018         var = ql_read32(qdev, ISR1);
2019         if (var & intr_context->irq_mask) {
2020                                 QPRINTK(qdev, INTR, INFO,
2021                         "Waking handler for rx_ring[0].\n");
2022                 ql_disable_completion_interrupt(qdev, intr_context->intr);
2023                                         napi_schedule(&rx_ring->napi);
2024                                 work_done++;
2025                         }
2026         ql_enable_completion_interrupt(qdev, intr_context->intr);
2027         return work_done ? IRQ_HANDLED : IRQ_NONE;
2028 }
2029
2030 static int ql_tso(struct sk_buff *skb, struct ob_mac_tso_iocb_req *mac_iocb_ptr)
2031 {
2032
2033         if (skb_is_gso(skb)) {
2034                 int err;
2035                 if (skb_header_cloned(skb)) {
2036                         err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
2037                         if (err)
2038                                 return err;
2039                 }
2040
2041                 mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
2042                 mac_iocb_ptr->flags3 |= OB_MAC_TSO_IOCB_IC;
2043                 mac_iocb_ptr->frame_len = cpu_to_le32((u32) skb->len);
2044                 mac_iocb_ptr->total_hdrs_len =
2045                     cpu_to_le16(skb_transport_offset(skb) + tcp_hdrlen(skb));
2046                 mac_iocb_ptr->net_trans_offset =
2047                     cpu_to_le16(skb_network_offset(skb) |
2048                                 skb_transport_offset(skb)
2049                                 << OB_MAC_TRANSPORT_HDR_SHIFT);
2050                 mac_iocb_ptr->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
2051                 mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_LSO;
2052                 if (likely(skb->protocol == htons(ETH_P_IP))) {
2053                         struct iphdr *iph = ip_hdr(skb);
2054                         iph->check = 0;
2055                         mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP4;
2056                         tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
2057                                                                  iph->daddr, 0,
2058                                                                  IPPROTO_TCP,
2059                                                                  0);
2060                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
2061                         mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP6;
2062                         tcp_hdr(skb)->check =
2063                             ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
2064                                              &ipv6_hdr(skb)->daddr,
2065                                              0, IPPROTO_TCP, 0);
2066                 }
2067                 return 1;
2068         }
2069         return 0;
2070 }
2071
2072 static void ql_hw_csum_setup(struct sk_buff *skb,
2073                              struct ob_mac_tso_iocb_req *mac_iocb_ptr)
2074 {
2075         int len;
2076         struct iphdr *iph = ip_hdr(skb);
2077         __sum16 *check;
2078         mac_iocb_ptr->opcode = OPCODE_OB_MAC_TSO_IOCB;
2079         mac_iocb_ptr->frame_len = cpu_to_le32((u32) skb->len);
2080         mac_iocb_ptr->net_trans_offset =
2081                 cpu_to_le16(skb_network_offset(skb) |
2082                 skb_transport_offset(skb) << OB_MAC_TRANSPORT_HDR_SHIFT);
2083
2084         mac_iocb_ptr->flags1 |= OB_MAC_TSO_IOCB_IP4;
2085         len = (ntohs(iph->tot_len) - (iph->ihl << 2));
2086         if (likely(iph->protocol == IPPROTO_TCP)) {
2087                 check = &(tcp_hdr(skb)->check);
2088                 mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_TC;
2089                 mac_iocb_ptr->total_hdrs_len =
2090                     cpu_to_le16(skb_transport_offset(skb) +
2091                                 (tcp_hdr(skb)->doff << 2));
2092         } else {
2093                 check = &(udp_hdr(skb)->check);
2094                 mac_iocb_ptr->flags2 |= OB_MAC_TSO_IOCB_UC;
2095                 mac_iocb_ptr->total_hdrs_len =
2096                     cpu_to_le16(skb_transport_offset(skb) +
2097                                 sizeof(struct udphdr));
2098         }
2099         *check = ~csum_tcpudp_magic(iph->saddr,
2100                                     iph->daddr, len, iph->protocol, 0);
2101 }
2102
2103 static netdev_tx_t qlge_send(struct sk_buff *skb, struct net_device *ndev)
2104 {
2105         struct tx_ring_desc *tx_ring_desc;
2106         struct ob_mac_iocb_req *mac_iocb_ptr;
2107         struct ql_adapter *qdev = netdev_priv(ndev);
2108         int tso;
2109         struct tx_ring *tx_ring;
2110         u32 tx_ring_idx = (u32) skb->queue_mapping;
2111
2112         tx_ring = &qdev->tx_ring[tx_ring_idx];
2113
2114         if (skb_padto(skb, ETH_ZLEN))
2115                 return NETDEV_TX_OK;
2116
2117         if (unlikely(atomic_read(&tx_ring->tx_count) < 2)) {
2118                 QPRINTK(qdev, TX_QUEUED, INFO,
2119                         "%s: shutting down tx queue %d du to lack of resources.\n",
2120                         __func__, tx_ring_idx);
2121                 netif_stop_subqueue(ndev, tx_ring->wq_id);
2122                 atomic_inc(&tx_ring->queue_stopped);
2123                 return NETDEV_TX_BUSY;
2124         }
2125         tx_ring_desc = &tx_ring->q[tx_ring->prod_idx];
2126         mac_iocb_ptr = tx_ring_desc->queue_entry;
2127         memset((void *)mac_iocb_ptr, 0, sizeof(*mac_iocb_ptr));
2128
2129         mac_iocb_ptr->opcode = OPCODE_OB_MAC_IOCB;
2130         mac_iocb_ptr->tid = tx_ring_desc->index;
2131         /* We use the upper 32-bits to store the tx queue for this IO.
2132          * When we get the completion we can use it to establish the context.
2133          */
2134         mac_iocb_ptr->txq_idx = tx_ring_idx;
2135         tx_ring_desc->skb = skb;
2136
2137         mac_iocb_ptr->frame_len = cpu_to_le16((u16) skb->len);
2138
2139         if (qdev->vlgrp && vlan_tx_tag_present(skb)) {
2140                 QPRINTK(qdev, TX_QUEUED, DEBUG, "Adding a vlan tag %d.\n",
2141                         vlan_tx_tag_get(skb));
2142                 mac_iocb_ptr->flags3 |= OB_MAC_IOCB_V;
2143                 mac_iocb_ptr->vlan_tci = cpu_to_le16(vlan_tx_tag_get(skb));
2144         }
2145         tso = ql_tso(skb, (struct ob_mac_tso_iocb_req *)mac_iocb_ptr);
2146         if (tso < 0) {
2147                 dev_kfree_skb_any(skb);
2148                 return NETDEV_TX_OK;
2149         } else if (unlikely(!tso) && (skb->ip_summed == CHECKSUM_PARTIAL)) {
2150                 ql_hw_csum_setup(skb,
2151                                  (struct ob_mac_tso_iocb_req *)mac_iocb_ptr);
2152         }
2153         if (ql_map_send(qdev, mac_iocb_ptr, skb, tx_ring_desc) !=
2154                         NETDEV_TX_OK) {
2155                 QPRINTK(qdev, TX_QUEUED, ERR,
2156                                 "Could not map the segments.\n");
2157                 return NETDEV_TX_BUSY;
2158         }
2159         QL_DUMP_OB_MAC_IOCB(mac_iocb_ptr);
2160         tx_ring->prod_idx++;
2161         if (tx_ring->prod_idx == tx_ring->wq_len)
2162                 tx_ring->prod_idx = 0;
2163         wmb();
2164
2165         ql_write_db_reg(tx_ring->prod_idx, tx_ring->prod_idx_db_reg);
2166         QPRINTK(qdev, TX_QUEUED, DEBUG, "tx queued, slot %d, len %d\n",
2167                 tx_ring->prod_idx, skb->len);
2168
2169         atomic_dec(&tx_ring->tx_count);
2170         return NETDEV_TX_OK;
2171 }
2172
2173 static void ql_free_shadow_space(struct ql_adapter *qdev)
2174 {
2175         if (qdev->rx_ring_shadow_reg_area) {
2176                 pci_free_consistent(qdev->pdev,
2177                                     PAGE_SIZE,
2178                                     qdev->rx_ring_shadow_reg_area,
2179                                     qdev->rx_ring_shadow_reg_dma);
2180                 qdev->rx_ring_shadow_reg_area = NULL;
2181         }
2182         if (qdev->tx_ring_shadow_reg_area) {
2183                 pci_free_consistent(qdev->pdev,
2184                                     PAGE_SIZE,
2185                                     qdev->tx_ring_shadow_reg_area,
2186                                     qdev->tx_ring_shadow_reg_dma);
2187                 qdev->tx_ring_shadow_reg_area = NULL;
2188         }
2189 }
2190
2191 static int ql_alloc_shadow_space(struct ql_adapter *qdev)
2192 {
2193         qdev->rx_ring_shadow_reg_area =
2194             pci_alloc_consistent(qdev->pdev,
2195                                  PAGE_SIZE, &qdev->rx_ring_shadow_reg_dma);
2196         if (qdev->rx_ring_shadow_reg_area == NULL) {
2197                 QPRINTK(qdev, IFUP, ERR,
2198                         "Allocation of RX shadow space failed.\n");
2199                 return -ENOMEM;
2200         }
2201         memset(qdev->rx_ring_shadow_reg_area, 0, PAGE_SIZE);
2202         qdev->tx_ring_shadow_reg_area =
2203             pci_alloc_consistent(qdev->pdev, PAGE_SIZE,
2204                                  &qdev->tx_ring_shadow_reg_dma);
2205         if (qdev->tx_ring_shadow_reg_area == NULL) {
2206                 QPRINTK(qdev, IFUP, ERR,
2207                         "Allocation of TX shadow space failed.\n");
2208                 goto err_wqp_sh_area;
2209         }
2210         memset(qdev->tx_ring_shadow_reg_area, 0, PAGE_SIZE);
2211         return 0;
2212
2213 err_wqp_sh_area:
2214         pci_free_consistent(qdev->pdev,
2215                             PAGE_SIZE,
2216                             qdev->rx_ring_shadow_reg_area,
2217                             qdev->rx_ring_shadow_reg_dma);
2218         return -ENOMEM;
2219 }
2220
2221 static void ql_init_tx_ring(struct ql_adapter *qdev, struct tx_ring *tx_ring)
2222 {
2223         struct tx_ring_desc *tx_ring_desc;
2224         int i;
2225         struct ob_mac_iocb_req *mac_iocb_ptr;
2226
2227         mac_iocb_ptr = tx_ring->wq_base;
2228         tx_ring_desc = tx_ring->q;
2229         for (i = 0; i < tx_ring->wq_len; i++) {
2230                 tx_ring_desc->index = i;
2231                 tx_ring_desc->skb = NULL;
2232                 tx_ring_desc->queue_entry = mac_iocb_ptr;
2233                 mac_iocb_ptr++;
2234                 tx_ring_desc++;
2235         }
2236         atomic_set(&tx_ring->tx_count, tx_ring->wq_len);
2237         atomic_set(&tx_ring->queue_stopped, 0);
2238 }
2239
2240 static void ql_free_tx_resources(struct ql_adapter *qdev,
2241                                  struct tx_ring *tx_ring)
2242 {
2243         if (tx_ring->wq_base) {
2244                 pci_free_consistent(qdev->pdev, tx_ring->wq_size,
2245                                     tx_ring->wq_base, tx_ring->wq_base_dma);
2246                 tx_ring->wq_base = NULL;
2247         }
2248         kfree(tx_ring->q);
2249         tx_ring->q = NULL;
2250 }
2251
2252 static int ql_alloc_tx_resources(struct ql_adapter *qdev,
2253                                  struct tx_ring *tx_ring)
2254 {
2255         tx_ring->wq_base =
2256             pci_alloc_consistent(qdev->pdev, tx_ring->wq_size,
2257                                  &tx_ring->wq_base_dma);
2258
2259         if ((tx_ring->wq_base == NULL)
2260                 || tx_ring->wq_base_dma & WQ_ADDR_ALIGN) {
2261                 QPRINTK(qdev, IFUP, ERR, "tx_ring alloc failed.\n");
2262                 return -ENOMEM;
2263         }
2264         tx_ring->q =
2265             kmalloc(tx_ring->wq_len * sizeof(struct tx_ring_desc), GFP_KERNEL);
2266         if (tx_ring->q == NULL)
2267                 goto err;
2268
2269         return 0;
2270 err:
2271         pci_free_consistent(qdev->pdev, tx_ring->wq_size,
2272                             tx_ring->wq_base, tx_ring->wq_base_dma);
2273         return -ENOMEM;
2274 }
2275
2276 static void ql_free_lbq_buffers(struct ql_adapter *qdev, struct rx_ring *rx_ring)
2277 {
2278         int i;
2279         struct bq_desc *lbq_desc;
2280
2281         for (i = 0; i < rx_ring->lbq_len; i++) {
2282                 lbq_desc = &rx_ring->lbq[i];
2283                 if (lbq_desc->p.lbq_page) {
2284                         pci_unmap_page(qdev->pdev,
2285                                        pci_unmap_addr(lbq_desc, mapaddr),
2286                                        pci_unmap_len(lbq_desc, maplen),
2287                                        PCI_DMA_FROMDEVICE);
2288
2289                         put_page(lbq_desc->p.lbq_page);
2290                         lbq_desc->p.lbq_page = NULL;
2291                 }
2292         }
2293 }
2294
2295 static void ql_free_sbq_buffers(struct ql_adapter *qdev, struct rx_ring *rx_ring)
2296 {
2297         int i;
2298         struct bq_desc *sbq_desc;
2299
2300         for (i = 0; i < rx_ring->sbq_len; i++) {
2301                 sbq_desc = &rx_ring->sbq[i];
2302                 if (sbq_desc == NULL) {
2303                         QPRINTK(qdev, IFUP, ERR, "sbq_desc %d is NULL.\n", i);
2304                         return;
2305                 }
2306                 if (sbq_desc->p.skb) {
2307                         pci_unmap_single(qdev->pdev,
2308                                          pci_unmap_addr(sbq_desc, mapaddr),
2309                                          pci_unmap_len(sbq_desc, maplen),
2310                                          PCI_DMA_FROMDEVICE);
2311                         dev_kfree_skb(sbq_desc->p.skb);
2312                         sbq_desc->p.skb = NULL;
2313                 }
2314         }
2315 }
2316
2317 /* Free all large and small rx buffers associated
2318  * with the completion queues for this device.
2319  */
2320 static void ql_free_rx_buffers(struct ql_adapter *qdev)
2321 {
2322         int i;
2323         struct rx_ring *rx_ring;
2324
2325         for (i = 0; i < qdev->rx_ring_count; i++) {
2326                 rx_ring = &qdev->rx_ring[i];
2327                 if (rx_ring->lbq)
2328                         ql_free_lbq_buffers(qdev, rx_ring);
2329                 if (rx_ring->sbq)
2330                         ql_free_sbq_buffers(qdev, rx_ring);
2331         }
2332 }
2333
2334 static void ql_alloc_rx_buffers(struct ql_adapter *qdev)
2335 {
2336         struct rx_ring *rx_ring;
2337         int i;
2338
2339         for (i = 0; i < qdev->rx_ring_count; i++) {
2340                 rx_ring = &qdev->rx_ring[i];
2341                 if (rx_ring->type != TX_Q)
2342                         ql_update_buffer_queues(qdev, rx_ring);
2343         }
2344 }
2345
2346 static void ql_init_lbq_ring(struct ql_adapter *qdev,
2347                                 struct rx_ring *rx_ring)
2348 {
2349         int i;
2350         struct bq_desc *lbq_desc;
2351         __le64 *bq = rx_ring->lbq_base;
2352
2353         memset(rx_ring->lbq, 0, rx_ring->lbq_len * sizeof(struct bq_desc));
2354         for (i = 0; i < rx_ring->lbq_len; i++) {
2355                 lbq_desc = &rx_ring->lbq[i];
2356                 memset(lbq_desc, 0, sizeof(*lbq_desc));
2357                 lbq_desc->index = i;
2358                 lbq_desc->addr = bq;
2359                 bq++;
2360         }
2361 }
2362
2363 static void ql_init_sbq_ring(struct ql_adapter *qdev,
2364                                 struct rx_ring *rx_ring)
2365 {
2366         int i;
2367         struct bq_desc *sbq_desc;
2368         __le64 *bq = rx_ring->sbq_base;
2369
2370         memset(rx_ring->sbq, 0, rx_ring->sbq_len * sizeof(struct bq_desc));
2371         for (i = 0; i < rx_ring->sbq_len; i++) {
2372                 sbq_desc = &rx_ring->sbq[i];
2373                 memset(sbq_desc, 0, sizeof(*sbq_desc));
2374                 sbq_desc->index = i;
2375                 sbq_desc->addr = bq;
2376                 bq++;
2377         }
2378 }
2379
2380 static void ql_free_rx_resources(struct ql_adapter *qdev,
2381                                  struct rx_ring *rx_ring)
2382 {
2383         /* Free the small buffer queue. */
2384         if (rx_ring->sbq_base) {
2385                 pci_free_consistent(qdev->pdev,
2386                                     rx_ring->sbq_size,
2387                                     rx_ring->sbq_base, rx_ring->sbq_base_dma);
2388                 rx_ring->sbq_base = NULL;
2389         }
2390
2391         /* Free the small buffer queue control blocks. */
2392         kfree(rx_ring->sbq);
2393         rx_ring->sbq = NULL;
2394
2395         /* Free the large buffer queue. */
2396         if (rx_ring->lbq_base) {
2397                 pci_free_consistent(qdev->pdev,
2398                                     rx_ring->lbq_size,
2399                                     rx_ring->lbq_base, rx_ring->lbq_base_dma);
2400                 rx_ring->lbq_base = NULL;
2401         }
2402
2403         /* Free the large buffer queue control blocks. */
2404         kfree(rx_ring->lbq);
2405         rx_ring->lbq = NULL;
2406
2407         /* Free the rx queue. */
2408         if (rx_ring->cq_base) {
2409                 pci_free_consistent(qdev->pdev,
2410                                     rx_ring->cq_size,
2411                                     rx_ring->cq_base, rx_ring->cq_base_dma);
2412                 rx_ring->cq_base = NULL;
2413         }
2414 }
2415
2416 /* Allocate queues and buffers for this completions queue based
2417  * on the values in the parameter structure. */
2418 static int ql_alloc_rx_resources(struct ql_adapter *qdev,
2419                                  struct rx_ring *rx_ring)
2420 {
2421
2422         /*
2423          * Allocate the completion queue for this rx_ring.
2424          */
2425         rx_ring->cq_base =
2426             pci_alloc_consistent(qdev->pdev, rx_ring->cq_size,
2427                                  &rx_ring->cq_base_dma);
2428
2429         if (rx_ring->cq_base == NULL) {
2430                 QPRINTK(qdev, IFUP, ERR, "rx_ring alloc failed.\n");
2431                 return -ENOMEM;
2432         }
2433
2434         if (rx_ring->sbq_len) {
2435                 /*
2436                  * Allocate small buffer queue.
2437                  */
2438                 rx_ring->sbq_base =
2439                     pci_alloc_consistent(qdev->pdev, rx_ring->sbq_size,
2440                                          &rx_ring->sbq_base_dma);
2441
2442                 if (rx_ring->sbq_base == NULL) {
2443                         QPRINTK(qdev, IFUP, ERR,
2444                                 "Small buffer queue allocation failed.\n");
2445                         goto err_mem;
2446                 }
2447
2448                 /*
2449                  * Allocate small buffer queue control blocks.
2450                  */
2451                 rx_ring->sbq =
2452                     kmalloc(rx_ring->sbq_len * sizeof(struct bq_desc),
2453                             GFP_KERNEL);
2454                 if (rx_ring->sbq == NULL) {
2455                         QPRINTK(qdev, IFUP, ERR,
2456                                 "Small buffer queue control block allocation failed.\n");
2457                         goto err_mem;
2458                 }
2459
2460                 ql_init_sbq_ring(qdev, rx_ring);
2461         }
2462
2463         if (rx_ring->lbq_len) {
2464                 /*
2465                  * Allocate large buffer queue.
2466                  */
2467                 rx_ring->lbq_base =
2468                     pci_alloc_consistent(qdev->pdev, rx_ring->lbq_size,
2469                                          &rx_ring->lbq_base_dma);
2470
2471                 if (rx_ring->lbq_base == NULL) {
2472                         QPRINTK(qdev, IFUP, ERR,
2473                                 "Large buffer queue allocation failed.\n");
2474                         goto err_mem;
2475                 }
2476                 /*
2477                  * Allocate large buffer queue control blocks.
2478                  */
2479                 rx_ring->lbq =
2480                     kmalloc(rx_ring->lbq_len * sizeof(struct bq_desc),
2481                             GFP_KERNEL);
2482                 if (rx_ring->lbq == NULL) {
2483                         QPRINTK(qdev, IFUP, ERR,
2484                                 "Large buffer queue control block allocation failed.\n");
2485                         goto err_mem;
2486                 }
2487
2488                 ql_init_lbq_ring(qdev, rx_ring);
2489         }
2490
2491         return 0;
2492
2493 err_mem:
2494         ql_free_rx_resources(qdev, rx_ring);
2495         return -ENOMEM;
2496 }
2497
2498 static void ql_tx_ring_clean(struct ql_adapter *qdev)
2499 {
2500         struct tx_ring *tx_ring;
2501         struct tx_ring_desc *tx_ring_desc;
2502         int i, j;
2503
2504         /*
2505          * Loop through all queues and free
2506          * any resources.
2507          */
2508         for (j = 0; j < qdev->tx_ring_count; j++) {
2509                 tx_ring = &qdev->tx_ring[j];
2510                 for (i = 0; i < tx_ring->wq_len; i++) {
2511                         tx_ring_desc = &tx_ring->q[i];
2512                         if (tx_ring_desc && tx_ring_desc->skb) {
2513                                 QPRINTK(qdev, IFDOWN, ERR,
2514                                 "Freeing lost SKB %p, from queue %d, index %d.\n",
2515                                         tx_ring_desc->skb, j,
2516                                         tx_ring_desc->index);
2517                                 ql_unmap_send(qdev, tx_ring_desc,
2518                                               tx_ring_desc->map_cnt);
2519                                 dev_kfree_skb(tx_ring_desc->skb);
2520                                 tx_ring_desc->skb = NULL;
2521                         }
2522                 }
2523         }
2524 }
2525
2526 static void ql_free_mem_resources(struct ql_adapter *qdev)
2527 {
2528         int i;
2529
2530         for (i = 0; i < qdev->tx_ring_count; i++)
2531                 ql_free_tx_resources(qdev, &qdev->tx_ring[i]);
2532         for (i = 0; i < qdev->rx_ring_count; i++)
2533                 ql_free_rx_resources(qdev, &qdev->rx_ring[i]);
2534         ql_free_shadow_space(qdev);
2535 }
2536
2537 static int ql_alloc_mem_resources(struct ql_adapter *qdev)
2538 {
2539         int i;
2540
2541         /* Allocate space for our shadow registers and such. */
2542         if (ql_alloc_shadow_space(qdev))
2543                 return -ENOMEM;
2544
2545         for (i = 0; i < qdev->rx_ring_count; i++) {
2546                 if (ql_alloc_rx_resources(qdev, &qdev->rx_ring[i]) != 0) {
2547                         QPRINTK(qdev, IFUP, ERR,
2548                                 "RX resource allocation failed.\n");
2549                         goto err_mem;
2550                 }
2551         }
2552         /* Allocate tx queue resources */
2553         for (i = 0; i < qdev->tx_ring_count; i++) {
2554                 if (ql_alloc_tx_resources(qdev, &qdev->tx_ring[i]) != 0) {
2555                         QPRINTK(qdev, IFUP, ERR,
2556                                 "TX resource allocation failed.\n");
2557                         goto err_mem;
2558                 }
2559         }
2560         return 0;
2561
2562 err_mem:
2563         ql_free_mem_resources(qdev);
2564         return -ENOMEM;
2565 }
2566
2567 /* Set up the rx ring control block and pass it to the chip.
2568  * The control block is defined as
2569  * "Completion Queue Initialization Control Block", or cqicb.
2570  */
2571 static int ql_start_rx_ring(struct ql_adapter *qdev, struct rx_ring *rx_ring)
2572 {
2573         struct cqicb *cqicb = &rx_ring->cqicb;
2574         void *shadow_reg = qdev->rx_ring_shadow_reg_area +
2575                 (rx_ring->cq_id * RX_RING_SHADOW_SPACE);
2576         u64 shadow_reg_dma = qdev->rx_ring_shadow_reg_dma +
2577                 (rx_ring->cq_id * RX_RING_SHADOW_SPACE);
2578         void __iomem *doorbell_area =
2579             qdev->doorbell_area + (DB_PAGE_SIZE * (128 + rx_ring->cq_id));
2580         int err = 0;
2581         u16 bq_len;
2582         u64 tmp;
2583         __le64 *base_indirect_ptr;
2584         int page_entries;
2585
2586         /* Set up the shadow registers for this ring. */
2587         rx_ring->prod_idx_sh_reg = shadow_reg;
2588         rx_ring->prod_idx_sh_reg_dma = shadow_reg_dma;
2589         shadow_reg += sizeof(u64);
2590         shadow_reg_dma += sizeof(u64);
2591         rx_ring->lbq_base_indirect = shadow_reg;
2592         rx_ring->lbq_base_indirect_dma = shadow_reg_dma;
2593         shadow_reg += (sizeof(u64) * MAX_DB_PAGES_PER_BQ(rx_ring->lbq_len));
2594         shadow_reg_dma += (sizeof(u64) * MAX_DB_PAGES_PER_BQ(rx_ring->lbq_len));
2595         rx_ring->sbq_base_indirect = shadow_reg;
2596         rx_ring->sbq_base_indirect_dma = shadow_reg_dma;
2597
2598         /* PCI doorbell mem area + 0x00 for consumer index register */
2599         rx_ring->cnsmr_idx_db_reg = (u32 __iomem *) doorbell_area;
2600         rx_ring->cnsmr_idx = 0;
2601         rx_ring->curr_entry = rx_ring->cq_base;
2602
2603         /* PCI doorbell mem area + 0x04 for valid register */
2604         rx_ring->valid_db_reg = doorbell_area + 0x04;
2605
2606         /* PCI doorbell mem area + 0x18 for large buffer consumer */
2607         rx_ring->lbq_prod_idx_db_reg = (u32 __iomem *) (doorbell_area + 0x18);
2608
2609         /* PCI doorbell mem area + 0x1c */
2610         rx_ring->sbq_prod_idx_db_reg = (u32 __iomem *) (doorbell_area + 0x1c);
2611
2612         memset((void *)cqicb, 0, sizeof(struct cqicb));
2613         cqicb->msix_vect = rx_ring->irq;
2614
2615         bq_len = (rx_ring->cq_len == 65536) ? 0 : (u16) rx_ring->cq_len;
2616         cqicb->len = cpu_to_le16(bq_len | LEN_V | LEN_CPP_CONT);
2617
2618         cqicb->addr = cpu_to_le64(rx_ring->cq_base_dma);
2619
2620         cqicb->prod_idx_addr = cpu_to_le64(rx_ring->prod_idx_sh_reg_dma);
2621
2622         /*
2623          * Set up the control block load flags.
2624          */
2625         cqicb->flags = FLAGS_LC |       /* Load queue base address */
2626             FLAGS_LV |          /* Load MSI-X vector */
2627             FLAGS_LI;           /* Load irq delay values */
2628         if (rx_ring->lbq_len) {
2629                 cqicb->flags |= FLAGS_LL;       /* Load lbq values */
2630                 tmp = (u64)rx_ring->lbq_base_dma;
2631                 base_indirect_ptr = (__le64 *) rx_ring->lbq_base_indirect;
2632                 page_entries = 0;
2633                 do {
2634                         *base_indirect_ptr = cpu_to_le64(tmp);
2635                         tmp += DB_PAGE_SIZE;
2636                         base_indirect_ptr++;
2637                         page_entries++;
2638                 } while (page_entries < MAX_DB_PAGES_PER_BQ(rx_ring->lbq_len));
2639                 cqicb->lbq_addr =
2640                     cpu_to_le64(rx_ring->lbq_base_indirect_dma);
2641                 bq_len = (rx_ring->lbq_buf_size == 65536) ? 0 :
2642                         (u16) rx_ring->lbq_buf_size;
2643                 cqicb->lbq_buf_size = cpu_to_le16(bq_len);
2644                 bq_len = (rx_ring->lbq_len == 65536) ? 0 :
2645                         (u16) rx_ring->lbq_len;
2646                 cqicb->lbq_len = cpu_to_le16(bq_len);
2647                 rx_ring->lbq_prod_idx = 0;
2648                 rx_ring->lbq_curr_idx = 0;
2649                 rx_ring->lbq_clean_idx = 0;
2650                 rx_ring->lbq_free_cnt = rx_ring->lbq_len;
2651         }
2652         if (rx_ring->sbq_len) {
2653                 cqicb->flags |= FLAGS_LS;       /* Load sbq values */
2654                 tmp = (u64)rx_ring->sbq_base_dma;
2655                 base_indirect_ptr = (__le64 *) rx_ring->sbq_base_indirect;
2656                 page_entries = 0;
2657                 do {
2658                         *base_indirect_ptr = cpu_to_le64(tmp);
2659                         tmp += DB_PAGE_SIZE;
2660                         base_indirect_ptr++;
2661                         page_entries++;
2662                 } while (page_entries < MAX_DB_PAGES_PER_BQ(rx_ring->sbq_len));
2663                 cqicb->sbq_addr =
2664                     cpu_to_le64(rx_ring->sbq_base_indirect_dma);
2665                 cqicb->sbq_buf_size =
2666                     cpu_to_le16((u16)(rx_ring->sbq_buf_size/2));
2667                 bq_len = (rx_ring->sbq_len == 65536) ? 0 :
2668                         (u16) rx_ring->sbq_len;
2669                 cqicb->sbq_len = cpu_to_le16(bq_len);
2670                 rx_ring->sbq_prod_idx = 0;
2671                 rx_ring->sbq_curr_idx = 0;
2672                 rx_ring->sbq_clean_idx = 0;
2673                 rx_ring->sbq_free_cnt = rx_ring->sbq_len;
2674         }
2675         switch (rx_ring->type) {
2676         case TX_Q:
2677                 cqicb->irq_delay = cpu_to_le16(qdev->tx_coalesce_usecs);
2678                 cqicb->pkt_delay = cpu_to_le16(qdev->tx_max_coalesced_frames);
2679                 break;
2680         case RX_Q:
2681                 /* Inbound completion handling rx_rings run in
2682                  * separate NAPI contexts.
2683                  */
2684                 netif_napi_add(qdev->ndev, &rx_ring->napi, ql_napi_poll_msix,
2685                                64);
2686                 cqicb->irq_delay = cpu_to_le16(qdev->rx_coalesce_usecs);
2687                 cqicb->pkt_delay = cpu_to_le16(qdev->rx_max_coalesced_frames);
2688                 break;
2689         default:
2690                 QPRINTK(qdev, IFUP, DEBUG, "Invalid rx_ring->type = %d.\n",
2691                         rx_ring->type);
2692         }
2693         QPRINTK(qdev, IFUP, DEBUG, "Initializing rx work queue.\n");
2694         err = ql_write_cfg(qdev, cqicb, sizeof(struct cqicb),
2695                            CFG_LCQ, rx_ring->cq_id);
2696         if (err) {
2697                 QPRINTK(qdev, IFUP, ERR, "Failed to load CQICB.\n");
2698                 return err;
2699         }
2700         return err;
2701 }
2702
2703 static int ql_start_tx_ring(struct ql_adapter *qdev, struct tx_ring *tx_ring)
2704 {
2705         struct wqicb *wqicb = (struct wqicb *)tx_ring;
2706         void __iomem *doorbell_area =
2707             qdev->doorbell_area + (DB_PAGE_SIZE * tx_ring->wq_id);
2708         void *shadow_reg = qdev->tx_ring_shadow_reg_area +
2709             (tx_ring->wq_id * sizeof(u64));
2710         u64 shadow_reg_dma = qdev->tx_ring_shadow_reg_dma +
2711             (tx_ring->wq_id * sizeof(u64));
2712         int err = 0;
2713
2714         /*
2715          * Assign doorbell registers for this tx_ring.
2716          */
2717         /* TX PCI doorbell mem area for tx producer index */
2718         tx_ring->prod_idx_db_reg = (u32 __iomem *) doorbell_area;
2719         tx_ring->prod_idx = 0;
2720         /* TX PCI doorbell mem area + 0x04 */
2721         tx_ring->valid_db_reg = doorbell_area + 0x04;
2722
2723         /*
2724          * Assign shadow registers for this tx_ring.
2725          */
2726         tx_ring->cnsmr_idx_sh_reg = shadow_reg;
2727         tx_ring->cnsmr_idx_sh_reg_dma = shadow_reg_dma;
2728
2729         wqicb->len = cpu_to_le16(tx_ring->wq_len | Q_LEN_V | Q_LEN_CPP_CONT);
2730         wqicb->flags = cpu_to_le16(Q_FLAGS_LC |
2731                                    Q_FLAGS_LB | Q_FLAGS_LI | Q_FLAGS_LO);
2732         wqicb->cq_id_rss = cpu_to_le16(tx_ring->cq_id);
2733         wqicb->rid = 0;
2734         wqicb->addr = cpu_to_le64(tx_ring->wq_base_dma);
2735
2736         wqicb->cnsmr_idx_addr = cpu_to_le64(tx_ring->cnsmr_idx_sh_reg_dma);
2737
2738         ql_init_tx_ring(qdev, tx_ring);
2739
2740         err = ql_write_cfg(qdev, wqicb, sizeof(*wqicb), CFG_LRQ,
2741                            (u16) tx_ring->wq_id);
2742         if (err) {
2743                 QPRINTK(qdev, IFUP, ERR, "Failed to load tx_ring.\n");
2744                 return err;
2745         }
2746         QPRINTK(qdev, IFUP, DEBUG, "Successfully loaded WQICB.\n");
2747         return err;
2748 }
2749
2750 static void ql_disable_msix(struct ql_adapter *qdev)
2751 {
2752         if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
2753                 pci_disable_msix(qdev->pdev);
2754                 clear_bit(QL_MSIX_ENABLED, &qdev->flags);
2755                 kfree(qdev->msi_x_entry);
2756                 qdev->msi_x_entry = NULL;
2757         } else if (test_bit(QL_MSI_ENABLED, &qdev->flags)) {
2758                 pci_disable_msi(qdev->pdev);
2759                 clear_bit(QL_MSI_ENABLED, &qdev->flags);
2760         }
2761 }
2762
2763 /* We start by trying to get the number of vectors
2764  * stored in qdev->intr_count. If we don't get that
2765  * many then we reduce the count and try again.
2766  */
2767 static void ql_enable_msix(struct ql_adapter *qdev)
2768 {
2769         int i, err;
2770
2771         /* Get the MSIX vectors. */
2772         if (irq_type == MSIX_IRQ) {
2773                 /* Try to alloc space for the msix struct,
2774                  * if it fails then go to MSI/legacy.
2775                  */
2776                 qdev->msi_x_entry = kcalloc(qdev->intr_count,
2777                                             sizeof(struct msix_entry),
2778                                             GFP_KERNEL);
2779                 if (!qdev->msi_x_entry) {
2780                         irq_type = MSI_IRQ;
2781                         goto msi;
2782                 }
2783
2784                 for (i = 0; i < qdev->intr_count; i++)
2785                         qdev->msi_x_entry[i].entry = i;
2786
2787                 /* Loop to get our vectors.  We start with
2788                  * what we want and settle for what we get.
2789                  */
2790                 do {
2791                         err = pci_enable_msix(qdev->pdev,
2792                                 qdev->msi_x_entry, qdev->intr_count);
2793                         if (err > 0)
2794                                 qdev->intr_count = err;
2795                 } while (err > 0);
2796
2797                 if (err < 0) {
2798                         kfree(qdev->msi_x_entry);
2799                         qdev->msi_x_entry = NULL;
2800                         QPRINTK(qdev, IFUP, WARNING,
2801                                 "MSI-X Enable failed, trying MSI.\n");
2802                         qdev->intr_count = 1;
2803                         irq_type = MSI_IRQ;
2804                 } else if (err == 0) {
2805                         set_bit(QL_MSIX_ENABLED, &qdev->flags);
2806                         QPRINTK(qdev, IFUP, INFO,
2807                                 "MSI-X Enabled, got %d vectors.\n",
2808                                 qdev->intr_count);
2809                         return;
2810                 }
2811         }
2812 msi:
2813         qdev->intr_count = 1;
2814         if (irq_type == MSI_IRQ) {
2815                 if (!pci_enable_msi(qdev->pdev)) {
2816                         set_bit(QL_MSI_ENABLED, &qdev->flags);
2817                         QPRINTK(qdev, IFUP, INFO,
2818                                 "Running with MSI interrupts.\n");
2819                         return;
2820                 }
2821         }
2822         irq_type = LEG_IRQ;
2823         QPRINTK(qdev, IFUP, DEBUG, "Running with legacy interrupts.\n");
2824 }
2825
2826 /* Each vector services 1 RSS ring and and 1 or more
2827  * TX completion rings.  This function loops through
2828  * the TX completion rings and assigns the vector that
2829  * will service it.  An example would be if there are
2830  * 2 vectors (so 2 RSS rings) and 8 TX completion rings.
2831  * This would mean that vector 0 would service RSS ring 0
2832  * and TX competion rings 0,1,2 and 3.  Vector 1 would
2833  * service RSS ring 1 and TX completion rings 4,5,6 and 7.
2834  */
2835 static void ql_set_tx_vect(struct ql_adapter *qdev)
2836 {
2837         int i, j, vect;
2838         u32 tx_rings_per_vector = qdev->tx_ring_count / qdev->intr_count;
2839
2840         if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
2841                 /* Assign irq vectors to TX rx_rings.*/
2842                 for (vect = 0, j = 0, i = qdev->rss_ring_count;
2843                                          i < qdev->rx_ring_count; i++) {
2844                         if (j == tx_rings_per_vector) {
2845                                 vect++;
2846                                 j = 0;
2847                         }
2848                         qdev->rx_ring[i].irq = vect;
2849                         j++;
2850                 }
2851         } else {
2852                 /* For single vector all rings have an irq
2853                  * of zero.
2854                  */
2855                 for (i = 0; i < qdev->rx_ring_count; i++)
2856                         qdev->rx_ring[i].irq = 0;
2857         }
2858 }
2859
2860 /* Set the interrupt mask for this vector.  Each vector
2861  * will service 1 RSS ring and 1 or more TX completion
2862  * rings.  This function sets up a bit mask per vector
2863  * that indicates which rings it services.
2864  */
2865 static void ql_set_irq_mask(struct ql_adapter *qdev, struct intr_context *ctx)
2866 {
2867         int j, vect = ctx->intr;
2868         u32 tx_rings_per_vector = qdev->tx_ring_count / qdev->intr_count;
2869
2870         if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
2871                 /* Add the RSS ring serviced by this vector
2872                  * to the mask.
2873                  */
2874                 ctx->irq_mask = (1 << qdev->rx_ring[vect].cq_id);
2875                 /* Add the TX ring(s) serviced by this vector
2876                  * to the mask. */
2877                 for (j = 0; j < tx_rings_per_vector; j++) {
2878                         ctx->irq_mask |=
2879                         (1 << qdev->rx_ring[qdev->rss_ring_count +
2880                         (vect * tx_rings_per_vector) + j].cq_id);
2881                 }
2882         } else {
2883                 /* For single vector we just shift each queue's
2884                  * ID into the mask.
2885                  */
2886                 for (j = 0; j < qdev->rx_ring_count; j++)
2887                         ctx->irq_mask |= (1 << qdev->rx_ring[j].cq_id);
2888         }
2889 }
2890
2891 /*
2892  * Here we build the intr_context structures based on
2893  * our rx_ring count and intr vector count.
2894  * The intr_context structure is used to hook each vector
2895  * to possibly different handlers.
2896  */
2897 static void ql_resolve_queues_to_irqs(struct ql_adapter *qdev)
2898 {
2899         int i = 0;
2900         struct intr_context *intr_context = &qdev->intr_context[0];
2901
2902         if (likely(test_bit(QL_MSIX_ENABLED, &qdev->flags))) {
2903                 /* Each rx_ring has it's
2904                  * own intr_context since we have separate
2905                  * vectors for each queue.
2906                  */
2907                 for (i = 0; i < qdev->intr_count; i++, intr_context++) {
2908                         qdev->rx_ring[i].irq = i;
2909                         intr_context->intr = i;
2910                         intr_context->qdev = qdev;
2911                         /* Set up this vector's bit-mask that indicates
2912                          * which queues it services.
2913                          */
2914                         ql_set_irq_mask(qdev, intr_context);
2915                         /*
2916                          * We set up each vectors enable/disable/read bits so
2917                          * there's no bit/mask calculations in the critical path.
2918                          */
2919                         intr_context->intr_en_mask =
2920                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
2921                             INTR_EN_TYPE_ENABLE | INTR_EN_IHD_MASK | INTR_EN_IHD
2922                             | i;
2923                         intr_context->intr_dis_mask =
2924                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
2925                             INTR_EN_TYPE_DISABLE | INTR_EN_IHD_MASK |
2926                             INTR_EN_IHD | i;
2927                         intr_context->intr_read_mask =
2928                             INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
2929                             INTR_EN_TYPE_READ | INTR_EN_IHD_MASK | INTR_EN_IHD |
2930                             i;
2931                         if (i == 0) {
2932                                 /* The first vector/queue handles
2933                                  * broadcast/multicast, fatal errors,
2934                                  * and firmware events.  This in addition
2935                                  * to normal inbound NAPI processing.
2936                                  */
2937                                 intr_context->handler = qlge_isr;
2938                                 sprintf(intr_context->name, "%s-rx-%d",
2939                                         qdev->ndev->name, i);
2940                         } else {
2941                                 /*
2942                                  * Inbound queues handle unicast frames only.
2943                                  */
2944                                 intr_context->handler = qlge_msix_rx_isr;
2945                                 sprintf(intr_context->name, "%s-rx-%d",
2946                                         qdev->ndev->name, i);
2947                         }
2948                 }
2949         } else {
2950                 /*
2951                  * All rx_rings use the same intr_context since
2952                  * there is only one vector.
2953                  */
2954                 intr_context->intr = 0;
2955                 intr_context->qdev = qdev;
2956                 /*
2957                  * We set up each vectors enable/disable/read bits so
2958                  * there's no bit/mask calculations in the critical path.
2959                  */
2960                 intr_context->intr_en_mask =
2961                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_ENABLE;
2962                 intr_context->intr_dis_mask =
2963                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK |
2964                     INTR_EN_TYPE_DISABLE;
2965                 intr_context->intr_read_mask =
2966                     INTR_EN_TYPE_MASK | INTR_EN_INTR_MASK | INTR_EN_TYPE_READ;
2967                 /*
2968                  * Single interrupt means one handler for all rings.
2969                  */
2970                 intr_context->handler = qlge_isr;
2971                 sprintf(intr_context->name, "%s-single_irq", qdev->ndev->name);
2972                 /* Set up this vector's bit-mask that indicates
2973                  * which queues it services. In this case there is
2974                  * a single vector so it will service all RSS and
2975                  * TX completion rings.
2976                  */
2977                 ql_set_irq_mask(qdev, intr_context);
2978         }
2979         /* Tell the TX completion rings which MSIx vector
2980          * they will be using.
2981          */
2982         ql_set_tx_vect(qdev);
2983 }
2984
2985 static void ql_free_irq(struct ql_adapter *qdev)
2986 {
2987         int i;
2988         struct intr_context *intr_context = &qdev->intr_context[0];
2989
2990         for (i = 0; i < qdev->intr_count; i++, intr_context++) {
2991                 if (intr_context->hooked) {
2992                         if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
2993                                 free_irq(qdev->msi_x_entry[i].vector,
2994                                          &qdev->rx_ring[i]);
2995                                 QPRINTK(qdev, IFDOWN, DEBUG,
2996                                         "freeing msix interrupt %d.\n", i);
2997                         } else {
2998                                 free_irq(qdev->pdev->irq, &qdev->rx_ring[0]);
2999                                 QPRINTK(qdev, IFDOWN, DEBUG,
3000                                         "freeing msi interrupt %d.\n", i);
3001                         }
3002                 }
3003         }
3004         ql_disable_msix(qdev);
3005 }
3006
3007 static int ql_request_irq(struct ql_adapter *qdev)
3008 {
3009         int i;
3010         int status = 0;
3011         struct pci_dev *pdev = qdev->pdev;
3012         struct intr_context *intr_context = &qdev->intr_context[0];
3013
3014         ql_resolve_queues_to_irqs(qdev);
3015
3016         for (i = 0; i < qdev->intr_count; i++, intr_context++) {
3017                 atomic_set(&intr_context->irq_cnt, 0);
3018                 if (test_bit(QL_MSIX_ENABLED, &qdev->flags)) {
3019                         status = request_irq(qdev->msi_x_entry[i].vector,
3020                                              intr_context->handler,
3021                                              0,
3022                                              intr_context->name,
3023                                              &qdev->rx_ring[i]);
3024                         if (status) {
3025                                 QPRINTK(qdev, IFUP, ERR,
3026                                         "Failed request for MSIX interrupt %d.\n",
3027                                         i);
3028                                 goto err_irq;
3029                         } else {
3030                                 QPRINTK(qdev, IFUP, DEBUG,
3031                                         "Hooked intr %d, queue type %s%s%s, with name %s.\n",
3032                                         i,
3033                                         qdev->rx_ring[i].type ==
3034                                         DEFAULT_Q ? "DEFAULT_Q" : "",
3035                                         qdev->rx_ring[i].type ==
3036                                         TX_Q ? "TX_Q" : "",
3037                                         qdev->rx_ring[i].type ==
3038                                         RX_Q ? "RX_Q" : "", intr_context->name);
3039                         }
3040                 } else {
3041                         QPRINTK(qdev, IFUP, DEBUG,
3042                                 "trying msi or legacy interrupts.\n");
3043                         QPRINTK(qdev, IFUP, DEBUG,
3044                                 "%s: irq = %d.\n", __func__, pdev->irq);
3045                         QPRINTK(qdev, IFUP, DEBUG,
3046                                 "%s: context->name = %s.\n", __func__,
3047                                intr_context->name);
3048                         QPRINTK(qdev, IFUP, DEBUG,
3049                                 "%s: dev_id = 0x%p.\n", __func__,
3050                                &qdev->rx_ring[0]);
3051                         status =
3052                             request_irq(pdev->irq, qlge_isr,
3053                                         test_bit(QL_MSI_ENABLED,
3054                                                  &qdev->
3055                                                  flags) ? 0 : IRQF_SHARED,
3056                                         intr_context->name, &qdev->rx_ring[0]);
3057                         if (status)
3058                                 goto err_irq;
3059
3060                         QPRINTK(qdev, IFUP, ERR,
3061                                 "Hooked intr %d, queue type %s%s%s, with name %s.\n",
3062                                 i,
3063                                 qdev->rx_ring[0].type ==
3064                                 DEFAULT_Q ? "DEFAULT_Q" : "",
3065                                 qdev->rx_ring[0].type == TX_Q ? "TX_Q" : "",
3066                                 qdev->rx_ring[0].type == RX_Q ? "RX_Q" : "",
3067                                 intr_context->name);
3068                 }
3069                 intr_context->hooked = 1;
3070         }
3071         return status;
3072 err_irq:
3073         QPRINTK(qdev, IFUP, ERR, "Failed to get the interrupts!!!/n");
3074         ql_free_irq(qdev);
3075         return status;
3076 }
3077
3078 static int ql_start_rss(struct ql_adapter *qdev)
3079 {
3080         struct ricb *ricb = &qdev->ricb;
3081         int status = 0;
3082         int i;
3083         u8 *hash_id = (u8 *) ricb->hash_cq_id;
3084
3085         memset((void *)ricb, 0, sizeof(*ricb));
3086
3087         ricb->base_cq = RSS_L4K;
3088         ricb->flags =
3089             (RSS_L6K | RSS_LI | RSS_LB | RSS_LM | RSS_RI4 | RSS_RI6 | RSS_RT4 |
3090              RSS_RT6);
3091         ricb->mask = cpu_to_le16(qdev->rss_ring_count - 1);
3092
3093         /*
3094          * Fill out the Indirection Table.
3095          */
3096         for (i = 0; i < 256; i++)
3097                 hash_id[i] = i & (qdev->rss_ring_count - 1);
3098
3099         /*
3100          * Random values for the IPv6 and IPv4 Hash Keys.
3101          */
3102         get_random_bytes((void *)&ricb->ipv6_hash_key[0], 40);
3103         get_random_bytes((void *)&ricb->ipv4_hash_key[0], 16);
3104
3105         QPRINTK(qdev, IFUP, DEBUG, "Initializing RSS.\n");
3106
3107         status = ql_write_cfg(qdev, ricb, sizeof(*ricb), CFG_LR, 0);
3108         if (status) {
3109                 QPRINTK(qdev, IFUP, ERR, "Failed to load RICB.\n");
3110                 return status;
3111         }
3112         QPRINTK(qdev, IFUP, DEBUG, "Successfully loaded RICB.\n");
3113         return status;
3114 }
3115
3116 static int ql_clear_routing_entries(struct ql_adapter *qdev)
3117 {
3118         int i, status = 0;
3119
3120         status = ql_sem_spinlock(qdev, SEM_RT_IDX_MASK);
3121         if (status)
3122                 return status;
3123         /* Clear all the entries in the routing table. */
3124         for (i = 0; i < 16; i++) {
3125                 status = ql_set_routing_reg(qdev, i, 0, 0);
3126                 if (status) {
3127                         QPRINTK(qdev, IFUP, ERR,
3128                                 "Failed to init routing register for CAM "
3129                                 "packets.\n");
3130                         break;
3131                 }
3132         }
3133         ql_sem_unlock(qdev, SEM_RT_IDX_MASK);
3134         return status;
3135 }
3136
3137 /* Initialize the frame-to-queue routing. */
3138 static int ql_route_initialize(struct ql_adapter *qdev)
3139 {
3140         int status = 0;
3141
3142         /* Clear all the entries in the routing table. */
3143         status = ql_clear_routing_entries(qdev);
3144         if (status)
3145                 return status;
3146
3147         status = ql_sem_spinlock(qdev, SEM_RT_IDX_MASK);
3148         if (status)
3149                 return status;
3150
3151         status = ql_set_routing_reg(qdev, RT_IDX_ALL_ERR_SLOT, RT_IDX_ERR, 1);
3152         if (status) {
3153                 QPRINTK(qdev, IFUP, ERR,
3154                         "Failed to init routing register for error packets.\n");
3155                 goto exit;
3156         }
3157         status = ql_set_routing_reg(qdev, RT_IDX_BCAST_SLOT, RT_IDX_BCAST, 1);
3158         if (status) {
3159                 QPRINTK(qdev, IFUP, ERR,
3160                         "Failed to init routing register for broadcast packets.\n");
3161                 goto exit;
3162         }
3163         /* If we have more than one inbound queue, then turn on RSS in the
3164          * routing block.
3165          */
3166         if (qdev->rss_ring_count > 1) {
3167                 status = ql_set_routing_reg(qdev, RT_IDX_RSS_MATCH_SLOT,
3168                                         RT_IDX_RSS_MATCH, 1);
3169                 if (status) {
3170                         QPRINTK(qdev, IFUP, ERR,
3171                                 "Failed to init routing register for MATCH RSS packets.\n");
3172                         goto exit;
3173                 }
3174         }
3175
3176         status = ql_set_routing_reg(qdev, RT_IDX_CAM_HIT_SLOT,
3177                                     RT_IDX_CAM_HIT, 1);
3178         if (status)
3179                 QPRINTK(qdev, IFUP, ERR,
3180                         "Failed to init routing register for CAM packets.\n");
3181 exit:
3182         ql_sem_unlock(qdev, SEM_RT_IDX_MASK);
3183         return status;
3184 }
3185
3186 int ql_cam_route_initialize(struct ql_adapter *qdev)
3187 {
3188         int status, set;
3189
3190         /* If check if the link is up and use to
3191          * determine if we are setting or clearing
3192          * the MAC address in the CAM.
3193          */
3194         set = ql_read32(qdev, STS);
3195         set &= qdev->port_link_up;
3196         status = ql_set_mac_addr(qdev, set);
3197         if (status) {
3198                 QPRINTK(qdev, IFUP, ERR, "Failed to init mac address.\n");
3199                 return status;
3200         }
3201
3202         status = ql_route_initialize(qdev);
3203         if (status)
3204                 QPRINTK(qdev, IFUP, ERR, "Failed to init routing table.\n");
3205
3206         return status;
3207 }
3208
3209 static int ql_adapter_initialize(struct ql_adapter *qdev)
3210 {
3211         u32 value, mask;
3212         int i;
3213         int status = 0;
3214
3215         /*
3216          * Set up the System register to halt on errors.
3217          */
3218         value = SYS_EFE | SYS_FAE;
3219         mask = value << 16;
3220         ql_write32(qdev, SYS, mask | value);
3221
3222         /* Set the default queue, and VLAN behavior. */
3223         value = NIC_RCV_CFG_DFQ | NIC_RCV_CFG_RV;
3224         mask = NIC_RCV_CFG_DFQ_MASK | (NIC_RCV_CFG_RV << 16);
3225         ql_write32(qdev, NIC_RCV_CFG, (mask | value));
3226
3227         /* Set the MPI interrupt to enabled. */
3228         ql_write32(qdev, INTR_MASK, (INTR_MASK_PI << 16) | INTR_MASK_PI);
3229
3230         /* Enable the function, set pagesize, enable error checking. */
3231         value = FSC_FE | FSC_EPC_INBOUND | FSC_EPC_OUTBOUND |
3232             FSC_EC | FSC_VM_PAGE_4K | FSC_SH;
3233
3234         /* Set/clear header splitting. */
3235         mask = FSC_VM_PAGESIZE_MASK |
3236             FSC_DBL_MASK | FSC_DBRST_MASK | (value << 16);
3237         ql_write32(qdev, FSC, mask | value);
3238
3239         ql_write32(qdev, SPLT_HDR, SPLT_HDR_EP |
3240                 min(SMALL_BUFFER_SIZE, MAX_SPLIT_SIZE));
3241
3242         /* Start up the rx queues. */
3243         for (i = 0; i < qdev->rx_ring_count; i++) {
3244                 status = ql_start_rx_ring(qdev, &qdev->rx_ring[i]);
3245                 if (status) {
3246                         QPRINTK(qdev, IFUP, ERR,
3247                                 "Failed to start rx ring[%d].\n", i);
3248                         return status;
3249                 }
3250         }
3251
3252         /* If there is more than one inbound completion queue
3253          * then download a RICB to configure RSS.
3254          */
3255         if (qdev->rss_ring_count > 1) {
3256                 status = ql_start_rss(qdev);
3257                 if (status) {
3258                         QPRINTK(qdev, IFUP, ERR, "Failed to start RSS.\n");
3259                         return status;
3260                 }
3261         }
3262
3263         /* Start up the tx queues. */
3264         for (i = 0; i < qdev->tx_ring_count; i++) {
3265                 status = ql_start_tx_ring(qdev, &qdev->tx_ring[i]);
3266                 if (status) {
3267                         QPRINTK(qdev, IFUP, ERR,
3268                                 "Failed to start tx ring[%d].\n", i);
3269                         return status;
3270                 }
3271         }
3272
3273         /* Initialize the port and set the max framesize. */
3274         status = qdev->nic_ops->port_initialize(qdev);
3275        if (status) {
3276               QPRINTK(qdev, IFUP, ERR, "Failed to start port.\n");
3277               return status;
3278        }
3279
3280         /* Set up the MAC address and frame routing filter. */
3281         status = ql_cam_route_initialize(qdev);
3282         if (status) {
3283                 QPRINTK(qdev, IFUP, ERR,
3284                                 "Failed to init CAM/Routing tables.\n");
3285                 return status;
3286         }
3287
3288         /* Start NAPI for the RSS queues. */
3289         for (i = 0; i < qdev->rss_ring_count; i++) {
3290                 QPRINTK(qdev, IFUP, DEBUG, "Enabling NAPI for rx_ring[%d].\n",
3291                         i);
3292                 napi_enable(&qdev->rx_ring[i].napi);
3293         }
3294
3295         return status;
3296 }
3297
3298 /* Issue soft reset to chip. */
3299 static int ql_adapter_reset(struct ql_adapter *qdev)
3300 {
3301         u32 value;
3302         int status = 0;
3303         unsigned long end_jiffies;
3304
3305         /* Clear all the entries in the routing table. */
3306         status = ql_clear_routing_entries(qdev);
3307         if (status) {
3308                 QPRINTK(qdev, IFUP, ERR, "Failed to clear routing bits.\n");
3309                 return status;
3310         }
3311
3312         end_jiffies = jiffies +
3313                 max((unsigned long)1, usecs_to_jiffies(30));
3314         ql_write32(qdev, RST_FO, (RST_FO_FR << 16) | RST_FO_FR);
3315
3316         do {
3317                 value = ql_read32(qdev, RST_FO);
3318                 if ((value & RST_FO_FR) == 0)
3319                         break;
3320                 cpu_relax();
3321         } while (time_before(jiffies, end_jiffies));
3322
3323         if (value & RST_FO_FR) {
3324                 QPRINTK(qdev, IFDOWN, ERR,
3325                         "ETIMEDOUT!!! errored out of resetting the chip!\n");
3326                 status = -ETIMEDOUT;
3327         }
3328
3329         return status;
3330 }
3331
3332 static void ql_display_dev_info(struct net_device *ndev)
3333 {
3334         struct ql_adapter *qdev = (struct ql_adapter *)netdev_priv(ndev);
3335
3336         QPRINTK(qdev, PROBE, INFO,
3337                 "Function #%d, Port %d, NIC Roll %d, NIC Rev = %d, "
3338                 "XG Roll = %d, XG Rev = %d.\n",
3339                 qdev->func,
3340                 qdev->port,
3341                 qdev->chip_rev_id & 0x0000000f,
3342                 qdev->chip_rev_id >> 4 & 0x0000000f,
3343                 qdev->chip_rev_id >> 8 & 0x0000000f,
3344                 qdev->chip_rev_id >> 12 & 0x0000000f);
3345         QPRINTK(qdev, PROBE, INFO, "MAC address %pM\n", ndev->dev_addr);
3346 }
3347
3348 static int ql_adapter_down(struct ql_adapter *qdev)
3349 {
3350         int i, status = 0;
3351
3352         ql_link_off(qdev);
3353
3354         /* Don't kill the reset worker thread if we
3355          * are in the process of recovery.
3356          */
3357         if (test_bit(QL_ADAPTER_UP, &qdev->flags))
3358                 cancel_delayed_work_sync(&qdev->asic_reset_work);
3359         cancel_delayed_work_sync(&qdev->mpi_reset_work);
3360         cancel_delayed_work_sync(&qdev->mpi_work);
3361         cancel_delayed_work_sync(&qdev->mpi_idc_work);
3362         cancel_delayed_work_sync(&qdev->mpi_port_cfg_work);
3363
3364         for (i = 0; i < qdev->rss_ring_count; i++)
3365                 napi_disable(&qdev->rx_ring[i].napi);
3366
3367         clear_bit(QL_ADAPTER_UP, &qdev->flags);
3368
3369         ql_disable_interrupts(qdev);
3370
3371         ql_tx_ring_clean(qdev);
3372
3373         /* Call netif_napi_del() from common point.
3374          */
3375         for (i = 0; i < qdev->rss_ring_count; i++)
3376                 netif_napi_del(&qdev->rx_ring[i].napi);
3377
3378         ql_free_rx_buffers(qdev);
3379
3380         status = ql_adapter_reset(qdev);
3381         if (status)
3382                 QPRINTK(qdev, IFDOWN, ERR, "reset(func #%d) FAILED!\n",
3383                         qdev->func);
3384         return status;
3385 }
3386
3387 static int ql_adapter_up(struct ql_adapter *qdev)
3388 {
3389         int err = 0;
3390
3391         err = ql_adapter_initialize(qdev);
3392         if (err) {
3393                 QPRINTK(qdev, IFUP, INFO, "Unable to initialize adapter.\n");
3394                 goto err_init;
3395         }
3396         set_bit(QL_ADAPTER_UP, &qdev->flags);
3397         ql_alloc_rx_buffers(qdev);
3398         /* If the port is initialized and the
3399          * link is up the turn on the carrier.
3400          */
3401         if ((ql_read32(qdev, STS) & qdev->port_init) &&
3402                         (ql_read32(qdev, STS) & qdev->port_link_up))
3403                 ql_link_on(qdev);
3404         ql_enable_interrupts(qdev);
3405         ql_enable_all_completion_interrupts(qdev);
3406         netif_tx_start_all_queues(qdev->ndev);
3407
3408         return 0;
3409 err_init:
3410         ql_adapter_reset(qdev);
3411         return err;
3412 }
3413
3414 static void ql_release_adapter_resources(struct ql_adapter *qdev)
3415 {
3416         ql_free_mem_resources(qdev);
3417         ql_free_irq(qdev);
3418 }
3419
3420 static int ql_get_adapter_resources(struct ql_adapter *qdev)
3421 {
3422         int status = 0;
3423
3424         if (ql_alloc_mem_resources(qdev)) {
3425                 QPRINTK(qdev, IFUP, ERR, "Unable to  allocate memory.\n");
3426                 return -ENOMEM;
3427         }
3428         status = ql_request_irq(qdev);
3429         return status;
3430 }
3431
3432 static int qlge_close(struct net_device *ndev)
3433 {
3434         struct ql_adapter *qdev = netdev_priv(ndev);
3435
3436         /*
3437          * Wait for device to recover from a reset.
3438          * (Rarely happens, but possible.)
3439          */
3440         while (!test_bit(QL_ADAPTER_UP, &qdev->flags))
3441                 msleep(1);
3442         ql_adapter_down(qdev);
3443         ql_release_adapter_resources(qdev);
3444         return 0;
3445 }
3446
3447 static int ql_configure_rings(struct ql_adapter *qdev)
3448 {
3449         int i;
3450         struct rx_ring *rx_ring;
3451         struct tx_ring *tx_ring;
3452         int cpu_cnt = min(MAX_CPUS, (int)num_online_cpus());
3453
3454         /* In a perfect world we have one RSS ring for each CPU
3455          * and each has it's own vector.  To do that we ask for
3456          * cpu_cnt vectors.  ql_enable_msix() will adjust the
3457          * vector count to what we actually get.  We then
3458          * allocate an RSS ring for each.
3459          * Essentially, we are doing min(cpu_count, msix_vector_count).
3460          */
3461         qdev->intr_count = cpu_cnt;
3462         ql_enable_msix(qdev);
3463         /* Adjust the RSS ring count to the actual vector count. */
3464         qdev->rss_ring_count = qdev->intr_count;
3465         qdev->tx_ring_count = cpu_cnt;
3466         qdev->rx_ring_count = qdev->tx_ring_count + qdev->rss_ring_count;
3467
3468         for (i = 0; i < qdev->tx_ring_count; i++) {
3469                 tx_ring = &qdev->tx_ring[i];
3470                 memset((void *)tx_ring, 0, sizeof(*tx_ring));
3471                 tx_ring->qdev = qdev;
3472                 tx_ring->wq_id = i;
3473                 tx_ring->wq_len = qdev->tx_ring_size;
3474                 tx_ring->wq_size =
3475                     tx_ring->wq_len * sizeof(struct ob_mac_iocb_req);
3476
3477                 /*
3478                  * The completion queue ID for the tx rings start
3479                  * immediately after the rss rings.
3480                  */
3481                 tx_ring->cq_id = qdev->rss_ring_count + i;
3482         }
3483
3484         for (i = 0; i < qdev->rx_ring_count; i++) {
3485                 rx_ring = &qdev->rx_ring[i];
3486                 memset((void *)rx_ring, 0, sizeof(*rx_ring));
3487                 rx_ring->qdev = qdev;
3488                 rx_ring->cq_id = i;
3489                 rx_ring->cpu = i % cpu_cnt;     /* CPU to run handler on. */
3490                 if (i < qdev->rss_ring_count) {
3491                         /*
3492                          * Inbound (RSS) queues.
3493                          */
3494                         rx_ring->cq_len = qdev->rx_ring_size;
3495                         rx_ring->cq_size =
3496                             rx_ring->cq_len * sizeof(struct ql_net_rsp_iocb);
3497                         rx_ring->lbq_len = NUM_LARGE_BUFFERS;
3498                         rx_ring->lbq_size =
3499                             rx_ring->lbq_len * sizeof(__le64);
3500                         rx_ring->lbq_buf_size = LARGE_BUFFER_SIZE;
3501                         rx_ring->sbq_len = NUM_SMALL_BUFFERS;
3502                         rx_ring->sbq_size =
3503                             rx_ring->sbq_len * sizeof(__le64);
3504                         rx_ring->sbq_buf_size = SMALL_BUFFER_SIZE * 2;
3505                         rx_ring->type = RX_Q;
3506                 } else {
3507                         /*
3508                          * Outbound queue handles outbound completions only.
3509                          */
3510                         /* outbound cq is same size as tx_ring it services. */
3511                         rx_ring->cq_len = qdev->tx_ring_size;
3512                         rx_ring->cq_size =
3513                             rx_ring->cq_len * sizeof(struct ql_net_rsp_iocb);
3514                         rx_ring->lbq_len = 0;
3515                         rx_ring->lbq_size = 0;
3516                         rx_ring->lbq_buf_size = 0;
3517                         rx_ring->sbq_len = 0;
3518                         rx_ring->sbq_size = 0;
3519                         rx_ring->sbq_buf_size = 0;
3520                         rx_ring->type = TX_Q;
3521                 }
3522         }
3523         return 0;
3524 }
3525
3526 static int qlge_open(struct net_device *ndev)
3527 {
3528         int err = 0;
3529         struct ql_adapter *qdev = netdev_priv(ndev);
3530
3531         err = ql_configure_rings(qdev);
3532         if (err)
3533                 return err;
3534
3535         err = ql_get_adapter_resources(qdev);
3536         if (err)
3537                 goto error_up;
3538
3539         err = ql_adapter_up(qdev);
3540         if (err)
3541                 goto error_up;
3542
3543         return err;
3544
3545 error_up:
3546         ql_release_adapter_resources(qdev);
3547         return err;
3548 }
3549
3550 static int qlge_change_mtu(struct net_device *ndev, int new_mtu)
3551 {
3552         struct ql_adapter *qdev = netdev_priv(ndev);
3553
3554         if (ndev->mtu == 1500 && new_mtu == 9000) {
3555                 QPRINTK(qdev, IFUP, ERR, "Changing to jumbo MTU.\n");
3556                 queue_delayed_work(qdev->workqueue,
3557                                 &qdev->mpi_port_cfg_work, 0);
3558         } else if (ndev->mtu == 9000 && new_mtu == 1500) {
3559                 QPRINTK(qdev, IFUP, ERR, "Changing to normal MTU.\n");
3560         } else if ((ndev->mtu == 1500 && new_mtu == 1500) ||
3561                    (ndev->mtu == 9000 && new_mtu == 9000)) {
3562                 return 0;
3563         } else
3564                 return -EINVAL;
3565         ndev->mtu = new_mtu;
3566         return 0;
3567 }
3568
3569 static struct net_device_stats *qlge_get_stats(struct net_device
3570                                                *ndev)
3571 {
3572         struct ql_adapter *qdev = netdev_priv(ndev);
3573         return &qdev->stats;
3574 }
3575
3576 static void qlge_set_multicast_list(struct net_device *ndev)
3577 {
3578         struct ql_adapter *qdev = (struct ql_adapter *)netdev_priv(ndev);
3579         struct dev_mc_list *mc_ptr;
3580         int i, status;
3581
3582         status = ql_sem_spinlock(qdev, SEM_RT_IDX_MASK);
3583         if (status)
3584                 return;
3585         /*
3586          * Set or clear promiscuous mode if a
3587          * transition is taking place.
3588          */
3589         if (ndev->flags & IFF_PROMISC) {
3590                 if (!test_bit(QL_PROMISCUOUS, &qdev->flags)) {
3591                         if (ql_set_routing_reg
3592                             (qdev, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 1)) {
3593                                 QPRINTK(qdev, HW, ERR,
3594                                         "Failed to set promiscous mode.\n");
3595                         } else {
3596                                 set_bit(QL_PROMISCUOUS, &qdev->flags);
3597                         }
3598                 }
3599         } else {
3600                 if (test_bit(QL_PROMISCUOUS, &qdev->flags)) {
3601                         if (ql_set_routing_reg
3602                             (qdev, RT_IDX_PROMISCUOUS_SLOT, RT_IDX_VALID, 0)) {
3603                                 QPRINTK(qdev, HW, ERR,
3604                                         "Failed to clear promiscous mode.\n");
3605                         } else {
3606                                 clear_bit(QL_PROMISCUOUS, &qdev->flags);
3607                         }
3608                 }
3609         }
3610
3611         /*
3612          * Set or clear all multicast mode if a
3613          * transition is taking place.
3614          */
3615         if ((ndev->flags & IFF_ALLMULTI) ||
3616             (ndev->mc_count > MAX_MULTICAST_ENTRIES)) {
3617                 if (!test_bit(QL_ALLMULTI, &qdev->flags)) {
3618                         if (ql_set_routing_reg
3619                             (qdev, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 1)) {
3620                                 QPRINTK(qdev, HW, ERR,
3621                                         "Failed to set all-multi mode.\n");
3622                         } else {
3623                                 set_bit(QL_ALLMULTI, &qdev->flags);
3624                         }
3625                 }
3626         } else {
3627                 if (test_bit(QL_ALLMULTI, &qdev->flags)) {
3628                         if (ql_set_routing_reg
3629                             (qdev, RT_IDX_ALLMULTI_SLOT, RT_IDX_MCAST, 0)) {
3630                                 QPRINTK(qdev, HW, ERR,
3631                                         "Failed to clear all-multi mode.\n");
3632                         } else {
3633                                 clear_bit(QL_ALLMULTI, &qdev->flags);
3634                         }
3635                 }
3636         }
3637
3638         if (ndev->mc_count) {
3639                 status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
3640                 if (status)
3641                         goto exit;
3642                 for (i = 0, mc_ptr = ndev->mc_list; mc_ptr;
3643                      i++, mc_ptr = mc_ptr->next)
3644                         if (ql_set_mac_addr_reg(qdev, (u8 *) mc_ptr->dmi_addr,
3645                                                 MAC_ADDR_TYPE_MULTI_MAC, i)) {
3646                                 QPRINTK(qdev, HW, ERR,
3647                                         "Failed to loadmulticast address.\n");
3648                                 ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
3649                                 goto exit;
3650                         }
3651                 ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
3652                 if (ql_set_routing_reg
3653                     (qdev, RT_IDX_MCAST_MATCH_SLOT, RT_IDX_MCAST_MATCH, 1)) {
3654                         QPRINTK(qdev, HW, ERR,
3655                                 "Failed to set multicast match mode.\n");
3656                 } else {
3657                         set_bit(QL_ALLMULTI, &qdev->flags);
3658                 }
3659         }
3660 exit:
3661         ql_sem_unlock(qdev, SEM_RT_IDX_MASK);
3662 }
3663
3664 static int qlge_set_mac_address(struct net_device *ndev, void *p)
3665 {
3666         struct ql_adapter *qdev = (struct ql_adapter *)netdev_priv(ndev);
3667         struct sockaddr *addr = p;
3668         int status;
3669
3670         if (netif_running(ndev))
3671                 return -EBUSY;
3672
3673         if (!is_valid_ether_addr(addr->sa_data))
3674                 return -EADDRNOTAVAIL;
3675         memcpy(ndev->dev_addr, addr->sa_data, ndev->addr_len);
3676
3677         status = ql_sem_spinlock(qdev, SEM_MAC_ADDR_MASK);
3678         if (status)
3679                 return status;
3680         status = ql_set_mac_addr_reg(qdev, (u8 *) ndev->dev_addr,
3681                         MAC_ADDR_TYPE_CAM_MAC, qdev->func * MAX_CQ);
3682         if (status)
3683                 QPRINTK(qdev, HW, ERR, "Failed to load MAC address.\n");
3684         ql_sem_unlock(qdev, SEM_MAC_ADDR_MASK);
3685         return status;
3686 }
3687
3688 static void qlge_tx_timeout(struct net_device *ndev)
3689 {
3690         struct ql_adapter *qdev = (struct ql_adapter *)netdev_priv(ndev);
3691         ql_queue_asic_error(qdev);
3692 }
3693
3694 static void ql_asic_reset_work(struct work_struct *work)
3695 {
3696         struct ql_adapter *qdev =
3697             container_of(work, struct ql_adapter, asic_reset_work.work);
3698         int status;
3699         rtnl_lock();
3700         status = ql_adapter_down(qdev);
3701         if (status)
3702                 goto error;
3703
3704         status = ql_adapter_up(qdev);
3705         if (status)
3706                 goto error;
3707         rtnl_unlock();
3708         return;
3709 error:
3710         QPRINTK(qdev, IFUP, ALERT,
3711                 "Driver up/down cycle failed, closing device\n");
3712
3713         set_bit(QL_ADAPTER_UP, &qdev->flags);
3714         dev_close(qdev->ndev);
3715         rtnl_unlock();
3716 }
3717
3718 static struct nic_operations qla8012_nic_ops = {
3719         .get_flash              = ql_get_8012_flash_params,
3720         .port_initialize        = ql_8012_port_initialize,
3721 };
3722
3723 static struct nic_operations qla8000_nic_ops = {
3724         .get_flash              = ql_get_8000_flash_params,
3725         .port_initialize        = ql_8000_port_initialize,
3726 };
3727
3728 /* Find the pcie function number for the other NIC
3729  * on this chip.  Since both NIC functions share a
3730  * common firmware we have the lowest enabled function
3731  * do any common work.  Examples would be resetting
3732  * after a fatal firmware error, or doing a firmware
3733  * coredump.
3734  */
3735 static int ql_get_alt_pcie_func(struct ql_adapter *qdev)
3736 {
3737         int status = 0;
3738         u32 temp;
3739         u32 nic_func1, nic_func2;
3740
3741         status = ql_read_mpi_reg(qdev, MPI_TEST_FUNC_PORT_CFG,
3742                         &temp);
3743         if (status)
3744                 return status;
3745
3746         nic_func1 = ((temp >> MPI_TEST_NIC1_FUNC_SHIFT) &
3747                         MPI_TEST_NIC_FUNC_MASK);
3748         nic_func2 = ((temp >> MPI_TEST_NIC2_FUNC_SHIFT) &
3749                         MPI_TEST_NIC_FUNC_MASK);
3750
3751         if (qdev->func == nic_func1)
3752                 qdev->alt_func = nic_func2;
3753         else if (qdev->func == nic_func2)
3754                 qdev->alt_func = nic_func1;
3755         else
3756                 status = -EIO;
3757
3758         return status;
3759 }
3760
3761 static int ql_get_board_info(struct ql_adapter *qdev)
3762 {
3763         int status;
3764         qdev->func =
3765             (ql_read32(qdev, STS) & STS_FUNC_ID_MASK) >> STS_FUNC_ID_SHIFT;
3766         if (qdev->func > 3)
3767                 return -EIO;
3768
3769         status = ql_get_alt_pcie_func(qdev);
3770         if (status)
3771                 return status;
3772
3773         qdev->port = (qdev->func < qdev->alt_func) ? 0 : 1;
3774         if (qdev->port) {
3775                 qdev->xg_sem_mask = SEM_XGMAC1_MASK;
3776                 qdev->port_link_up = STS_PL1;
3777                 qdev->port_init = STS_PI1;
3778                 qdev->mailbox_in = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC2_MBI;
3779                 qdev->mailbox_out = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC2_MBO;
3780         } else {
3781                 qdev->xg_sem_mask = SEM_XGMAC0_MASK;
3782                 qdev->port_link_up = STS_PL0;
3783                 qdev->port_init = STS_PI0;
3784                 qdev->mailbox_in = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC0_MBI;
3785                 qdev->mailbox_out = PROC_ADDR_MPI_RISC | PROC_ADDR_FUNC0_MBO;
3786         }
3787         qdev->chip_rev_id = ql_read32(qdev, REV_ID);
3788         qdev->device_id = qdev->pdev->device;
3789         if (qdev->device_id == QLGE_DEVICE_ID_8012)
3790                 qdev->nic_ops = &qla8012_nic_ops;
3791         else if (qdev->device_id == QLGE_DEVICE_ID_8000)
3792                 qdev->nic_ops = &qla8000_nic_ops;
3793         return status;
3794 }
3795
3796 static void ql_release_all(struct pci_dev *pdev)
3797 {
3798         struct net_device *ndev = pci_get_drvdata(pdev);
3799         struct ql_adapter *qdev = netdev_priv(ndev);
3800
3801         if (qdev->workqueue) {
3802                 destroy_workqueue(qdev->workqueue);
3803                 qdev->workqueue = NULL;
3804         }
3805
3806         if (qdev->reg_base)
3807                 iounmap(qdev->reg_base);
3808         if (qdev->doorbell_area)
3809                 iounmap(qdev->doorbell_area);
3810         pci_release_regions(pdev);
3811         pci_set_drvdata(pdev, NULL);
3812 }
3813
3814 static int __devinit ql_init_device(struct pci_dev *pdev,
3815                                     struct net_device *ndev, int cards_found)
3816 {
3817         struct ql_adapter *qdev = netdev_priv(ndev);
3818         int pos, err = 0;
3819         u16 val16;
3820
3821         memset((void *)qdev, 0, sizeof(*qdev));
3822         err = pci_enable_device(pdev);
3823         if (err) {
3824                 dev_err(&pdev->dev, "PCI device enable failed.\n");
3825                 return err;
3826         }
3827
3828         qdev->ndev = ndev;
3829         qdev->pdev = pdev;
3830         pci_set_drvdata(pdev, ndev);
3831         pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
3832         if (pos <= 0) {
3833                 dev_err(&pdev->dev, PFX "Cannot find PCI Express capability, "
3834                         "aborting.\n");
3835                 return pos;
3836         } else {
3837                 pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &val16);
3838                 val16 &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
3839                 val16 |= (PCI_EXP_DEVCTL_CERE |
3840                           PCI_EXP_DEVCTL_NFERE |
3841                           PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE);
3842                 pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, val16);
3843         }
3844
3845         err = pci_request_regions(pdev, DRV_NAME);
3846         if (err) {
3847                 dev_err(&pdev->dev, "PCI region request failed.\n");
3848                 return err;
3849         }
3850
3851         pci_set_master(pdev);
3852         if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3853                 set_bit(QL_DMA64, &qdev->flags);
3854                 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3855         } else {
3856                 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3857                 if (!err)
3858                        err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
3859         }
3860
3861         if (err) {
3862                 dev_err(&pdev->dev, "No usable DMA configuration.\n");
3863                 goto err_out;
3864         }
3865
3866         qdev->reg_base =
3867             ioremap_nocache(pci_resource_start(pdev, 1),
3868                             pci_resource_len(pdev, 1));
3869         if (!qdev->reg_base) {
3870                 dev_err(&pdev->dev, "Register mapping failed.\n");
3871                 err = -ENOMEM;
3872                 goto err_out;
3873         }
3874
3875         qdev->doorbell_area_size = pci_resource_len(pdev, 3);
3876         qdev->doorbell_area =
3877             ioremap_nocache(pci_resource_start(pdev, 3),
3878                             pci_resource_len(pdev, 3));
3879         if (!qdev->doorbell_area) {
3880                 dev_err(&pdev->dev, "Doorbell register mapping failed.\n");
3881                 err = -ENOMEM;
3882                 goto err_out;
3883         }
3884
3885         err = ql_get_board_info(qdev);
3886         if (err) {
3887                 dev_err(&pdev->dev, "Register access failed.\n");
3888                 err = -EIO;
3889                 goto err_out;
3890         }
3891         qdev->msg_enable = netif_msg_init(debug, default_msg);
3892         spin_lock_init(&qdev->hw_lock);
3893         spin_lock_init(&qdev->stats_lock);
3894
3895         /* make sure the EEPROM is good */
3896         err = qdev->nic_ops->get_flash(qdev);
3897         if (err) {
3898                 dev_err(&pdev->dev, "Invalid FLASH.\n");
3899                 goto err_out;
3900         }
3901
3902         memcpy(ndev->perm_addr, ndev->dev_addr, ndev->addr_len);
3903
3904         /* Set up the default ring sizes. */
3905         qdev->tx_ring_size = NUM_TX_RING_ENTRIES;
3906         qdev->rx_ring_size = NUM_RX_RING_ENTRIES;
3907
3908         /* Set up the coalescing parameters. */
3909         qdev->rx_coalesce_usecs = DFLT_COALESCE_WAIT;
3910         qdev->tx_coalesce_usecs = DFLT_COALESCE_WAIT;
3911         qdev->rx_max_coalesced_frames = DFLT_INTER_FRAME_WAIT;
3912         qdev->tx_max_coalesced_frames = DFLT_INTER_FRAME_WAIT;
3913
3914         /*
3915          * Set up the operating parameters.
3916          */
3917         qdev->rx_csum = 1;
3918         qdev->workqueue = create_singlethread_workqueue(ndev->name);
3919         INIT_DELAYED_WORK(&qdev->asic_reset_work, ql_asic_reset_work);
3920         INIT_DELAYED_WORK(&qdev->mpi_reset_work, ql_mpi_reset_work);
3921         INIT_DELAYED_WORK(&qdev->mpi_work, ql_mpi_work);
3922         INIT_DELAYED_WORK(&qdev->mpi_port_cfg_work, ql_mpi_port_cfg_work);
3923         INIT_DELAYED_WORK(&qdev->mpi_idc_work, ql_mpi_idc_work);
3924         init_completion(&qdev->ide_completion);
3925
3926         if (!cards_found) {
3927                 dev_info(&pdev->dev, "%s\n", DRV_STRING);
3928                 dev_info(&pdev->dev, "Driver name: %s, Version: %s.\n",
3929                          DRV_NAME, DRV_VERSION);
3930         }
3931         return 0;
3932 err_out:
3933         ql_release_all(pdev);
3934         pci_disable_device(pdev);
3935         return err;
3936 }
3937
3938
3939 static const struct net_device_ops qlge_netdev_ops = {
3940         .ndo_open               = qlge_open,
3941         .ndo_stop               = qlge_close,
3942         .ndo_start_xmit         = qlge_send,
3943         .ndo_change_mtu         = qlge_change_mtu,
3944         .ndo_get_stats          = qlge_get_stats,
3945         .ndo_set_multicast_list = qlge_set_multicast_list,
3946         .ndo_set_mac_address    = qlge_set_mac_address,
3947         .ndo_validate_addr      = eth_validate_addr,
3948         .ndo_tx_timeout         = qlge_tx_timeout,
3949         .ndo_vlan_rx_register   = ql_vlan_rx_register,
3950         .ndo_vlan_rx_add_vid    = ql_vlan_rx_add_vid,
3951         .ndo_vlan_rx_kill_vid   = ql_vlan_rx_kill_vid,
3952 };
3953
3954 static int __devinit qlge_probe(struct pci_dev *pdev,
3955                                 const struct pci_device_id *pci_entry)
3956 {
3957         struct net_device *ndev = NULL;
3958         struct ql_adapter *qdev = NULL;
3959         static int cards_found = 0;
3960         int err = 0;
3961
3962         ndev = alloc_etherdev_mq(sizeof(struct ql_adapter),
3963                         min(MAX_CPUS, (int)num_online_cpus()));
3964         if (!ndev)
3965                 return -ENOMEM;
3966
3967         err = ql_init_device(pdev, ndev, cards_found);
3968         if (err < 0) {
3969                 free_netdev(ndev);
3970                 return err;
3971         }
3972
3973         qdev = netdev_priv(ndev);
3974         SET_NETDEV_DEV(ndev, &pdev->dev);
3975         ndev->features = (0
3976                           | NETIF_F_IP_CSUM
3977                           | NETIF_F_SG
3978                           | NETIF_F_TSO
3979                           | NETIF_F_TSO6
3980                           | NETIF_F_TSO_ECN
3981                           | NETIF_F_HW_VLAN_TX
3982                           | NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER);
3983         ndev->features |= NETIF_F_GRO;
3984
3985         if (test_bit(QL_DMA64, &qdev->flags))
3986                 ndev->features |= NETIF_F_HIGHDMA;
3987
3988         /*
3989          * Set up net_device structure.
3990          */
3991         ndev->tx_queue_len = qdev->tx_ring_size;
3992         ndev->irq = pdev->irq;
3993
3994         ndev->netdev_ops = &qlge_netdev_ops;
3995         SET_ETHTOOL_OPS(ndev, &qlge_ethtool_ops);
3996         ndev->watchdog_timeo = 10 * HZ;
3997
3998         err = register_netdev(ndev);
3999         if (err) {
4000                 dev_err(&pdev->dev, "net device registration failed.\n");
4001                 ql_release_all(pdev);
4002                 pci_disable_device(pdev);
4003                 return err;
4004         }
4005         ql_link_off(qdev);
4006         ql_display_dev_info(ndev);
4007         cards_found++;
4008         return 0;
4009 }
4010
4011 static void __devexit qlge_remove(struct pci_dev *pdev)
4012 {
4013         struct net_device *ndev = pci_get_drvdata(pdev);
4014         unregister_netdev(ndev);
4015         ql_release_all(pdev);
4016         pci_disable_device(pdev);
4017         free_netdev(ndev);
4018 }
4019
4020 /*
4021  * This callback is called by the PCI subsystem whenever
4022  * a PCI bus error is detected.
4023  */
4024 static pci_ers_result_t qlge_io_error_detected(struct pci_dev *pdev,
4025                                                enum pci_channel_state state)
4026 {
4027         struct net_device *ndev = pci_get_drvdata(pdev);
4028         struct ql_adapter *qdev = netdev_priv(ndev);
4029
4030         netif_device_detach(ndev);
4031
4032         if (state == pci_channel_io_perm_failure)
4033                 return PCI_ERS_RESULT_DISCONNECT;
4034
4035         if (netif_running(ndev))
4036                 ql_adapter_down(qdev);
4037
4038         pci_disable_device(pdev);
4039
4040         /* Request a slot reset. */
4041         return PCI_ERS_RESULT_NEED_RESET;
4042 }
4043
4044 /*
4045  * This callback is called after the PCI buss has been reset.
4046  * Basically, this tries to restart the card from scratch.
4047  * This is a shortened version of the device probe/discovery code,
4048  * it resembles the first-half of the () routine.
4049  */
4050 static pci_ers_result_t qlge_io_slot_reset(struct pci_dev *pdev)
4051 {
4052         struct net_device *ndev = pci_get_drvdata(pdev);
4053         struct ql_adapter *qdev = netdev_priv(ndev);
4054
4055         if (pci_enable_device(pdev)) {
4056                 QPRINTK(qdev, IFUP, ERR,
4057                         "Cannot re-enable PCI device after reset.\n");
4058                 return PCI_ERS_RESULT_DISCONNECT;
4059         }
4060
4061         pci_set_master(pdev);
4062
4063         netif_carrier_off(ndev);
4064         ql_adapter_reset(qdev);
4065
4066         /* Make sure the EEPROM is good */
4067         memcpy(ndev->perm_addr, ndev->dev_addr, ndev->addr_len);
4068
4069         if (!is_valid_ether_addr(ndev->perm_addr)) {
4070                 QPRINTK(qdev, IFUP, ERR, "After reset, invalid MAC address.\n");
4071                 return PCI_ERS_RESULT_DISCONNECT;
4072         }
4073
4074         return PCI_ERS_RESULT_RECOVERED;
4075 }
4076
4077 static void qlge_io_resume(struct pci_dev *pdev)
4078 {
4079         struct net_device *ndev = pci_get_drvdata(pdev);
4080         struct ql_adapter *qdev = netdev_priv(ndev);
4081
4082         pci_set_master(pdev);
4083
4084         if (netif_running(ndev)) {
4085                 if (ql_adapter_up(qdev)) {
4086                         QPRINTK(qdev, IFUP, ERR,
4087                                 "Device initialization failed after reset.\n");
4088                         return;
4089                 }
4090         }
4091
4092         netif_device_attach(ndev);
4093 }
4094
4095 static struct pci_error_handlers qlge_err_handler = {
4096         .error_detected = qlge_io_error_detected,
4097         .slot_reset = qlge_io_slot_reset,
4098         .resume = qlge_io_resume,
4099 };
4100
4101 static int qlge_suspend(struct pci_dev *pdev, pm_message_t state)
4102 {
4103         struct net_device *ndev = pci_get_drvdata(pdev);
4104         struct ql_adapter *qdev = netdev_priv(ndev);
4105         int err;
4106
4107         netif_device_detach(ndev);
4108
4109         if (netif_running(ndev)) {
4110                 err = ql_adapter_down(qdev);
4111                 if (!err)
4112                         return err;
4113         }
4114
4115         err = pci_save_state(pdev);
4116         if (err)
4117                 return err;
4118
4119         pci_disable_device(pdev);
4120
4121         pci_set_power_state(pdev, pci_choose_state(pdev, state));
4122
4123         return 0;
4124 }
4125
4126 #ifdef CONFIG_PM
4127 static int qlge_resume(struct pci_dev *pdev)
4128 {
4129         struct net_device *ndev = pci_get_drvdata(pdev);
4130         struct ql_adapter *qdev = netdev_priv(ndev);
4131         int err;
4132
4133         pci_set_power_state(pdev, PCI_D0);
4134         pci_restore_state(pdev);
4135         err = pci_enable_device(pdev);
4136         if (err) {
4137                 QPRINTK(qdev, IFUP, ERR, "Cannot enable PCI device from suspend\n");
4138                 return err;
4139         }
4140         pci_set_master(pdev);
4141
4142         pci_enable_wake(pdev, PCI_D3hot, 0);
4143         pci_enable_wake(pdev, PCI_D3cold, 0);
4144
4145         if (netif_running(ndev)) {
4146                 err = ql_adapter_up(qdev);
4147                 if (err)
4148                         return err;
4149         }
4150
4151         netif_device_attach(ndev);
4152
4153         return 0;
4154 }
4155 #endif /* CONFIG_PM */
4156
4157 static void qlge_shutdown(struct pci_dev *pdev)
4158 {
4159         qlge_suspend(pdev, PMSG_SUSPEND);
4160 }
4161
4162 static struct pci_driver qlge_driver = {
4163         .name = DRV_NAME,
4164         .id_table = qlge_pci_tbl,
4165         .probe = qlge_probe,
4166         .remove = __devexit_p(qlge_remove),
4167 #ifdef CONFIG_PM
4168         .suspend = qlge_suspend,
4169         .resume = qlge_resume,
4170 #endif
4171         .shutdown = qlge_shutdown,
4172         .err_handler = &qlge_err_handler
4173 };
4174
4175 static int __init qlge_init_module(void)
4176 {
4177         return pci_register_driver(&qlge_driver);
4178 }
4179
4180 static void __exit qlge_exit(void)
4181 {
4182         pci_unregister_driver(&qlge_driver);
4183 }
4184
4185 module_init(qlge_init_module);
4186 module_exit(qlge_exit);