drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static u32 crit_err_count;
  50 u32 int_mod_timer_init;
  51 u32 int_mod_cq_depth_256;
  52 u32 int_mod_cq_depth_128;
  53 u32 int_mod_cq_depth_32;
  54 u32 int_mod_cq_depth_24;
  55 u32 int_mod_cq_depth_16;
  56 u32 int_mod_cq_depth_4;
  57 u32 int_mod_cq_depth_1;
  58 static const u8 nes_max_critical_error_count = 100;
  59 #include "nes_cm.h"
  60
  61 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  62 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  63 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  64                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  65 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  66 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  67 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  68 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  69                                    struct nes_hw_aeqe *aeqe);
  70 static void process_critical_error(struct nes_device *nesdev);
  71 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  72 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  73
  74 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  75 static unsigned char *nes_iwarp_state_str[] = {
  76         "Non-Existant",
  77         "Idle",
  78         "RTS",
  79         "Closing",
  80         "RSVD1",
  81         "Terminate",
  82         "Error",
  83         "RSVD2",
  84 };
  85
  86 static unsigned char *nes_tcp_state_str[] = {
  87         "Non-Existant",
  88         "Closed",
  89         "Listen",
  90         "SYN Sent",
  91         "SYN Rcvd",
  92         "Established",
  93         "Close Wait",
  94         "FIN Wait 1",
  95         "Closing",
  96         "Last Ack",
  97         "FIN Wait 2",
  98         "Time Wait",
  99         "RSVD1",
 100         "RSVD2",
 101         "RSVD3",
 102         "RSVD4",
 103 };
 104 #endif
 105
 106
 107 /**
 108  * nes_nic_init_timer_defaults
 109  */
 110 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 111 {
 112         unsigned long flags;
 113         struct nes_adapter *nesadapter = nesdev->nesadapter;
 114         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 115
 116         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 117
 118         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 119         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 120         if (jumbomode) {
 121                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 122                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 123                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 124         } else {
 125                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 126                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 127                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 128         }
 129
 130         /* todo use netdev->mtu to set thresholds */
 131         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 132 }
 133
 134
 135 /**
 136  * nes_nic_init_timer
 137  */
 138 static void  nes_nic_init_timer(struct nes_device *nesdev)
 139 {
 140         unsigned long flags;
 141         struct nes_adapter *nesadapter = nesdev->nesadapter;
 142         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 143
 144         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 145
 146         if (shared_timer->timer_in_use_old == 0) {
 147                 nesdev->deepcq_count = 0;
 148                 shared_timer->timer_direction_upward = 0;
 149                 shared_timer->timer_direction_downward = 0;
 150                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 151                 shared_timer->timer_in_use_old = 0;
 152
 153         }
 154         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 155                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 156                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 157                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 158         }
 159         /* todo use netdev->mtu to set thresholds */
 160         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 161 }
 162
 163
 164 /**
 165  * nes_nic_tune_timer
 166  */
 167 static void nes_nic_tune_timer(struct nes_device *nesdev)
 168 {
 169         unsigned long flags;
 170         struct nes_adapter *nesadapter = nesdev->nesadapter;
 171         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 172         u16 cq_count = nesdev->currcq_count;
 173
 174         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 175
 176         if (shared_timer->cq_count_old <= cq_count)
 177                 shared_timer->cq_direction_downward = 0;
 178         else
 179                 shared_timer->cq_direction_downward++;
 180         shared_timer->cq_count_old = cq_count;
 181         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 182                 if (cq_count <= shared_timer->threshold_low &&
 183                     shared_timer->threshold_low > 4) {
 184                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 185                         shared_timer->cq_direction_downward=0;
 186                         nesdev->currcq_count = 0;
 187                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 188                         return;
 189                 }
 190         }
 191
 192         if (cq_count > 1) {
 193                 nesdev->deepcq_count += cq_count;
 194                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 195                         shared_timer->timer_direction_upward++;
 196                         shared_timer->timer_direction_downward = 0;
 197                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 198                         shared_timer->timer_direction_upward = 0;
 199                         shared_timer->timer_direction_downward = 0;
 200                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 201                         shared_timer->timer_direction_downward++;
 202                         shared_timer->timer_direction_upward = 0;
 203                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 204                         shared_timer->timer_in_use -= 2;
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward++;
 207                 } else {
 208                         shared_timer->timer_in_use -= 4;
 209                         shared_timer->timer_direction_upward = 0;
 210                         shared_timer->timer_direction_downward++;
 211                 }
 212
 213                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 214                         shared_timer->timer_in_use += 3;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward = 0;
 217                 }
 218                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 219                         shared_timer->timer_in_use -= 4 ;
 220                         shared_timer->timer_direction_downward = 0;
 221                         shared_timer->timer_direction_upward = 0;
 222                 }
 223         }
 224
 225         /* boundary checking */
 226         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 227                 shared_timer->timer_in_use = shared_timer->threshold_high;
 228         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 229                 shared_timer->timer_in_use = shared_timer->threshold_low;
 230
 231         nesdev->currcq_count = 0;
 232
 233         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 234 }
 235
 236
 237 /**
 238  * nes_init_adapter - initialize adapter
 239  */
 240 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 241         struct nes_adapter *nesadapter = NULL;
 242         unsigned long num_pds;
 243         u32 u32temp;
 244         u32 port_count;
 245         u16 max_rq_wrs;
 246         u16 max_sq_wrs;
 247         u32 max_mr;
 248         u32 max_256pbl;
 249         u32 max_4kpbl;
 250         u32 max_qp;
 251         u32 max_irrq;
 252         u32 max_cq;
 253         u32 hte_index_mask;
 254         u32 adapter_size;
 255         u32 arp_table_size;
 256         u16 vendor_id;
 257         u8  OneG_Mode;
 258         u8  func_index;
 259
 260         /* search the list of existing adapters */
 261         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 262                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 263                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 264                                 nesdev->pcidev->devfn,
 265                                 PCI_SLOT(nesadapter->devfn),
 266                                 nesadapter->bus_number,
 267                                 PCI_SLOT(nesdev->pcidev->devfn),
 268                                 nesdev->pcidev->bus->number );
 269                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 270                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 271                         nesadapter->ref_count++;
 272                         return nesadapter;
 273                 }
 274         }
 275
 276         /* no adapter found */
 277         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 278         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 279                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 280                                 hw_rev);
 281                 return NULL;
 282         }
 283
 284         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 285                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 286                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 287                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 288                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 289
 290         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 291
 292
 293         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 294                 return NULL;
 295
 296         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 297         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 298
 299         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 300         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 301                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 302                                 max_qp, u32temp);
 303                 max_qp = (u32)1 << (u32temp & 0x001f);
 304         }
 305
 306         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 307         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 308                         max_qp, hte_index_mask);
 309
 310         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 311
 312         max_irrq = 1 << (u32temp & 0x001f);
 313
 314         if (max_qp > max_irrq) {
 315                 max_qp = max_irrq;
 316                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 317                                 max_qp);
 318         }
 319
 320         /* there should be no reason to allocate more pds than qps */
 321         if (num_pds > max_qp)
 322                 num_pds = max_qp;
 323
 324         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 325         max_mr = (u32)8192 << (u32temp & 0x7);
 326
 327         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 328         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 329         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 330         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 333         arp_table_size = 1 << u32temp;
 334
 335         adapter_size = (sizeof(struct nes_adapter) +
 336                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 337         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 338         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 339         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 340         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 341         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 342         adapter_size += sizeof(struct nes_qp **) * max_qp;
 343
 344         /* allocate a new adapter struct */
 345         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 346         if (nesadapter == NULL) {
 347                 return NULL;
 348         }
 349
 350         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 351                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 352
 353         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 354                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 355                 kfree(nesadapter);
 356                 return NULL;
 357         }
 358
 359         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 360                                                         OneG_Mode)) {
 361                 kfree(nesadapter);
 362                 return NULL;
 363         }
 364         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 365
 366         memset(nesadapter->pft_mcast_map, 255,
 367                sizeof nesadapter->pft_mcast_map);
 368
 369         /* populate the new nesadapter */
 370         nesadapter->devfn = nesdev->pcidev->devfn;
 371         nesadapter->bus_number = nesdev->pcidev->bus->number;
 372         nesadapter->ref_count = 1;
 373         nesadapter->timer_int_req = 0xffff0000;
 374         nesadapter->OneG_Mode = OneG_Mode;
 375         nesadapter->doorbell_start = nesdev->doorbell_region;
 376
 377         /* nesadapter->tick_delta = clk_divisor; */
 378         nesadapter->hw_rev = hw_rev;
 379         nesadapter->port_count = port_count;
 380
 381         nesadapter->max_qp = max_qp;
 382         nesadapter->hte_index_mask = hte_index_mask;
 383         nesadapter->max_irrq = max_irrq;
 384         nesadapter->max_mr = max_mr;
 385         nesadapter->max_256pbl = max_256pbl - 1;
 386         nesadapter->max_4kpbl = max_4kpbl - 1;
 387         nesadapter->max_cq = max_cq;
 388         nesadapter->free_256pbl = max_256pbl - 1;
 389         nesadapter->free_4kpbl = max_4kpbl - 1;
 390         nesadapter->max_pd = num_pds;
 391         nesadapter->arp_table_size = arp_table_size;
 392
 393         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 394         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 395                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 396                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 397                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 398         } else {
 399                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 400                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 401                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 402                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 403         }
 404         /* Setup and enable the periodic timer */
 405         if (nesadapter->et_rx_coalesce_usecs_irq)
 406                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 407                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 408         else
 409                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 410
 411         nesadapter->base_pd = 1;
 412
 413         nesadapter->device_cap_flags =
 414                 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
 415
 416         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 417                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 418         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 419         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 420         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 421         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 422         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 423
 424
 425         /* mark the usual suspect QPs and CQs as in use */
 426         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 427                 set_bit(u32temp, nesadapter->allocated_qps);
 428                 set_bit(u32temp, nesadapter->allocated_cqs);
 429         }
 430
 431         for (u32temp = 0; u32temp < 20; u32temp++)
 432                 set_bit(u32temp, nesadapter->allocated_pds);
 433         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 434
 435         max_rq_wrs = ((u32temp >> 8) & 3);
 436         switch (max_rq_wrs) {
 437                 case 0:
 438                         max_rq_wrs = 4;
 439                         break;
 440                 case 1:
 441                         max_rq_wrs = 16;
 442                         break;
 443                 case 2:
 444                         max_rq_wrs = 32;
 445                         break;
 446                 case 3:
 447                         max_rq_wrs = 512;
 448                         break;
 449         }
 450
 451         max_sq_wrs = (u32temp & 3);
 452         switch (max_sq_wrs) {
 453                 case 0:
 454                         max_sq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_sq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_sq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_sq_wrs = 512;
 464                         break;
 465         }
 466         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 467         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 468
 469         nesadapter->max_sge = 4;
 470         nesadapter->max_cqe = 32767;
 471
 472         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 473                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 474                 kfree(nesadapter);
 475                 return NULL;
 476         }
 477
 478         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 479         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 480                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 481
 482         /* setup port configuration */
 483         if (nesadapter->port_count == 1) {
 484                 nesadapter->log_port = 0x00000000;
 485                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 486                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 487                 else
 488                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 489         } else {
 490                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 491                         nesadapter->log_port = 0x000000D8;
 492                 } else {
 493                         if (nesadapter->port_count == 2)
 494                                 nesadapter->log_port = 0x00000044;
 495                         else
 496                                 nesadapter->log_port = 0x000000e4;
 497                 }
 498                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 499         }
 500
 501         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 502                                                 nesadapter->log_port);
 503         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 504                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 505
 506         spin_lock_init(&nesadapter->resource_lock);
 507         spin_lock_init(&nesadapter->phy_lock);
 508         spin_lock_init(&nesadapter->pbl_lock);
 509         spin_lock_init(&nesadapter->periodic_timer_lock);
 510
 511         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 512         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 513         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 514         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 515
 516         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 517                 u32 pcs_control_status0, pcs_control_status1;
 518                 u32 reset_value;
 519                 u32 i = 0;
 520                 u32 int_cnt = 0;
 521                 u32 ext_cnt = 0;
 522                 unsigned long flags;
 523                 u32 j = 0;
 524
 525                 pcs_control_status0 = nes_read_indexed(nesdev,
 526                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 527                 pcs_control_status1 = nes_read_indexed(nesdev,
 528                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 529
 530                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 531                         pcs_control_status0 = nes_read_indexed(nesdev,
 532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 533                         pcs_control_status1 = nes_read_indexed(nesdev,
 534                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 535                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 536                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 537                                 int_cnt++;
 538                         msleep(1);
 539                 }
 540                 if (int_cnt > 1) {
 541                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 542                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 543                         mh_detected++;
 544                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 545                         reset_value |= 0x0000003d;
 546                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 547
 548                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 549                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 550                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 551
 552                         pcs_control_status0 = nes_read_indexed(nesdev,
 553                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 554                         pcs_control_status1 = nes_read_indexed(nesdev,
 555                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 556
 557                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 558                                 pcs_control_status0 = nes_read_indexed(nesdev,
 559                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 560                                 pcs_control_status1 = nes_read_indexed(nesdev,
 561                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 562                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 563                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 564                                         if (++ext_cnt > int_cnt) {
 565                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 566                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 567                                                                 0x0000F0C8);
 568                                                 mh_detected++;
 569                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 570                                                 reset_value |= 0x0000003d;
 571                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 572
 573                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 574                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 575                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 576                                                 break;
 577                                         }
 578                                 }
 579                                 msleep(1);
 580                         }
 581                 }
 582         }
 583
 584         if (nesadapter->hw_rev == NE020_REV) {
 585                 init_timer(&nesadapter->mh_timer);
 586                 nesadapter->mh_timer.function = nes_mh_fix;
 587                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 588                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 589                 add_timer(&nesadapter->mh_timer);
 590         } else {
 591                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 592         }
 593
 594         init_timer(&nesadapter->lc_timer);
 595         nesadapter->lc_timer.function = nes_clc;
 596         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 597         nesadapter->lc_timer.data = (unsigned long)nesdev;
 598         add_timer(&nesadapter->lc_timer);
 599
 600         list_add_tail(&nesadapter->list, &nes_adapter_list);
 601
 602         for (func_index = 0; func_index < 8; func_index++) {
 603                 pci_bus_read_config_word(nesdev->pcidev->bus,
 604                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 605                                         func_index), 0, &vendor_id);
 606                 if (vendor_id == 0xffff)
 607                         break;
 608         }
 609         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 610                 func_index, pci_name(nesdev->pcidev));
 611         nesadapter->adapter_fcn_count = func_index;
 612
 613         return nesadapter;
 614 }
 615
 616
 617 /**
 618  * nes_reset_adapter_ne020
 619  */
 620 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 621 {
 622         u32 port_count;
 623         u32 u32temp;
 624         u32 i;
 625
 626         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 627         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 628         /* TODO: assuming that both SERDES are set the same for now */
 629         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 630         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 631                         u32temp, port_count);
 632         if (*OneG_Mode)
 633                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 634         u32temp &= 0xff00ffc0;
 635         switch (port_count) {
 636                 case 1:
 637                         u32temp |= 0x00ee0000;
 638                         break;
 639                 case 2:
 640                         u32temp |= 0x00cc0000;
 641                         break;
 642                 case 4:
 643                         u32temp |= 0x00000000;
 644                         break;
 645                 default:
 646                         return 0;
 647                         break;
 648         }
 649
 650         /* check and do full reset if needed */
 651         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 652                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 653                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 654
 655                 i = 0;
 656                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 657                         mdelay(1);
 658                 if (i >= 10000) {
 659                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 660                         return 0;
 661                 }
 662
 663                 i = 0;
 664                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 665                         mdelay(1);
 666                 if (i >= 10000) {
 667                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 668                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 669                         return 0;
 670                 }
 671         }
 672
 673         /* port reset */
 674         switch (port_count) {
 675                 case 1:
 676                         u32temp |= 0x00ee0010;
 677                         break;
 678                 case 2:
 679                         u32temp |= 0x00cc0030;
 680                         break;
 681                 case 4:
 682                         u32temp |= 0x00000030;
 683                         break;
 684         }
 685
 686         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 687         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 688
 689         i = 0;
 690         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 691                 mdelay(1);
 692         if (i >= 10000) {
 693                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 694                 return 0;
 695         }
 696
 697         /* serdes 0 */
 698         i = 0;
 699         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 700                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 701                 mdelay(1);
 702         if (i >= 5000) {
 703                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 704                 return 0;
 705         }
 706
 707         /* serdes 1 */
 708         if (port_count > 1) {
 709                 i = 0;
 710                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 711                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 712                         mdelay(1);
 713                 if (i >= 5000) {
 714                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 715                         return 0;
 716                 }
 717         }
 718
 719         return port_count;
 720 }
 721
 722
 723 /**
 724  * nes_init_serdes
 725  */
 726 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 727                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 728 {
 729         int i;
 730         u32 u32temp;
 731         u32 serdes_common_control;
 732
 733         if (hw_rev != NE020_REV) {
 734                 /* init serdes 0 */
 735
 736                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 737                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 738                         serdes_common_control = nes_read_indexed(nesdev,
 739                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 740                         serdes_common_control |= 0x000000100;
 741                         nes_write_indexed(nesdev,
 742                                         NES_IDX_ETH_SERDES_COMMON_CONTROL0,
 743                                         serdes_common_control);
 744                 } else if (!OneG_Mode) {
 745                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 746                 }
 747                 if (((port_count > 1) &&
 748                         (nesadapter->phy_type[0] != NES_PHY_TYPE_PUMA_1G)) ||
 749                         ((port_count > 2) &&
 750                         (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G))) {
 751                         /* init serdes 1 */
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 753                         if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 754                                 serdes_common_control = nes_read_indexed(nesdev,
 755                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 756                                 serdes_common_control |= 0x000000100;
 757                                 nes_write_indexed(nesdev,
 758                                         NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 759                                         serdes_common_control);
 760                         } else if (!OneG_Mode) {
 761                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 762                         }
 763                 }
 764         } else {
 765                 /* init serdes 0 */
 766                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 767                 i = 0;
 768                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 769                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 770                         mdelay(1);
 771                 if (i >= 5000) {
 772                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 773                         return 1;
 774                 }
 775                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 776                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 777                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 778                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 779                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 780                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 781                 if (OneG_Mode)
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 783                 else
 784                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 785
 786                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 787                 if (port_count > 1) {
 788                         /* init serdes 1 */
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 790                         i = 0;
 791                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 792                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 793                                 mdelay(1);
 794                         if (i >= 5000) {
 795                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 796                                 /* return 1; */
 797                         }
 798                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 799                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 800                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 801                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 804                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 805                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 806                 }
 807         }
 808         return 0;
 809 }
 810
 811
 812 /**
 813  * nes_init_csr_ne020
 814  * Initialize registers for ne020 hardware
 815  */
 816 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 817 {
 818         u32 u32temp;
 819
 820         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 821
 822         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 823         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 824         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 825         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 826         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 827         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 828         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 829         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 830
 831         /* TODO: move these MAC register settings to NIC bringup */
 832         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 833         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 834         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 835         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 836         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 837         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 838         if (port_count > 1) {
 839                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 840                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 841                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 842                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 843                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 844                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 845                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 846         }
 847         if (port_count > 2) {
 848                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 849                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 850                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 851                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 852                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 853                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 854                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 855
 856                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 857                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 858                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 859                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 860                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 861                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 862                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 863         }
 864
 865         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 866         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 867         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 868                                                          0x00000001);
 869         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 870         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 871         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 872         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 873         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 874
 875         /* TODO: move this to code, get from EEPROM */
 876         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 877         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 878         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 879
 880         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 881         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 882
 883         if (hw_rev != NE020_REV) {
 884                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 885                 u32temp |= 0x80000000;
 886                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 887                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 888                 u32temp &= 0x7fffffff;
 889                 u32temp |= 0x7fff0010;
 890                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 891         }
 892 }
 893
 894
 895 /**
 896  * nes_destroy_adapter - destroy the adapter structure
 897  */
 898 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 899 {
 900         struct nes_adapter *tmp_adapter;
 901
 902         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 903                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 904                                 tmp_adapter);
 905         }
 906
 907         nesadapter->ref_count--;
 908         if (!nesadapter->ref_count) {
 909                 if (nesadapter->hw_rev == NE020_REV) {
 910                         del_timer(&nesadapter->mh_timer);
 911                 }
 912                 del_timer(&nesadapter->lc_timer);
 913
 914                 list_del(&nesadapter->list);
 915                 kfree(nesadapter);
 916         }
 917 }
 918
 919
 920 /**
 921  * nes_init_cqp
 922  */
 923 int nes_init_cqp(struct nes_device *nesdev)
 924 {
 925         struct nes_adapter *nesadapter = nesdev->nesadapter;
 926         struct nes_hw_cqp_qp_context *cqp_qp_context;
 927         struct nes_hw_cqp_wqe *cqp_wqe;
 928         struct nes_hw_ceq *ceq;
 929         struct nes_hw_ceq *nic_ceq;
 930         struct nes_hw_aeq *aeq;
 931         void *vmem;
 932         dma_addr_t pmem;
 933         u32 count=0;
 934         u32 cqp_head;
 935         u64 u64temp;
 936         u32 u32temp;
 937
 938         /* allocate CQP memory */
 939         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 940         /* SQ is 512 byte aligned, others are 256 byte aligned */
 941         nesdev->cqp_mem_size = 512 +
 942                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 943                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 944                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 945                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 946                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 947                         sizeof(struct nes_hw_cqp_qp_context);
 948
 949         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 950                         &nesdev->cqp_pbase);
 951         if (!nesdev->cqp_vbase) {
 952                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 953                 return -ENOMEM;
 954         }
 955         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 956
 957         /* Allocate a twice the number of CQP requests as the SQ size */
 958         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 959                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 960         if (nesdev->nes_cqp_requests == NULL) {
 961                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 962                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 963                                 nesdev->cqp.sq_pbase);
 964                 return -ENOMEM;
 965         }
 966
 967         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 968                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 969
 970         spin_lock_init(&nesdev->cqp.lock);
 971         init_waitqueue_head(&nesdev->cqp.waitq);
 972
 973         /* Setup Various Structures */
 974         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 975                         ~(unsigned long)(512 - 1));
 976         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 977                         ~(unsigned long long)(512 - 1));
 978
 979         nesdev->cqp.sq_vbase = vmem;
 980         nesdev->cqp.sq_pbase = pmem;
 981         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 982         nesdev->cqp.sq_head = 0;
 983         nesdev->cqp.sq_tail = 0;
 984         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 985
 986         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 987         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 988
 989         nesdev->ccq.cq_vbase = vmem;
 990         nesdev->ccq.cq_pbase = pmem;
 991         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 992         nesdev->ccq.cq_head = 0;
 993         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 994         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 995
 996         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 997         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 998
 999         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1000         ceq = &nesadapter->ceq[nesdev->ceq_index];
1001         ceq->ceq_vbase = vmem;
1002         ceq->ceq_pbase = pmem;
1003         ceq->ceq_size = NES_CCEQ_SIZE;
1004         ceq->ceq_head = 0;
1005
1006         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1007         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1008
1009         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1010         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1011         nic_ceq->ceq_vbase = vmem;
1012         nic_ceq->ceq_pbase = pmem;
1013         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1014         nic_ceq->ceq_head = 0;
1015
1016         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1017         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1018
1019         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1020         aeq->aeq_vbase = vmem;
1021         aeq->aeq_pbase = pmem;
1022         aeq->aeq_size = nesadapter->max_qp;
1023         aeq->aeq_head = 0;
1024
1025         /* Setup QP Context */
1026         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1027         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1028
1029         cqp_qp_context = vmem;
1030         cqp_qp_context->context_words[0] =
1031                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1032         cqp_qp_context->context_words[1] = 0;
1033         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1034         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1035
1036
1037         /* Write the address to Create CQP */
1038         if ((sizeof(dma_addr_t) > 4)) {
1039                 nes_write_indexed(nesdev,
1040                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1041                                 ((u64)pmem) >> 32);
1042         } else {
1043                 nes_write_indexed(nesdev,
1044                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1045         }
1046         nes_write_indexed(nesdev,
1047                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1048                         (u32)pmem);
1049
1050         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1051         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1052
1053         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1054                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1055                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1056         }
1057
1058         /* Write Create CCQ WQE */
1059         cqp_head = nesdev->cqp.sq_head++;
1060         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1061         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1062         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1063                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1064                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1065         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1066                             (nesdev->ccq.cq_number |
1067                              ((u32)nesdev->ceq_index << 16)));
1068         u64temp = (u64)nesdev->ccq.cq_pbase;
1069         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1070         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1071         u64temp = (unsigned long)&nesdev->ccq;
1072         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1073                         cpu_to_le32((u32)(u64temp >> 1));
1074         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1075                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1076         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1077
1078         /* Write Create CEQ WQE */
1079         cqp_head = nesdev->cqp.sq_head++;
1080         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1081         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1082         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1083                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1084         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1085         u64temp = (u64)ceq->ceq_pbase;
1086         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1087
1088         /* Write Create AEQ WQE */
1089         cqp_head = nesdev->cqp.sq_head++;
1090         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1091         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1092         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1093                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1094         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1095         u64temp = (u64)aeq->aeq_pbase;
1096         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1097
1098         /* Write Create NIC CEQ WQE */
1099         cqp_head = nesdev->cqp.sq_head++;
1100         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1101         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1102         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1103                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1104         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1105         u64temp = (u64)nic_ceq->ceq_pbase;
1106         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1107
1108         /* Poll until CCQP done */
1109         count = 0;
1110         do {
1111                 if (count++ > 1000) {
1112                         printk(KERN_ERR PFX "Error creating CQP\n");
1113                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1114                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1115                         return -1;
1116                 }
1117                 udelay(10);
1118         } while (!(nes_read_indexed(nesdev,
1119                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1120
1121         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1122                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1123
1124         u32temp = 0x04800000;
1125         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1126
1127         /* wait for the CCQ, CEQ, and AEQ to get created */
1128         count = 0;
1129         do {
1130                 if (count++ > 1000) {
1131                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1132                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1133                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1134                         return -1;
1135                 }
1136                 udelay(10);
1137         } while (((nes_read_indexed(nesdev,
1138                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1139
1140         /* dump the QP status value */
1141         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1142                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1143
1144         nesdev->cqp.sq_tail++;
1145
1146         return 0;
1147 }
1148
1149
1150 /**
1151  * nes_destroy_cqp
1152  */
1153 int nes_destroy_cqp(struct nes_device *nesdev)
1154 {
1155         struct nes_hw_cqp_wqe *cqp_wqe;
1156         u32 count = 0;
1157         u32 cqp_head;
1158         unsigned long flags;
1159
1160         do {
1161                 if (count++ > 1000)
1162                         break;
1163                 udelay(10);
1164         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1165
1166         /* Reset CCQ */
1167         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1168                         nesdev->ccq.cq_number);
1169
1170         /* Disable device interrupts */
1171         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1172
1173         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1174
1175         /* Destroy the AEQ */
1176         cqp_head = nesdev->cqp.sq_head++;
1177         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1178         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1179         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1180                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1181         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1182
1183         /* Destroy the NIC CEQ */
1184         cqp_head = nesdev->cqp.sq_head++;
1185         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1186         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1187         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1188                         ((u32)nesdev->nic_ceq_index << 8));
1189
1190         /* Destroy the CEQ */
1191         cqp_head = nesdev->cqp.sq_head++;
1192         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1193         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1194         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1195                         (nesdev->ceq_index << 8));
1196
1197         /* Destroy the CCQ */
1198         cqp_head = nesdev->cqp.sq_head++;
1199         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1200         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1201         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1202         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1203                         ((u32)nesdev->ceq_index << 16));
1204
1205         /* Destroy CQP */
1206         cqp_head = nesdev->cqp.sq_head++;
1207         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1208         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1209         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1210                         NES_CQP_QP_TYPE_CQP);
1211         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1212
1213         barrier();
1214         /* Ring doorbell (5 WQEs) */
1215         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1216
1217         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1218
1219         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1220         count = 0;
1221         do {
1222                 if (count++ > 1000) {
1223                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1224                                         PCI_FUNC(nesdev->pcidev->devfn));
1225                         break;
1226                 }
1227                 udelay(10);
1228         } while (((nes_read_indexed(nesdev,
1229                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1230
1231         /* dump the QP status value */
1232         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1233                         PCI_FUNC(nesdev->pcidev->devfn),
1234                         nes_read_indexed(nesdev,
1235                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1236
1237         kfree(nesdev->nes_cqp_requests);
1238
1239         /* Free the control structures */
1240         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1241                         nesdev->cqp.sq_pbase);
1242
1243         return 0;
1244 }
1245
1246
1247 /**
1248  * nes_init_phy
1249  */
1250 int nes_init_phy(struct nes_device *nesdev)
1251 {
1252         struct nes_adapter *nesadapter = nesdev->nesadapter;
1253         u32 counter = 0;
1254         u32 sds_common_control0;
1255         u32 mac_index = nesdev->mac_index;
1256         u32 tx_config = 0;
1257         u16 phy_data;
1258         u32 temp_phy_data = 0;
1259         u32 temp_phy_data2 = 0;
1260         u32 i = 0;
1261
1262         if ((nesadapter->OneG_Mode) &&
1263             (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
1264                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1265                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1266                         printk(PFX "%s: Programming mdc config for 1G\n", __func__);
1267                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1268                         tx_config &= 0xFFFFFFE3;
1269                         tx_config |= 0x04;
1270                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1271                 }
1272
1273                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1274                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1275                                 nesadapter->phy_index[mac_index], phy_data);
1276                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index], 0xb000);
1277
1278                 /* Reset the PHY */
1279                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1280                 udelay(100);
1281                 counter = 0;
1282                 do {
1283                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1284                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1285                         if (counter++ > 100) break;
1286                 } while (phy_data & 0x8000);
1287
1288                 /* Setting no phy loopback */
1289                 phy_data &= 0xbfff;
1290                 phy_data |= 0x1140;
1291                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1292                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1293                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1294
1295                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1296                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1297
1298                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1299                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1300
1301                 /* Setting the interrupt mask */
1302                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1303                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1304                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1305
1306                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1307                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1308
1309                 /* turning on flow control */
1310                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1311                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1312                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1313                                 (phy_data & ~(0x03E0)) | 0xc00);
1314                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1315                                 phy_data | 0xc00); */
1316                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1317                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1318
1319                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1320                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1321                 /* Clear Half duplex */
1322                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1323                                 phy_data & ~(0x0100));
1324                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1325                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1326
1327                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1328                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1329         } else {
1330                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) ||
1331                     (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1332                         /* setup 10G MDIO operation */
1333                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1334                         tx_config &= 0xFFFFFFE3;
1335                         tx_config |= 0x15;
1336                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1337                 }
1338                 if ((nesadapter->phy_type[mac_index] == NES_PHY_TYPE_ARGUS)) {
1339                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1340
1341                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1342                         mdelay(10);
1343                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1344                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1345
1346                         /*
1347                          * if firmware is already running (like from a
1348                          * driver un-load/load, don't do anything.
1349                          */
1350                         if (temp_phy_data == temp_phy_data2) {
1351                                 /* configure QT2505 AMCC PHY */
1352                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0x0000, 0x8000);
1353                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0000);
1354                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc302, 0x0044);
1355                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc318, 0x0052);
1356                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc319, 0x0008);
1357                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc31a, 0x0098);
1358                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0026, 0x0E00);
1359                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0027, 0x0001);
1360                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0x0028, 0xA528);
1361
1362                                 /*
1363                                  * remove micro from reset; chip boots from ROM,
1364                                  * uploads EEPROM f/w image, uC executes f/w
1365                                  */
1366                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc300, 0x0002);
1367
1368                                 /*
1369                                  * wait for heart beat to start to
1370                                  * know loading is done
1371                                  */
1372                                 counter = 0;
1373                                 do {
1374                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1375                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1376                                         if (counter++ > 1000) {
1377                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from heartbeat check <this is bad!!!> \n");
1378                                                 break;
1379                                         }
1380                                         mdelay(100);
1381                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7ee);
1382                                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1383                                 } while ((temp_phy_data2 == temp_phy_data));
1384
1385                                 /*
1386                                  * wait for tracking to start to know
1387                                  * f/w is good to go
1388                                  */
1389                                 counter = 0;
1390                                 do {
1391                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x3, 0xd7fd);
1392                                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1393                                         if (counter++ > 1000) {
1394                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from status check <this is bad!!!> \n");
1395                                                 break;
1396                                         }
1397                                         mdelay(1000);
1398                                         /*
1399                                          * nes_debug(NES_DBG_PHY, "AMCC PHY- phy_status not ready yet = 0x%02X\n",
1400                                          *                      temp_phy_data);
1401                                          */
1402                                 } while (((temp_phy_data & 0xff) != 0x50) && ((temp_phy_data & 0xff) != 0x70));
1403
1404                                 /* set LOS Control invert RXLOSB_I_PADINV */
1405                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd003, 0x0000);
1406                                 /* set LOS Control to mask of RXLOSB_I */
1407                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xc314, 0x0042);
1408                                 /* set LED1 to input mode (LED1 and LED2 share same LED) */
1409                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd006, 0x0007);
1410                                 /* set LED2 to RX link_status and activity */
1411                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd007, 0x000A);
1412                                 /* set LED3 to RX link_status */
1413                                 nes_write_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 0x1, 0xd008, 0x0009);
1414
1415                                 /*
1416                                  * reset the res-calibration on t2
1417                                  * serdes; ensures it is stable after
1418                                  * the amcc phy is stable
1419                                  */
1420
1421                                 sds_common_control0  = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
1422                                 sds_common_control0 |= 0x1;
1423                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1424
1425                                 /* release the res-calibration reset */
1426                                 sds_common_control0 &= 0xfffffffe;
1427                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds_common_control0);
1428
1429                                 i = 0;
1430                                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1431                                                 && (i++ < 5000)) {
1432                                         /* mdelay(1); */
1433                                 }
1434
1435                                 /*
1436                                  * wait for link train done before moving on,
1437                                  * or will get an interupt storm
1438                                  */
1439                                 counter = 0;
1440                                 do {
1441                                         temp_phy_data = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 +
1442                                                                 (0x200 * (nesdev->mac_index & 1)));
1443                                         if (counter++ > 1000) {
1444                                                 nes_debug(NES_DBG_PHY, "AMCC PHY- breaking from link train wait <this is bad, link didnt train!!!>\n");
1445                                                 break;
1446                                         }
1447                                         mdelay(1);
1448                                 } while (((temp_phy_data & 0x0f1f0000) != 0x0f0f0000));
1449                         }
1450                 }
1451         }
1452         return 0;
1453 }
1454
1455
1456 /**
1457  * nes_replenish_nic_rq
1458  */
1459 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1460 {
1461         unsigned long flags;
1462         dma_addr_t bus_address;
1463         struct sk_buff *skb;
1464         struct nes_hw_nic_rq_wqe *nic_rqe;
1465         struct nes_hw_nic *nesnic;
1466         struct nes_device *nesdev;
1467         u32 rx_wqes_posted = 0;
1468
1469         nesnic = &nesvnic->nic;
1470         nesdev = nesvnic->nesdev;
1471         spin_lock_irqsave(&nesnic->rq_lock, flags);
1472         if (nesnic->replenishing_rq !=0) {
1473                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1474                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1475                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1476                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1477                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1478                         add_timer(&nesvnic->rq_wqes_timer);
1479                 } else
1480                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1481                 return;
1482         }
1483         nesnic->replenishing_rq = 1;
1484         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1485         do {
1486                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1487                 if (skb) {
1488                         skb->dev = nesvnic->netdev;
1489
1490                         bus_address = pci_map_single(nesdev->pcidev,
1491                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1492
1493                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1494                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1495                                         cpu_to_le32(nesvnic->max_frame_size);
1496                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1497                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1498                                         cpu_to_le32((u32)bus_address);
1499                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1500                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1501                         nesnic->rx_skb[nesnic->rq_head] = skb;
1502                         nesnic->rq_head++;
1503                         nesnic->rq_head &= nesnic->rq_size - 1;
1504                         atomic_dec(&nesvnic->rx_skbs_needed);
1505                         barrier();
1506                         if (++rx_wqes_posted == 255) {
1507                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1508                                 rx_wqes_posted = 0;
1509                         }
1510                 } else {
1511                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1512                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1513                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1514                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1515                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1516                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1517                                 add_timer(&nesvnic->rq_wqes_timer);
1518                         } else
1519                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1520                         break;
1521                 }
1522         } while (atomic_read(&nesvnic->rx_skbs_needed));
1523         barrier();
1524         if (rx_wqes_posted)
1525                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1526         nesnic->replenishing_rq = 0;
1527 }
1528
1529
1530 /**
1531  * nes_rq_wqes_timeout
1532  */
1533 static void nes_rq_wqes_timeout(unsigned long parm)
1534 {
1535         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1536         printk("%s: Timer fired.\n", __func__);
1537         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1538         if (atomic_read(&nesvnic->rx_skbs_needed))
1539                 nes_replenish_nic_rq(nesvnic);
1540 }
1541
1542
1543 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1544                                void **tcph, u64 *hdr_flags, void *priv)
1545 {
1546         unsigned int ip_len;
1547         struct iphdr *iph;
1548         skb_reset_network_header(skb);
1549         iph = ip_hdr(skb);
1550         if (iph->protocol != IPPROTO_TCP)
1551                 return -1;
1552         ip_len = ip_hdrlen(skb);
1553         skb_set_transport_header(skb, ip_len);
1554         *tcph = tcp_hdr(skb);
1555
1556         *hdr_flags = LRO_IPV4 | LRO_TCP;
1557         *iphdr = iph;
1558         return 0;
1559 }
1560
1561
1562 /**
1563  * nes_init_nic_qp
1564  */
1565 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1566 {
1567         struct nes_hw_cqp_wqe *cqp_wqe;
1568         struct nes_hw_nic_sq_wqe *nic_sqe;
1569         struct nes_hw_nic_qp_context *nic_context;
1570         struct sk_buff *skb;
1571         struct nes_hw_nic_rq_wqe *nic_rqe;
1572         struct nes_vnic *nesvnic = netdev_priv(netdev);
1573         unsigned long flags;
1574         void *vmem;
1575         dma_addr_t pmem;
1576         u64 u64temp;
1577         int ret;
1578         u32 cqp_head;
1579         u32 counter;
1580         u32 wqe_count;
1581         u8 jumbomode=0;
1582
1583         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1584         nesvnic->nic_mem_size = 256 +
1585                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1586                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1587                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1588                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1589                         sizeof(struct nes_hw_nic_qp_context);
1590
1591         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1592                         &nesvnic->nic_pbase);
1593         if (!nesvnic->nic_vbase) {
1594                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1595                 return -ENOMEM;
1596         }
1597         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1598         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1599                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1600
1601         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1602                         ~(unsigned long)(256 - 1));
1603         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1604                         ~(unsigned long long)(256 - 1));
1605
1606         /* Setup the first Fragment buffers */
1607         nesvnic->nic.first_frag_vbase = vmem;
1608
1609         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1610                 nesvnic->nic.frag_paddr[counter] = pmem;
1611                 pmem += sizeof(struct nes_first_frag);
1612         }
1613
1614         /* setup the SQ */
1615         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1616
1617         nesvnic->nic.sq_vbase = (void *)vmem;
1618         nesvnic->nic.sq_pbase = pmem;
1619         nesvnic->nic.sq_head = 0;
1620         nesvnic->nic.sq_tail = 0;
1621         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1622         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1623                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1624                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1625                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1626                                 NES_NIC_SQ_WQE_COMPLETION);
1627                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1628                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1629                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1630                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1631                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1632                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1633         }
1634
1635         nesvnic->get_cqp_request = nes_get_cqp_request;
1636         nesvnic->post_cqp_request = nes_post_cqp_request;
1637         nesvnic->mcrq_mcast_filter = NULL;
1638
1639         spin_lock_init(&nesvnic->nic.sq_lock);
1640         spin_lock_init(&nesvnic->nic.rq_lock);
1641
1642         /* setup the RQ */
1643         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1644         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1645
1646
1647         nesvnic->nic.rq_vbase = vmem;
1648         nesvnic->nic.rq_pbase = pmem;
1649         nesvnic->nic.rq_head = 0;
1650         nesvnic->nic.rq_tail = 0;
1651         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1652
1653         /* setup the CQ */
1654         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1655         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1656
1657         if (nesdev->nesadapter->netdev_count > 2)
1658                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1659         else
1660                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1661
1662         nesvnic->nic_cq.cq_vbase = vmem;
1663         nesvnic->nic_cq.cq_pbase = pmem;
1664         nesvnic->nic_cq.cq_head = 0;
1665         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1666
1667         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1668
1669         /* Send CreateCQ request to CQP */
1670         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1671         cqp_head = nesdev->cqp.sq_head;
1672
1673         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1674         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1675
1676         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1677                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1678                         ((u32)nesvnic->nic_cq.cq_size << 16));
1679         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1680                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1681         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1682         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1683         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1684         u64temp = (unsigned long)&nesvnic->nic_cq;
1685         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1686         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1687                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1688         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1689         if (++cqp_head >= nesdev->cqp.sq_size)
1690                 cqp_head = 0;
1691         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1692         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1693
1694         /* Send CreateQP request to CQP */
1695         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1696         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1697                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1698                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1699         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1700                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1701                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1702         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1703                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1704         }
1705
1706         u64temp = (u64)nesvnic->nic.sq_pbase;
1707         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1708         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1709         u64temp = (u64)nesvnic->nic.rq_pbase;
1710         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1711         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1712
1713         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1714                         NES_CQP_QP_TYPE_NIC);
1715         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1716         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1717                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1718         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1719
1720         if (++cqp_head >= nesdev->cqp.sq_size)
1721                 cqp_head = 0;
1722         nesdev->cqp.sq_head = cqp_head;
1723
1724         barrier();
1725
1726         /* Ring doorbell (2 WQEs) */
1727         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1728
1729         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1730         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1731                         nesvnic->nic.qp_id);
1732
1733         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1734                         NES_EVENT_TIMEOUT);
1735         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1736                         nesvnic->nic.qp_id, ret);
1737         if (!ret) {
1738                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1739                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1740                                 nesvnic->nic_pbase);
1741                 return -EIO;
1742         }
1743
1744         /* Populate the RQ */
1745         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1746                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1747                 if (!skb) {
1748                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1749
1750                         nes_destroy_nic_qp(nesvnic);
1751                         return -ENOMEM;
1752                 }
1753
1754                 skb->dev = netdev;
1755
1756                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1757                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1758
1759                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1760                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1761                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1762                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1763                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1764                 nesvnic->nic.rx_skb[counter] = skb;
1765         }
1766
1767         wqe_count = NES_NIC_WQ_SIZE - 1;
1768         nesvnic->nic.rq_head = wqe_count;
1769         barrier();
1770         do {
1771                 counter = min(wqe_count, ((u32)255));
1772                 wqe_count -= counter;
1773                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1774         } while (wqe_count);
1775         init_timer(&nesvnic->rq_wqes_timer);
1776         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1777         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1778         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1779         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1780         {
1781                 nes_nic_init_timer(nesdev);
1782                 if (netdev->mtu > 1500)
1783                         jumbomode = 1;
1784                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1785         }
1786         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1787         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1788         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1789         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1790         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1791         nesvnic->lro_mgr.dev            = netdev;
1792         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1793         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1794         return 0;
1795 }
1796
1797
1798 /**
1799  * nes_destroy_nic_qp
1800  */
1801 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1802 {
1803         u64 u64temp;
1804         dma_addr_t bus_address;
1805         struct nes_device *nesdev = nesvnic->nesdev;
1806         struct nes_hw_cqp_wqe *cqp_wqe;
1807         struct nes_hw_nic_sq_wqe *nic_sqe;
1808         struct nes_hw_nic_rq_wqe *nic_rqe;
1809         __le16 *wqe_fragment_length;
1810         u16  wqe_fragment_index;
1811         u64 wqe_frag;
1812         u32 cqp_head;
1813         unsigned long flags;
1814         int ret;
1815
1816         /* Free remaining NIC receive buffers */
1817         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1818                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1819                 wqe_frag  = (u64)le32_to_cpu(
1820                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1821                 wqe_frag |= ((u64)le32_to_cpu(
1822                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1823                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1824                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1825                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1826                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1827         }
1828
1829         /* Free remaining NIC transmit buffers */
1830         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1831                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1832                 wqe_fragment_index = 1;
1833                 wqe_fragment_length = (__le16 *)
1834                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1835                 /* bump past the vlan tag */
1836                 wqe_fragment_length++;
1837                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1838                         u64temp = (u64)le32_to_cpu(
1839                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1840                                 wqe_fragment_index*2]);
1841                         u64temp += ((u64)le32_to_cpu(
1842                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1843                                 + wqe_fragment_index*2]))<<32;
1844                         bus_address = (dma_addr_t)u64temp;
1845                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1846                                         nesvnic->nic.first_frag_overflow)) {
1847                                 pci_unmap_single(nesdev->pcidev,
1848                                                 bus_address,
1849                                                 le16_to_cpu(wqe_fragment_length[
1850                                                         wqe_fragment_index++]),
1851                                                 PCI_DMA_TODEVICE);
1852                         }
1853                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1854                                 if (wqe_fragment_length[wqe_fragment_index]) {
1855                                         u64temp = le32_to_cpu(
1856                                                 nic_sqe->wqe_words[
1857                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1858                                                 wqe_fragment_index*2]);
1859                                         u64temp += ((u64)le32_to_cpu(
1860                                                 nic_sqe->wqe_words[
1861                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1862                                                 wqe_fragment_index*2]))<<32;
1863                                         bus_address = (dma_addr_t)u64temp;
1864                                         pci_unmap_page(nesdev->pcidev,
1865                                                         bus_address,
1866                                                         le16_to_cpu(
1867                                                         wqe_fragment_length[
1868                                                         wqe_fragment_index]),
1869                                                         PCI_DMA_TODEVICE);
1870                                 } else
1871                                         break;
1872                         }
1873                 }
1874                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1875                         dev_kfree_skb(
1876                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1877
1878                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1879                                         & (nesvnic->nic.sq_size - 1);
1880         }
1881
1882         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1883
1884         /* Destroy NIC QP */
1885         cqp_head = nesdev->cqp.sq_head;
1886         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1887         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1888
1889         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1890                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1891         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1892                 nesvnic->nic.qp_id);
1893
1894         if (++cqp_head >= nesdev->cqp.sq_size)
1895                 cqp_head = 0;
1896
1897         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1898
1899         /* Destroy NIC CQ */
1900         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1901         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1902                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1903         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1904                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1905
1906         if (++cqp_head >= nesdev->cqp.sq_size)
1907                 cqp_head = 0;
1908
1909         nesdev->cqp.sq_head = cqp_head;
1910         barrier();
1911
1912         /* Ring doorbell (2 WQEs) */
1913         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1914
1915         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1916         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1917                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1918                         cqp_head, nesdev->cqp.sq_head,
1919                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1920
1921         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1922                         NES_EVENT_TIMEOUT);
1923
1924         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1925                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1926                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1927         if (!ret) {
1928                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1929                                 nesvnic->nic.qp_id);
1930         }
1931
1932         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1933                         nesvnic->nic_pbase);
1934 }
1935
1936 /**
1937  * nes_napi_isr
1938  */
1939 int nes_napi_isr(struct nes_device *nesdev)
1940 {
1941         struct nes_adapter *nesadapter = nesdev->nesadapter;
1942         u32 int_stat;
1943
1944         if (nesdev->napi_isr_ran) {
1945                 /* interrupt status has already been read in ISR */
1946                 int_stat = nesdev->int_stat;
1947         } else {
1948                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1949                 nesdev->int_stat = int_stat;
1950                 nesdev->napi_isr_ran = 1;
1951         }
1952
1953         int_stat &= nesdev->int_req;
1954         /* iff NIC, process here, else wait for DPC */
1955         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1956                 nesdev->napi_isr_ran = 0;
1957                 nes_write32(nesdev->regs + NES_INT_STAT,
1958                         (int_stat &
1959                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1960
1961                 /* Process the CEQs */
1962                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1963
1964                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1965                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1966                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1967                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1968                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1969                                 /* Enable Periodic timer interrupts */
1970                                 nesdev->int_req |= NES_INT_TIMER;
1971                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1972                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1973                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1974                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1975                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1976                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1977                         }
1978
1979                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1980                         {
1981                                 nes_nic_init_timer(nesdev);
1982                         }
1983                         /* Enable interrupts, except CEQs */
1984                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1985                 } else {
1986                         /* Enable interrupts, make sure timer is off */
1987                         nesdev->int_req &= ~NES_INT_TIMER;
1988                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1989                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1990                 }
1991                 nesdev->deepcq_count = 0;
1992                 return 1;
1993         } else {
1994                 return 0;
1995         }
1996 }
1997
1998 static void process_critical_error(struct nes_device *nesdev)
1999 {
2000         u32 debug_error;
2001         u32 nes_idx_debug_error_masks0 = 0;
2002         u16 error_module = 0;
2003
2004         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2005         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2006                         (u16)debug_error);
2007         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2008                         0x01010000 | (debug_error & 0x0000ffff));
2009         if (crit_err_count++ > 10)
2010                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2011         error_module = (u16) (debug_error & 0x1F00) >> 8;
2012         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2013                         nes_max_critical_error_count) {
2014                 printk(KERN_ERR PFX "Masking off critical error for module "
2015                         "0x%02X\n", (u16)error_module);
2016                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2017                         NES_IDX_DEBUG_ERROR_MASKS0);
2018                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2019                         nes_idx_debug_error_masks0 | (1 << error_module));
2020         }
2021 }
2022 /**
2023  * nes_dpc
2024  */
2025 void nes_dpc(unsigned long param)
2026 {
2027         struct nes_device *nesdev = (struct nes_device *)param;
2028         struct nes_adapter *nesadapter = nesdev->nesadapter;
2029         u32 counter;
2030         u32 loop_counter = 0;
2031         u32 int_status_bit;
2032         u32 int_stat;
2033         u32 timer_stat;
2034         u32 temp_int_stat;
2035         u32 intf_int_stat;
2036         u32 processed_intf_int = 0;
2037         u16 processed_timer_int = 0;
2038         u16 completion_ints = 0;
2039         u16 timer_ints = 0;
2040
2041         /* nes_debug(NES_DBG_ISR, "\n"); */
2042
2043         do {
2044                 timer_stat = 0;
2045                 if (nesdev->napi_isr_ran) {
2046                         nesdev->napi_isr_ran = 0;
2047                         int_stat = nesdev->int_stat;
2048                 } else
2049                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2050                 if (processed_intf_int != 0)
2051                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2052                 else
2053                         int_stat &= nesdev->int_req;
2054                 if (processed_timer_int == 0) {
2055                         processed_timer_int = 1;
2056                         if (int_stat & NES_INT_TIMER) {
2057                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2058                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2059                                         int_stat &= ~NES_INT_TIMER;
2060                                 }
2061                         }
2062                 } else {
2063                         int_stat &= ~NES_INT_TIMER;
2064                 }
2065
2066                 if (int_stat) {
2067                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2068                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2069                                 /* Ack the interrupts */
2070                                 nes_write32(nesdev->regs+NES_INT_STAT,
2071                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2072                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2073                         }
2074
2075                         temp_int_stat = int_stat;
2076                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2077                                 if (int_stat & int_status_bit) {
2078                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2079                                         temp_int_stat &= ~int_status_bit;
2080                                         completion_ints = 1;
2081                                 }
2082                                 if (!(temp_int_stat & 0x0000ffff))
2083                                         break;
2084                                 int_status_bit <<= 1;
2085                         }
2086
2087                         /* Process the AEQ for this pci function */
2088                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2089                         if (int_stat & int_status_bit) {
2090                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2091                         }
2092
2093                         /* Process the MAC interrupt for this pci function */
2094                         int_status_bit = 1 << (24 + nesdev->mac_index);
2095                         if (int_stat & int_status_bit) {
2096                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2097                         }
2098
2099                         if (int_stat & NES_INT_TIMER) {
2100                                 if (timer_stat & nesdev->timer_int_req) {
2101                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2102                                                         (timer_stat & nesdev->timer_int_req) |
2103                                                         ~(nesdev->nesadapter->timer_int_req));
2104                                         timer_ints = 1;
2105                                 }
2106                         }
2107
2108                         if (int_stat & NES_INT_INTF) {
2109                                 processed_intf_int = 1;
2110                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2111                                 intf_int_stat &= nesdev->intf_int_req;
2112                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2113                                         process_critical_error(nesdev);
2114                                 }
2115                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2116                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2117                                         BUG();
2118                                 }
2119                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2120                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2121                                         BUG();
2122                                 }
2123                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2124                         }
2125
2126                         if (int_stat & NES_INT_TSW) {
2127                         }
2128                 }
2129                 /* Don't use the interface interrupt bit stay in loop */
2130                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2131                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2132         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2133
2134         if (timer_ints == 1) {
2135                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2136                         if (completion_ints == 0) {
2137                                 nesdev->timer_only_int_count++;
2138                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2139                                         nesdev->timer_only_int_count = 0;
2140                                         nesdev->int_req &= ~NES_INT_TIMER;
2141                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2142                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2143                                 } else {
2144                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2145                                 }
2146                         } else {
2147                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2148                                 {
2149                                         nes_nic_init_timer(nesdev);
2150                                 }
2151                                 nesdev->timer_only_int_count = 0;
2152                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2153                         }
2154                 } else {
2155                         nesdev->timer_only_int_count = 0;
2156                         nesdev->int_req &= ~NES_INT_TIMER;
2157                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2158                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2159                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2160                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2161                 }
2162         } else {
2163                 if ( (completion_ints == 1) &&
2164                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2165                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2166                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2167                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2168                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2169                         nesdev->timer_only_int_count = 0;
2170                         nesdev->int_req |= NES_INT_TIMER;
2171                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2172                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2173                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2174                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2175                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2176                 } else {
2177                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2178                 }
2179         }
2180         nesdev->deepcq_count = 0;
2181 }
2182
2183
2184 /**
2185  * nes_process_ceq
2186  */
2187 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2188 {
2189         u64 u64temp;
2190         struct nes_hw_cq *cq;
2191         u32 head;
2192         u32 ceq_size;
2193
2194         /* nes_debug(NES_DBG_CQ, "\n"); */
2195         head = ceq->ceq_head;
2196         ceq_size = ceq->ceq_size;
2197
2198         do {
2199                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2200                                 NES_CEQE_VALID) {
2201                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2202                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2203                         u64temp <<= 1;
2204                         cq = *((struct nes_hw_cq **)&u64temp);
2205                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2206                         barrier();
2207                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2208
2209                         /* call the event handler */
2210                         cq->ce_handler(nesdev, cq);
2211
2212                         if (++head >= ceq_size)
2213                                 head = 0;
2214                 } else {
2215                         break;
2216                 }
2217
2218         } while (1);
2219
2220         ceq->ceq_head = head;
2221 }
2222
2223
2224 /**
2225  * nes_process_aeq
2226  */
2227 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2228 {
2229         /* u64 u64temp; */
2230         u32 head;
2231         u32 aeq_size;
2232         u32 aeqe_misc;
2233         u32 aeqe_cq_id;
2234         struct nes_hw_aeqe volatile *aeqe;
2235
2236         head = aeq->aeq_head;
2237         aeq_size = aeq->aeq_size;
2238
2239         do {
2240                 aeqe = &aeq->aeq_vbase[head];
2241                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2242                         break;
2243                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2244                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2245                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2246                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2247                                 /* dealing with an accelerated QP related AE */
2248                                 /*
2249                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2250                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2251                                  */
2252                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2253                         } else {
2254                                 /* TODO: dealing with a CQP related AE */
2255                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2256                                                 (u16)(aeqe_misc >> 16));
2257                         }
2258                 }
2259
2260                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2261
2262                 if (++head >= aeq_size)
2263                         head = 0;
2264         }
2265         while (1);
2266         aeq->aeq_head = head;
2267 }
2268
2269 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2270 {
2271         struct nes_adapter *nesadapter = nesdev->nesadapter;
2272         u32 reset_value;
2273         u32 i=0;
2274         u32 u32temp;
2275
2276         if (nesadapter->hw_rev == NE020_REV) {
2277                 return;
2278         }
2279         mh_detected++;
2280
2281         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2282
2283         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2284                 reset_value |= 0x0000001d;
2285         else
2286                 reset_value |= 0x0000002d;
2287
2288         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2289                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2290                         nesadapter->link_interrupt_count[0] = 0;
2291                         nesadapter->link_interrupt_count[1] = 0;
2292                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2293                         if (0x00000040 & u32temp)
2294                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2295                         else
2296                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2297
2298                         reset_value |= 0x0000003d;
2299                 }
2300                 nesadapter->link_interrupt_count[mac_index] = 0;
2301         }
2302
2303         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2304
2305         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2306                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2307
2308         if (0x0000003d == (reset_value & 0x0000003d)) {
2309                 u32 pcs_control_status0, pcs_control_status1;
2310
2311                 for (i = 0; i < 10; i++) {
2312                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2313                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2314                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2315                              && (pcs_control_status0 & 0x00100000))
2316                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2317                                 && (pcs_control_status1 & 0x00100000)))
2318                                 continue;
2319                         else
2320                                 break;
2321                 }
2322                 if (10 == i) {
2323                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2324                         if (0x00000040 & u32temp)
2325                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2326                         else
2327                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2328
2329                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2330
2331                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2332                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2333                 }
2334         }
2335 }
2336
2337 /**
2338  * nes_process_mac_intr
2339  */
2340 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2341 {
2342         unsigned long flags;
2343         u32 pcs_control_status;
2344         struct nes_adapter *nesadapter = nesdev->nesadapter;
2345         struct nes_vnic *nesvnic;
2346         u32 mac_status;
2347         u32 mac_index = nesdev->mac_index;
2348         u32 u32temp;
2349         u16 phy_data;
2350         u16 temp_phy_data;
2351         u32 pcs_val  = 0x0f0f0000;
2352         u32 pcs_mask = 0x0f1f0000;
2353
2354         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2355         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2356                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2357                 return;
2358         }
2359         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2360         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2361
2362         /* ack the MAC interrupt */
2363         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2364         /* Clear the interrupt */
2365         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2366
2367         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2368
2369         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2370                 nesdev->link_status_interrupts++;
2371                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2372                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2373                         nes_reset_link(nesdev, mac_index);
2374                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2375                 }
2376                 /* read the PHY interrupt status register */
2377                 if ((nesadapter->OneG_Mode) &&
2378                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2379                         do {
2380                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2381                                                 nesadapter->phy_index[mac_index], &phy_data);
2382                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2383                                                 nesadapter->phy_index[mac_index], phy_data);
2384                         } while (phy_data&0x8000);
2385
2386                         temp_phy_data = 0;
2387                         do {
2388                                 nes_read_1G_phy_reg(nesdev, 0x11,
2389                                                 nesadapter->phy_index[mac_index], &phy_data);
2390                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2391                                                 nesadapter->phy_index[mac_index], phy_data);
2392                                 if (temp_phy_data == phy_data)
2393                                         break;
2394                                 temp_phy_data = phy_data;
2395                         } while (1);
2396
2397                         nes_read_1G_phy_reg(nesdev, 0x1e,
2398                                         nesadapter->phy_index[mac_index], &phy_data);
2399                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2400                                         nesadapter->phy_index[mac_index], phy_data);
2401
2402                         nes_read_1G_phy_reg(nesdev, 1,
2403                                         nesadapter->phy_index[mac_index], &phy_data);
2404                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2405                                         nesadapter->phy_index[mac_index], phy_data);
2406
2407                         if (temp_phy_data & 0x1000) {
2408                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2409                                 phy_data = 4;
2410                         } else {
2411                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2412                         }
2413                 }
2414                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2415                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2416                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2417
2418                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2419                         switch (mac_index) {
2420                         case 1:
2421                         case 3:
2422                                 pcs_control_status = nes_read_indexed(nesdev,
2423                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2424                                 break;
2425                         default:
2426                                 pcs_control_status = nes_read_indexed(nesdev,
2427                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2428                                 break;
2429                         }
2430                 } else {
2431                         pcs_control_status = nes_read_indexed(nesdev,
2432                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2433                         pcs_control_status = nes_read_indexed(nesdev,
2434                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2435                 }
2436
2437                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2438                                 mac_index, pcs_control_status);
2439                 if ((nesadapter->OneG_Mode) &&
2440                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2441                         u32temp = 0x01010000;
2442                         if (nesadapter->port_count > 2) {
2443                                 u32temp |= 0x02020000;
2444                         }
2445                         if ((pcs_control_status & u32temp)!= u32temp) {
2446                                 phy_data = 0;
2447                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2448                         }
2449                 } else {
2450                         switch (nesadapter->phy_type[mac_index]) {
2451                         case NES_PHY_TYPE_IRIS:
2452                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2453                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2454                                 u32temp = 20;
2455                                 do {
2456                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2457                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2458                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2459                                                 break;
2460                                         temp_phy_data = phy_data;
2461                                 } while (1);
2462                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2463                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2464                                 break;
2465
2466                         case NES_PHY_TYPE_ARGUS:
2467                                 /* clear the alarms */
2468                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2469                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2470                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2471                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2472                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2473                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2474                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2475                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2476                                 /* check link status */
2477                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2478                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2479                                 u32temp = 100;
2480                                 do {
2481                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2482
2483                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2484                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2485                                                 break;
2486                                         temp_phy_data = phy_data;
2487                                 } while (1);
2488                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2489                                         __func__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2490                                 break;
2491
2492                         case NES_PHY_TYPE_PUMA_1G:
2493                                 if (mac_index < 2)
2494                                         pcs_val = pcs_mask = 0x01010000;
2495                                 else
2496                                         pcs_val = pcs_mask = 0x02020000;
2497                                 /* fall through */
2498                         default:
2499                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2500                                 break;
2501                         }
2502                 }
2503
2504                 if (phy_data & 0x0004) {
2505                         nesadapter->mac_link_down[mac_index] = 0;
2506                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2507                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2508                                                 nesvnic->linkup);
2509                                 if (nesvnic->linkup == 0) {
2510                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2511                                                         nesvnic->netdev->name, nesvnic->netdev);
2512                                         if (netif_queue_stopped(nesvnic->netdev))
2513                                                 netif_start_queue(nesvnic->netdev);
2514                                         nesvnic->linkup = 1;
2515                                         netif_carrier_on(nesvnic->netdev);
2516                                 }
2517                         }
2518                 } else {
2519                         nesadapter->mac_link_down[mac_index] = 1;
2520                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2521                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2522                                                 nesvnic->linkup);
2523                                 if (nesvnic->linkup == 1) {
2524                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2525                                                         nesvnic->netdev->name, nesvnic->netdev);
2526                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2527                                                 netif_stop_queue(nesvnic->netdev);
2528                                         nesvnic->linkup = 0;
2529                                         netif_carrier_off(nesvnic->netdev);
2530                                 }
2531                         }
2532                 }
2533         }
2534
2535         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2536 }
2537
2538
2539
2540 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2541 {
2542         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2543
2544         netif_rx_schedule(&nesvnic->napi);
2545 }
2546
2547
2548 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2549 * getting out of nic_ce_handler
2550 */
2551 #define MAX_RQES_TO_PROCESS     384
2552
2553 /**
2554  * nes_nic_ce_handler
2555  */
2556 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2557 {
2558         u64 u64temp;
2559         dma_addr_t bus_address;
2560         struct nes_hw_nic *nesnic;
2561         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2562         struct nes_adapter *nesadapter = nesdev->nesadapter;
2563         struct nes_hw_nic_rq_wqe *nic_rqe;
2564         struct nes_hw_nic_sq_wqe *nic_sqe;
2565         struct sk_buff *skb;
2566         struct sk_buff *rx_skb;
2567         __le16 *wqe_fragment_length;
2568         u32 head;
2569         u32 cq_size;
2570         u32 rx_pkt_size;
2571         u32 cqe_count=0;
2572         u32 cqe_errv;
2573         u32 cqe_misc;
2574         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2575         u16 vlan_tag;
2576         u16 pkt_type;
2577         u16 rqes_processed = 0;
2578         u8 sq_cqes = 0;
2579         u8 nes_use_lro = 0;
2580
2581         head = cq->cq_head;
2582         cq_size = cq->cq_size;
2583         cq->cqes_pending = 1;
2584         if (nesvnic->netdev->features & NETIF_F_LRO)
2585                 nes_use_lro = 1;
2586         do {
2587                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2588                                 NES_NIC_CQE_VALID) {
2589                         nesnic = &nesvnic->nic;
2590                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2591                         if (cqe_misc & NES_NIC_CQE_SQ) {
2592                                 sq_cqes++;
2593                                 wqe_fragment_index = 1;
2594                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2595                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2596                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2597                                 /* bump past the vlan tag */
2598                                 wqe_fragment_length++;
2599                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2600                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2601                                                         wqe_fragment_index * 2]);
2602                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2603                                                         wqe_fragment_index * 2])) << 32;
2604                                         bus_address = (dma_addr_t)u64temp;
2605                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2606                                                 pci_unmap_single(nesdev->pcidev,
2607                                                                 bus_address,
2608                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2609                                                                 PCI_DMA_TODEVICE);
2610                                         }
2611                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2612                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2613                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2614                                                                                 wqe_fragment_index * 2]);
2615                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2616                                                                                 + wqe_fragment_index * 2])) <<32;
2617                                                         bus_address = (dma_addr_t)u64temp;
2618                                                         pci_unmap_page(nesdev->pcidev,
2619                                                                         bus_address,
2620                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2621                                                                         PCI_DMA_TODEVICE);
2622                                                 } else
2623                                                         break;
2624                                         }
2625                                         if (skb)
2626                                                 dev_kfree_skb_any(skb);
2627                                 }
2628                                 nesnic->sq_tail++;
2629                                 nesnic->sq_tail &= nesnic->sq_size-1;
2630                                 if (sq_cqes > 128) {
2631                                         barrier();
2632                                 /* restart the queue if it had been stopped */
2633                                 if (netif_queue_stopped(nesvnic->netdev))
2634                                         netif_wake_queue(nesvnic->netdev);
2635                                         sq_cqes = 0;
2636                                 }
2637                         } else {
2638                                 rqes_processed ++;
2639
2640                                 cq->rx_cqes_completed++;
2641                                 cq->rx_pkts_indicated++;
2642                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2643                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2644                                 /* Get the skb */
2645                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2646                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2647                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2648                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2649                                 pci_unmap_single(nesdev->pcidev, bus_address,
2650                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2651                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2652                                 /* rx_skb->len = rx_pkt_size; */
2653                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2654                                 skb_put(rx_skb, rx_pkt_size);
2655                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2656                                 nesnic->rq_tail++;
2657                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2658
2659                                 atomic_inc(&nesvnic->rx_skbs_needed);
2660                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2661                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2662                                                         cq->cq_number | (cqe_count << 16));
2663                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2664                                         nesdev->currcq_count += cqe_count;
2665                                         cqe_count = 0;
2666                                         nes_replenish_nic_rq(nesvnic);
2667                                 }
2668                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2669                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2670                                 rx_skb->ip_summed = CHECKSUM_NONE;
2671
2672                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2673                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2674                                         if ((cqe_errv &
2675                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2676                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2677                                                 if (nesvnic->rx_checksum_disabled == 0) {
2678                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2679                                                 }
2680                                         } else
2681                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2682                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2683                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2684
2685                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2686                                         if ((cqe_errv &
2687                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2688                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2689                                                 if (nesvnic->rx_checksum_disabled == 0) {
2690                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2691                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2692                                                                   nesvnic->netdev->name); */
2693                                                 }
2694                                         } else
2695                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2696                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2697                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2698                                         }
2699                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2700                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2701
2702                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2703                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2704                                                 rx_skb = NULL;
2705                                 }
2706                                 if (rx_skb == NULL)
2707                                         goto skip_rx_indicate0;
2708
2709
2710                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2711                                     (nesvnic->vlan_grp != NULL)) {
2712                                         vlan_tag = (u16)(le32_to_cpu(
2713                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2714                                                         >> 16);
2715                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2716                                                         nesvnic->netdev->name, vlan_tag);
2717                                         if (nes_use_lro)
2718                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2719                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2720                                         else
2721                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2722                                 } else {
2723                                         if (nes_use_lro)
2724                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2725                                         else
2726                                                 nes_netif_rx(rx_skb);
2727                                 }
2728
2729 skip_rx_indicate0:
2730                                 nesvnic->netdev->last_rx = jiffies;
2731                                 /* nesvnic->netstats.rx_packets++; */
2732                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2733                         }
2734
2735                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2736                         /* Accounting... */
2737                         cqe_count++;
2738                         if (++head >= cq_size)
2739                                 head = 0;
2740                         if (cqe_count == 255) {
2741                                 /* Replenish Nic CQ */
2742                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2743                                                 cq->cq_number | (cqe_count << 16));
2744                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2745                                 nesdev->currcq_count += cqe_count;
2746                                 cqe_count = 0;
2747                         }
2748
2749                         if (cq->rx_cqes_completed >= nesvnic->budget)
2750                                 break;
2751                 } else {
2752                         cq->cqes_pending = 0;
2753                         break;
2754                 }
2755
2756         } while (1);
2757
2758         if (nes_use_lro)
2759                 lro_flush_all(&nesvnic->lro_mgr);
2760         if (sq_cqes) {
2761                 barrier();
2762                 /* restart the queue if it had been stopped */
2763                 if (netif_queue_stopped(nesvnic->netdev))
2764                         netif_wake_queue(nesvnic->netdev);
2765         }
2766         cq->cq_head = head;
2767         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2768                         cq->cq_number, cqe_count, cq->cq_head); */
2769         cq->cqe_allocs_pending = cqe_count;
2770         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2771         {
2772                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2773                 nesdev->currcq_count += cqe_count;
2774                 nes_nic_tune_timer(nesdev);
2775         }
2776         if (atomic_read(&nesvnic->rx_skbs_needed))
2777                 nes_replenish_nic_rq(nesvnic);
2778 }
2779
2780
2781 /**
2782  * nes_cqp_ce_handler
2783  */
2784 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2785 {
2786         u64 u64temp;
2787         unsigned long flags;
2788         struct nes_hw_cqp *cqp = NULL;
2789         struct nes_cqp_request *cqp_request;
2790         struct nes_hw_cqp_wqe *cqp_wqe;
2791         u32 head;
2792         u32 cq_size;
2793         u32 cqe_count=0;
2794         u32 error_code;
2795         /* u32 counter; */
2796
2797         head = cq->cq_head;
2798         cq_size = cq->cq_size;
2799
2800         do {
2801                 /* process the CQE */
2802                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2803                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2804
2805                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2806                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2807                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2808                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2809                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2810                         cqp = *((struct nes_hw_cqp **)&u64temp);
2811
2812                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2813                         if (error_code) {
2814                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2815                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2816                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2817                                                 (u16)(error_code >> 16),
2818                                                 (u16)error_code);
2819                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2820                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2821                         }
2822
2823                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2824                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2825                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2826                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2827                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2828                         if (cqp_request) {
2829                                 if (cqp_request->waiting) {
2830                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2831                                         cqp_request->major_code = (u16)(error_code >> 16);
2832                                         cqp_request->minor_code = (u16)error_code;
2833                                         barrier();
2834                                         cqp_request->request_done = 1;
2835                                         wake_up(&cqp_request->waitq);
2836                                         nes_put_cqp_request(nesdev, cqp_request);
2837                                 } else {
2838                                         if (cqp_request->callback)
2839                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2840                                         nes_free_cqp_request(nesdev, cqp_request);
2841                                 }
2842                         } else {
2843                                 wake_up(&nesdev->cqp.waitq);
2844                         }
2845
2846                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2847                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2848                         if (++cqp->sq_tail >= cqp->sq_size)
2849                                 cqp->sq_tail = 0;
2850
2851                         /* Accounting... */
2852                         cqe_count++;
2853                         if (++head >= cq_size)
2854                                 head = 0;
2855                 } else {
2856                         break;
2857                 }
2858         } while (1);
2859         cq->cq_head = head;
2860
2861         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2862         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2863                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2864                         (nesdev->cqp.sq_size - 1)) != 1)) {
2865                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2866                                 struct nes_cqp_request, list);
2867                 list_del_init(&cqp_request->list);
2868                 head = nesdev->cqp.sq_head++;
2869                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2870                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2871                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2872                 barrier();
2873                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2874                         cpu_to_le32((u32)((unsigned long)cqp_request));
2875                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2876                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2877                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2878                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2879                 /* Ring doorbell (1 WQEs) */
2880                 barrier();
2881                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2882         }
2883         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2884
2885         /* Arm the CCQ */
2886         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2887                         cq->cq_number);
2888         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2889 }
2890
2891
2892 /**
2893  * nes_process_iwarp_aeqe
2894  */
2895 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
2896                                    struct nes_hw_aeqe *aeqe)
2897 {
2898         u64 context;
2899         u64 aeqe_context = 0;
2900         unsigned long flags;
2901         struct nes_qp *nesqp;
2902         int resource_allocated;
2903         /* struct iw_cm_id *cm_id; */
2904         struct nes_adapter *nesadapter = nesdev->nesadapter;
2905         struct ib_event ibevent;
2906         /* struct iw_cm_event cm_event; */
2907         u32 aeq_info;
2908         u32 next_iwarp_state = 0;
2909         u16 async_event_id;
2910         u8 tcp_state;
2911         u8 iwarp_state;
2912
2913         nes_debug(NES_DBG_AEQ, "\n");
2914         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2915         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2916                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2917                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2918         } else {
2919                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2920                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2921                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2922                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
2923                 BUG_ON(!context);
2924         }
2925
2926         async_event_id = (u16)aeq_info;
2927         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2928         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2929         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2930                         " Tcp state = %s, iWARP state = %s\n",
2931                         async_event_id,
2932                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2933                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2934
2935         switch (async_event_id) {
2936                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2937                         nesqp = *((struct nes_qp **)&context);
2938                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2939                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2940                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2941                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2942                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2943                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2944                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2945                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2946                                                 async_event_id, nesqp->last_aeq, tcp_state);
2947                         }
2948                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2949                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2950                                 /* FIN Received but tcp state or IB state moved on,
2951                                                 should expect a close complete */
2952                                 return;
2953                         }
2954                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2955                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2956                 case NES_AEQE_AEID_TERMINATE_SENT:
2957                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2958                 case NES_AEQE_AEID_RESET_SENT:
2959                         nesqp = *((struct nes_qp **)&context);
2960                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2961                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2962                         }
2963                         spin_lock_irqsave(&nesqp->lock, flags);
2964                         nesqp->hw_iwarp_state = iwarp_state;
2965                         nesqp->hw_tcp_state = tcp_state;
2966                         nesqp->last_aeq = async_event_id;
2967
2968                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2969                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2970                                 nesqp->hte_added = 0;
2971                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2972                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2973                                                 nesqp->hwqp.qp_id);
2974                                 nes_hw_modify_qp(nesdev, nesqp,
2975                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2976                                 spin_lock_irqsave(&nesqp->lock, flags);
2977                         }
2978
2979                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2980                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2981                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2982                                 switch (nesqp->hw_iwarp_state) {
2983                                         case NES_AEQE_IWARP_STATE_RTS:
2984                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2985                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2986                                                 break;
2987                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2988                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2989                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2990                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2991                                                         next_iwarp_state |= 0x02000000;
2992                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2993                                                 }
2994                                                 break;
2995                                         default:
2996                                                 next_iwarp_state = 0;
2997                                 }
2998                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2999                                 if (next_iwarp_state) {
3000                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3001                                                         " also added another reference\n",
3002                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3003                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3004                                 }
3005                                 nes_cm_disconn(nesqp);
3006                         } else {
3007                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3008                                         /* FIN Received but ib state not RTS,
3009                                                         close complete will be on its way */
3010                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3011                                         return;
3012                                 }
3013                                 spin_unlock_irqrestore(&nesqp->lock, flags);
3014                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
3015                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
3016                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3017                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
3018                                                         " also added another reference\n",
3019                                                         nesqp->hwqp.qp_id, next_iwarp_state);
3020                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3021                                 }
3022                                 nes_cm_disconn(nesqp);
3023                         }
3024                         break;
3025                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3026                         nesqp = *((struct nes_qp **)&context);
3027                         spin_lock_irqsave(&nesqp->lock, flags);
3028                         nesqp->hw_iwarp_state = iwarp_state;
3029                         nesqp->hw_tcp_state = tcp_state;
3030                         nesqp->last_aeq = async_event_id;
3031                         spin_unlock_irqrestore(&nesqp->lock, flags);
3032                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
3033                                         " event on QP%u \n  Q2 Data:\n",
3034                                         nesqp->hwqp.qp_id);
3035                         if (nesqp->ibqp.event_handler) {
3036                                 ibevent.device = nesqp->ibqp.device;
3037                                 ibevent.element.qp = &nesqp->ibqp;
3038                                 ibevent.event = IB_EVENT_QP_FATAL;
3039                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3040                         }
3041                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3042                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
3043                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3044                                 nes_cm_disconn(nesqp);
3045                         } else {
3046                                 nesqp->in_disconnect = 0;
3047                                 wake_up(&nesqp->kick_waitq);
3048                         }
3049                         break;
3050                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3051                         nesqp = *((struct nes_qp **)&context);
3052                         spin_lock_irqsave(&nesqp->lock, flags);
3053                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
3054                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3055                         nesqp->last_aeq = async_event_id;
3056                         if (nesqp->cm_id) {
3057                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3058                                                 " event on QP%u, remote IP = 0x%08X \n",
3059                                                 nesqp->hwqp.qp_id,
3060                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
3061                         } else {
3062                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
3063                                                 " event on QP%u \n",
3064                                                 nesqp->hwqp.qp_id);
3065                         }
3066                         spin_unlock_irqrestore(&nesqp->lock, flags);
3067                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
3068                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
3069                         if (nesqp->ibqp.event_handler) {
3070                                 ibevent.device = nesqp->ibqp.device;
3071                                 ibevent.element.qp = &nesqp->ibqp;
3072                                 ibevent.event = IB_EVENT_QP_FATAL;
3073                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3074                         }
3075                         break;
3076                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3077                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
3078                                 nesqp = nesadapter->qp_table[le32_to_cpu(
3079                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3080                         } else {
3081                                 /* TODO: get the actual WQE and mask off wqe index */
3082                                 context &= ~((u64)511);
3083                                 nesqp = *((struct nes_qp **)&context);
3084                         }
3085                         spin_lock_irqsave(&nesqp->lock, flags);
3086                         nesqp->hw_iwarp_state = iwarp_state;
3087                         nesqp->hw_tcp_state = tcp_state;
3088                         nesqp->last_aeq = async_event_id;
3089                         spin_unlock_irqrestore(&nesqp->lock, flags);
3090                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
3091                                         nesqp->hwqp.qp_id);
3092                         if (nesqp->ibqp.event_handler) {
3093                                 ibevent.device = nesqp->ibqp.device;
3094                                 ibevent.element.qp = &nesqp->ibqp;
3095                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3096                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3097                         }
3098                         break;
3099                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3100                         nesqp = *((struct nes_qp **)&context);
3101                         spin_lock_irqsave(&nesqp->lock, flags);
3102                         nesqp->hw_iwarp_state = iwarp_state;
3103                         nesqp->hw_tcp_state = tcp_state;
3104                         nesqp->last_aeq = async_event_id;
3105                         spin_unlock_irqrestore(&nesqp->lock, flags);
3106                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
3107                                         nesqp->hwqp.qp_id);
3108                         if (nesqp->ibqp.event_handler) {
3109                                 ibevent.device = nesqp->ibqp.device;
3110                                 ibevent.element.qp = &nesqp->ibqp;
3111                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3112                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3113                         }
3114                         break;
3115                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3116                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
3117                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3118                         spin_lock_irqsave(&nesqp->lock, flags);
3119                         nesqp->hw_iwarp_state = iwarp_state;
3120                         nesqp->hw_tcp_state = tcp_state;
3121                         nesqp->last_aeq = async_event_id;
3122                         spin_unlock_irqrestore(&nesqp->lock, flags);
3123                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
3124                                         " nesqp = %p, AE reported %p\n",
3125                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
3126                         if (nesqp->ibqp.event_handler) {
3127                                 ibevent.device = nesqp->ibqp.device;
3128                                 ibevent.element.qp = &nesqp->ibqp;
3129                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3130                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3131                         }
3132                         break;
3133                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3134                         context <<= 1;
3135                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3136                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3137                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3138                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3139                         if (resource_allocated) {
3140                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3141                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3142                         }
3143                         break;
3144                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3145                         nesqp = nesadapter->qp_table[le32_to_cpu(
3146                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
3147                         spin_lock_irqsave(&nesqp->lock, flags);
3148                         nesqp->hw_iwarp_state = iwarp_state;
3149                         nesqp->hw_tcp_state = tcp_state;
3150                         nesqp->last_aeq = async_event_id;
3151                         spin_unlock_irqrestore(&nesqp->lock, flags);
3152                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
3153                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
3154                                         nesqp->hwqp.qp_id);
3155                         if (nesqp->ibqp.event_handler) {
3156                                 ibevent.device = nesqp->ibqp.device;
3157                                 ibevent.element.qp = &nesqp->ibqp;
3158                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3159                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3160                         }
3161                         /* tell cm to disconnect, cm will queue work to thread */
3162                         nes_cm_disconn(nesqp);
3163                         break;
3164                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3165                         nesqp = *((struct nes_qp **)&context);
3166                         spin_lock_irqsave(&nesqp->lock, flags);
3167                         nesqp->hw_iwarp_state = iwarp_state;
3168                         nesqp->hw_tcp_state = tcp_state;
3169                         nesqp->last_aeq = async_event_id;
3170                         spin_unlock_irqrestore(&nesqp->lock, flags);
3171                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
3172                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
3173                                         nesqp->hwqp.qp_id);
3174                         if (nesqp->ibqp.event_handler) {
3175                                 ibevent.device = nesqp->ibqp.device;
3176                                 ibevent.element.qp = &nesqp->ibqp;
3177                                 ibevent.event = IB_EVENT_QP_FATAL;
3178                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3179                         }
3180                         /* tell cm to disconnect, cm will queue work to thread */
3181                         nes_cm_disconn(nesqp);
3182                         break;
3183                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3184                         nesqp = *((struct nes_qp **)&context);
3185                         spin_lock_irqsave(&nesqp->lock, flags);
3186                         nesqp->hw_iwarp_state = iwarp_state;
3187                         nesqp->hw_tcp_state = tcp_state;
3188                         nesqp->last_aeq = async_event_id;
3189                         spin_unlock_irqrestore(&nesqp->lock, flags);
3190                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
3191                                         " event on QP%u \n  Q2 Data:\n",
3192                                         nesqp->hwqp.qp_id);
3193                         if (nesqp->ibqp.event_handler) {
3194                                 ibevent.device = nesqp->ibqp.device;
3195                                 ibevent.element.qp = &nesqp->ibqp;
3196                                 ibevent.event = IB_EVENT_QP_FATAL;
3197                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
3198                         }
3199                         /* tell cm to disconnect, cm will queue work to thread */
3200                         nes_cm_disconn(nesqp);
3201                         break;
3202                         /* TODO: additional AEs need to be here */
3203                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3204                         nesqp = *((struct nes_qp **)&context);
3205                         spin_lock_irqsave(&nesqp->lock, flags);
3206                         nesqp->hw_iwarp_state = iwarp_state;
3207                         nesqp->hw_tcp_state = tcp_state;
3208                         nesqp->last_aeq = async_event_id;
3209                         spin_unlock_irqrestore(&nesqp->lock, flags);
3210                         if (nesqp->ibqp.event_handler) {
3211                                 ibevent.device = nesqp->ibqp.device;
3212                                 ibevent.element.qp = &nesqp->ibqp;
3213                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
3214                                 nesqp->ibqp.event_handler(&ibevent,
3215                                                 nesqp->ibqp.qp_context);
3216                         }
3217                         nes_cm_disconn(nesqp);
3218                         break;
3219                 default:
3220                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3221                                         async_event_id);
3222                         break;
3223         }
3224
3225 }
3226
3227
3228 /**
3229  * nes_iwarp_ce_handler
3230  */
3231 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3232 {
3233         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3234
3235         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3236                         nescq->hw_cq.cq_number); */
3237         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3238
3239         if (nescq->ibcq.comp_handler)
3240                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3241
3242         return;
3243 }
3244
3245
3246 /**
3247  * nes_manage_apbvt()
3248  */
3249 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3250                 u32 nic_index, u32 add_port)
3251 {
3252         struct nes_device *nesdev = nesvnic->nesdev;
3253         struct nes_hw_cqp_wqe *cqp_wqe;
3254         struct nes_cqp_request *cqp_request;
3255         int ret = 0;
3256         u16 major_code;
3257
3258         /* Send manage APBVT request to CQP */
3259         cqp_request = nes_get_cqp_request(nesdev);
3260         if (cqp_request == NULL) {
3261                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3262                 return -ENOMEM;
3263         }
3264         cqp_request->waiting = 1;
3265         cqp_wqe = &cqp_request->cqp_wqe;
3266
3267         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3268                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3269                         accel_local_port, accel_local_port, nic_index);
3270
3271         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3272         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3273                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3274         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3275                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3276
3277         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3278
3279         atomic_set(&cqp_request->refcount, 2);
3280         nes_post_cqp_request(nesdev, cqp_request);
3281
3282         if (add_port == NES_MANAGE_APBVT_ADD)
3283                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3284                                 NES_EVENT_TIMEOUT);
3285         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3286                         ret, cqp_request->major_code, cqp_request->minor_code);
3287         major_code = cqp_request->major_code;
3288
3289         nes_put_cqp_request(nesdev, cqp_request);
3290
3291         if (!ret)
3292                 return -ETIME;
3293         else if (major_code)
3294                 return -EIO;
3295         else
3296                 return 0;
3297 }
3298
3299
3300 /**
3301  * nes_manage_arp_cache
3302  */
3303 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3304                 u32 ip_addr, u32 action)
3305 {
3306         struct nes_hw_cqp_wqe *cqp_wqe;
3307         struct nes_vnic *nesvnic = netdev_priv(netdev);
3308         struct nes_device *nesdev;
3309         struct nes_cqp_request *cqp_request;
3310         int arp_index;
3311
3312         nesdev = nesvnic->nesdev;
3313         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3314         if (arp_index == -1) {
3315                 return;
3316         }
3317
3318         /* update the ARP entry */
3319         cqp_request = nes_get_cqp_request(nesdev);
3320         if (cqp_request == NULL) {
3321                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3322                 return;
3323         }
3324         cqp_request->waiting = 0;
3325         cqp_wqe = &cqp_request->cqp_wqe;
3326         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3327
3328         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3329                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3330         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3331                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3332         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3333
3334         if (action == NES_ARP_ADD) {
3335                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3336                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3337                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3338                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3339                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3340                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3341         } else {
3342                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3343                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3344         }
3345
3346         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3347                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3348
3349         atomic_set(&cqp_request->refcount, 1);
3350         nes_post_cqp_request(nesdev, cqp_request);
3351 }
3352
3353
3354 /**
3355  * flush_wqes
3356  */
3357 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3358                 u32 which_wq, u32 wait_completion)
3359 {
3360         struct nes_cqp_request *cqp_request;
3361         struct nes_hw_cqp_wqe *cqp_wqe;
3362         int ret;
3363
3364         cqp_request = nes_get_cqp_request(nesdev);
3365         if (cqp_request == NULL) {
3366                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3367                 return;
3368         }
3369         if (wait_completion) {
3370                 cqp_request->waiting = 1;
3371                 atomic_set(&cqp_request->refcount, 2);
3372         } else {
3373                 cqp_request->waiting = 0;
3374         }
3375         cqp_wqe = &cqp_request->cqp_wqe;
3376         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3377
3378         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3379                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3380         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3381
3382         nes_post_cqp_request(nesdev, cqp_request);
3383
3384         if (wait_completion) {
3385                 /* Wait for CQP */
3386                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3387                                 NES_EVENT_TIMEOUT);
3388                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3389                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3390                                 ret, cqp_request->major_code, cqp_request->minor_code);
3391                 nes_put_cqp_request(nesdev, cqp_request);
3392         }
3393 }