drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2008 NetEffect, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41
  42 #include "nes.h"
  43
  44 u32 crit_err_count = 0;
  45 u32 int_mod_timer_init;
  46 u32 int_mod_cq_depth_256;
  47 u32 int_mod_cq_depth_128;
  48 u32 int_mod_cq_depth_32;
  49 u32 int_mod_cq_depth_24;
  50 u32 int_mod_cq_depth_16;
  51 u32 int_mod_cq_depth_4;
  52 u32 int_mod_cq_depth_1;
  53
  54 #include "nes_cm.h"
  55
  56
  57 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  58 static unsigned char *nes_iwarp_state_str[] = {
  59         "Non-Existant",
  60         "Idle",
  61         "RTS",
  62         "Closing",
  63         "RSVD1",
  64         "Terminate",
  65         "Error",
  66         "RSVD2",
  67 };
  68
  69 static unsigned char *nes_tcp_state_str[] = {
  70         "Non-Existant",
  71         "Closed",
  72         "Listen",
  73         "SYN Sent",
  74         "SYN Rcvd",
  75         "Established",
  76         "Close Wait",
  77         "FIN Wait 1",
  78         "Closing",
  79         "Last Ack",
  80         "FIN Wait 2",
  81         "Time Wait",
  82         "RSVD1",
  83         "RSVD2",
  84         "RSVD3",
  85         "RSVD4",
  86 };
  87 #endif
  88
  89
  90 /**
  91  * nes_nic_init_timer_defaults
  92  */
  93 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
  94 {
  95         unsigned long flags;
  96         struct nes_adapter *nesadapter = nesdev->nesadapter;
  97         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
  98
  99         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 100
 101         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 102         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 103         if (jumbomode) {
 104                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 105                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 106                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 107         } else {
 108                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 109                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 110                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 111         }
 112
 113         /* todo use netdev->mtu to set thresholds */
 114         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 115 }
 116
 117
 118 /**
 119  * nes_nic_init_timer
 120  */
 121 static void  nes_nic_init_timer(struct nes_device *nesdev)
 122 {
 123         unsigned long flags;
 124         struct nes_adapter *nesadapter = nesdev->nesadapter;
 125         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 126
 127         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 128
 129         if (shared_timer->timer_in_use_old == 0) {
 130                 nesdev->deepcq_count = 0;
 131                 shared_timer->timer_direction_upward = 0;
 132                 shared_timer->timer_direction_downward = 0;
 133                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 134                 shared_timer->timer_in_use_old = 0;
 135
 136         }
 137         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 138                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 139                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 140                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 141         }
 142         /* todo use netdev->mtu to set thresholds */
 143         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 144 }
 145
 146
 147 /**
 148  * nes_nic_tune_timer
 149  */
 150 static void nes_nic_tune_timer(struct nes_device *nesdev)
 151 {
 152         unsigned long flags;
 153         struct nes_adapter *nesadapter = nesdev->nesadapter;
 154         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 155         u16 cq_count = nesdev->currcq_count;
 156
 157         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 158
 159         if (shared_timer->cq_count_old < cq_count) {
 160                 if (cq_count > shared_timer->threshold_low)
 161                         shared_timer->cq_direction_downward=0;
 162         }
 163         if (shared_timer->cq_count_old >= cq_count)
 164                 shared_timer->cq_direction_downward++;
 165         shared_timer->cq_count_old = cq_count;
 166         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 167                 if (cq_count <= shared_timer->threshold_low) {
 168                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 169                         shared_timer->cq_direction_downward=0;
 170                         nesdev->currcq_count = 0;
 171                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 172                         return;
 173                 }
 174         }
 175
 176         if (cq_count > 1) {
 177                 nesdev->deepcq_count += cq_count;
 178                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 179                         shared_timer->timer_direction_upward++;
 180                         shared_timer->timer_direction_downward = 0;
 181                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 182                         shared_timer->timer_direction_upward = 0;
 183                         shared_timer->timer_direction_downward = 0;
 184                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 185                         shared_timer->timer_direction_downward++;
 186                         shared_timer->timer_direction_upward = 0;
 187                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 188                         shared_timer->timer_in_use -= 2;
 189                         shared_timer->timer_direction_upward = 0;
 190                         shared_timer->timer_direction_downward++;
 191                 } else {
 192                         shared_timer->timer_in_use -= 4;
 193                         shared_timer->timer_direction_upward = 0;
 194                         shared_timer->timer_direction_downward++;
 195                 }
 196
 197                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 198                         shared_timer->timer_in_use += 3;
 199                         shared_timer->timer_direction_upward = 0;
 200                         shared_timer->timer_direction_downward = 0;
 201                 }
 202                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 203                         shared_timer->timer_in_use -= 4 ;
 204                         shared_timer->timer_direction_downward = 0;
 205                         shared_timer->timer_direction_upward = 0;
 206                 }
 207         }
 208
 209         /* boundary checking */
 210         if (shared_timer->timer_in_use > NES_NIC_FAST_TIMER_HIGH)
 211                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_HIGH;
 212         else if (shared_timer->timer_in_use < NES_NIC_FAST_TIMER_LOW) {
 213                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER_LOW;
 214         }
 215
 216         nesdev->currcq_count = 0;
 217
 218         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 219 }
 220
 221
 222 /**
 223  * nes_init_adapter - initialize adapter
 224  */
 225 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 226         struct nes_adapter *nesadapter = NULL;
 227         unsigned long num_pds;
 228         u32 u32temp;
 229         u32 port_count;
 230         u16 max_rq_wrs;
 231         u16 max_sq_wrs;
 232         u32 max_mr;
 233         u32 max_256pbl;
 234         u32 max_4kpbl;
 235         u32 max_qp;
 236         u32 max_irrq;
 237         u32 max_cq;
 238         u32 hte_index_mask;
 239         u32 adapter_size;
 240         u32 arp_table_size;
 241         u16 vendor_id;
 242         u8  OneG_Mode;
 243         u8  func_index;
 244
 245         /* search the list of existing adapters */
 246         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 247                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 248                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 249                                 nesdev->pcidev->devfn,
 250                                 PCI_SLOT(nesadapter->devfn),
 251                                 nesadapter->bus_number,
 252                                 PCI_SLOT(nesdev->pcidev->devfn),
 253                                 nesdev->pcidev->bus->number );
 254                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 255                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 256                         nesadapter->ref_count++;
 257                         return nesadapter;
 258                 }
 259         }
 260
 261         /* no adapter found */
 262         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 263         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 264                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 265                                 hw_rev);
 266                 return NULL;
 267         }
 268
 269         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 270                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 271                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 272                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 273                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 274
 275         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 276
 277
 278         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 279                 return NULL;
 280         if (nes_init_serdes(nesdev, hw_rev, port_count, OneG_Mode))
 281                 return NULL;
 282         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 283
 284         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 285         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 286
 287         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 288         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 289                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 290                                 max_qp, u32temp);
 291                 max_qp = (u32)1 << (u32temp & 0x001f);
 292         }
 293
 294         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 295         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 296                         max_qp, hte_index_mask);
 297
 298         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 299
 300         max_irrq = 1 << (u32temp & 0x001f);
 301
 302         if (max_qp > max_irrq) {
 303                 max_qp = max_irrq;
 304                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 305                                 max_qp);
 306         }
 307
 308         /* there should be no reason to allocate more pds than qps */
 309         if (num_pds > max_qp)
 310                 num_pds = max_qp;
 311
 312         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 313         max_mr = (u32)8192 << (u32temp & 0x7);
 314
 315         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 316         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 317         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 318         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 319
 320         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 321         arp_table_size = 1 << u32temp;
 322
 323         adapter_size = (sizeof(struct nes_adapter) +
 324                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 325         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 326         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 327         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 328         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 329         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 330         adapter_size += sizeof(struct nes_qp **) * max_qp;
 331
 332         /* allocate a new adapter struct */
 333         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 334         if (nesadapter == NULL) {
 335                 return NULL;
 336         }
 337
 338         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 339                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 340
 341         /* populate the new nesadapter */
 342         nesadapter->devfn = nesdev->pcidev->devfn;
 343         nesadapter->bus_number = nesdev->pcidev->bus->number;
 344         nesadapter->ref_count = 1;
 345         nesadapter->timer_int_req = 0xffff0000;
 346         nesadapter->OneG_Mode = OneG_Mode;
 347         nesadapter->doorbell_start = nesdev->doorbell_region;
 348
 349         /* nesadapter->tick_delta = clk_divisor; */
 350         nesadapter->hw_rev = hw_rev;
 351         nesadapter->port_count = port_count;
 352
 353         nesadapter->max_qp = max_qp;
 354         nesadapter->hte_index_mask = hte_index_mask;
 355         nesadapter->max_irrq = max_irrq;
 356         nesadapter->max_mr = max_mr;
 357         nesadapter->max_256pbl = max_256pbl - 1;
 358         nesadapter->max_4kpbl = max_4kpbl - 1;
 359         nesadapter->max_cq = max_cq;
 360         nesadapter->free_256pbl = max_256pbl - 1;
 361         nesadapter->free_4kpbl = max_4kpbl - 1;
 362         nesadapter->max_pd = num_pds;
 363         nesadapter->arp_table_size = arp_table_size;
 364
 365         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 366         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 367                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 368                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 369                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 370         } else {
 371                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 372                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 373                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 374                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __FUNCTION__);
 375         }
 376         /* Setup and enable the periodic timer */
 377         if (nesadapter->et_rx_coalesce_usecs_irq)
 378                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 379                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 380         else
 381                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 382
 383         nesadapter->base_pd = 1;
 384
 385         nesadapter->device_cap_flags =
 386                         IB_DEVICE_ZERO_STAG | IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW;
 387
 388         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 389                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 390         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 391         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 392         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 393         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 394         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 395
 396
 397         /* mark the usual suspect QPs and CQs as in use */
 398         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 399                 set_bit(u32temp, nesadapter->allocated_qps);
 400                 set_bit(u32temp, nesadapter->allocated_cqs);
 401         }
 402
 403         for (u32temp = 0; u32temp < 20; u32temp++)
 404                 set_bit(u32temp, nesadapter->allocated_pds);
 405         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 406
 407         max_rq_wrs = ((u32temp >> 8) & 3);
 408         switch (max_rq_wrs) {
 409                 case 0:
 410                         max_rq_wrs = 4;
 411                         break;
 412                 case 1:
 413                         max_rq_wrs = 16;
 414                         break;
 415                 case 2:
 416                         max_rq_wrs = 32;
 417                         break;
 418                 case 3:
 419                         max_rq_wrs = 512;
 420                         break;
 421         }
 422
 423         max_sq_wrs = (u32temp & 3);
 424         switch (max_sq_wrs) {
 425                 case 0:
 426                         max_sq_wrs = 4;
 427                         break;
 428                 case 1:
 429                         max_sq_wrs = 16;
 430                         break;
 431                 case 2:
 432                         max_sq_wrs = 32;
 433                         break;
 434                 case 3:
 435                         max_sq_wrs = 512;
 436                         break;
 437         }
 438         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 439         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 440
 441         nesadapter->max_sge = 4;
 442         nesadapter->max_cqe = 32767;
 443
 444         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 445                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 446                 kfree(nesadapter);
 447                 return NULL;
 448         }
 449
 450         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 451         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 452                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 453
 454         /* setup port configuration */
 455         if (nesadapter->port_count == 1) {
 456                 u32temp = 0x00000000;
 457                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 458                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 459                 else
 460                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 461         } else {
 462                 if (nesadapter->port_count == 2)
 463                         u32temp = 0x00000044;
 464                 else
 465                         u32temp = 0x000000e4;
 466                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 467         }
 468
 469         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, u32temp);
 470         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 471                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 472
 473         spin_lock_init(&nesadapter->resource_lock);
 474         spin_lock_init(&nesadapter->phy_lock);
 475         spin_lock_init(&nesadapter->pbl_lock);
 476         spin_lock_init(&nesadapter->periodic_timer_lock);
 477
 478         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 479         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 480         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 481         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 482
 483         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 484                 u32 pcs_control_status0, pcs_control_status1;
 485                 u32 reset_value;
 486                 u32 i = 0;
 487                 u32 int_cnt = 0;
 488                 u32 ext_cnt = 0;
 489                 unsigned long flags;
 490                 u32 j = 0;
 491
 492                 pcs_control_status0 = nes_read_indexed(nesdev,
 493                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 494                 pcs_control_status1 = nes_read_indexed(nesdev,
 495                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 496
 497                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 498                         pcs_control_status0 = nes_read_indexed(nesdev,
 499                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 500                         pcs_control_status1 = nes_read_indexed(nesdev,
 501                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 502                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 503                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 504                                 int_cnt++;
 505                         msleep(1);
 506                 }
 507                 if (int_cnt > 1) {
 508                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 509                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
 510                         mh_detected++;
 511                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 512                         reset_value |= 0x0000003d;
 513                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 514
 515                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 516                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 517                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 518
 519                         pcs_control_status0 = nes_read_indexed(nesdev,
 520                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 521                         pcs_control_status1 = nes_read_indexed(nesdev,
 522                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 523
 524                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 525                                 pcs_control_status0 = nes_read_indexed(nesdev,
 526                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 527                                 pcs_control_status1 = nes_read_indexed(nesdev,
 528                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 529                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 530                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 531                                         if (++ext_cnt > int_cnt) {
 532                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 533                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 534                                                                 0x0000F0C8);
 535                                                 mh_detected++;
 536                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 537                                                 reset_value |= 0x0000003d;
 538                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 539
 540                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 541                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 542                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 543                                                 break;
 544                                         }
 545                                 }
 546                                 msleep(1);
 547                         }
 548                 }
 549         }
 550
 551         if (nesadapter->hw_rev == NE020_REV) {
 552                 init_timer(&nesadapter->mh_timer);
 553                 nesadapter->mh_timer.function = nes_mh_fix;
 554                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 555                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 556                 add_timer(&nesadapter->mh_timer);
 557         } else {
 558                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 559         }
 560
 561         init_timer(&nesadapter->lc_timer);
 562         nesadapter->lc_timer.function = nes_clc;
 563         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 564         nesadapter->lc_timer.data = (unsigned long)nesdev;
 565         add_timer(&nesadapter->lc_timer);
 566
 567         list_add_tail(&nesadapter->list, &nes_adapter_list);
 568
 569         for (func_index = 0; func_index < 8; func_index++) {
 570                 pci_bus_read_config_word(nesdev->pcidev->bus,
 571                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 572                                         func_index), 0, &vendor_id);
 573                 if (vendor_id == 0xffff)
 574                         break;
 575         }
 576         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __FUNCTION__,
 577                 func_index, pci_name(nesdev->pcidev));
 578         nesadapter->adapter_fcn_count = func_index;
 579
 580         return nesadapter;
 581 }
 582
 583
 584 /**
 585  * nes_reset_adapter_ne020
 586  */
 587 unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 588 {
 589         u32 port_count;
 590         u32 u32temp;
 591         u32 i;
 592
 593         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 594         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 595         /* TODO: assuming that both SERDES are set the same for now */
 596         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 597         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 598                         u32temp, port_count);
 599         if (*OneG_Mode)
 600                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 601         u32temp &= 0xff00ffc0;
 602         switch (port_count) {
 603                 case 1:
 604                         u32temp |= 0x00ee0000;
 605                         break;
 606                 case 2:
 607                         u32temp |= 0x00cc0000;
 608                         break;
 609                 case 4:
 610                         u32temp |= 0x00000000;
 611                         break;
 612                 default:
 613                         return 0;
 614                         break;
 615         }
 616
 617         /* check and do full reset if needed */
 618         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 619                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 620                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 621
 622                 i = 0;
 623                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 624                         mdelay(1);
 625                 if (i >= 10000) {
 626                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 627                         return 0;
 628                 }
 629         }
 630
 631         /* port reset */
 632         switch (port_count) {
 633                 case 1:
 634                         u32temp |= 0x00ee0010;
 635                         break;
 636                 case 2:
 637                         u32temp |= 0x00cc0030;
 638                         break;
 639                 case 4:
 640                         u32temp |= 0x00000030;
 641                         break;
 642         }
 643
 644         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 645         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 646
 647         i = 0;
 648         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 649                 mdelay(1);
 650         if (i >= 10000) {
 651                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 652                 return 0;
 653         }
 654
 655         /* serdes 0 */
 656         i = 0;
 657         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 658                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 659                 mdelay(1);
 660         if (i >= 5000) {
 661                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 662                 return 0;
 663         }
 664
 665         /* serdes 1 */
 666         if (port_count > 1) {
 667                 i = 0;
 668                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 669                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 670                         mdelay(1);
 671                 if (i >= 5000) {
 672                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 673                         return 0;
 674                 }
 675         }
 676
 677
 678
 679         i = 0;
 680         while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                 mdelay(1);
 682         if (i >= 10000) {
 683                 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                 return 0;
 686         }
 687
 688         return port_count;
 689 }
 690
 691
 692 /**
 693  * nes_init_serdes
 694  */
 695 int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, u8  OneG_Mode)
 696 {
 697         int i;
 698         u32 u32temp;
 699
 700         if (hw_rev != NE020_REV) {
 701                 /* init serdes 0 */
 702
 703                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 704                 if (!OneG_Mode)
 705                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 706                 if (port_count > 1) {
 707                         /* init serdes 1 */
 708                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 709                         if (!OneG_Mode)
 710                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 711                         }
 712         } else {
 713                 /* init serdes 0 */
 714                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 715                 i = 0;
 716                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 717                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 718                         mdelay(1);
 719                 if (i >= 5000) {
 720                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 721                         return 1;
 722                 }
 723                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 724                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 725                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 726                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 727                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 728                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 729                 if (OneG_Mode)
 730                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 731                 else
 732                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 733
 734                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 735                 if (port_count > 1) {
 736                         /* init serdes 1 */
 737                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 738                         i = 0;
 739                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 740                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 741                                 mdelay(1);
 742                         if (i >= 5000) {
 743                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __FUNCTION__, u32temp);
 744                                 /* return 1; */
 745                         }
 746                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 747                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 748                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 749                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 750                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 751                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 753                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 754                 }
 755         }
 756         return 0;
 757 }
 758
 759
 760 /**
 761  * nes_init_csr_ne020
 762  * Initialize registers for ne020 hardware
 763  */
 764 void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 765 {
 766         u32 u32temp;
 767
 768         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 769
 770         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 771         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 772         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 773         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 774         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 775         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 776         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 777         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 778
 779         /* TODO: move these MAC register settings to NIC bringup */
 780         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 781         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 782         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 783         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 784         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 785         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 786         if (port_count > 1) {
 787                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 788                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 789                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 790                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 791                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 792                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 793                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 794         }
 795         if (port_count > 2) {
 796                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 797                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 798                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 799                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 800                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 801                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 802                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 803
 804                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 805                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 806                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 807                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 808                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 809                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 810                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 811         }
 812
 813         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 814         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 815         nes_write_indexed(nesdev, 0x00005004, 0x00020001);
 816         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 817         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 818         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 819         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 820         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 821
 822         /* TODO: move this to code, get from EEPROM */
 823         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 824         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 825         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 826                                                                                                                 //
 827         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 828         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 829
 830         if (hw_rev != NE020_REV) {
 831                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 832                 u32temp |= 0x80000000;
 833                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 834                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 835                 u32temp &= 0x7fffffff;
 836                 u32temp |= 0x7fff0010;
 837                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 838         }
 839 }
 840
 841
 842 /**
 843  * nes_destroy_adapter - destroy the adapter structure
 844  */
 845 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 846 {
 847         struct nes_adapter *tmp_adapter;
 848
 849         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 850                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 851                                 tmp_adapter);
 852         }
 853
 854         nesadapter->ref_count--;
 855         if (!nesadapter->ref_count) {
 856                 if (nesadapter->hw_rev == NE020_REV) {
 857                         del_timer(&nesadapter->mh_timer);
 858                 }
 859                 del_timer(&nesadapter->lc_timer);
 860
 861                 list_del(&nesadapter->list);
 862                 kfree(nesadapter);
 863         }
 864 }
 865
 866
 867 /**
 868  * nes_init_cqp
 869  */
 870 int nes_init_cqp(struct nes_device *nesdev)
 871 {
 872         struct nes_adapter *nesadapter = nesdev->nesadapter;
 873         struct nes_hw_cqp_qp_context *cqp_qp_context;
 874         struct nes_hw_cqp_wqe *cqp_wqe;
 875         struct nes_hw_ceq *ceq;
 876         struct nes_hw_ceq *nic_ceq;
 877         struct nes_hw_aeq *aeq;
 878         void *vmem;
 879         dma_addr_t pmem;
 880         u32 count=0;
 881         u32 cqp_head;
 882         u64 u64temp;
 883         u32 u32temp;
 884
 885         /* allocate CQP memory */
 886         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 887         /* SQ is 512 byte aligned, others are 256 byte aligned */
 888         nesdev->cqp_mem_size = 512 +
 889                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 890                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 891                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 892                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 893                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 894                         sizeof(struct nes_hw_cqp_qp_context);
 895
 896         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 897                         &nesdev->cqp_pbase);
 898         if (!nesdev->cqp_vbase) {
 899                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 900                 return -ENOMEM;
 901         }
 902         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 903
 904         /* Allocate a twice the number of CQP requests as the SQ size */
 905         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 906                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 907         if (nesdev->nes_cqp_requests == NULL) {
 908                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 909                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 910                                 nesdev->cqp.sq_pbase);
 911                 return -ENOMEM;
 912         }
 913
 914         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
 915                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
 916
 917         spin_lock_init(&nesdev->cqp.lock);
 918         init_waitqueue_head(&nesdev->cqp.waitq);
 919
 920         /* Setup Various Structures */
 921         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
 922                         ~(unsigned long)(512 - 1));
 923         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
 924                         ~(unsigned long long)(512 - 1));
 925
 926         nesdev->cqp.sq_vbase = vmem;
 927         nesdev->cqp.sq_pbase = pmem;
 928         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
 929         nesdev->cqp.sq_head = 0;
 930         nesdev->cqp.sq_tail = 0;
 931         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
 932
 933         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 934         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
 935
 936         nesdev->ccq.cq_vbase = vmem;
 937         nesdev->ccq.cq_pbase = pmem;
 938         nesdev->ccq.cq_size = NES_CCQ_SIZE;
 939         nesdev->ccq.cq_head = 0;
 940         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
 941         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
 942
 943         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 944         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
 945
 946         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
 947         ceq = &nesadapter->ceq[nesdev->ceq_index];
 948         ceq->ceq_vbase = vmem;
 949         ceq->ceq_pbase = pmem;
 950         ceq->ceq_size = NES_CCEQ_SIZE;
 951         ceq->ceq_head = 0;
 952
 953         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 954         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
 955
 956         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
 957         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
 958         nic_ceq->ceq_vbase = vmem;
 959         nic_ceq->ceq_pbase = pmem;
 960         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
 961         nic_ceq->ceq_head = 0;
 962
 963         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 964         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
 965
 966         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
 967         aeq->aeq_vbase = vmem;
 968         aeq->aeq_pbase = pmem;
 969         aeq->aeq_size = nesadapter->max_qp;
 970         aeq->aeq_head = 0;
 971
 972         /* Setup QP Context */
 973         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 974         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
 975
 976         cqp_qp_context = vmem;
 977         cqp_qp_context->context_words[0] =
 978                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
 979         cqp_qp_context->context_words[1] = 0;
 980         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
 981         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
 982
 983
 984         /* Write the address to Create CQP */
 985         if ((sizeof(dma_addr_t) > 4)) {
 986                 nes_write_indexed(nesdev,
 987                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 988                                 ((u64)pmem) >> 32);
 989         } else {
 990                 nes_write_indexed(nesdev,
 991                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
 992         }
 993         nes_write_indexed(nesdev,
 994                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
 995                         (u32)pmem);
 996
 997         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
 998         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
 999
1000         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1001                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1002                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1003         }
1004
1005         /* Write Create CCQ WQE */
1006         cqp_head = nesdev->cqp.sq_head++;
1007         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1008         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1009         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1010                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1011                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1012         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1013                             (nesdev->ccq.cq_number |
1014                              ((u32)nesdev->ceq_index << 16)));
1015         u64temp = (u64)nesdev->ccq.cq_pbase;
1016         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1017         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1018         u64temp = (unsigned long)&nesdev->ccq;
1019         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1020                         cpu_to_le32((u32)(u64temp >> 1));
1021         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1022                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1023         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1024
1025         /* Write Create CEQ WQE */
1026         cqp_head = nesdev->cqp.sq_head++;
1027         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1028         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1029         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1030                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1031         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1032         u64temp = (u64)ceq->ceq_pbase;
1033         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1034
1035         /* Write Create AEQ WQE */
1036         cqp_head = nesdev->cqp.sq_head++;
1037         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1038         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1039         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1040                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1041         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1042         u64temp = (u64)aeq->aeq_pbase;
1043         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1044
1045         /* Write Create NIC CEQ WQE */
1046         cqp_head = nesdev->cqp.sq_head++;
1047         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1048         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1049         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1050                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1051         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1052         u64temp = (u64)nic_ceq->ceq_pbase;
1053         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1054
1055         /* Poll until CCQP done */
1056         count = 0;
1057         do {
1058                 if (count++ > 1000) {
1059                         printk(KERN_ERR PFX "Error creating CQP\n");
1060                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1061                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1062                         return -1;
1063                 }
1064                 udelay(10);
1065         } while (!(nes_read_indexed(nesdev,
1066                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1067
1068         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1069                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1070
1071         u32temp = 0x04800000;
1072         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1073
1074         /* wait for the CCQ, CEQ, and AEQ to get created */
1075         count = 0;
1076         do {
1077                 if (count++ > 1000) {
1078                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1079                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1080                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1081                         return -1;
1082                 }
1083                 udelay(10);
1084         } while (((nes_read_indexed(nesdev,
1085                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1086
1087         /* dump the QP status value */
1088         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1089                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1090
1091         nesdev->cqp.sq_tail++;
1092
1093         return 0;
1094 }
1095
1096
1097 /**
1098  * nes_destroy_cqp
1099  */
1100 int nes_destroy_cqp(struct nes_device *nesdev)
1101 {
1102         struct nes_hw_cqp_wqe *cqp_wqe;
1103         u32 count = 0;
1104         u32 cqp_head;
1105         unsigned long flags;
1106
1107         do {
1108                 if (count++ > 1000)
1109                         break;
1110                 udelay(10);
1111         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1112
1113         /* Reset CCQ */
1114         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1115                         nesdev->ccq.cq_number);
1116
1117         /* Disable device interrupts */
1118         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1119
1120         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1121
1122         /* Destroy the AEQ */
1123         cqp_head = nesdev->cqp.sq_head++;
1124         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1125         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1126         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1127                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1128         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1129
1130         /* Destroy the NIC CEQ */
1131         cqp_head = nesdev->cqp.sq_head++;
1132         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1133         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1134         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1135                         ((u32)nesdev->nic_ceq_index << 8));
1136
1137         /* Destroy the CEQ */
1138         cqp_head = nesdev->cqp.sq_head++;
1139         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1140         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1141         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1142                         (nesdev->ceq_index << 8));
1143
1144         /* Destroy the CCQ */
1145         cqp_head = nesdev->cqp.sq_head++;
1146         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1147         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1148         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1149         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1150                         ((u32)nesdev->ceq_index << 16));
1151
1152         /* Destroy CQP */
1153         cqp_head = nesdev->cqp.sq_head++;
1154         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1155         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1156         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1157                         NES_CQP_QP_TYPE_CQP);
1158         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1159
1160         barrier();
1161         /* Ring doorbell (5 WQEs) */
1162         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1163
1164         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1165
1166         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1167         count = 0;
1168         do {
1169                 if (count++ > 1000) {
1170                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1171                                         PCI_FUNC(nesdev->pcidev->devfn));
1172                         break;
1173                 }
1174                 udelay(10);
1175         } while (((nes_read_indexed(nesdev,
1176                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1177
1178         /* dump the QP status value */
1179         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1180                         PCI_FUNC(nesdev->pcidev->devfn),
1181                         nes_read_indexed(nesdev,
1182                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1183
1184         kfree(nesdev->nes_cqp_requests);
1185
1186         /* Free the control structures */
1187         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1188                         nesdev->cqp.sq_pbase);
1189
1190         return 0;
1191 }
1192
1193
1194 /**
1195  * nes_init_phy
1196  */
1197 int nes_init_phy(struct nes_device *nesdev)
1198 {
1199         struct nes_adapter *nesadapter = nesdev->nesadapter;
1200         u32 counter = 0;
1201         u32 mac_index = nesdev->mac_index;
1202         u32 tx_config;
1203         u16 phy_data;
1204
1205         if (nesadapter->OneG_Mode) {
1206                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1207                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_1G) {
1208                         printk(PFX "%s: Programming mdc config for 1G\n", __FUNCTION__);
1209                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1210                         tx_config |= 0x04;
1211                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1212                 }
1213
1214                 nes_read_1G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index], &phy_data);
1215                 nes_debug(NES_DBG_PHY, "Phy data from register 1 phy address %u = 0x%X.\n",
1216                                 nesadapter->phy_index[mac_index], phy_data);
1217                 nes_write_1G_phy_reg(nesdev, 23, nesadapter->phy_index[mac_index],  0xb000);
1218
1219                 /* Reset the PHY */
1220                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], 0x8000);
1221                 udelay(100);
1222                 counter = 0;
1223                 do {
1224                         nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1225                         nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1226                         if (counter++ > 100) break;
1227                 } while (phy_data & 0x8000);
1228
1229                 /* Setting no phy loopback */
1230                 phy_data &= 0xbfff;
1231                 phy_data |= 0x1140;
1232                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index],  phy_data);
1233                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1234                 nes_debug(NES_DBG_PHY, "Phy data from register 0 = 0x%X.\n", phy_data);
1235
1236                 nes_read_1G_phy_reg(nesdev, 0x17, nesadapter->phy_index[mac_index], &phy_data);
1237                 nes_debug(NES_DBG_PHY, "Phy data from register 0x17 = 0x%X.\n", phy_data);
1238
1239                 nes_read_1G_phy_reg(nesdev, 0x1e, nesadapter->phy_index[mac_index], &phy_data);
1240                 nes_debug(NES_DBG_PHY, "Phy data from register 0x1e = 0x%X.\n", phy_data);
1241
1242                 /* Setting the interrupt mask */
1243                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1244                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1245                 nes_write_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], 0xffee);
1246
1247                 nes_read_1G_phy_reg(nesdev, 0x19, nesadapter->phy_index[mac_index], &phy_data);
1248                 nes_debug(NES_DBG_PHY, "Phy data from register 0x19 = 0x%X.\n", phy_data);
1249
1250                 /* turning on flow control */
1251                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1252                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1253                 nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1254                                 (phy_data & ~(0x03E0)) | 0xc00);
1255                 /* nes_write_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index],
1256                                 phy_data | 0xc00); */
1257                 nes_read_1G_phy_reg(nesdev, 4, nesadapter->phy_index[mac_index], &phy_data);
1258                 nes_debug(NES_DBG_PHY, "Phy data from register 0x4 = 0x%X.\n", phy_data);
1259
1260                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1261                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1262                 /* Clear Half duplex */
1263                 nes_write_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index],
1264                                 phy_data & ~(0x0100));
1265                 nes_read_1G_phy_reg(nesdev, 9, nesadapter->phy_index[mac_index], &phy_data);
1266                 nes_debug(NES_DBG_PHY, "Phy data from register 0x9 = 0x%X.\n", phy_data);
1267
1268                 nes_read_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], &phy_data);
1269                 nes_write_1G_phy_reg(nesdev, 0, nesadapter->phy_index[mac_index], phy_data | 0x0300);
1270         } else {
1271                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
1272                         /* setup 10G MDIO operation */
1273                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1274                         tx_config |= 0x14;
1275                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1276                 }
1277         }
1278         return 0;
1279 }
1280
1281
1282 /**
1283  * nes_replenish_nic_rq
1284  */
1285 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1286 {
1287         unsigned long flags;
1288         dma_addr_t bus_address;
1289         struct sk_buff *skb;
1290         struct nes_hw_nic_rq_wqe *nic_rqe;
1291         struct nes_hw_nic *nesnic;
1292         struct nes_device *nesdev;
1293         u32 rx_wqes_posted = 0;
1294
1295         nesnic = &nesvnic->nic;
1296         nesdev = nesvnic->nesdev;
1297         spin_lock_irqsave(&nesnic->rq_lock, flags);
1298         if (nesnic->replenishing_rq !=0) {
1299                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1300                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1301                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1302                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1303                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1304                         add_timer(&nesvnic->rq_wqes_timer);
1305                 } else
1306                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1307                 return;
1308         }
1309         nesnic->replenishing_rq = 1;
1310         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1311         do {
1312                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1313                 if (skb) {
1314                         skb->dev = nesvnic->netdev;
1315
1316                         bus_address = pci_map_single(nesdev->pcidev,
1317                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1318
1319                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1320                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1321                                         cpu_to_le32(nesvnic->max_frame_size);
1322                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1323                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1324                                         cpu_to_le32((u32)bus_address);
1325                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1326                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1327                         nesnic->rx_skb[nesnic->rq_head] = skb;
1328                         nesnic->rq_head++;
1329                         nesnic->rq_head &= nesnic->rq_size - 1;
1330                         atomic_dec(&nesvnic->rx_skbs_needed);
1331                         barrier();
1332                         if (++rx_wqes_posted == 255) {
1333                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1334                                 rx_wqes_posted = 0;
1335                         }
1336                 } else {
1337                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1338                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1339                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1340                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1341                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1342                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1343                                 add_timer(&nesvnic->rq_wqes_timer);
1344                         } else
1345                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1346                         break;
1347                 }
1348         } while (atomic_read(&nesvnic->rx_skbs_needed));
1349         barrier();
1350         if (rx_wqes_posted)
1351                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1352         nesnic->replenishing_rq = 0;
1353 }
1354
1355
1356 /**
1357  * nes_rq_wqes_timeout
1358  */
1359 static void nes_rq_wqes_timeout(unsigned long parm)
1360 {
1361         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1362         printk("%s: Timer fired.\n", __FUNCTION__);
1363         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1364         if (atomic_read(&nesvnic->rx_skbs_needed))
1365                 nes_replenish_nic_rq(nesvnic);
1366 }
1367
1368
1369 /**
1370  * nes_init_nic_qp
1371  */
1372 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1373 {
1374         struct nes_hw_cqp_wqe *cqp_wqe;
1375         struct nes_hw_nic_sq_wqe *nic_sqe;
1376         struct nes_hw_nic_qp_context *nic_context;
1377         struct sk_buff *skb;
1378         struct nes_hw_nic_rq_wqe *nic_rqe;
1379         struct nes_vnic *nesvnic = netdev_priv(netdev);
1380         unsigned long flags;
1381         void *vmem;
1382         dma_addr_t pmem;
1383         u64 u64temp;
1384         int ret;
1385         u32 cqp_head;
1386         u32 counter;
1387         u32 wqe_count;
1388         u8 jumbomode=0;
1389
1390         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1391         nesvnic->nic_mem_size = 256 +
1392                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1393                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1394                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1395                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1396                         sizeof(struct nes_hw_nic_qp_context);
1397
1398         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1399                         &nesvnic->nic_pbase);
1400         if (!nesvnic->nic_vbase) {
1401                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1402                 return -ENOMEM;
1403         }
1404         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1405         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1406                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1407
1408         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1409                         ~(unsigned long)(256 - 1));
1410         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1411                         ~(unsigned long long)(256 - 1));
1412
1413         /* Setup the first Fragment buffers */
1414         nesvnic->nic.first_frag_vbase = vmem;
1415
1416         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1417                 nesvnic->nic.frag_paddr[counter] = pmem;
1418                 pmem += sizeof(struct nes_first_frag);
1419         }
1420
1421         /* setup the SQ */
1422         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1423
1424         nesvnic->nic.sq_vbase = (void *)vmem;
1425         nesvnic->nic.sq_pbase = pmem;
1426         nesvnic->nic.sq_head = 0;
1427         nesvnic->nic.sq_tail = 0;
1428         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1429         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1430                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1431                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1432                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1433                                 NES_NIC_SQ_WQE_COMPLETION);
1434                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1435                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1436                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1437                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1438                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1439                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1440         }
1441
1442         nesvnic->get_cqp_request = nes_get_cqp_request;
1443         nesvnic->post_cqp_request = nes_post_cqp_request;
1444         nesvnic->mcrq_mcast_filter = NULL;
1445
1446         spin_lock_init(&nesvnic->nic.sq_lock);
1447         spin_lock_init(&nesvnic->nic.rq_lock);
1448
1449         /* setup the RQ */
1450         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1451         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1452
1453
1454         nesvnic->nic.rq_vbase = vmem;
1455         nesvnic->nic.rq_pbase = pmem;
1456         nesvnic->nic.rq_head = 0;
1457         nesvnic->nic.rq_tail = 0;
1458         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1459
1460         /* setup the CQ */
1461         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1462         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1463
1464         if (nesdev->nesadapter->netdev_count > 2)
1465                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1466         else
1467                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1468
1469         nesvnic->nic_cq.cq_vbase = vmem;
1470         nesvnic->nic_cq.cq_pbase = pmem;
1471         nesvnic->nic_cq.cq_head = 0;
1472         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1473
1474         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1475
1476         /* Send CreateCQ request to CQP */
1477         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1478         cqp_head = nesdev->cqp.sq_head;
1479
1480         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1481         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1482
1483         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1484                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1485                         ((u32)nesvnic->nic_cq.cq_size << 16));
1486         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1487                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1488         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1489         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1490         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1491         u64temp = (unsigned long)&nesvnic->nic_cq;
1492         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1493         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1494                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1495         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1496         if (++cqp_head >= nesdev->cqp.sq_size)
1497                 cqp_head = 0;
1498         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1499         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1500
1501         /* Send CreateQP request to CQP */
1502         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1503         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1504                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1505                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1506         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1507                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1508                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1509         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1510                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1511         }
1512
1513         u64temp = (u64)nesvnic->nic.sq_pbase;
1514         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1515         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1516         u64temp = (u64)nesvnic->nic.rq_pbase;
1517         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1518         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1519
1520         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1521                         NES_CQP_QP_TYPE_NIC);
1522         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1523         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1524                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1525         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1526
1527         if (++cqp_head >= nesdev->cqp.sq_size)
1528                 cqp_head = 0;
1529         nesdev->cqp.sq_head = cqp_head;
1530
1531         barrier();
1532
1533         /* Ring doorbell (2 WQEs) */
1534         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1535
1536         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1537         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1538                         nesvnic->nic.qp_id);
1539
1540         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1541                         NES_EVENT_TIMEOUT);
1542         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1543                         nesvnic->nic.qp_id, ret);
1544         if (!ret) {
1545                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1546                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1547                                 nesvnic->nic_pbase);
1548                 return -EIO;
1549         }
1550
1551         /* Populate the RQ */
1552         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1553                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1554                 if (!skb) {
1555                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1556
1557                         nes_destroy_nic_qp(nesvnic);
1558                         return -ENOMEM;
1559                 }
1560
1561                 skb->dev = netdev;
1562
1563                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1564                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1565
1566                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1567                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1568                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1569                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1570                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1571                 nesvnic->nic.rx_skb[counter] = skb;
1572         }
1573
1574         wqe_count = NES_NIC_WQ_SIZE - 1;
1575         nesvnic->nic.rq_head = wqe_count;
1576         barrier();
1577         do {
1578                 counter = min(wqe_count, ((u32)255));
1579                 wqe_count -= counter;
1580                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1581         } while (wqe_count);
1582         init_timer(&nesvnic->rq_wqes_timer);
1583         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1584         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1585         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1586
1587         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1588         {
1589                 nes_nic_init_timer(nesdev);
1590                 if (netdev->mtu > 1500)
1591                         jumbomode = 1;
1592                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1593         }
1594
1595         return 0;
1596 }
1597
1598
1599 /**
1600  * nes_destroy_nic_qp
1601  */
1602 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1603 {
1604         struct nes_device *nesdev = nesvnic->nesdev;
1605         struct nes_hw_cqp_wqe *cqp_wqe;
1606         struct nes_hw_nic_rq_wqe *nic_rqe;
1607         u64 wqe_frag;
1608         u32 cqp_head;
1609         unsigned long flags;
1610         int ret;
1611
1612         /* Free remaining NIC receive buffers */
1613         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1614                 nic_rqe = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1615                 wqe_frag = (u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1616                 wqe_frag |= ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
1617                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1618                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1619                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1620                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1621         }
1622
1623         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1624
1625         /* Destroy NIC QP */
1626         cqp_head = nesdev->cqp.sq_head;
1627         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1628         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1629
1630         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1631                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1632         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1633                 nesvnic->nic.qp_id);
1634
1635         if (++cqp_head >= nesdev->cqp.sq_size)
1636                 cqp_head = 0;
1637
1638         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1639
1640         /* Destroy NIC CQ */
1641         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1642         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1643                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1644         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1645                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1646
1647         if (++cqp_head >= nesdev->cqp.sq_size)
1648                 cqp_head = 0;
1649
1650         nesdev->cqp.sq_head = cqp_head;
1651         barrier();
1652
1653         /* Ring doorbell (2 WQEs) */
1654         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1655
1656         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1657         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1658                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1659                         cqp_head, nesdev->cqp.sq_head,
1660                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1661
1662         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1663                         NES_EVENT_TIMEOUT);
1664
1665         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1666                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1667                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1668         if (!ret) {
1669                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1670                                 nesvnic->nic.qp_id);
1671         }
1672
1673         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1674                         nesvnic->nic_pbase);
1675 }
1676
1677 /**
1678  * nes_napi_isr
1679  */
1680 int nes_napi_isr(struct nes_device *nesdev)
1681 {
1682         struct nes_adapter *nesadapter = nesdev->nesadapter;
1683         u32 int_stat;
1684
1685         if (nesdev->napi_isr_ran) {
1686                 /* interrupt status has already been read in ISR */
1687                 int_stat = nesdev->int_stat;
1688         } else {
1689                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1690                 nesdev->int_stat = int_stat;
1691                 nesdev->napi_isr_ran = 1;
1692         }
1693
1694         int_stat &= nesdev->int_req;
1695         /* iff NIC, process here, else wait for DPC */
1696         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1697                 nesdev->napi_isr_ran = 0;
1698                 nes_write32(nesdev->regs+NES_INT_STAT,
1699                                 (int_stat &
1700                                 ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1701
1702                 /* Process the CEQs */
1703                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1704
1705                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1706                                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1707                                           ((nesadapter->et_use_adaptive_rx_coalesce) &&
1708                                            (nesdev->deepcq_count > nesadapter->et_pkt_rate_low)))) ) {
1709                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1710                                 /* Enable Periodic timer interrupts */
1711                                 nesdev->int_req |= NES_INT_TIMER;
1712                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1713                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1714                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1715                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1716                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1717                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1718                         }
1719
1720                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1721                         {
1722                                 nes_nic_init_timer(nesdev);
1723                         }
1724                         /* Enable interrupts, except CEQs */
1725                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1726                 } else {
1727                         /* Enable interrupts, make sure timer is off */
1728                         nesdev->int_req &= ~NES_INT_TIMER;
1729                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1730                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1731                         nesadapter->tune_timer.timer_in_use_old = 0;
1732                 }
1733                 nesdev->deepcq_count = 0;
1734                 return 1;
1735         } else {
1736                 return 0;
1737         }
1738 }
1739
1740
1741 /**
1742  * nes_dpc
1743  */
1744 void nes_dpc(unsigned long param)
1745 {
1746         struct nes_device *nesdev = (struct nes_device *)param;
1747         struct nes_adapter *nesadapter = nesdev->nesadapter;
1748         u32 counter;
1749         u32 loop_counter = 0;
1750         u32 int_status_bit;
1751         u32 int_stat;
1752         u32 timer_stat;
1753         u32 temp_int_stat;
1754         u32 intf_int_stat;
1755         u32 debug_error;
1756         u32 processed_intf_int = 0;
1757         u16 processed_timer_int = 0;
1758         u16 completion_ints = 0;
1759         u16 timer_ints = 0;
1760
1761         /* nes_debug(NES_DBG_ISR, "\n"); */
1762
1763         do {
1764                 timer_stat = 0;
1765                 if (nesdev->napi_isr_ran) {
1766                         nesdev->napi_isr_ran = 0;
1767                         int_stat = nesdev->int_stat;
1768                 } else
1769                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
1770                 if (processed_intf_int != 0)
1771                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
1772                 else
1773                         int_stat &= nesdev->int_req;
1774                 if (processed_timer_int == 0) {
1775                         processed_timer_int = 1;
1776                         if (int_stat & NES_INT_TIMER) {
1777                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
1778                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
1779                                         int_stat &= ~NES_INT_TIMER;
1780                                 }
1781                         }
1782                 } else {
1783                         int_stat &= ~NES_INT_TIMER;
1784                 }
1785
1786                 if (int_stat) {
1787                         if (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1788                                         NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)) {
1789                                 /* Ack the interrupts */
1790                                 nes_write32(nesdev->regs+NES_INT_STAT,
1791                                                 (int_stat & ~(NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1792                                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3)));
1793                         }
1794
1795                         temp_int_stat = int_stat;
1796                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
1797                                 if (int_stat & int_status_bit) {
1798                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
1799                                         temp_int_stat &= ~int_status_bit;
1800                                         completion_ints = 1;
1801                                 }
1802                                 if (!(temp_int_stat & 0x0000ffff))
1803                                         break;
1804                                 int_status_bit <<= 1;
1805                         }
1806
1807                         /* Process the AEQ for this pci function */
1808                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
1809                         if (int_stat & int_status_bit) {
1810                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
1811                         }
1812
1813                         /* Process the MAC interrupt for this pci function */
1814                         int_status_bit = 1 << (24 + nesdev->mac_index);
1815                         if (int_stat & int_status_bit) {
1816                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
1817                         }
1818
1819                         if (int_stat & NES_INT_TIMER) {
1820                                 if (timer_stat & nesdev->timer_int_req) {
1821                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
1822                                                         (timer_stat & nesdev->timer_int_req) |
1823                                                         ~(nesdev->nesadapter->timer_int_req));
1824                                         timer_ints = 1;
1825                                 }
1826                         }
1827
1828                         if (int_stat & NES_INT_INTF) {
1829                                 processed_intf_int = 1;
1830                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
1831                                 intf_int_stat &= nesdev->intf_int_req;
1832                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
1833                                         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
1834                                         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
1835                                                         (u16)debug_error);
1836                                         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
1837                                                         0x01010000 | (debug_error & 0x0000ffff));
1838                                         /* BUG(); */
1839                                         if (crit_err_count++ > 10)
1840                                                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
1841                                 }
1842                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
1843                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
1844                                         BUG();
1845                                 }
1846                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
1847                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
1848                                         BUG();
1849                                 }
1850                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
1851                         }
1852
1853                         if (int_stat & NES_INT_TSW) {
1854                         }
1855                 }
1856                 /* Don't use the interface interrupt bit stay in loop */
1857                 int_stat &= ~NES_INT_INTF|NES_INT_TIMER|NES_INT_MAC0|
1858                                 NES_INT_MAC1|NES_INT_MAC2|NES_INT_MAC3;
1859         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
1860
1861         if (timer_ints == 1) {
1862                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
1863                         if (completion_ints == 0) {
1864                                 nesdev->timer_only_int_count++;
1865                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
1866                                         nesdev->timer_only_int_count = 0;
1867                                         nesdev->int_req &= ~NES_INT_TIMER;
1868                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1869                                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1870                                         nesdev->nesadapter->tune_timer.timer_in_use_old = 0;
1871                                 } else {
1872                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1873                                 }
1874                         } else {
1875                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1876                                 {
1877                                         nes_nic_init_timer(nesdev);
1878                                 }
1879                                 nesdev->timer_only_int_count = 0;
1880                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff|(~nesdev->int_req));
1881                         }
1882                 } else {
1883                         nesdev->timer_only_int_count = 0;
1884                         nesdev->int_req &= ~NES_INT_TIMER;
1885                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1886                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1887                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1888                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1889                 }
1890         } else {
1891                 if ( (completion_ints == 1) &&
1892                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
1893                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1894                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
1895                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
1896                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
1897                         nesdev->timer_only_int_count = 0;
1898                         nesdev->int_req |= NES_INT_TIMER;
1899                         nes_write32(nesdev->regs+NES_TIMER_STAT,
1900                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
1901                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1902                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1903                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1904                 } else {
1905                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1906                 }
1907         }
1908         nesdev->deepcq_count = 0;
1909 }
1910
1911
1912 /**
1913  * nes_process_ceq
1914  */
1915 void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
1916 {
1917         u64 u64temp;
1918         struct nes_hw_cq *cq;
1919         u32 head;
1920         u32 ceq_size;
1921
1922         /* nes_debug(NES_DBG_CQ, "\n"); */
1923         head = ceq->ceq_head;
1924         ceq_size = ceq->ceq_size;
1925
1926         do {
1927                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
1928                                 NES_CEQE_VALID) {
1929                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX])))<<32) |
1930                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
1931                         u64temp <<= 1;
1932                         cq = *((struct nes_hw_cq **)&u64temp);
1933                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
1934                         barrier();
1935                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
1936
1937                         /* call the event handler */
1938                         cq->ce_handler(nesdev, cq);
1939
1940                         if (++head >= ceq_size)
1941                                 head = 0;
1942                 } else {
1943                         break;
1944                 }
1945
1946         } while (1);
1947
1948         ceq->ceq_head = head;
1949 }
1950
1951
1952 /**
1953  * nes_process_aeq
1954  */
1955 void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
1956 {
1957 //      u64 u64temp;
1958         u32 head;
1959         u32 aeq_size;
1960         u32 aeqe_misc;
1961         u32 aeqe_cq_id;
1962         struct nes_hw_aeqe volatile *aeqe;
1963
1964         head = aeq->aeq_head;
1965         aeq_size = aeq->aeq_size;
1966
1967         do {
1968                 aeqe = &aeq->aeq_vbase[head];
1969                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
1970                         break;
1971                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
1972                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
1973                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
1974                         if (aeqe_cq_id >= NES_FIRST_QPN) {
1975                                 /* dealing with an accelerated QP related AE */
1976 //                              u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])))<<32) |
1977 //                                      ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
1978                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
1979                         } else {
1980                                 /* TODO: dealing with a CQP related AE */
1981                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
1982                                                 (u16)(aeqe_misc >> 16));
1983                         }
1984                 }
1985
1986                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
1987
1988                 if (++head >= aeq_size)
1989                         head = 0;
1990         }
1991         while (1);
1992         aeq->aeq_head = head;
1993 }
1994
1995 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
1996 {
1997         struct nes_adapter *nesadapter = nesdev->nesadapter;
1998         u32 reset_value;
1999         u32 i=0;
2000         u32 u32temp;
2001
2002         if (nesadapter->hw_rev == NE020_REV) {
2003                 return;
2004         }
2005         mh_detected++;
2006
2007         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2008
2009         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2010                 reset_value |= 0x0000001d;
2011         else
2012                 reset_value |= 0x0000002d;
2013
2014         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2015                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2016                         nesadapter->link_interrupt_count[0] = 0;
2017                         nesadapter->link_interrupt_count[1] = 0;
2018                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2019                         if (0x00000040 & u32temp)
2020                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2021                         else
2022                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2023
2024                         reset_value |= 0x0000003d;
2025                 }
2026                 nesadapter->link_interrupt_count[mac_index] = 0;
2027         }
2028
2029         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2030
2031         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2032                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2033
2034         if (0x0000003d == (reset_value & 0x0000003d)) {
2035                 u32 pcs_control_status0, pcs_control_status1;
2036
2037                 for (i = 0; i < 10; i++) {
2038                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2039                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2040                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2041                              && (pcs_control_status0 & 0x00100000))
2042                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2043                                 && (pcs_control_status1 & 0x00100000)))
2044                                 continue;
2045                         else
2046                                 break;
2047                 }
2048                 if (10 == i) {
2049                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2050                         if (0x00000040 & u32temp)
2051                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2052                         else
2053                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2054
2055                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2056
2057                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2058                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2059                 }
2060         }
2061 }
2062
2063 /**
2064  * nes_process_mac_intr
2065  */
2066 void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2067 {
2068         unsigned long flags;
2069         u32 pcs_control_status;
2070         struct nes_adapter *nesadapter = nesdev->nesadapter;
2071         struct nes_vnic *nesvnic;
2072         u32 mac_status;
2073         u32 mac_index = nesdev->mac_index;
2074         u32 u32temp;
2075         u16 phy_data;
2076         u16 temp_phy_data;
2077
2078         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2079         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2080                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2081                 return;
2082         }
2083         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2084         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2085
2086         /* ack the MAC interrupt */
2087         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2088         /* Clear the interrupt */
2089         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2090
2091         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2092
2093         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2094                 nesdev->link_status_interrupts++;
2095                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2096                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2097                         nes_reset_link(nesdev, mac_index);
2098                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2099                 }
2100                 /* read the PHY interrupt status register */
2101                 if (nesadapter->OneG_Mode) {
2102                         do {
2103                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2104                                                 nesadapter->phy_index[mac_index], &phy_data);
2105                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2106                                                 nesadapter->phy_index[mac_index], phy_data);
2107                         } while (phy_data&0x8000);
2108
2109                         temp_phy_data = 0;
2110                         do {
2111                                 nes_read_1G_phy_reg(nesdev, 0x11,
2112                                                 nesadapter->phy_index[mac_index], &phy_data);
2113                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2114                                                 nesadapter->phy_index[mac_index], phy_data);
2115                                 if (temp_phy_data == phy_data)
2116                                         break;
2117                                 temp_phy_data = phy_data;
2118                         } while (1);
2119
2120                         nes_read_1G_phy_reg(nesdev, 0x1e,
2121                                         nesadapter->phy_index[mac_index], &phy_data);
2122                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2123                                         nesadapter->phy_index[mac_index], phy_data);
2124
2125                         nes_read_1G_phy_reg(nesdev, 1,
2126                                         nesadapter->phy_index[mac_index], &phy_data);
2127                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2128                                         nesadapter->phy_index[mac_index], phy_data);
2129
2130                         if (temp_phy_data & 0x1000) {
2131                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2132                                 phy_data = 4;
2133                         } else {
2134                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2135                         }
2136                 }
2137                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2138                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2139                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2140                 pcs_control_status = nes_read_indexed(nesdev,
2141                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2142                 pcs_control_status = nes_read_indexed(nesdev,
2143                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index&1)*0x200));
2144                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2145                                 mac_index, pcs_control_status);
2146                 if (nesadapter->OneG_Mode) {
2147                         u32temp = 0x01010000;
2148                         if (nesadapter->port_count > 2) {
2149                                 u32temp |= 0x02020000;
2150                         }
2151                         if ((pcs_control_status & u32temp)!= u32temp) {
2152                                 phy_data = 0;
2153                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2154                         }
2155                 } else if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_IRIS) {
2156                         nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2157                         temp_phy_data = (u16)nes_read_indexed(nesdev,
2158                                                                 NES_IDX_MAC_MDIO_CONTROL);
2159                         u32temp = 20;
2160                         do {
2161                                 nes_read_10G_phy_reg(nesdev, 1, nesadapter->phy_index[mac_index]);
2162                                 phy_data = (u16)nes_read_indexed(nesdev,
2163                                                                 NES_IDX_MAC_MDIO_CONTROL);
2164                                 if ((phy_data == temp_phy_data) || (!(--u32temp)))
2165                                         break;
2166                                 temp_phy_data = phy_data;
2167                         } while (1);
2168                         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2169                                 __FUNCTION__, phy_data, nesadapter->mac_link_down ? "DOWN" : "UP");
2170
2171                 } else {
2172                         phy_data = (0x0f0f0000 == (pcs_control_status & 0x0f1f0000)) ? 4 : 0;
2173                 }
2174
2175                 if (phy_data & 0x0004) {
2176                         nesadapter->mac_link_down[mac_index] = 0;
2177                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2178                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2179                                                 nesvnic->linkup);
2180                                 if (nesvnic->linkup == 0) {
2181                                         printk(PFX "The Link is now up for port %u, netdev %p.\n",
2182                                                         mac_index, nesvnic->netdev);
2183                                         if (netif_queue_stopped(nesvnic->netdev))
2184                                                 netif_start_queue(nesvnic->netdev);
2185                                         nesvnic->linkup = 1;
2186                                         netif_carrier_on(nesvnic->netdev);
2187                                 }
2188                         }
2189                 } else {
2190                         nesadapter->mac_link_down[mac_index] = 1;
2191                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2192                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2193                                                 nesvnic->linkup);
2194                                 if (nesvnic->linkup == 1) {
2195                                         printk(PFX "The Link is now down for port %u, netdev %p.\n",
2196                                                         mac_index, nesvnic->netdev);
2197                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2198                                                 netif_stop_queue(nesvnic->netdev);
2199                                         nesvnic->linkup = 0;
2200                                         netif_carrier_off(nesvnic->netdev);
2201                                 }
2202                         }
2203                 }
2204         }
2205
2206         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2207 }
2208
2209
2210
2211 void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2212 {
2213         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2214
2215         netif_rx_schedule(nesdev->netdev[nesvnic->netdev_index], &nesvnic->napi);
2216 }
2217
2218
2219 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2220 * getting out of nic_ce_handler
2221 */
2222 #define MAX_RQES_TO_PROCESS     384
2223
2224 /**
2225  * nes_nic_ce_handler
2226  */
2227 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2228 {
2229         u64 u64temp;
2230         dma_addr_t bus_address;
2231         struct nes_hw_nic *nesnic;
2232         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2233         struct nes_adapter *nesadapter = nesdev->nesadapter;
2234         struct nes_hw_nic_rq_wqe *nic_rqe;
2235         struct nes_hw_nic_sq_wqe *nic_sqe;
2236         struct sk_buff *skb;
2237         struct sk_buff *rx_skb;
2238         __le16 *wqe_fragment_length;
2239         u32 head;
2240         u32 cq_size;
2241         u32 rx_pkt_size;
2242         u32 cqe_count=0;
2243         u32 cqe_errv;
2244         u32 cqe_misc;
2245         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2246         u16 vlan_tag;
2247         u16 pkt_type;
2248         u16 rqes_processed = 0;
2249         u8 sq_cqes = 0;
2250
2251         head = cq->cq_head;
2252         cq_size = cq->cq_size;
2253         cq->cqes_pending = 1;
2254         do {
2255                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2256                                 NES_NIC_CQE_VALID) {
2257                         nesnic = &nesvnic->nic;
2258                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2259                         if (cqe_misc & NES_NIC_CQE_SQ) {
2260                                 sq_cqes++;
2261                                 wqe_fragment_index = 1;
2262                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2263                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2264                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2265                                 /* bump past the vlan tag */
2266                                 wqe_fragment_length++;
2267                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2268                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2269                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2270                                         bus_address = (dma_addr_t)u64temp;
2271                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2272                                                 pci_unmap_single(nesdev->pcidev,
2273                                                                 bus_address,
2274                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2275                                                                 PCI_DMA_TODEVICE);
2276                                         }
2277                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2278                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2279                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+wqe_fragment_index*2]);
2280                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+wqe_fragment_index*2]))<<32;
2281                                                         bus_address = (dma_addr_t)u64temp;
2282                                                         pci_unmap_page(nesdev->pcidev,
2283                                                                         bus_address,
2284                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2285                                                                         PCI_DMA_TODEVICE);
2286                                                 } else
2287                                                         break;
2288                                         }
2289                                         if (skb)
2290                                                 dev_kfree_skb_any(skb);
2291                                 }
2292                                 nesnic->sq_tail++;
2293                                 nesnic->sq_tail &= nesnic->sq_size-1;
2294                                 if (sq_cqes > 128) {
2295                                         barrier();
2296                                 /* restart the queue if it had been stopped */
2297                                 if (netif_queue_stopped(nesvnic->netdev))
2298                                         netif_wake_queue(nesvnic->netdev);
2299                                         sq_cqes = 0;
2300                                 }
2301                         } else {
2302                                 rqes_processed ++;
2303
2304                                 cq->rx_cqes_completed++;
2305                                 cq->rx_pkts_indicated++;
2306                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2307                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2308                                 /* Get the skb */
2309                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2310                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2311                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2312                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2313                                 pci_unmap_single(nesdev->pcidev, bus_address,
2314                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2315                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2316                                 /* rx_skb->len = rx_pkt_size; */
2317                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2318                                 skb_put(rx_skb, rx_pkt_size);
2319                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2320                                 nesnic->rq_tail++;
2321                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2322
2323                                 atomic_inc(&nesvnic->rx_skbs_needed);
2324                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2325                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2326                                                         cq->cq_number | (cqe_count << 16));
2327 //                                      nesadapter->tune_timer.cq_count += cqe_count;
2328                                         nesdev->currcq_count += cqe_count;
2329                                         cqe_count = 0;
2330                                         nes_replenish_nic_rq(nesvnic);
2331                                 }
2332                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2333                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2334                                 rx_skb->ip_summed = CHECKSUM_NONE;
2335
2336                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2337                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2338                                         if ((cqe_errv &
2339                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2340                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2341                                                 if (nesvnic->rx_checksum_disabled == 0) {
2342                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2343                                                 }
2344                                         } else
2345                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2346                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2347                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2348
2349                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2350                                         if ((cqe_errv &
2351                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2352                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2353                                                 if (nesvnic->rx_checksum_disabled == 0) {
2354                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2355                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2356                                                                   nesvnic->netdev->name); */
2357                                                 }
2358                                         } else
2359                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2360                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2361                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2362                                         }
2363                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2364                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2365
2366                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2367                                         nes_cm_recv(rx_skb, nesvnic->netdev);
2368                                 } else {
2369                                         if ((cqe_misc & NES_NIC_CQE_TAG_VALID) && (nesvnic->vlan_grp != NULL)) {
2370                                                 vlan_tag = (u16)(le32_to_cpu(
2371                                                                 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2372                                                                 >> 16);
2373                                                 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2374                                                                 nesvnic->netdev->name, vlan_tag);
2375                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2376                                         } else {
2377                                                 nes_netif_rx(rx_skb);
2378                                         }
2379                                 }
2380
2381                                 nesvnic->netdev->last_rx = jiffies;
2382                                 /* nesvnic->netstats.rx_packets++; */
2383                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2384                         }
2385
2386                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2387                         /* Accounting... */
2388                         cqe_count++;
2389                         if (++head >= cq_size)
2390                                 head = 0;
2391                         if (cqe_count == 255) {
2392                                 /* Replenish Nic CQ */
2393                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2394                                                 cq->cq_number | (cqe_count << 16));
2395 //                              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2396                                 nesdev->currcq_count += cqe_count;
2397                                 cqe_count = 0;
2398                         }
2399
2400                         if (cq->rx_cqes_completed >= nesvnic->budget)
2401                                 break;
2402                 } else {
2403                         cq->cqes_pending = 0;
2404                         break;
2405                 }
2406
2407         } while (1);
2408
2409         if (sq_cqes) {
2410                 barrier();
2411                 /* restart the queue if it had been stopped */
2412                 if (netif_queue_stopped(nesvnic->netdev))
2413                         netif_wake_queue(nesvnic->netdev);
2414         }
2415
2416         cq->cq_head = head;
2417         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2418                         cq->cq_number, cqe_count, cq->cq_head); */
2419         cq->cqe_allocs_pending = cqe_count;
2420         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2421         {
2422 //              nesdev->nesadapter->tune_timer.cq_count += cqe_count;
2423                 nesdev->currcq_count += cqe_count;
2424                 nes_nic_tune_timer(nesdev);
2425         }
2426         if (atomic_read(&nesvnic->rx_skbs_needed))
2427                 nes_replenish_nic_rq(nesvnic);
2428         }
2429
2430
2431 /**
2432  * nes_cqp_ce_handler
2433  */
2434 void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2435 {
2436         u64 u64temp;
2437         unsigned long flags;
2438         struct nes_hw_cqp *cqp = NULL;
2439         struct nes_cqp_request *cqp_request;
2440         struct nes_hw_cqp_wqe *cqp_wqe;
2441         u32 head;
2442         u32 cq_size;
2443         u32 cqe_count=0;
2444         u32 error_code;
2445         /* u32 counter; */
2446
2447         head = cq->cq_head;
2448         cq_size = cq->cq_size;
2449
2450         do {
2451                 /* process the CQE */
2452                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2453                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2454
2455                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2456                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2457                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) |
2458                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2459                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2460                         cqp = *((struct nes_hw_cqp **)&u64temp);
2461
2462                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2463                         if (error_code) {
2464                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2465                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2466                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2467                                                 (u16)(error_code >> 16),
2468                                                 (u16)error_code);
2469                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2470                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2471                         }
2472
2473                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2474                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX])))<<32) |
2475                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2476                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2477                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2478                         if (cqp_request) {
2479                                 if (cqp_request->waiting) {
2480                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2481                                         cqp_request->major_code = (u16)(error_code >> 16);
2482                                         cqp_request->minor_code = (u16)error_code;
2483                                         barrier();
2484                                         cqp_request->request_done = 1;
2485                                         wake_up(&cqp_request->waitq);
2486                                         if (atomic_dec_and_test(&cqp_request->refcount)) {
2487                                                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2488                                                                 cqp_request,
2489                                                                 le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2490                                                 if (cqp_request->dynamic) {
2491                                                         kfree(cqp_request);
2492                                                 } else {
2493                                                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2494                                                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2495                                                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2496                                                 }
2497                                         }
2498                                 } else if (cqp_request->callback) {
2499                                         /* Envoke the callback routine */
2500                                         cqp_request->cqp_callback(nesdev, cqp_request);
2501                                         if (cqp_request->dynamic) {
2502                                                 kfree(cqp_request);
2503                                         } else {
2504                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2505                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2506                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2507                                         }
2508                                 } else {
2509                                         nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n",
2510                                                         cqp_request,
2511                                                         le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f);
2512                                         if (cqp_request->dynamic) {
2513                                                 kfree(cqp_request);
2514                                         } else {
2515                                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2516                                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2517                                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2518                                         }
2519                                 }
2520                         } else {
2521                                 wake_up(&nesdev->cqp.waitq);
2522                         }
2523
2524                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2525                         nes_write32(nesdev->regs+NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2526                         if (++cqp->sq_tail >= cqp->sq_size)
2527                                 cqp->sq_tail = 0;
2528
2529                         /* Accounting... */
2530                         cqe_count++;
2531                         if (++head >= cq_size)
2532                                 head = 0;
2533                 } else {
2534                         break;
2535                 }
2536         } while (1);
2537         cq->cq_head = head;
2538
2539         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2540         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2541                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2542                         (nesdev->cqp.sq_size - 1)) != 1)) {
2543                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2544                                 struct nes_cqp_request, list);
2545                 list_del_init(&cqp_request->list);
2546                 head = nesdev->cqp.sq_head++;
2547                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2548                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2549                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2550                 barrier();
2551                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2552                         cpu_to_le32((u32)((unsigned long)cqp_request));
2553                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2554                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2555                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2556                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2557                 /* Ring doorbell (1 WQEs) */
2558                 barrier();
2559                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2560         }
2561         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2562
2563         /* Arm the CCQ */
2564         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2565                         cq->cq_number);
2566         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2567 }
2568
2569
2570 /**
2571  * nes_process_iwarp_aeqe
2572  */
2573 void nes_process_iwarp_aeqe(struct nes_device *nesdev, struct nes_hw_aeqe *aeqe)
2574 {
2575         u64 context;
2576         u64 aeqe_context = 0;
2577         unsigned long flags;
2578         struct nes_qp *nesqp;
2579         int resource_allocated;
2580         /* struct iw_cm_id *cm_id; */
2581         struct nes_adapter *nesadapter = nesdev->nesadapter;
2582         struct ib_event ibevent;
2583         /* struct iw_cm_event cm_event; */
2584         u32 aeq_info;
2585         u32 next_iwarp_state = 0;
2586         u16 async_event_id;
2587         u8 tcp_state;
2588         u8 iwarp_state;
2589
2590         nes_debug(NES_DBG_AEQ, "\n");
2591         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2592         if ((NES_AEQE_INBOUND_RDMA&aeq_info) || (!(NES_AEQE_QP&aeq_info))) {
2593                 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2594                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2595         } else {
2596                 aeqe_context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
2597                 aeqe_context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
2598                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
2599                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2600                 BUG_ON(!context);
2601         }
2602
2603         async_event_id = (u16)aeq_info;
2604         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
2605         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
2606         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
2607                         " Tcp state = %s, iWARP state = %s\n",
2608                         async_event_id,
2609                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
2610                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
2611
2612
2613         switch (async_event_id) {
2614                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
2615                         nesqp = *((struct nes_qp **)&context);
2616                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
2617                                 nesqp->cm_id->add_ref(nesqp->cm_id);
2618                                 nes_add_ref(&nesqp->ibqp);
2619                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
2620                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
2621                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
2622                                                 " need ae to finish up, original_last_aeq = 0x%04X."
2623                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
2624                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
2625                                                 async_event_id, nesqp->last_aeq, tcp_state);
2626                         }
2627                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2628                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
2629                                 /* FIN Received but tcp state or IB state moved on,
2630                                                 should expect a close complete */
2631                                 return;
2632                         }
2633                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
2634                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
2635                 case NES_AEQE_AEID_TERMINATE_SENT:
2636                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
2637                 case NES_AEQE_AEID_RESET_SENT:
2638                         nesqp = *((struct nes_qp **)&context);
2639                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
2640                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2641                         }
2642                         nes_add_ref(&nesqp->ibqp);
2643                         spin_lock_irqsave(&nesqp->lock, flags);
2644                         nesqp->hw_iwarp_state = iwarp_state;
2645                         nesqp->hw_tcp_state = tcp_state;
2646                         nesqp->last_aeq = async_event_id;
2647
2648                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
2649                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
2650                                 nesqp->hte_added = 0;
2651                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2652                                 nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u to remove hte\n",
2653                                                 nesqp->hwqp.qp_id);
2654                                 nes_hw_modify_qp(nesdev, nesqp,
2655                                                 NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE, 0);
2656                                 spin_lock_irqsave(&nesqp->lock, flags);
2657                         }
2658
2659                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
2660                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2661                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2662                                 switch (nesqp->hw_iwarp_state) {
2663                                         case NES_AEQE_IWARP_STATE_RTS:
2664                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
2665                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
2666                                                 break;
2667                                         case NES_AEQE_IWARP_STATE_TERMINATE:
2668                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE;
2669                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;
2670                                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2671                                                         next_iwarp_state |= 0x02000000;
2672                                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2673                                                 }
2674                                                 break;
2675                                         default:
2676                                                 next_iwarp_state = 0;
2677                                 }
2678                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2679                                 if (next_iwarp_state) {
2680                                         nes_add_ref(&nesqp->ibqp);
2681                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2682                                                         " also added another reference\n",
2683                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2684                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2685                                 }
2686                                 nes_cm_disconn(nesqp);
2687                         } else {
2688                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
2689                                         /* FIN Received but ib state not RTS,
2690                                                         close complete will be on its way */
2691                                         spin_unlock_irqrestore(&nesqp->lock, flags);
2692                                         nes_rem_ref(&nesqp->ibqp);
2693                                         return;
2694                                 }
2695                                 spin_unlock_irqrestore(&nesqp->lock, flags);
2696                                 if (async_event_id == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) {
2697                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000;
2698                                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2699                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X,"
2700                                                         " also added another reference\n",
2701                                                         nesqp->hwqp.qp_id, next_iwarp_state);
2702                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2703                                 }
2704                                 nes_cm_disconn(nesqp);
2705                         }
2706                         break;
2707                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
2708                         nesqp = *((struct nes_qp **)&context);
2709                         spin_lock_irqsave(&nesqp->lock, flags);
2710                         nesqp->hw_iwarp_state = iwarp_state;
2711                         nesqp->hw_tcp_state = tcp_state;
2712                         nesqp->last_aeq = async_event_id;
2713                         spin_unlock_irqrestore(&nesqp->lock, flags);
2714                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TERMINATE_RECEIVED"
2715                                         " event on QP%u \n  Q2 Data:\n",
2716                                         nesqp->hwqp.qp_id);
2717                         if (nesqp->ibqp.event_handler) {
2718                                 ibevent.device = nesqp->ibqp.device;
2719                                 ibevent.element.qp = &nesqp->ibqp;
2720                                 ibevent.event = IB_EVENT_QP_FATAL;
2721                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2722                         }
2723                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
2724                                         ((nesqp->ibqp_state == IB_QPS_RTS)&&
2725                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
2726                                 nes_add_ref(&nesqp->ibqp);
2727                                 nes_cm_disconn(nesqp);
2728                         } else {
2729                                 nesqp->in_disconnect = 0;
2730                                 wake_up(&nesqp->kick_waitq);
2731                         }
2732                         break;
2733                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
2734                         nesqp = *((struct nes_qp **)&context);
2735                         nes_add_ref(&nesqp->ibqp);
2736                         spin_lock_irqsave(&nesqp->lock, flags);
2737                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;
2738                         nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;
2739                         nesqp->last_aeq = async_event_id;
2740                         if (nesqp->cm_id) {
2741                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2742                                                 " event on QP%u, remote IP = 0x%08X \n",
2743                                                 nesqp->hwqp.qp_id,
2744                                                 ntohl(nesqp->cm_id->remote_addr.sin_addr.s_addr));
2745                         } else {
2746                                 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_TOO_MANY_RETRIES"
2747                                                 " event on QP%u \n",
2748                                                 nesqp->hwqp.qp_id);
2749                         }
2750                         spin_unlock_irqrestore(&nesqp->lock, flags);
2751                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_RESET;
2752                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0);
2753                         if (nesqp->ibqp.event_handler) {
2754                                 ibevent.device = nesqp->ibqp.device;
2755                                 ibevent.element.qp = &nesqp->ibqp;
2756                                 ibevent.event = IB_EVENT_QP_FATAL;
2757                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2758                         }
2759                         break;
2760                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
2761                         if (NES_AEQE_INBOUND_RDMA&aeq_info) {
2762                                 nesqp = nesadapter->qp_table[le32_to_cpu(
2763                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2764                         } else {
2765                                 /* TODO: get the actual WQE and mask off wqe index */
2766                                 context &= ~((u64)511);
2767                                 nesqp = *((struct nes_qp **)&context);
2768                         }
2769                         spin_lock_irqsave(&nesqp->lock, flags);
2770                         nesqp->hw_iwarp_state = iwarp_state;
2771                         nesqp->hw_tcp_state = tcp_state;
2772                         nesqp->last_aeq = async_event_id;
2773                         spin_unlock_irqrestore(&nesqp->lock, flags);
2774                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_BAD_STAG_INDEX event on QP%u\n",
2775                                         nesqp->hwqp.qp_id);
2776                         if (nesqp->ibqp.event_handler) {
2777                                 ibevent.device = nesqp->ibqp.device;
2778                                 ibevent.element.qp = &nesqp->ibqp;
2779                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2780                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2781                         }
2782                         break;
2783                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
2784                         nesqp = *((struct nes_qp **)&context);
2785                         spin_lock_irqsave(&nesqp->lock, flags);
2786                         nesqp->hw_iwarp_state = iwarp_state;
2787                         nesqp->hw_tcp_state = tcp_state;
2788                         nesqp->last_aeq = async_event_id;
2789                         spin_unlock_irqrestore(&nesqp->lock, flags);
2790                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_AMP_UNALLOCATED_STAG event on QP%u\n",
2791                                         nesqp->hwqp.qp_id);
2792                         if (nesqp->ibqp.event_handler) {
2793                                 ibevent.device = nesqp->ibqp.device;
2794                                 ibevent.element.qp = &nesqp->ibqp;
2795                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2796                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2797                         }
2798                         break;
2799                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
2800                         nesqp = nesadapter->qp_table[le32_to_cpu(aeqe->aeqe_words
2801                                         [NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2802                         spin_lock_irqsave(&nesqp->lock, flags);
2803                         nesqp->hw_iwarp_state = iwarp_state;
2804                         nesqp->hw_tcp_state = tcp_state;
2805                         nesqp->last_aeq = async_event_id;
2806                         spin_unlock_irqrestore(&nesqp->lock, flags);
2807                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_PRIV_OPERATION_DENIED event on QP%u,"
2808                                         " nesqp = %p, AE reported %p\n",
2809                                         nesqp->hwqp.qp_id, nesqp, *((struct nes_qp **)&context));
2810                         if (nesqp->ibqp.event_handler) {
2811                                 ibevent.device = nesqp->ibqp.device;
2812                                 ibevent.element.qp = &nesqp->ibqp;
2813                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2814                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2815                         }
2816                         break;
2817                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
2818                         context <<= 1;
2819                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
2820                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
2821                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
2822                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2823                         if (resource_allocated) {
2824                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
2825                                                 __FUNCTION__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
2826                         }
2827                         break;
2828                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
2829                         nesqp = nesadapter->qp_table[le32_to_cpu(
2830                                         aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])-NES_FIRST_QPN];
2831                         spin_lock_irqsave(&nesqp->lock, flags);
2832                         nesqp->hw_iwarp_state = iwarp_state;
2833                         nesqp->hw_tcp_state = tcp_state;
2834                         nesqp->last_aeq = async_event_id;
2835                         spin_unlock_irqrestore(&nesqp->lock, flags);
2836                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG"
2837                                         "_FOR_AVAILABLE_BUFFER event on QP%u\n",
2838                                         nesqp->hwqp.qp_id);
2839                         if (nesqp->ibqp.event_handler) {
2840                                 ibevent.device = nesqp->ibqp.device;
2841                                 ibevent.element.qp = &nesqp->ibqp;
2842                                 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2843                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2844                         }
2845                         /* tell cm to disconnect, cm will queue work to thread */
2846                         nes_add_ref(&nesqp->ibqp);
2847                         nes_cm_disconn(nesqp);
2848                         break;
2849                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
2850                         nesqp = *((struct nes_qp **)&context);
2851                         spin_lock_irqsave(&nesqp->lock, flags);
2852                         nesqp->hw_iwarp_state = iwarp_state;
2853                         nesqp->hw_tcp_state = tcp_state;
2854                         nesqp->last_aeq = async_event_id;
2855                         spin_unlock_irqrestore(&nesqp->lock, flags);
2856                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_DDP_UBE_INVALID_MSN"
2857                                         "_NO_BUFFER_AVAILABLE event on QP%u\n",
2858                                         nesqp->hwqp.qp_id);
2859                         if (nesqp->ibqp.event_handler) {
2860                                 ibevent.device = nesqp->ibqp.device;
2861                                 ibevent.element.qp = &nesqp->ibqp;
2862                                 ibevent.event = IB_EVENT_QP_FATAL;
2863                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2864                         }
2865                         /* tell cm to disconnect, cm will queue work to thread */
2866                         nes_add_ref(&nesqp->ibqp);
2867                         nes_cm_disconn(nesqp);
2868                         break;
2869                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
2870                         nesqp = *((struct nes_qp **)&context);
2871                         spin_lock_irqsave(&nesqp->lock, flags);
2872                         nesqp->hw_iwarp_state = iwarp_state;
2873                         nesqp->hw_tcp_state = tcp_state;
2874                         nesqp->last_aeq = async_event_id;
2875                         spin_unlock_irqrestore(&nesqp->lock, flags);
2876                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR"
2877                                         " event on QP%u \n  Q2 Data:\n",
2878                                         nesqp->hwqp.qp_id);
2879                         if (nesqp->ibqp.event_handler) {
2880                                 ibevent.device = nesqp->ibqp.device;
2881                                 ibevent.element.qp = &nesqp->ibqp;
2882                                 ibevent.event = IB_EVENT_QP_FATAL;
2883                                 nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context);
2884                         }
2885                         /* tell cm to disconnect, cm will queue work to thread */
2886                         nes_add_ref(&nesqp->ibqp);
2887                         nes_cm_disconn(nesqp);
2888                         break;
2889                         /* TODO: additional AEs need to be here */
2890                 default:
2891                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
2892                                         async_event_id);
2893                         break;
2894         }
2895
2896 }
2897
2898
2899 /**
2900  * nes_iwarp_ce_handler
2901  */
2902 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
2903 {
2904         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
2905
2906         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
2907                         nescq->hw_cq.cq_number); */
2908         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
2909
2910         if (nescq->ibcq.comp_handler)
2911                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
2912
2913         return;
2914 }
2915
2916
2917 /**
2918  * nes_manage_apbvt()
2919  */
2920 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
2921                 u32 nic_index, u32 add_port)
2922 {
2923         struct nes_device *nesdev = nesvnic->nesdev;
2924         struct nes_hw_cqp_wqe *cqp_wqe;
2925         unsigned long flags;
2926         struct nes_cqp_request *cqp_request;
2927         int ret = 0;
2928         u16 major_code;
2929
2930         /* Send manage APBVT request to CQP */
2931         cqp_request = nes_get_cqp_request(nesdev);
2932         if (cqp_request == NULL) {
2933                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
2934                 return -ENOMEM;
2935         }
2936         cqp_request->waiting = 1;
2937         cqp_wqe = &cqp_request->cqp_wqe;
2938
2939         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
2940                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
2941                         accel_local_port, accel_local_port, nic_index);
2942
2943         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2944         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
2945                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
2946         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2947                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
2948
2949         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
2950
2951         atomic_set(&cqp_request->refcount, 2);
2952         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
2953
2954         if (add_port == NES_MANAGE_APBVT_ADD)
2955                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
2956                                 NES_EVENT_TIMEOUT);
2957         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
2958                         ret, cqp_request->major_code, cqp_request->minor_code);
2959         major_code = cqp_request->major_code;
2960         if (atomic_dec_and_test(&cqp_request->refcount)) {
2961                 if (cqp_request->dynamic) {
2962                         kfree(cqp_request);
2963                 } else {
2964                         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2965                         list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
2966                         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2967                 }
2968         }
2969         if (!ret)
2970                 return -ETIME;
2971         else if (major_code)
2972                 return -EIO;
2973         else
2974                 return 0;
2975 }
2976
2977
2978 /**
2979  * nes_manage_arp_cache
2980  */
2981 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
2982                 u32 ip_addr, u32 action)
2983 {
2984         struct nes_hw_cqp_wqe *cqp_wqe;
2985         struct nes_vnic *nesvnic = netdev_priv(netdev);
2986         struct nes_device *nesdev;
2987         struct nes_cqp_request *cqp_request;
2988         int arp_index;
2989
2990         nesdev = nesvnic->nesdev;
2991         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
2992         if (arp_index == -1) {
2993                 return;
2994         }
2995
2996         /* update the ARP entry */
2997         cqp_request = nes_get_cqp_request(nesdev);
2998         if (cqp_request == NULL) {
2999                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3000                 return;
3001         }
3002         cqp_request->waiting = 0;
3003         cqp_wqe = &cqp_request->cqp_wqe;
3004         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3005
3006         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3007                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3008         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3009                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3010         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3011
3012         if (action == NES_ARP_ADD) {
3013                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3014                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3015                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3016                                 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3017                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3018                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3019         } else {
3020                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3021                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3022         }
3023
3024         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3025                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3026
3027         atomic_set(&cqp_request->refcount, 1);
3028         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3029 }
3030
3031
3032 /**
3033  * flush_wqes
3034  */
3035 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3036                 u32 which_wq, u32 wait_completion)
3037 {
3038         unsigned long flags;
3039         struct nes_cqp_request *cqp_request;
3040         struct nes_hw_cqp_wqe *cqp_wqe;
3041         int ret;
3042
3043         cqp_request = nes_get_cqp_request(nesdev);
3044         if (cqp_request == NULL) {
3045                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3046                 return;
3047         }
3048         if (wait_completion) {
3049                 cqp_request->waiting = 1;
3050                 atomic_set(&cqp_request->refcount, 2);
3051         } else {
3052                 cqp_request->waiting = 0;
3053         }
3054         cqp_wqe = &cqp_request->cqp_wqe;
3055         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3056
3057         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3058                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3059         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3060
3061         nes_post_cqp_request(nesdev, cqp_request, NES_CQP_REQUEST_RING_DOORBELL);
3062
3063         if (wait_completion) {
3064                 /* Wait for CQP */
3065                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3066                                 NES_EVENT_TIMEOUT);
3067                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3068                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3069                                 ret, cqp_request->major_code, cqp_request->minor_code);
3070                 if (atomic_dec_and_test(&cqp_request->refcount)) {
3071                         if (cqp_request->dynamic) {
3072                                 kfree(cqp_request);
3073                         } else {
3074                                 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3075                                 list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs);
3076                                 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3077                         }
3078                 }
3079         }
3080 }