drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42 #include <linux/slab.h>
  43
  44 #include "nes.h"
  45
  46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  47 module_param(nes_lro_max_aggr, uint, 0444);
  48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  49
  50 static int wide_ppm_offset;
  51 module_param(wide_ppm_offset, int, 0644);
  52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  53
  54 static u32 crit_err_count;
  55 u32 int_mod_timer_init;
  56 u32 int_mod_cq_depth_256;
  57 u32 int_mod_cq_depth_128;
  58 u32 int_mod_cq_depth_32;
  59 u32 int_mod_cq_depth_24;
  60 u32 int_mod_cq_depth_16;
  61 u32 int_mod_cq_depth_4;
  62 u32 int_mod_cq_depth_1;
  63 static const u8 nes_max_critical_error_count = 100;
  64 #include "nes_cm.h"
  65
  66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  69                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  74                                    struct nes_hw_aeqe *aeqe);
  75 static void process_critical_error(struct nes_device *nesdev);
  76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  78 static void nes_terminate_timeout(unsigned long context);
  79 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  80
  81 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  82 static unsigned char *nes_iwarp_state_str[] = {
  83         "Non-Existent",
  84         "Idle",
  85         "RTS",
  86         "Closing",
  87         "RSVD1",
  88         "Terminate",
  89         "Error",
  90         "RSVD2",
  91 };
  92
  93 static unsigned char *nes_tcp_state_str[] = {
  94         "Non-Existent",
  95         "Closed",
  96         "Listen",
  97         "SYN Sent",
  98         "SYN Rcvd",
  99         "Established",
 100         "Close Wait",
 101         "FIN Wait 1",
 102         "Closing",
 103         "Last Ack",
 104         "FIN Wait 2",
 105         "Time Wait",
 106         "RSVD1",
 107         "RSVD2",
 108         "RSVD3",
 109         "RSVD4",
 110 };
 111 #endif
 112
 113
 114 /**
 115  * nes_nic_init_timer_defaults
 116  */
 117 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 118 {
 119         unsigned long flags;
 120         struct nes_adapter *nesadapter = nesdev->nesadapter;
 121         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 122
 123         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 124
 125         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 126         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 127         if (jumbomode) {
 128                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 129                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 130                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 131         } else {
 132                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 133                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 134                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 135         }
 136
 137         /* todo use netdev->mtu to set thresholds */
 138         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 139 }
 140
 141
 142 /**
 143  * nes_nic_init_timer
 144  */
 145 static void  nes_nic_init_timer(struct nes_device *nesdev)
 146 {
 147         unsigned long flags;
 148         struct nes_adapter *nesadapter = nesdev->nesadapter;
 149         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 150
 151         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 152
 153         if (shared_timer->timer_in_use_old == 0) {
 154                 nesdev->deepcq_count = 0;
 155                 shared_timer->timer_direction_upward = 0;
 156                 shared_timer->timer_direction_downward = 0;
 157                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 158                 shared_timer->timer_in_use_old = 0;
 159
 160         }
 161         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 162                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 163                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 164                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 165         }
 166         /* todo use netdev->mtu to set thresholds */
 167         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 168 }
 169
 170
 171 /**
 172  * nes_nic_tune_timer
 173  */
 174 static void nes_nic_tune_timer(struct nes_device *nesdev)
 175 {
 176         unsigned long flags;
 177         struct nes_adapter *nesadapter = nesdev->nesadapter;
 178         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 179         u16 cq_count = nesdev->currcq_count;
 180
 181         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 182
 183         if (shared_timer->cq_count_old <= cq_count)
 184                 shared_timer->cq_direction_downward = 0;
 185         else
 186                 shared_timer->cq_direction_downward++;
 187         shared_timer->cq_count_old = cq_count;
 188         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 189                 if (cq_count <= shared_timer->threshold_low &&
 190                     shared_timer->threshold_low > 4) {
 191                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 192                         shared_timer->cq_direction_downward=0;
 193                         nesdev->currcq_count = 0;
 194                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 195                         return;
 196                 }
 197         }
 198
 199         if (cq_count > 1) {
 200                 nesdev->deepcq_count += cq_count;
 201                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 202                         shared_timer->timer_direction_upward++;
 203                         shared_timer->timer_direction_downward = 0;
 204                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 205                         shared_timer->timer_direction_upward = 0;
 206                         shared_timer->timer_direction_downward = 0;
 207                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 208                         shared_timer->timer_direction_downward++;
 209                         shared_timer->timer_direction_upward = 0;
 210                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 211                         shared_timer->timer_in_use -= 2;
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward++;
 214                 } else {
 215                         shared_timer->timer_in_use -= 4;
 216                         shared_timer->timer_direction_upward = 0;
 217                         shared_timer->timer_direction_downward++;
 218                 }
 219
 220                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 221                         shared_timer->timer_in_use += 3;
 222                         shared_timer->timer_direction_upward = 0;
 223                         shared_timer->timer_direction_downward = 0;
 224                 }
 225                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 226                         shared_timer->timer_in_use -= 4 ;
 227                         shared_timer->timer_direction_downward = 0;
 228                         shared_timer->timer_direction_upward = 0;
 229                 }
 230         }
 231
 232         /* boundary checking */
 233         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 234                 shared_timer->timer_in_use = shared_timer->threshold_high;
 235         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 236                 shared_timer->timer_in_use = shared_timer->threshold_low;
 237
 238         nesdev->currcq_count = 0;
 239
 240         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 241 }
 242
 243
 244 /**
 245  * nes_init_adapter - initialize adapter
 246  */
 247 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 248         struct nes_adapter *nesadapter = NULL;
 249         unsigned long num_pds;
 250         u32 u32temp;
 251         u32 port_count;
 252         u16 max_rq_wrs;
 253         u16 max_sq_wrs;
 254         u32 max_mr;
 255         u32 max_256pbl;
 256         u32 max_4kpbl;
 257         u32 max_qp;
 258         u32 max_irrq;
 259         u32 max_cq;
 260         u32 hte_index_mask;
 261         u32 adapter_size;
 262         u32 arp_table_size;
 263         u16 vendor_id;
 264         u16 device_id;
 265         u8  OneG_Mode;
 266         u8  func_index;
 267
 268         /* search the list of existing adapters */
 269         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 270                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 271                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 272                                 nesdev->pcidev->devfn,
 273                                 PCI_SLOT(nesadapter->devfn),
 274                                 nesadapter->bus_number,
 275                                 PCI_SLOT(nesdev->pcidev->devfn),
 276                                 nesdev->pcidev->bus->number );
 277                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 278                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 279                         nesadapter->ref_count++;
 280                         return nesadapter;
 281                 }
 282         }
 283
 284         /* no adapter found */
 285         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 286         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 287                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 288                                 hw_rev);
 289                 return NULL;
 290         }
 291
 292         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 293                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 296                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 297
 298         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 299
 300
 301         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 302                 return NULL;
 303
 304         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 305         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 306
 307         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 308         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 309                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 310                                 max_qp, u32temp);
 311                 max_qp = (u32)1 << (u32temp & 0x001f);
 312         }
 313
 314         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 315         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 316                         max_qp, hte_index_mask);
 317
 318         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 319
 320         max_irrq = 1 << (u32temp & 0x001f);
 321
 322         if (max_qp > max_irrq) {
 323                 max_qp = max_irrq;
 324                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 325                                 max_qp);
 326         }
 327
 328         /* there should be no reason to allocate more pds than qps */
 329         if (num_pds > max_qp)
 330                 num_pds = max_qp;
 331
 332         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 333         max_mr = (u32)8192 << (u32temp & 0x7);
 334
 335         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 336         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 337         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 338         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 339
 340         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 341         arp_table_size = 1 << u32temp;
 342
 343         adapter_size = (sizeof(struct nes_adapter) +
 344                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 349         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 350         adapter_size += sizeof(struct nes_qp **) * max_qp;
 351
 352         /* allocate a new adapter struct */
 353         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 354         if (nesadapter == NULL) {
 355                 return NULL;
 356         }
 357
 358         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 359                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 360
 361         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 362                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 363                 kfree(nesadapter);
 364                 return NULL;
 365         }
 366
 367         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 368                                 (nesadapter->mac_addr_low >> 24);
 369
 370         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 371                                  PCI_DEVICE_ID, &device_id);
 372         nesadapter->vendor_part_id = device_id;
 373
 374         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 375                                                         OneG_Mode)) {
 376                 kfree(nesadapter);
 377                 return NULL;
 378         }
 379         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 380
 381         memset(nesadapter->pft_mcast_map, 255,
 382                sizeof nesadapter->pft_mcast_map);
 383
 384         /* populate the new nesadapter */
 385         nesadapter->devfn = nesdev->pcidev->devfn;
 386         nesadapter->bus_number = nesdev->pcidev->bus->number;
 387         nesadapter->ref_count = 1;
 388         nesadapter->timer_int_req = 0xffff0000;
 389         nesadapter->OneG_Mode = OneG_Mode;
 390         nesadapter->doorbell_start = nesdev->doorbell_region;
 391
 392         /* nesadapter->tick_delta = clk_divisor; */
 393         nesadapter->hw_rev = hw_rev;
 394         nesadapter->port_count = port_count;
 395
 396         nesadapter->max_qp = max_qp;
 397         nesadapter->hte_index_mask = hte_index_mask;
 398         nesadapter->max_irrq = max_irrq;
 399         nesadapter->max_mr = max_mr;
 400         nesadapter->max_256pbl = max_256pbl - 1;
 401         nesadapter->max_4kpbl = max_4kpbl - 1;
 402         nesadapter->max_cq = max_cq;
 403         nesadapter->free_256pbl = max_256pbl - 1;
 404         nesadapter->free_4kpbl = max_4kpbl - 1;
 405         nesadapter->max_pd = num_pds;
 406         nesadapter->arp_table_size = arp_table_size;
 407
 408         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 409         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 410                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 411                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 412                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 413         } else {
 414                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 415                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 416                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 417                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 418         }
 419         /* Setup and enable the periodic timer */
 420         if (nesadapter->et_rx_coalesce_usecs_irq)
 421                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 422                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 423         else
 424                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 425
 426         nesadapter->base_pd = 1;
 427
 428         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 429                                        IB_DEVICE_MEM_WINDOW |
 430                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 431
 432         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 433                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 434         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 435         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 436         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 437         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 438         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 439
 440
 441         /* mark the usual suspect QPs, MR and CQs as in use */
 442         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 443                 set_bit(u32temp, nesadapter->allocated_qps);
 444                 set_bit(u32temp, nesadapter->allocated_cqs);
 445         }
 446         set_bit(0, nesadapter->allocated_mrs);
 447
 448         for (u32temp = 0; u32temp < 20; u32temp++)
 449                 set_bit(u32temp, nesadapter->allocated_pds);
 450         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 451
 452         max_rq_wrs = ((u32temp >> 8) & 3);
 453         switch (max_rq_wrs) {
 454                 case 0:
 455                         max_rq_wrs = 4;
 456                         break;
 457                 case 1:
 458                         max_rq_wrs = 16;
 459                         break;
 460                 case 2:
 461                         max_rq_wrs = 32;
 462                         break;
 463                 case 3:
 464                         max_rq_wrs = 512;
 465                         break;
 466         }
 467
 468         max_sq_wrs = (u32temp & 3);
 469         switch (max_sq_wrs) {
 470                 case 0:
 471                         max_sq_wrs = 4;
 472                         break;
 473                 case 1:
 474                         max_sq_wrs = 16;
 475                         break;
 476                 case 2:
 477                         max_sq_wrs = 32;
 478                         break;
 479                 case 3:
 480                         max_sq_wrs = 512;
 481                         break;
 482         }
 483         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 484         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 485
 486         nesadapter->max_sge = 4;
 487         nesadapter->max_cqe = 32766;
 488
 489         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 490                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 491                 kfree(nesadapter);
 492                 return NULL;
 493         }
 494
 495         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 496         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 497                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 498
 499         /* setup port configuration */
 500         if (nesadapter->port_count == 1) {
 501                 nesadapter->log_port = 0x00000000;
 502                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 503                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 504                 else
 505                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 506         } else {
 507                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 508                         nesadapter->log_port = 0x000000D8;
 509                 } else {
 510                         if (nesadapter->port_count == 2)
 511                                 nesadapter->log_port = 0x00000044;
 512                         else
 513                                 nesadapter->log_port = 0x000000e4;
 514                 }
 515                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 516         }
 517
 518         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 519                                                 nesadapter->log_port);
 520         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 521                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 522
 523         spin_lock_init(&nesadapter->resource_lock);
 524         spin_lock_init(&nesadapter->phy_lock);
 525         spin_lock_init(&nesadapter->pbl_lock);
 526         spin_lock_init(&nesadapter->periodic_timer_lock);
 527
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 531         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 532
 533         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 534                 u32 pcs_control_status0, pcs_control_status1;
 535                 u32 reset_value;
 536                 u32 i = 0;
 537                 u32 int_cnt = 0;
 538                 u32 ext_cnt = 0;
 539                 unsigned long flags;
 540                 u32 j = 0;
 541
 542                 pcs_control_status0 = nes_read_indexed(nesdev,
 543                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 544                 pcs_control_status1 = nes_read_indexed(nesdev,
 545                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 546
 547                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 548                         pcs_control_status0 = nes_read_indexed(nesdev,
 549                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 550                         pcs_control_status1 = nes_read_indexed(nesdev,
 551                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 552                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 553                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 554                                 int_cnt++;
 555                         msleep(1);
 556                 }
 557                 if (int_cnt > 1) {
 558                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 559                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 560                         mh_detected++;
 561                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 562                         reset_value |= 0x0000003d;
 563                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 564
 565                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 566                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 567                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 568
 569                         pcs_control_status0 = nes_read_indexed(nesdev,
 570                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 571                         pcs_control_status1 = nes_read_indexed(nesdev,
 572                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 573
 574                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 575                                 pcs_control_status0 = nes_read_indexed(nesdev,
 576                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 577                                 pcs_control_status1 = nes_read_indexed(nesdev,
 578                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 579                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 580                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 581                                         if (++ext_cnt > int_cnt) {
 582                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 583                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 584                                                                 0x0000F088);
 585                                                 mh_detected++;
 586                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 587                                                 reset_value |= 0x0000003d;
 588                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 589
 590                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 591                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 592                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 593                                                 break;
 594                                         }
 595                                 }
 596                                 msleep(1);
 597                         }
 598                 }
 599         }
 600
 601         if (nesadapter->hw_rev == NE020_REV) {
 602                 init_timer(&nesadapter->mh_timer);
 603                 nesadapter->mh_timer.function = nes_mh_fix;
 604                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 605                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 606                 add_timer(&nesadapter->mh_timer);
 607         } else {
 608                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 609         }
 610
 611         init_timer(&nesadapter->lc_timer);
 612         nesadapter->lc_timer.function = nes_clc;
 613         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 614         nesadapter->lc_timer.data = (unsigned long)nesdev;
 615         add_timer(&nesadapter->lc_timer);
 616
 617         list_add_tail(&nesadapter->list, &nes_adapter_list);
 618
 619         for (func_index = 0; func_index < 8; func_index++) {
 620                 pci_bus_read_config_word(nesdev->pcidev->bus,
 621                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 622                                         func_index), 0, &vendor_id);
 623                 if (vendor_id == 0xffff)
 624                         break;
 625         }
 626         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 627                 func_index, pci_name(nesdev->pcidev));
 628         nesadapter->adapter_fcn_count = func_index;
 629
 630         return nesadapter;
 631 }
 632
 633
 634 /**
 635  * nes_reset_adapter_ne020
 636  */
 637 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 638 {
 639         u32 port_count;
 640         u32 u32temp;
 641         u32 i;
 642
 643         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 644         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 645         /* TODO: assuming that both SERDES are set the same for now */
 646         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 647         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 648                         u32temp, port_count);
 649         if (*OneG_Mode)
 650                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 651         u32temp &= 0xff00ffc0;
 652         switch (port_count) {
 653                 case 1:
 654                         u32temp |= 0x00ee0000;
 655                         break;
 656                 case 2:
 657                         u32temp |= 0x00cc0000;
 658                         break;
 659                 case 4:
 660                         u32temp |= 0x00000000;
 661                         break;
 662                 default:
 663                         return 0;
 664                         break;
 665         }
 666
 667         /* check and do full reset if needed */
 668         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 669                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 670                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 671
 672                 i = 0;
 673                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 674                         mdelay(1);
 675                 if (i > 10000) {
 676                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 677                         return 0;
 678                 }
 679
 680                 i = 0;
 681                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 682                         mdelay(1);
 683                 if (i > 10000) {
 684                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 685                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 686                         return 0;
 687                 }
 688         }
 689
 690         /* port reset */
 691         switch (port_count) {
 692                 case 1:
 693                         u32temp |= 0x00ee0010;
 694                         break;
 695                 case 2:
 696                         u32temp |= 0x00cc0030;
 697                         break;
 698                 case 4:
 699                         u32temp |= 0x00000030;
 700                         break;
 701         }
 702
 703         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 704         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 705
 706         i = 0;
 707         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 708                 mdelay(1);
 709         if (i > 10000) {
 710                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 711                 return 0;
 712         }
 713
 714         /* serdes 0 */
 715         i = 0;
 716         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 717                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 718                 mdelay(1);
 719         if (i > 5000) {
 720                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 721                 return 0;
 722         }
 723
 724         /* serdes 1 */
 725         if (port_count > 1) {
 726                 i = 0;
 727                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 728                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 729                         mdelay(1);
 730                 if (i > 5000) {
 731                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 732                         return 0;
 733                 }
 734         }
 735
 736         return port_count;
 737 }
 738
 739
 740 /**
 741  * nes_init_serdes
 742  */
 743 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 744                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 745 {
 746         int i;
 747         u32 u32temp;
 748         u32 sds;
 749
 750         if (hw_rev != NE020_REV) {
 751                 /* init serdes 0 */
 752                 switch (nesadapter->phy_type[0]) {
 753                 case NES_PHY_TYPE_CX4:
 754                         if (wide_ppm_offset)
 755                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 756                         else
 757                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 758                         break;
 759                 case NES_PHY_TYPE_KR:
 760                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 761                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 762                         break;
 763                 case NES_PHY_TYPE_PUMA_1G:
 764                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 765                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 766                         sds |= 0x00000100;
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 768                         break;
 769                 default:
 770                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 771                         break;
 772                 }
 773
 774                 if (!OneG_Mode)
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 776
 777                 if (port_count < 2)
 778                         return 0;
 779
 780                 /* init serdes 1 */
 781                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 783
 784                 switch (nesadapter->phy_type[1]) {
 785                 case NES_PHY_TYPE_ARGUS:
 786                 case NES_PHY_TYPE_SFP_D:
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 788                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 789                         break;
 790                 case NES_PHY_TYPE_CX4:
 791                         if (wide_ppm_offset)
 792                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 793                         break;
 794                 case NES_PHY_TYPE_KR:
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 796                         break;
 797                 case NES_PHY_TYPE_PUMA_1G:
 798                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 799                         sds |= 0x000000100;
 800                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 801                 }
 802                 if (!OneG_Mode) {
 803                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 804                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 805                         sds &= 0xFFFFFFBF;
 806                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 807                 }
 808         } else {
 809                 /* init serdes 0 */
 810                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 811                 i = 0;
 812                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 813                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 814                         mdelay(1);
 815                 if (i > 5000) {
 816                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 817                         return 1;
 818                 }
 819                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 820                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 821                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 822                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 823                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 824                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 825                 if (OneG_Mode)
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 827                 else
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 829
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 831                 if (port_count > 1) {
 832                         /* init serdes 1 */
 833                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 834                         i = 0;
 835                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 836                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 837                                 mdelay(1);
 838                         if (i > 5000) {
 839                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 840                                 /* return 1; */
 841                         }
 842                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 843                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 844                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 845                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 846                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 847                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 848                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 849                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 850                 }
 851         }
 852         return 0;
 853 }
 854
 855
 856 /**
 857  * nes_init_csr_ne020
 858  * Initialize registers for ne020 hardware
 859  */
 860 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 861 {
 862         u32 u32temp;
 863
 864         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 865
 866         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 867         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 868         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 869         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 870         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 871         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 872         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 873         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 874
 875         /* TODO: move these MAC register settings to NIC bringup */
 876         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 877         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 878         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 879         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 880         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 881         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 882         if (port_count > 1) {
 883                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 884                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 885                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 886                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 887                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 888                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 889                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 890         }
 891         if (port_count > 2) {
 892                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 893                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 894                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 895                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 896                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 897                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 898                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 899
 900                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 902                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 903                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 904                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 905                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 906                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 907         }
 908
 909         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 910         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 911         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 912                                                          0x00000001);
 913         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 914         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 915         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 916         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 917         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 918
 919         /* TODO: move this to code, get from EEPROM */
 920         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 921         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 922         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 923
 924         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 925         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 926
 927         if (hw_rev != NE020_REV) {
 928                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 929                 u32temp |= 0x80000000;
 930                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 931                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 932                 u32temp &= 0x7fffffff;
 933                 u32temp |= 0x7fff0010;
 934                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 935                 if (port_count > 1) {
 936                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 937                         u32temp &= 0x7fffffff;
 938                         u32temp |= 0x7fff0010;
 939                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 940                 }
 941         }
 942 }
 943
 944
 945 /**
 946  * nes_destroy_adapter - destroy the adapter structure
 947  */
 948 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 949 {
 950         struct nes_adapter *tmp_adapter;
 951
 952         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 953                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 954                                 tmp_adapter);
 955         }
 956
 957         nesadapter->ref_count--;
 958         if (!nesadapter->ref_count) {
 959                 if (nesadapter->hw_rev == NE020_REV) {
 960                         del_timer(&nesadapter->mh_timer);
 961                 }
 962                 del_timer(&nesadapter->lc_timer);
 963
 964                 list_del(&nesadapter->list);
 965                 kfree(nesadapter);
 966         }
 967 }
 968
 969
 970 /**
 971  * nes_init_cqp
 972  */
 973 int nes_init_cqp(struct nes_device *nesdev)
 974 {
 975         struct nes_adapter *nesadapter = nesdev->nesadapter;
 976         struct nes_hw_cqp_qp_context *cqp_qp_context;
 977         struct nes_hw_cqp_wqe *cqp_wqe;
 978         struct nes_hw_ceq *ceq;
 979         struct nes_hw_ceq *nic_ceq;
 980         struct nes_hw_aeq *aeq;
 981         void *vmem;
 982         dma_addr_t pmem;
 983         u32 count=0;
 984         u32 cqp_head;
 985         u64 u64temp;
 986         u32 u32temp;
 987
 988         /* allocate CQP memory */
 989         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 990         /* SQ is 512 byte aligned, others are 256 byte aligned */
 991         nesdev->cqp_mem_size = 512 +
 992                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 993                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 994                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 995                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 996                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 997                         sizeof(struct nes_hw_cqp_qp_context);
 998
 999         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1000                         &nesdev->cqp_pbase);
1001         if (!nesdev->cqp_vbase) {
1002                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1003                 return -ENOMEM;
1004         }
1005         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1006
1007         /* Allocate a twice the number of CQP requests as the SQ size */
1008         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1009                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1010         if (nesdev->nes_cqp_requests == NULL) {
1011                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1012                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1013                                 nesdev->cqp.sq_pbase);
1014                 return -ENOMEM;
1015         }
1016
1017         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1018                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1019
1020         spin_lock_init(&nesdev->cqp.lock);
1021         init_waitqueue_head(&nesdev->cqp.waitq);
1022
1023         /* Setup Various Structures */
1024         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1025                         ~(unsigned long)(512 - 1));
1026         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1027                         ~(unsigned long long)(512 - 1));
1028
1029         nesdev->cqp.sq_vbase = vmem;
1030         nesdev->cqp.sq_pbase = pmem;
1031         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1032         nesdev->cqp.sq_head = 0;
1033         nesdev->cqp.sq_tail = 0;
1034         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1035
1036         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1037         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1038
1039         nesdev->ccq.cq_vbase = vmem;
1040         nesdev->ccq.cq_pbase = pmem;
1041         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1042         nesdev->ccq.cq_head = 0;
1043         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1044         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1045
1046         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1047         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1048
1049         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1050         ceq = &nesadapter->ceq[nesdev->ceq_index];
1051         ceq->ceq_vbase = vmem;
1052         ceq->ceq_pbase = pmem;
1053         ceq->ceq_size = NES_CCEQ_SIZE;
1054         ceq->ceq_head = 0;
1055
1056         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1057         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1058
1059         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1060         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1061         nic_ceq->ceq_vbase = vmem;
1062         nic_ceq->ceq_pbase = pmem;
1063         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1064         nic_ceq->ceq_head = 0;
1065
1066         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1067         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1068
1069         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1070         aeq->aeq_vbase = vmem;
1071         aeq->aeq_pbase = pmem;
1072         aeq->aeq_size = nesadapter->max_qp;
1073         aeq->aeq_head = 0;
1074
1075         /* Setup QP Context */
1076         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1077         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1078
1079         cqp_qp_context = vmem;
1080         cqp_qp_context->context_words[0] =
1081                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1082         cqp_qp_context->context_words[1] = 0;
1083         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1084         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1085
1086
1087         /* Write the address to Create CQP */
1088         if ((sizeof(dma_addr_t) > 4)) {
1089                 nes_write_indexed(nesdev,
1090                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1091                                 ((u64)pmem) >> 32);
1092         } else {
1093                 nes_write_indexed(nesdev,
1094                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1095         }
1096         nes_write_indexed(nesdev,
1097                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098                         (u32)pmem);
1099
1100         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1101         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1102
1103         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1104                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1105                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1106         }
1107
1108         /* Write Create CCQ WQE */
1109         cqp_head = nesdev->cqp.sq_head++;
1110         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1111         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1112         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1113                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1114                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1115         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1116                             (nesdev->ccq.cq_number |
1117                              ((u32)nesdev->ceq_index << 16)));
1118         u64temp = (u64)nesdev->ccq.cq_pbase;
1119         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1120         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1121         u64temp = (unsigned long)&nesdev->ccq;
1122         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1123                         cpu_to_le32((u32)(u64temp >> 1));
1124         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1125                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1126         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1127
1128         /* Write Create CEQ WQE */
1129         cqp_head = nesdev->cqp.sq_head++;
1130         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1131         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1132         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1133                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1134         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1135         u64temp = (u64)ceq->ceq_pbase;
1136         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1137
1138         /* Write Create AEQ WQE */
1139         cqp_head = nesdev->cqp.sq_head++;
1140         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1141         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1142         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1143                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1144         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1145         u64temp = (u64)aeq->aeq_pbase;
1146         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1147
1148         /* Write Create NIC CEQ WQE */
1149         cqp_head = nesdev->cqp.sq_head++;
1150         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1151         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1152         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1153                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1154         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1155         u64temp = (u64)nic_ceq->ceq_pbase;
1156         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1157
1158         /* Poll until CCQP done */
1159         count = 0;
1160         do {
1161                 if (count++ > 1000) {
1162                         printk(KERN_ERR PFX "Error creating CQP\n");
1163                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1164                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1165                         return -1;
1166                 }
1167                 udelay(10);
1168         } while (!(nes_read_indexed(nesdev,
1169                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1170
1171         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1172                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1173
1174         u32temp = 0x04800000;
1175         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1176
1177         /* wait for the CCQ, CEQ, and AEQ to get created */
1178         count = 0;
1179         do {
1180                 if (count++ > 1000) {
1181                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1182                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1183                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1184                         return -1;
1185                 }
1186                 udelay(10);
1187         } while (((nes_read_indexed(nesdev,
1188                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1189
1190         /* dump the QP status value */
1191         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1192                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1193
1194         nesdev->cqp.sq_tail++;
1195
1196         return 0;
1197 }
1198
1199
1200 /**
1201  * nes_destroy_cqp
1202  */
1203 int nes_destroy_cqp(struct nes_device *nesdev)
1204 {
1205         struct nes_hw_cqp_wqe *cqp_wqe;
1206         u32 count = 0;
1207         u32 cqp_head;
1208         unsigned long flags;
1209
1210         do {
1211                 if (count++ > 1000)
1212                         break;
1213                 udelay(10);
1214         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1215
1216         /* Reset CCQ */
1217         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1218                         nesdev->ccq.cq_number);
1219
1220         /* Disable device interrupts */
1221         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1222
1223         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1224
1225         /* Destroy the AEQ */
1226         cqp_head = nesdev->cqp.sq_head++;
1227         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1228         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1229         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1230                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1231         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1232
1233         /* Destroy the NIC CEQ */
1234         cqp_head = nesdev->cqp.sq_head++;
1235         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1238                         ((u32)nesdev->nic_ceq_index << 8));
1239
1240         /* Destroy the CEQ */
1241         cqp_head = nesdev->cqp.sq_head++;
1242         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245                         (nesdev->ceq_index << 8));
1246
1247         /* Destroy the CCQ */
1248         cqp_head = nesdev->cqp.sq_head++;
1249         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1252         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1253                         ((u32)nesdev->ceq_index << 16));
1254
1255         /* Destroy CQP */
1256         cqp_head = nesdev->cqp.sq_head++;
1257         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1260                         NES_CQP_QP_TYPE_CQP);
1261         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1262
1263         barrier();
1264         /* Ring doorbell (5 WQEs) */
1265         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1266
1267         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1268
1269         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1270         count = 0;
1271         do {
1272                 if (count++ > 1000) {
1273                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1274                                         PCI_FUNC(nesdev->pcidev->devfn));
1275                         break;
1276                 }
1277                 udelay(10);
1278         } while (((nes_read_indexed(nesdev,
1279                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1280
1281         /* dump the QP status value */
1282         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1283                         PCI_FUNC(nesdev->pcidev->devfn),
1284                         nes_read_indexed(nesdev,
1285                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1286
1287         kfree(nesdev->nes_cqp_requests);
1288
1289         /* Free the control structures */
1290         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1291                         nesdev->cqp.sq_pbase);
1292
1293         return 0;
1294 }
1295
1296
1297 /**
1298  * nes_init_1g_phy
1299  */
1300 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1301 {
1302         u32 counter = 0;
1303         u16 phy_data;
1304         int ret = 0;
1305
1306         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1307         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1308
1309         /* Reset the PHY */
1310         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1311         udelay(100);
1312         counter = 0;
1313         do {
1314                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1315                 if (counter++ > 100) {
1316                         ret = -1;
1317                         break;
1318                 }
1319         } while (phy_data & 0x8000);
1320
1321         /* Setting no phy loopback */
1322         phy_data &= 0xbfff;
1323         phy_data |= 0x1140;
1324         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1325         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1326         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1327         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1328
1329         /* Setting the interrupt mask */
1330         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1331         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1332         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1333
1334         /* turning on flow control */
1335         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1336         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1337         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1338
1339         /* Clear Half duplex */
1340         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1341         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1342         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1343
1344         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1345         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1346
1347         return ret;
1348 }
1349
1350
1351 /**
1352  * nes_init_2025_phy
1353  */
1354 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1355 {
1356         u32 temp_phy_data = 0;
1357         u32 temp_phy_data2 = 0;
1358         u32 counter = 0;
1359         u32 sds;
1360         u32 mac_index = nesdev->mac_index;
1361         int ret = 0;
1362         unsigned int first_attempt = 1;
1363
1364         /* Check firmware heartbeat */
1365         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1366         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1367         udelay(1500);
1368         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1369         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1370
1371         if (temp_phy_data != temp_phy_data2) {
1372                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1373                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374                 if ((temp_phy_data & 0xff) > 0x20)
1375                         return 0;
1376                 printk(PFX "Reinitialize external PHY\n");
1377         }
1378
1379         /* no heartbeat, configure the PHY */
1380         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1381         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1382         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1383         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1384
1385         switch (phy_type) {
1386         case NES_PHY_TYPE_ARGUS:
1387                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1388                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1389                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1390                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1392                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1393                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1394
1395                 /* setup LEDs */
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1399                 break;
1400
1401         case NES_PHY_TYPE_SFP_D:
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1406                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1407                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1408                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1409
1410                 /* setup LEDs */
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1414                 break;
1415
1416         case NES_PHY_TYPE_KR:
1417                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1421                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1422                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1423                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1424
1425                 /* setup LEDs */
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1429
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1431                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1432                 break;
1433         }
1434
1435         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1436
1437         /* Bring PHY out of reset */
1438         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1439
1440         /* Check for heartbeat */
1441         counter = 0;
1442         mdelay(690);
1443         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1444         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1445         do {
1446                 if (counter++ > 150) {
1447                         printk(PFX "No PHY heartbeat\n");
1448                         break;
1449                 }
1450                 mdelay(1);
1451                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453         } while ((temp_phy_data2 == temp_phy_data));
1454
1455         /* wait for tracking */
1456         counter = 0;
1457         do {
1458                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1459                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460                 if (counter++ > 300) {
1461                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1462                                 first_attempt = 0;
1463                                 counter = 0;
1464                                 /* reset AMCC PHY and try again */
1465                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1466                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1467                                 continue;
1468                         } else {
1469                                 ret = 1;
1470                                 break;
1471                         }
1472                 }
1473                 mdelay(10);
1474         } while ((temp_phy_data & 0xff) < 0x30);
1475
1476         /* setup signal integrity */
1477         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1478         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1479         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1480         if (phy_type == NES_PHY_TYPE_KR) {
1481                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1482         } else {
1483                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1484                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1485         }
1486
1487         /* reset serdes */
1488         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1489         sds |= 0x1;
1490         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1491         sds &= 0xfffffffe;
1492         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1493
1494         counter = 0;
1495         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1496                         && (counter++ < 5000))
1497                 ;
1498
1499         return ret;
1500 }
1501
1502
1503 /**
1504  * nes_init_phy
1505  */
1506 int nes_init_phy(struct nes_device *nesdev)
1507 {
1508         struct nes_adapter *nesadapter = nesdev->nesadapter;
1509         u32 mac_index = nesdev->mac_index;
1510         u32 tx_config = 0;
1511         unsigned long flags;
1512         u8  phy_type = nesadapter->phy_type[mac_index];
1513         u8  phy_index = nesadapter->phy_index[mac_index];
1514         int ret = 0;
1515
1516         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1517         if (phy_type == NES_PHY_TYPE_1G) {
1518                 /* setup 1G MDIO operation */
1519                 tx_config &= 0xFFFFFFE3;
1520                 tx_config |= 0x04;
1521         } else {
1522                 /* setup 10G MDIO operation */
1523                 tx_config &= 0xFFFFFFE3;
1524                 tx_config |= 0x15;
1525         }
1526         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1527
1528         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1529
1530         switch (phy_type) {
1531         case NES_PHY_TYPE_1G:
1532                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1533                 break;
1534         case NES_PHY_TYPE_ARGUS:
1535         case NES_PHY_TYPE_SFP_D:
1536         case NES_PHY_TYPE_KR:
1537                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1538                 break;
1539         }
1540
1541         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1542
1543         return ret;
1544 }
1545
1546
1547 /**
1548  * nes_replenish_nic_rq
1549  */
1550 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1551 {
1552         unsigned long flags;
1553         dma_addr_t bus_address;
1554         struct sk_buff *skb;
1555         struct nes_hw_nic_rq_wqe *nic_rqe;
1556         struct nes_hw_nic *nesnic;
1557         struct nes_device *nesdev;
1558         u32 rx_wqes_posted = 0;
1559
1560         nesnic = &nesvnic->nic;
1561         nesdev = nesvnic->nesdev;
1562         spin_lock_irqsave(&nesnic->rq_lock, flags);
1563         if (nesnic->replenishing_rq !=0) {
1564                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1565                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1566                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1567                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1568                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1569                         add_timer(&nesvnic->rq_wqes_timer);
1570                 } else
1571                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1572                 return;
1573         }
1574         nesnic->replenishing_rq = 1;
1575         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1576         do {
1577                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1578                 if (skb) {
1579                         skb->dev = nesvnic->netdev;
1580
1581                         bus_address = pci_map_single(nesdev->pcidev,
1582                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1583
1584                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1585                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1586                                         cpu_to_le32(nesvnic->max_frame_size);
1587                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1588                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1589                                         cpu_to_le32((u32)bus_address);
1590                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1591                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1592                         nesnic->rx_skb[nesnic->rq_head] = skb;
1593                         nesnic->rq_head++;
1594                         nesnic->rq_head &= nesnic->rq_size - 1;
1595                         atomic_dec(&nesvnic->rx_skbs_needed);
1596                         barrier();
1597                         if (++rx_wqes_posted == 255) {
1598                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1599                                 rx_wqes_posted = 0;
1600                         }
1601                 } else {
1602                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1603                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1604                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1605                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1606                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1607                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1608                                 add_timer(&nesvnic->rq_wqes_timer);
1609                         } else
1610                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1611                         break;
1612                 }
1613         } while (atomic_read(&nesvnic->rx_skbs_needed));
1614         barrier();
1615         if (rx_wqes_posted)
1616                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1617         nesnic->replenishing_rq = 0;
1618 }
1619
1620
1621 /**
1622  * nes_rq_wqes_timeout
1623  */
1624 static void nes_rq_wqes_timeout(unsigned long parm)
1625 {
1626         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1627         printk("%s: Timer fired.\n", __func__);
1628         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1629         if (atomic_read(&nesvnic->rx_skbs_needed))
1630                 nes_replenish_nic_rq(nesvnic);
1631 }
1632
1633
1634 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1635                                void **tcph, u64 *hdr_flags, void *priv)
1636 {
1637         unsigned int ip_len;
1638         struct iphdr *iph;
1639         skb_reset_network_header(skb);
1640         iph = ip_hdr(skb);
1641         if (iph->protocol != IPPROTO_TCP)
1642                 return -1;
1643         ip_len = ip_hdrlen(skb);
1644         skb_set_transport_header(skb, ip_len);
1645         *tcph = tcp_hdr(skb);
1646
1647         *hdr_flags = LRO_IPV4 | LRO_TCP;
1648         *iphdr = iph;
1649         return 0;
1650 }
1651
1652
1653 /**
1654  * nes_init_nic_qp
1655  */
1656 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1657 {
1658         struct nes_hw_cqp_wqe *cqp_wqe;
1659         struct nes_hw_nic_sq_wqe *nic_sqe;
1660         struct nes_hw_nic_qp_context *nic_context;
1661         struct sk_buff *skb;
1662         struct nes_hw_nic_rq_wqe *nic_rqe;
1663         struct nes_vnic *nesvnic = netdev_priv(netdev);
1664         unsigned long flags;
1665         void *vmem;
1666         dma_addr_t pmem;
1667         u64 u64temp;
1668         int ret;
1669         u32 cqp_head;
1670         u32 counter;
1671         u32 wqe_count;
1672         u8 jumbomode=0;
1673
1674         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1675         nesvnic->nic_mem_size = 256 +
1676                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1677                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1678                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1679                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1680                         sizeof(struct nes_hw_nic_qp_context);
1681
1682         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1683                         &nesvnic->nic_pbase);
1684         if (!nesvnic->nic_vbase) {
1685                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1686                 return -ENOMEM;
1687         }
1688         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1689         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1690                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1691
1692         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1693                         ~(unsigned long)(256 - 1));
1694         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1695                         ~(unsigned long long)(256 - 1));
1696
1697         /* Setup the first Fragment buffers */
1698         nesvnic->nic.first_frag_vbase = vmem;
1699
1700         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1701                 nesvnic->nic.frag_paddr[counter] = pmem;
1702                 pmem += sizeof(struct nes_first_frag);
1703         }
1704
1705         /* setup the SQ */
1706         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1707
1708         nesvnic->nic.sq_vbase = (void *)vmem;
1709         nesvnic->nic.sq_pbase = pmem;
1710         nesvnic->nic.sq_head = 0;
1711         nesvnic->nic.sq_tail = 0;
1712         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1713         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1714                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1715                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1716                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1717                                 NES_NIC_SQ_WQE_COMPLETION);
1718                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1719                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1720                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1721                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1722                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1723                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1724         }
1725
1726         nesvnic->get_cqp_request = nes_get_cqp_request;
1727         nesvnic->post_cqp_request = nes_post_cqp_request;
1728         nesvnic->mcrq_mcast_filter = NULL;
1729
1730         spin_lock_init(&nesvnic->nic.rq_lock);
1731
1732         /* setup the RQ */
1733         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1734         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1735
1736
1737         nesvnic->nic.rq_vbase = vmem;
1738         nesvnic->nic.rq_pbase = pmem;
1739         nesvnic->nic.rq_head = 0;
1740         nesvnic->nic.rq_tail = 0;
1741         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1742
1743         /* setup the CQ */
1744         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1745         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1746
1747         if (nesdev->nesadapter->netdev_count > 2)
1748                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1749         else
1750                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1751
1752         nesvnic->nic_cq.cq_vbase = vmem;
1753         nesvnic->nic_cq.cq_pbase = pmem;
1754         nesvnic->nic_cq.cq_head = 0;
1755         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1756
1757         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1758
1759         /* Send CreateCQ request to CQP */
1760         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1761         cqp_head = nesdev->cqp.sq_head;
1762
1763         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1764         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1765
1766         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1767                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1768                         ((u32)nesvnic->nic_cq.cq_size << 16));
1769         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1770                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1771         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1772         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1773         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1774         u64temp = (unsigned long)&nesvnic->nic_cq;
1775         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1776         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1777                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1778         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1779         if (++cqp_head >= nesdev->cqp.sq_size)
1780                 cqp_head = 0;
1781         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1782         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1783
1784         /* Send CreateQP request to CQP */
1785         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1786         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1787                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1788                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1789         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1790                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1791                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1792         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1793                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1794         }
1795
1796         u64temp = (u64)nesvnic->nic.sq_pbase;
1797         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1798         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1799         u64temp = (u64)nesvnic->nic.rq_pbase;
1800         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1801         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1802
1803         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1804                         NES_CQP_QP_TYPE_NIC);
1805         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1806         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1807                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1808         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1809
1810         if (++cqp_head >= nesdev->cqp.sq_size)
1811                 cqp_head = 0;
1812         nesdev->cqp.sq_head = cqp_head;
1813
1814         barrier();
1815
1816         /* Ring doorbell (2 WQEs) */
1817         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1818
1819         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1820         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1821                         nesvnic->nic.qp_id);
1822
1823         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1824                         NES_EVENT_TIMEOUT);
1825         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1826                         nesvnic->nic.qp_id, ret);
1827         if (!ret) {
1828                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1829                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1830                                 nesvnic->nic_pbase);
1831                 return -EIO;
1832         }
1833
1834         /* Populate the RQ */
1835         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1836                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1837                 if (!skb) {
1838                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1839
1840                         nes_destroy_nic_qp(nesvnic);
1841                         return -ENOMEM;
1842                 }
1843
1844                 skb->dev = netdev;
1845
1846                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1847                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1848
1849                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1850                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1851                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1852                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1853                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1854                 nesvnic->nic.rx_skb[counter] = skb;
1855         }
1856
1857         wqe_count = NES_NIC_WQ_SIZE - 1;
1858         nesvnic->nic.rq_head = wqe_count;
1859         barrier();
1860         do {
1861                 counter = min(wqe_count, ((u32)255));
1862                 wqe_count -= counter;
1863                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1864         } while (wqe_count);
1865         init_timer(&nesvnic->rq_wqes_timer);
1866         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1867         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1868         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1869         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1870         {
1871                 nes_nic_init_timer(nesdev);
1872                 if (netdev->mtu > 1500)
1873                         jumbomode = 1;
1874                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1875         }
1876         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1877         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1878         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1879         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1880         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1881         nesvnic->lro_mgr.dev            = netdev;
1882         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1883         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1884         return 0;
1885 }
1886
1887
1888 /**
1889  * nes_destroy_nic_qp
1890  */
1891 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1892 {
1893         u64 u64temp;
1894         dma_addr_t bus_address;
1895         struct nes_device *nesdev = nesvnic->nesdev;
1896         struct nes_hw_cqp_wqe *cqp_wqe;
1897         struct nes_hw_nic_sq_wqe *nic_sqe;
1898         struct nes_hw_nic_rq_wqe *nic_rqe;
1899         __le16 *wqe_fragment_length;
1900         u16  wqe_fragment_index;
1901         u64 wqe_frag;
1902         u32 cqp_head;
1903         u32 wqm_cfg0;
1904         unsigned long flags;
1905         int ret;
1906
1907         /* clear wqe stall before destroying NIC QP */
1908         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1909         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1910
1911         /* Free remaining NIC receive buffers */
1912         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1913                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1914                 wqe_frag  = (u64)le32_to_cpu(
1915                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1916                 wqe_frag |= ((u64)le32_to_cpu(
1917                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1918                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1919                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1920                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1921                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1922         }
1923
1924         /* Free remaining NIC transmit buffers */
1925         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1926                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1927                 wqe_fragment_index = 1;
1928                 wqe_fragment_length = (__le16 *)
1929                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1930                 /* bump past the vlan tag */
1931                 wqe_fragment_length++;
1932                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1933                         u64temp = (u64)le32_to_cpu(
1934                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1935                                 wqe_fragment_index*2]);
1936                         u64temp += ((u64)le32_to_cpu(
1937                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1938                                 + wqe_fragment_index*2]))<<32;
1939                         bus_address = (dma_addr_t)u64temp;
1940                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1941                                         nesvnic->nic.first_frag_overflow)) {
1942                                 pci_unmap_single(nesdev->pcidev,
1943                                                 bus_address,
1944                                                 le16_to_cpu(wqe_fragment_length[
1945                                                         wqe_fragment_index++]),
1946                                                 PCI_DMA_TODEVICE);
1947                         }
1948                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1949                                 if (wqe_fragment_length[wqe_fragment_index]) {
1950                                         u64temp = le32_to_cpu(
1951                                                 nic_sqe->wqe_words[
1952                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1953                                                 wqe_fragment_index*2]);
1954                                         u64temp += ((u64)le32_to_cpu(
1955                                                 nic_sqe->wqe_words[
1956                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1957                                                 wqe_fragment_index*2]))<<32;
1958                                         bus_address = (dma_addr_t)u64temp;
1959                                         pci_unmap_page(nesdev->pcidev,
1960                                                         bus_address,
1961                                                         le16_to_cpu(
1962                                                         wqe_fragment_length[
1963                                                         wqe_fragment_index]),
1964                                                         PCI_DMA_TODEVICE);
1965                                 } else
1966                                         break;
1967                         }
1968                 }
1969                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1970                         dev_kfree_skb(
1971                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1972
1973                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1974                                         & (nesvnic->nic.sq_size - 1);
1975         }
1976
1977         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1978
1979         /* Destroy NIC QP */
1980         cqp_head = nesdev->cqp.sq_head;
1981         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1982         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1983
1984         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1985                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1986         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1987                 nesvnic->nic.qp_id);
1988
1989         if (++cqp_head >= nesdev->cqp.sq_size)
1990                 cqp_head = 0;
1991
1992         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1993
1994         /* Destroy NIC CQ */
1995         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1996         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1997                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1998         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1999                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2000
2001         if (++cqp_head >= nesdev->cqp.sq_size)
2002                 cqp_head = 0;
2003
2004         nesdev->cqp.sq_head = cqp_head;
2005         barrier();
2006
2007         /* Ring doorbell (2 WQEs) */
2008         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2009
2010         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2011         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2012                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2013                         cqp_head, nesdev->cqp.sq_head,
2014                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2015
2016         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2017                         NES_EVENT_TIMEOUT);
2018
2019         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2020                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2021                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2022         if (!ret) {
2023                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2024                                 nesvnic->nic.qp_id);
2025         }
2026
2027         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2028                         nesvnic->nic_pbase);
2029
2030         /* restore old wqm_cfg0 value */
2031         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2032 }
2033
2034 /**
2035  * nes_napi_isr
2036  */
2037 int nes_napi_isr(struct nes_device *nesdev)
2038 {
2039         struct nes_adapter *nesadapter = nesdev->nesadapter;
2040         u32 int_stat;
2041
2042         if (nesdev->napi_isr_ran) {
2043                 /* interrupt status has already been read in ISR */
2044                 int_stat = nesdev->int_stat;
2045         } else {
2046                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2047                 nesdev->int_stat = int_stat;
2048                 nesdev->napi_isr_ran = 1;
2049         }
2050
2051         int_stat &= nesdev->int_req;
2052         /* iff NIC, process here, else wait for DPC */
2053         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2054                 nesdev->napi_isr_ran = 0;
2055                 nes_write32(nesdev->regs + NES_INT_STAT,
2056                         (int_stat &
2057                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2058
2059                 /* Process the CEQs */
2060                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2061
2062                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2063                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2064                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2065                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2066                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2067                                 /* Enable Periodic timer interrupts */
2068                                 nesdev->int_req |= NES_INT_TIMER;
2069                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2070                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2071                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2072                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2073                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2074                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2075                         }
2076
2077                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2078                         {
2079                                 nes_nic_init_timer(nesdev);
2080                         }
2081                         /* Enable interrupts, except CEQs */
2082                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2083                 } else {
2084                         /* Enable interrupts, make sure timer is off */
2085                         nesdev->int_req &= ~NES_INT_TIMER;
2086                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2087                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2088                 }
2089                 nesdev->deepcq_count = 0;
2090                 return 1;
2091         } else {
2092                 return 0;
2093         }
2094 }
2095
2096 static void process_critical_error(struct nes_device *nesdev)
2097 {
2098         u32 debug_error;
2099         u32 nes_idx_debug_error_masks0 = 0;
2100         u16 error_module = 0;
2101
2102         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2103         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2104                         (u16)debug_error);
2105         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2106                         0x01010000 | (debug_error & 0x0000ffff));
2107         if (crit_err_count++ > 10)
2108                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2109         error_module = (u16) (debug_error & 0x1F00) >> 8;
2110         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2111                         nes_max_critical_error_count) {
2112                 printk(KERN_ERR PFX "Masking off critical error for module "
2113                         "0x%02X\n", (u16)error_module);
2114                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2115                         NES_IDX_DEBUG_ERROR_MASKS0);
2116                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2117                         nes_idx_debug_error_masks0 | (1 << error_module));
2118         }
2119 }
2120 /**
2121  * nes_dpc
2122  */
2123 void nes_dpc(unsigned long param)
2124 {
2125         struct nes_device *nesdev = (struct nes_device *)param;
2126         struct nes_adapter *nesadapter = nesdev->nesadapter;
2127         u32 counter;
2128         u32 loop_counter = 0;
2129         u32 int_status_bit;
2130         u32 int_stat;
2131         u32 timer_stat;
2132         u32 temp_int_stat;
2133         u32 intf_int_stat;
2134         u32 processed_intf_int = 0;
2135         u16 processed_timer_int = 0;
2136         u16 completion_ints = 0;
2137         u16 timer_ints = 0;
2138
2139         /* nes_debug(NES_DBG_ISR, "\n"); */
2140
2141         do {
2142                 timer_stat = 0;
2143                 if (nesdev->napi_isr_ran) {
2144                         nesdev->napi_isr_ran = 0;
2145                         int_stat = nesdev->int_stat;
2146                 } else
2147                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2148                 if (processed_intf_int != 0)
2149                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2150                 else
2151                         int_stat &= nesdev->int_req;
2152                 if (processed_timer_int == 0) {
2153                         processed_timer_int = 1;
2154                         if (int_stat & NES_INT_TIMER) {
2155                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2156                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2157                                         int_stat &= ~NES_INT_TIMER;
2158                                 }
2159                         }
2160                 } else {
2161                         int_stat &= ~NES_INT_TIMER;
2162                 }
2163
2164                 if (int_stat) {
2165                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2166                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2167                                 /* Ack the interrupts */
2168                                 nes_write32(nesdev->regs+NES_INT_STAT,
2169                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2170                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2171                         }
2172
2173                         temp_int_stat = int_stat;
2174                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2175                                 if (int_stat & int_status_bit) {
2176                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2177                                         temp_int_stat &= ~int_status_bit;
2178                                         completion_ints = 1;
2179                                 }
2180                                 if (!(temp_int_stat & 0x0000ffff))
2181                                         break;
2182                                 int_status_bit <<= 1;
2183                         }
2184
2185                         /* Process the AEQ for this pci function */
2186                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2187                         if (int_stat & int_status_bit) {
2188                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2189                         }
2190
2191                         /* Process the MAC interrupt for this pci function */
2192                         int_status_bit = 1 << (24 + nesdev->mac_index);
2193                         if (int_stat & int_status_bit) {
2194                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2195                         }
2196
2197                         if (int_stat & NES_INT_TIMER) {
2198                                 if (timer_stat & nesdev->timer_int_req) {
2199                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2200                                                         (timer_stat & nesdev->timer_int_req) |
2201                                                         ~(nesdev->nesadapter->timer_int_req));
2202                                         timer_ints = 1;
2203                                 }
2204                         }
2205
2206                         if (int_stat & NES_INT_INTF) {
2207                                 processed_intf_int = 1;
2208                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2209                                 intf_int_stat &= nesdev->intf_int_req;
2210                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2211                                         process_critical_error(nesdev);
2212                                 }
2213                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2214                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2215                                         BUG();
2216                                 }
2217                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2218                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2219                                         BUG();
2220                                 }
2221                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2222                         }
2223
2224                         if (int_stat & NES_INT_TSW) {
2225                         }
2226                 }
2227                 /* Don't use the interface interrupt bit stay in loop */
2228                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2229                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2230         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2231
2232         if (timer_ints == 1) {
2233                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2234                         if (completion_ints == 0) {
2235                                 nesdev->timer_only_int_count++;
2236                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2237                                         nesdev->timer_only_int_count = 0;
2238                                         nesdev->int_req &= ~NES_INT_TIMER;
2239                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2240                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2241                                 } else {
2242                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2243                                 }
2244                         } else {
2245                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2246                                 {
2247                                         nes_nic_init_timer(nesdev);
2248                                 }
2249                                 nesdev->timer_only_int_count = 0;
2250                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2251                         }
2252                 } else {
2253                         nesdev->timer_only_int_count = 0;
2254                         nesdev->int_req &= ~NES_INT_TIMER;
2255                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2256                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2257                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2258                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2259                 }
2260         } else {
2261                 if ( (completion_ints == 1) &&
2262                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2263                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2264                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2265                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2266                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2267                         nesdev->timer_only_int_count = 0;
2268                         nesdev->int_req |= NES_INT_TIMER;
2269                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2270                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2271                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2272                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2273                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2274                 } else {
2275                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2276                 }
2277         }
2278         nesdev->deepcq_count = 0;
2279 }
2280
2281
2282 /**
2283  * nes_process_ceq
2284  */
2285 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2286 {
2287         u64 u64temp;
2288         struct nes_hw_cq *cq;
2289         u32 head;
2290         u32 ceq_size;
2291
2292         /* nes_debug(NES_DBG_CQ, "\n"); */
2293         head = ceq->ceq_head;
2294         ceq_size = ceq->ceq_size;
2295
2296         do {
2297                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2298                                 NES_CEQE_VALID) {
2299                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2300                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2301                         u64temp <<= 1;
2302                         cq = *((struct nes_hw_cq **)&u64temp);
2303                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2304                         barrier();
2305                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2306
2307                         /* call the event handler */
2308                         cq->ce_handler(nesdev, cq);
2309
2310                         if (++head >= ceq_size)
2311                                 head = 0;
2312                 } else {
2313                         break;
2314                 }
2315
2316         } while (1);
2317
2318         ceq->ceq_head = head;
2319 }
2320
2321
2322 /**
2323  * nes_process_aeq
2324  */
2325 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2326 {
2327         /* u64 u64temp; */
2328         u32 head;
2329         u32 aeq_size;
2330         u32 aeqe_misc;
2331         u32 aeqe_cq_id;
2332         struct nes_hw_aeqe volatile *aeqe;
2333
2334         head = aeq->aeq_head;
2335         aeq_size = aeq->aeq_size;
2336
2337         do {
2338                 aeqe = &aeq->aeq_vbase[head];
2339                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2340                         break;
2341                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2342                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2343                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2344                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2345                                 /* dealing with an accelerated QP related AE */
2346                                 /*
2347                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2348                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2349                                  */
2350                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2351                         } else {
2352                                 /* TODO: dealing with a CQP related AE */
2353                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2354                                                 (u16)(aeqe_misc >> 16));
2355                         }
2356                 }
2357
2358                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2359
2360                 if (++head >= aeq_size)
2361                         head = 0;
2362
2363                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2364         }
2365         while (1);
2366         aeq->aeq_head = head;
2367 }
2368
2369 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2370 {
2371         struct nes_adapter *nesadapter = nesdev->nesadapter;
2372         u32 reset_value;
2373         u32 i=0;
2374         u32 u32temp;
2375
2376         if (nesadapter->hw_rev == NE020_REV) {
2377                 return;
2378         }
2379         mh_detected++;
2380
2381         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2382
2383         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2384                 reset_value |= 0x0000001d;
2385         else
2386                 reset_value |= 0x0000002d;
2387
2388         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2389                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2390                         nesadapter->link_interrupt_count[0] = 0;
2391                         nesadapter->link_interrupt_count[1] = 0;
2392                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2393                         if (0x00000040 & u32temp)
2394                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2395                         else
2396                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2397
2398                         reset_value |= 0x0000003d;
2399                 }
2400                 nesadapter->link_interrupt_count[mac_index] = 0;
2401         }
2402
2403         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2404
2405         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2406                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2407
2408         if (0x0000003d == (reset_value & 0x0000003d)) {
2409                 u32 pcs_control_status0, pcs_control_status1;
2410
2411                 for (i = 0; i < 10; i++) {
2412                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2413                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2414                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2415                              && (pcs_control_status0 & 0x00100000))
2416                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2417                                 && (pcs_control_status1 & 0x00100000)))
2418                                 continue;
2419                         else
2420                                 break;
2421                 }
2422                 if (10 == i) {
2423                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2424                         if (0x00000040 & u32temp)
2425                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2426                         else
2427                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2428
2429                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2430
2431                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2432                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2433                 }
2434         }
2435 }
2436
2437 /**
2438  * nes_process_mac_intr
2439  */
2440 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2441 {
2442         unsigned long flags;
2443         u32 pcs_control_status;
2444         struct nes_adapter *nesadapter = nesdev->nesadapter;
2445         struct nes_vnic *nesvnic;
2446         u32 mac_status;
2447         u32 mac_index = nesdev->mac_index;
2448         u32 u32temp;
2449         u16 phy_data;
2450         u16 temp_phy_data;
2451         u32 pcs_val  = 0x0f0f0000;
2452         u32 pcs_mask = 0x0f1f0000;
2453         u32 cdr_ctrl;
2454
2455         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2456         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2457                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2458                 return;
2459         }
2460         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2461
2462         /* ack the MAC interrupt */
2463         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2464         /* Clear the interrupt */
2465         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2466
2467         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2468
2469         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2470                 nesdev->link_status_interrupts++;
2471                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2472                         nes_reset_link(nesdev, mac_index);
2473
2474                 /* read the PHY interrupt status register */
2475                 if ((nesadapter->OneG_Mode) &&
2476                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2477                         do {
2478                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2479                                                 nesadapter->phy_index[mac_index], &phy_data);
2480                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2481                                                 nesadapter->phy_index[mac_index], phy_data);
2482                         } while (phy_data&0x8000);
2483
2484                         temp_phy_data = 0;
2485                         do {
2486                                 nes_read_1G_phy_reg(nesdev, 0x11,
2487                                                 nesadapter->phy_index[mac_index], &phy_data);
2488                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2489                                                 nesadapter->phy_index[mac_index], phy_data);
2490                                 if (temp_phy_data == phy_data)
2491                                         break;
2492                                 temp_phy_data = phy_data;
2493                         } while (1);
2494
2495                         nes_read_1G_phy_reg(nesdev, 0x1e,
2496                                         nesadapter->phy_index[mac_index], &phy_data);
2497                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2498                                         nesadapter->phy_index[mac_index], phy_data);
2499
2500                         nes_read_1G_phy_reg(nesdev, 1,
2501                                         nesadapter->phy_index[mac_index], &phy_data);
2502                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2503                                         nesadapter->phy_index[mac_index], phy_data);
2504
2505                         if (temp_phy_data & 0x1000) {
2506                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2507                                 phy_data = 4;
2508                         } else {
2509                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2510                         }
2511                 }
2512                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2513                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2514                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2515
2516                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2517                         switch (mac_index) {
2518                         case 1:
2519                         case 3:
2520                                 pcs_control_status = nes_read_indexed(nesdev,
2521                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2522                                 break;
2523                         default:
2524                                 pcs_control_status = nes_read_indexed(nesdev,
2525                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2526                                 break;
2527                         }
2528                 } else {
2529                         pcs_control_status = nes_read_indexed(nesdev,
2530                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2531                         pcs_control_status = nes_read_indexed(nesdev,
2532                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2533                 }
2534
2535                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2536                                 mac_index, pcs_control_status);
2537                 if ((nesadapter->OneG_Mode) &&
2538                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2539                         u32temp = 0x01010000;
2540                         if (nesadapter->port_count > 2) {
2541                                 u32temp |= 0x02020000;
2542                         }
2543                         if ((pcs_control_status & u32temp)!= u32temp) {
2544                                 phy_data = 0;
2545                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2546                         }
2547                 } else {
2548                         switch (nesadapter->phy_type[mac_index]) {
2549                         case NES_PHY_TYPE_ARGUS:
2550                         case NES_PHY_TYPE_SFP_D:
2551                         case NES_PHY_TYPE_KR:
2552                                 /* clear the alarms */
2553                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2554                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2555                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2556                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2557                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2558                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2559                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2560                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2561                                 /* check link status */
2562                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2563                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2564
2565                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2566                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2567                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2568                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2569
2570                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2571
2572                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2573                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2574                                 break;
2575
2576                         case NES_PHY_TYPE_PUMA_1G:
2577                                 if (mac_index < 2)
2578                                         pcs_val = pcs_mask = 0x01010000;
2579                                 else
2580                                         pcs_val = pcs_mask = 0x02020000;
2581                                 /* fall through */
2582                         default:
2583                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2584                                 break;
2585                         }
2586                 }
2587
2588                 if (phy_data & 0x0004) {
2589                         if (wide_ppm_offset &&
2590                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2591                             (nesadapter->hw_rev != NE020_REV)) {
2592                                 cdr_ctrl = nes_read_indexed(nesdev,
2593                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2594                                                             mac_index * 0x200);
2595                                 nes_write_indexed(nesdev,
2596                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2597                                                   mac_index * 0x200,
2598                                                   cdr_ctrl | 0x000F0000);
2599                         }
2600                         nesadapter->mac_link_down[mac_index] = 0;
2601                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2602                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2603                                                 nesvnic->linkup);
2604                                 if (nesvnic->linkup == 0) {
2605                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2606                                                         nesvnic->netdev->name, nesvnic->netdev);
2607                                         if (netif_queue_stopped(nesvnic->netdev))
2608                                                 netif_start_queue(nesvnic->netdev);
2609                                         nesvnic->linkup = 1;
2610                                         netif_carrier_on(nesvnic->netdev);
2611
2612                                         spin_lock(&nesvnic->port_ibevent_lock);
2613                                         if (nesvnic->of_device_registered) {
2614                                                 if (nesdev->iw_status == 0) {
2615                                                         nesdev->iw_status = 1;
2616                                                         nes_port_ibevent(nesvnic);
2617                                                 }
2618                                         }
2619                                         spin_unlock(&nesvnic->port_ibevent_lock);
2620                                 }
2621                         }
2622                 } else {
2623                         if (wide_ppm_offset &&
2624                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2625                             (nesadapter->hw_rev != NE020_REV)) {
2626                                 cdr_ctrl = nes_read_indexed(nesdev,
2627                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2628                                                             mac_index * 0x200);
2629                                 nes_write_indexed(nesdev,
2630                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2631                                                   mac_index * 0x200,
2632                                                   cdr_ctrl & 0xFFF0FFFF);
2633                         }
2634                         nesadapter->mac_link_down[mac_index] = 1;
2635                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2636                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2637                                                 nesvnic->linkup);
2638                                 if (nesvnic->linkup == 1) {
2639                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2640                                                         nesvnic->netdev->name, nesvnic->netdev);
2641                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2642                                                 netif_stop_queue(nesvnic->netdev);
2643                                         nesvnic->linkup = 0;
2644                                         netif_carrier_off(nesvnic->netdev);
2645
2646                                         spin_lock(&nesvnic->port_ibevent_lock);
2647                                         if (nesvnic->of_device_registered) {
2648                                                 if (nesdev->iw_status == 1) {
2649                                                         nesdev->iw_status = 0;
2650                                                         nes_port_ibevent(nesvnic);
2651                                                 }
2652                                         }
2653                                         spin_unlock(&nesvnic->port_ibevent_lock);
2654                                 }
2655                         }
2656                 }
2657                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2658                         if (nesdev->link_recheck)
2659                                 cancel_delayed_work(&nesdev->work);
2660                         nesdev->link_recheck = 1;
2661                         schedule_delayed_work(&nesdev->work,
2662                                               NES_LINK_RECHECK_DELAY);
2663                 }
2664         }
2665
2666         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2667
2668         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2669 }
2670
2671 void nes_recheck_link_status(struct work_struct *work)
2672 {
2673         unsigned long flags;
2674         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2675         struct nes_adapter *nesadapter = nesdev->nesadapter;
2676         struct nes_vnic *nesvnic;
2677         u32 mac_index = nesdev->mac_index;
2678         u16 phy_data;
2679         u16 temp_phy_data;
2680
2681         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2682
2683         /* check link status */
2684         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2685         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2686
2687         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2688         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2689         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2690         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2691
2692         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2693
2694         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2695                 __func__, phy_data,
2696                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2697
2698         if (phy_data & 0x0004) {
2699                 nesadapter->mac_link_down[mac_index] = 0;
2700                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2701                         if (nesvnic->linkup == 0) {
2702                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2703                                                 nesvnic->netdev->name, nesvnic->netdev);
2704                                 if (netif_queue_stopped(nesvnic->netdev))
2705                                         netif_start_queue(nesvnic->netdev);
2706                                 nesvnic->linkup = 1;
2707                                 netif_carrier_on(nesvnic->netdev);
2708
2709                                 spin_lock(&nesvnic->port_ibevent_lock);
2710                                 if (nesvnic->of_device_registered) {
2711                                         if (nesdev->iw_status == 0) {
2712                                                 nesdev->iw_status = 1;
2713                                                 nes_port_ibevent(nesvnic);
2714                                         }
2715                                 }
2716                                 spin_unlock(&nesvnic->port_ibevent_lock);
2717                         }
2718                 }
2719
2720         } else {
2721                 nesadapter->mac_link_down[mac_index] = 1;
2722                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2723                         if (nesvnic->linkup == 1) {
2724                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2725                                                 nesvnic->netdev->name, nesvnic->netdev);
2726                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2727                                         netif_stop_queue(nesvnic->netdev);
2728                                 nesvnic->linkup = 0;
2729                                 netif_carrier_off(nesvnic->netdev);
2730
2731                                 spin_lock(&nesvnic->port_ibevent_lock);
2732                                 if (nesvnic->of_device_registered) {
2733                                         if (nesdev->iw_status == 1) {
2734                                                 nesdev->iw_status = 0;
2735                                                 nes_port_ibevent(nesvnic);
2736                                         }
2737                                 }
2738                                 spin_unlock(&nesvnic->port_ibevent_lock);
2739                         }
2740                 }
2741         }
2742         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2743                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2744         else
2745                 nesdev->link_recheck = 0;
2746
2747         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2748 }
2749
2750
2751 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2752 {
2753         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2754
2755         napi_schedule(&nesvnic->napi);
2756 }
2757
2758
2759 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2760 * getting out of nic_ce_handler
2761 */
2762 #define MAX_RQES_TO_PROCESS     384
2763
2764 /**
2765  * nes_nic_ce_handler
2766  */
2767 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2768 {
2769         u64 u64temp;
2770         dma_addr_t bus_address;
2771         struct nes_hw_nic *nesnic;
2772         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2773         struct nes_adapter *nesadapter = nesdev->nesadapter;
2774         struct nes_hw_nic_rq_wqe *nic_rqe;
2775         struct nes_hw_nic_sq_wqe *nic_sqe;
2776         struct sk_buff *skb;
2777         struct sk_buff *rx_skb;
2778         __le16 *wqe_fragment_length;
2779         u32 head;
2780         u32 cq_size;
2781         u32 rx_pkt_size;
2782         u32 cqe_count=0;
2783         u32 cqe_errv;
2784         u32 cqe_misc;
2785         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2786         u16 vlan_tag;
2787         u16 pkt_type;
2788         u16 rqes_processed = 0;
2789         u8 sq_cqes = 0;
2790         u8 nes_use_lro = 0;
2791
2792         head = cq->cq_head;
2793         cq_size = cq->cq_size;
2794         cq->cqes_pending = 1;
2795         if (nesvnic->netdev->features & NETIF_F_LRO)
2796                 nes_use_lro = 1;
2797         do {
2798                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2799                                 NES_NIC_CQE_VALID) {
2800                         nesnic = &nesvnic->nic;
2801                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2802                         if (cqe_misc & NES_NIC_CQE_SQ) {
2803                                 sq_cqes++;
2804                                 wqe_fragment_index = 1;
2805                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2806                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2807                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2808                                 /* bump past the vlan tag */
2809                                 wqe_fragment_length++;
2810                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2811                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2812                                                         wqe_fragment_index * 2]);
2813                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2814                                                         wqe_fragment_index * 2])) << 32;
2815                                         bus_address = (dma_addr_t)u64temp;
2816                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2817                                                 pci_unmap_single(nesdev->pcidev,
2818                                                                 bus_address,
2819                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2820                                                                 PCI_DMA_TODEVICE);
2821                                         }
2822                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2823                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2824                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2825                                                                                 wqe_fragment_index * 2]);
2826                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2827                                                                                 + wqe_fragment_index * 2])) <<32;
2828                                                         bus_address = (dma_addr_t)u64temp;
2829                                                         pci_unmap_page(nesdev->pcidev,
2830                                                                         bus_address,
2831                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2832                                                                         PCI_DMA_TODEVICE);
2833                                                 } else
2834                                                         break;
2835                                         }
2836                                 }
2837                                 if (skb)
2838                                         dev_kfree_skb_any(skb);
2839                                 nesnic->sq_tail++;
2840                                 nesnic->sq_tail &= nesnic->sq_size-1;
2841                                 if (sq_cqes > 128) {
2842                                         barrier();
2843                                         /* restart the queue if it had been stopped */
2844                                         if (netif_queue_stopped(nesvnic->netdev))
2845                                                 netif_wake_queue(nesvnic->netdev);
2846                                         sq_cqes = 0;
2847                                 }
2848                         } else {
2849                                 rqes_processed ++;
2850
2851                                 cq->rx_cqes_completed++;
2852                                 cq->rx_pkts_indicated++;
2853                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2854                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2855                                 /* Get the skb */
2856                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2857                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2858                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2859                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2860                                 pci_unmap_single(nesdev->pcidev, bus_address,
2861                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2862                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2863                                 /* rx_skb->len = rx_pkt_size; */
2864                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2865                                 skb_put(rx_skb, rx_pkt_size);
2866                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2867                                 nesnic->rq_tail++;
2868                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2869
2870                                 atomic_inc(&nesvnic->rx_skbs_needed);
2871                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2872                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2873                                                         cq->cq_number | (cqe_count << 16));
2874                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2875                                         nesdev->currcq_count += cqe_count;
2876                                         cqe_count = 0;
2877                                         nes_replenish_nic_rq(nesvnic);
2878                                 }
2879                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2880                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2881                                 rx_skb->ip_summed = CHECKSUM_NONE;
2882
2883                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2884                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2885                                         if ((cqe_errv &
2886                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2887                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2888                                                 if (nesvnic->rx_checksum_disabled == 0) {
2889                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2890                                                 }
2891                                         } else
2892                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2893                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2894                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2895
2896                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2897                                         if ((cqe_errv &
2898                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2899                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2900                                                 if (nesvnic->rx_checksum_disabled == 0) {
2901                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2902                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2903                                                                   nesvnic->netdev->name); */
2904                                                 }
2905                                         } else
2906                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2907                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2908                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2909                                         }
2910                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2911                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2912
2913                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2914                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2915                                                 rx_skb = NULL;
2916                                 }
2917                                 if (rx_skb == NULL)
2918                                         goto skip_rx_indicate0;
2919
2920
2921                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2922                                     (nesvnic->vlan_grp != NULL)) {
2923                                         vlan_tag = (u16)(le32_to_cpu(
2924                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2925                                                         >> 16);
2926                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2927                                                         nesvnic->netdev->name, vlan_tag);
2928                                         if (nes_use_lro)
2929                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2930                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2931                                         else
2932                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2933                                 } else {
2934                                         if (nes_use_lro)
2935                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2936                                         else
2937                                                 nes_netif_rx(rx_skb);
2938                                 }
2939
2940 skip_rx_indicate0:
2941                                 ;
2942                                 /* nesvnic->netstats.rx_packets++; */
2943                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2944                         }
2945
2946                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2947                         /* Accounting... */
2948                         cqe_count++;
2949                         if (++head >= cq_size)
2950                                 head = 0;
2951                         if (cqe_count == 255) {
2952                                 /* Replenish Nic CQ */
2953                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2954                                                 cq->cq_number | (cqe_count << 16));
2955                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2956                                 nesdev->currcq_count += cqe_count;
2957                                 cqe_count = 0;
2958                         }
2959
2960                         if (cq->rx_cqes_completed >= nesvnic->budget)
2961                                 break;
2962                 } else {
2963                         cq->cqes_pending = 0;
2964                         break;
2965                 }
2966
2967         } while (1);
2968
2969         if (nes_use_lro)
2970                 lro_flush_all(&nesvnic->lro_mgr);
2971         if (sq_cqes) {
2972                 barrier();
2973                 /* restart the queue if it had been stopped */
2974                 if (netif_queue_stopped(nesvnic->netdev))
2975                         netif_wake_queue(nesvnic->netdev);
2976         }
2977         cq->cq_head = head;
2978         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2979                         cq->cq_number, cqe_count, cq->cq_head); */
2980         cq->cqe_allocs_pending = cqe_count;
2981         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2982         {
2983                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2984                 nesdev->currcq_count += cqe_count;
2985                 nes_nic_tune_timer(nesdev);
2986         }
2987         if (atomic_read(&nesvnic->rx_skbs_needed))
2988                 nes_replenish_nic_rq(nesvnic);
2989 }
2990
2991
2992 /**
2993  * nes_cqp_ce_handler
2994  */
2995 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2996 {
2997         u64 u64temp;
2998         unsigned long flags;
2999         struct nes_hw_cqp *cqp = NULL;
3000         struct nes_cqp_request *cqp_request;
3001         struct nes_hw_cqp_wqe *cqp_wqe;
3002         u32 head;
3003         u32 cq_size;
3004         u32 cqe_count=0;
3005         u32 error_code;
3006         /* u32 counter; */
3007
3008         head = cq->cq_head;
3009         cq_size = cq->cq_size;
3010
3011         do {
3012                 /* process the CQE */
3013                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3014                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3015
3016                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
3017                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3018                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3019                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3020                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3021                         cqp = *((struct nes_hw_cqp **)&u64temp);
3022
3023                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3024                         if (error_code) {
3025                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3026                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3027                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3028                                                 (u16)(error_code >> 16),
3029                                                 (u16)error_code);
3030                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
3031                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
3032                         }
3033
3034                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
3035                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
3036                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
3037                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
3038                         cqp_request = *((struct nes_cqp_request **)&u64temp);
3039                         if (cqp_request) {
3040                                 if (cqp_request->waiting) {
3041                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3042                                         cqp_request->major_code = (u16)(error_code >> 16);
3043                                         cqp_request->minor_code = (u16)error_code;
3044                                         barrier();
3045                                         cqp_request->request_done = 1;
3046                                         wake_up(&cqp_request->waitq);
3047                                         nes_put_cqp_request(nesdev, cqp_request);
3048                                 } else {
3049                                         if (cqp_request->callback)
3050                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3051                                         nes_free_cqp_request(nesdev, cqp_request);
3052                                 }
3053                         } else {
3054                                 wake_up(&nesdev->cqp.waitq);
3055                         }
3056
3057                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3058                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3059                         if (++cqp->sq_tail >= cqp->sq_size)
3060                                 cqp->sq_tail = 0;
3061
3062                         /* Accounting... */
3063                         cqe_count++;
3064                         if (++head >= cq_size)
3065                                 head = 0;
3066                 } else {
3067                         break;
3068                 }
3069         } while (1);
3070         cq->cq_head = head;
3071
3072         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3073         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3074                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3075                         (nesdev->cqp.sq_size - 1)) != 1)) {
3076                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3077                                 struct nes_cqp_request, list);
3078                 list_del_init(&cqp_request->list);
3079                 head = nesdev->cqp.sq_head++;
3080                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3081                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3082                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3083                 barrier();
3084                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
3085                         cpu_to_le32((u32)((unsigned long)cqp_request));
3086                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
3087                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3088                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3089                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3090                 /* Ring doorbell (1 WQEs) */
3091                 barrier();
3092                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3093         }
3094         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3095
3096         /* Arm the CCQ */
3097         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3098                         cq->cq_number);
3099         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3100 }
3101
3102
3103 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3104 {
3105         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3106                 /* skip over ethernet header */
3107                 pkt += ETH_HLEN;
3108
3109                 /* Skip over IP and TCP headers */
3110                 pkt += 4 * (pkt[0] & 0x0f);
3111                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3112         }
3113         return pkt;
3114 }
3115
3116 /* Determine if incoming error pkt is rdma layer */
3117 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3118 {
3119         u8 *pkt;
3120         u16 *mpa;
3121         u32 opcode = 0xffffffff;
3122
3123         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3124                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3125                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3126                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3127         }
3128
3129         return opcode;
3130 }
3131
3132 /* Build iWARP terminate header */
3133 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3134 {
3135         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3136         u16 ddp_seg_len;
3137         int copy_len = 0;
3138         u8 is_tagged = 0;
3139         u8 flush_code = 0;
3140         struct nes_terminate_hdr *termhdr;
3141
3142         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3143         memset(termhdr, 0, 64);
3144
3145         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3146
3147                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3148                 pkt = locate_mpa(pkt, aeq_info);
3149                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3150                 if (ddp_seg_len) {
3151                         copy_len = 2;
3152                         termhdr->hdrct = DDP_LEN_FLAG;
3153                         if (pkt[2] & 0x80) {
3154                                 is_tagged = 1;
3155                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3156                                         copy_len += TERM_DDP_LEN_TAGGED;
3157                                         termhdr->hdrct |= DDP_HDR_FLAG;
3158                                 }
3159                         } else {
3160                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3161                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3162                                         termhdr->hdrct |= DDP_HDR_FLAG;
3163                                 }
3164
3165                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3166                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3167                                                 copy_len += TERM_RDMA_LEN;
3168                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3169                                         }
3170                                 }
3171                         }
3172                 }
3173         }
3174
3175         switch (async_event_id) {
3176         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3177                 switch (iwarp_opcode(nesqp, aeq_info)) {
3178                 case IWARP_OPCODE_WRITE:
3179                         flush_code = IB_WC_LOC_PROT_ERR;
3180                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3181                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3182                         break;
3183                 default:
3184                         flush_code = IB_WC_REM_ACCESS_ERR;
3185                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3186                         termhdr->error_code = RDMAP_INV_STAG;
3187                 }
3188                 break;
3189         case NES_AEQE_AEID_AMP_INVALID_STAG:
3190                 flush_code = IB_WC_REM_ACCESS_ERR;
3191                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3192                 termhdr->error_code = RDMAP_INV_STAG;
3193                 break;
3194         case NES_AEQE_AEID_AMP_BAD_QP:
3195                 flush_code = IB_WC_LOC_QP_OP_ERR;
3196                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3197                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3198                 break;
3199         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3200         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3201                 switch (iwarp_opcode(nesqp, aeq_info)) {
3202                 case IWARP_OPCODE_SEND_INV:
3203                 case IWARP_OPCODE_SEND_SE_INV:
3204                         flush_code = IB_WC_REM_OP_ERR;
3205                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3206                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3207                         break;
3208                 default:
3209                         flush_code = IB_WC_REM_ACCESS_ERR;
3210                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3211                         termhdr->error_code = RDMAP_INV_STAG;
3212                 }
3213                 break;
3214         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3215                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3216                         flush_code = IB_WC_LOC_PROT_ERR;
3217                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3218                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3219                 } else {
3220                         flush_code = IB_WC_REM_ACCESS_ERR;
3221                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3222                         termhdr->error_code = RDMAP_INV_BOUNDS;
3223                 }
3224                 break;
3225         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3226         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3227         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3228                 flush_code = IB_WC_REM_ACCESS_ERR;
3229                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3230                 termhdr->error_code = RDMAP_ACCESS;
3231                 break;
3232         case NES_AEQE_AEID_AMP_TO_WRAP:
3233                 flush_code = IB_WC_REM_ACCESS_ERR;
3234                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3235                 termhdr->error_code = RDMAP_TO_WRAP;
3236                 break;
3237         case NES_AEQE_AEID_AMP_BAD_PD:
3238                 switch (iwarp_opcode(nesqp, aeq_info)) {
3239                 case IWARP_OPCODE_WRITE:
3240                         flush_code = IB_WC_LOC_PROT_ERR;
3241                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3242                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3243                         break;
3244                 case IWARP_OPCODE_SEND_INV:
3245                 case IWARP_OPCODE_SEND_SE_INV:
3246                         flush_code = IB_WC_REM_ACCESS_ERR;
3247                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3248                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3249                         break;
3250                 default:
3251                         flush_code = IB_WC_REM_ACCESS_ERR;
3252                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3253                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3254                 }
3255                 break;
3256         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3257                 flush_code = IB_WC_LOC_LEN_ERR;
3258                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3259                 termhdr->error_code = MPA_MARKER;
3260                 break;
3261         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3262                 flush_code = IB_WC_GENERAL_ERR;
3263                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3264                 termhdr->error_code = MPA_CRC;
3265                 break;
3266         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3267         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3268                 flush_code = IB_WC_LOC_LEN_ERR;
3269                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3270                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3271                 break;
3272         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3273         case NES_AEQE_AEID_DDP_NO_L_BIT:
3274                 flush_code = IB_WC_FATAL_ERR;
3275                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3276                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3277                 break;
3278         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3279         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3280                 flush_code = IB_WC_GENERAL_ERR;
3281                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3282                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3283                 break;
3284         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3285                 flush_code = IB_WC_LOC_LEN_ERR;
3286                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3287                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3288                 break;
3289         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3290                 flush_code = IB_WC_GENERAL_ERR;
3291                 if (is_tagged) {
3292                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3293                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3294                 } else {
3295                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3296                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3297                 }
3298                 break;
3299         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3300                 flush_code = IB_WC_GENERAL_ERR;
3301                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3302                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3303                 break;
3304         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3305                 flush_code = IB_WC_REM_OP_ERR;
3306                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3307                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3308                 break;
3309         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3310                 flush_code = IB_WC_GENERAL_ERR;
3311                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3312                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3313                 break;
3314         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3315                 flush_code = IB_WC_GENERAL_ERR;
3316                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3317                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3318                 break;
3319         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3320                 flush_code = IB_WC_LOC_QP_OP_ERR;
3321                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3322                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3323                 break;
3324         default:
3325                 flush_code = IB_WC_FATAL_ERR;
3326                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3327                 termhdr->error_code = RDMAP_UNSPECIFIED;
3328                 break;
3329         }
3330
3331         if (copy_len)
3332                 memcpy(termhdr + 1, pkt, copy_len);
3333
3334         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3335                 if (aeq_info & NES_AEQE_SQ)
3336                         nesqp->term_sq_flush_code = flush_code;
3337                 else
3338                         nesqp->term_rq_flush_code = flush_code;
3339         }
3340
3341         return sizeof(struct nes_terminate_hdr) + copy_len;
3342 }
3343
3344 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3345                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3346 {
3347         u64 context;
3348         unsigned long flags;
3349         u32 aeq_info;
3350         u16 async_event_id;
3351         u8 tcp_state;
3352         u8 iwarp_state;
3353         u32 termlen = 0;
3354         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3355                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3356         struct nes_adapter *nesadapter = nesdev->nesadapter;
3357
3358         if (nesqp->term_flags & NES_TERM_SENT)
3359                 return; /* Sanity check */
3360
3361         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3362         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3363         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3364         async_event_id = (u16)aeq_info;
3365
3366         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3367                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3368         if (!context) {
3369                 WARN_ON(!context);
3370                 return;
3371         }
3372
3373         nesqp = (struct nes_qp *)(unsigned long)context;
3374         spin_lock_irqsave(&nesqp->lock, flags);
3375         nesqp->hw_iwarp_state = iwarp_state;
3376         nesqp->hw_tcp_state = tcp_state;
3377         nesqp->last_aeq = async_event_id;
3378         nesqp->terminate_eventtype = eventtype;
3379         spin_unlock_irqrestore(&nesqp->lock, flags);
3380
3381         if (nesadapter->send_term_ok)
3382                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3383         else
3384                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3385
3386         if (!nesdev->iw_status)  {
3387                 nesqp->term_flags = NES_TERM_DONE;
3388                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3389                 nes_cm_disconn(nesqp);
3390         } else {
3391                 nes_terminate_start_timer(nesqp);
3392                 nesqp->term_flags |= NES_TERM_SENT;
3393                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3394         }
3395 }
3396
3397 static void nes_terminate_send_fin(struct nes_device *nesdev,
3398                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3399 {
3400         u32 aeq_info;
3401         u16 async_event_id;
3402         u8 tcp_state;
3403         u8 iwarp_state;
3404         unsigned long flags;
3405
3406         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3407         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3408         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3409         async_event_id = (u16)aeq_info;
3410
3411         spin_lock_irqsave(&nesqp->lock, flags);
3412         nesqp->hw_iwarp_state = iwarp_state;
3413         nesqp->hw_tcp_state = tcp_state;
3414         nesqp->last_aeq = async_event_id;
3415         spin_unlock_irqrestore(&nesqp->lock, flags);
3416
3417         /* Send the fin only */
3418         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3419                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3420 }
3421
3422 /* Cleanup after a terminate sent or received */
3423 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3424 {
3425         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3426         unsigned long flags;
3427         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3428         struct nes_device *nesdev = nesvnic->nesdev;
3429         u8 first_time = 0;
3430
3431         spin_lock_irqsave(&nesqp->lock, flags);
3432         if (nesqp->hte_added) {
3433                 nesqp->hte_added = 0;
3434                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3435         }
3436
3437         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3438         nesqp->term_flags |= NES_TERM_DONE;
3439         spin_unlock_irqrestore(&nesqp->lock, flags);
3440
3441         /* Make sure we go through this only once */
3442         if (first_time) {
3443                 if (timeout_occurred == 0)
3444                         del_timer(&nesqp->terminate_timer);
3445                 else
3446                         next_iwarp_state |= NES_CQP_QP_RESET;
3447
3448                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3449                 nes_cm_disconn(nesqp);
3450         }
3451 }
3452
3453 static void nes_terminate_received(struct nes_device *nesdev,
3454                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3455 {
3456         u32 aeq_info;
3457         u8 *pkt;
3458         u32 *mpa;
3459         u8 ddp_ctl;
3460         u8 rdma_ctl;
3461         u16 aeq_id = 0;
3462
3463         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3464         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3465                 /* Terminate is not a performance path so the silicon */
3466                 /* did not validate the frame - do it now */
3467                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3468                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3469                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3470                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3471                 if ((ddp_ctl & 0xc0) != 0x40)
3472                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3473                 else if ((ddp_ctl & 0x03) != 1)
3474                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3475                 else if (be32_to_cpu(mpa[2]) != 2)
3476                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3477                 else if (be32_to_cpu(mpa[3]) != 1)
3478                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3479                 else if (be32_to_cpu(mpa[4]) != 0)
3480                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3481                 else if ((rdma_ctl & 0xc0) != 0x40)
3482                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3483
3484                 if (aeq_id) {
3485                         /* Bad terminate recvd - send back a terminate */
3486                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3487                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3488                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3489                         return;
3490                 }
3491         }
3492
3493         nesqp->term_flags |= NES_TERM_RCVD;
3494         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3495         nes_terminate_start_timer(nesqp);
3496         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3497 }
3498
3499 /* Timeout routine in case terminate fails to complete */
3500 static void nes_terminate_timeout(unsigned long context)
3501 {
3502         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3503
3504         nes_terminate_done(nesqp, 1);
3505 }
3506
3507 /* Set a timer in case hw cannot complete the terminate sequence */
3508 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3509 {
3510         init_timer(&nesqp->terminate_timer);
3511         nesqp->terminate_timer.function = nes_terminate_timeout;
3512         nesqp->terminate_timer.expires = jiffies + HZ;
3513         nesqp->terminate_timer.data = (unsigned long)nesqp;
3514         add_timer(&nesqp->terminate_timer);
3515 }
3516
3517 /**
3518  * nes_process_iwarp_aeqe
3519  */
3520 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3521                                    struct nes_hw_aeqe *aeqe)
3522 {
3523         u64 context;
3524         unsigned long flags;
3525         struct nes_qp *nesqp;
3526         struct nes_hw_cq *hw_cq;
3527         struct nes_cq *nescq;
3528         int resource_allocated;
3529         struct nes_adapter *nesadapter = nesdev->nesadapter;
3530         u32 aeq_info;
3531         u32 next_iwarp_state = 0;
3532         u32 aeqe_cq_id;
3533         u16 async_event_id;
3534         u8 tcp_state;
3535         u8 iwarp_state;
3536         struct ib_event ibevent;
3537
3538         nes_debug(NES_DBG_AEQ, "\n");
3539         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3540         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3541                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3542                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3543         } else {
3544                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3545                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3546                 BUG_ON(!context);
3547         }
3548
3549         /* context is nesqp unless async_event_id == CQ ERROR */
3550         nesqp = (struct nes_qp *)(unsigned long)context;
3551         async_event_id = (u16)aeq_info;
3552         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3553         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3554         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3555                         " Tcp state = %s, iWARP state = %s\n",
3556                         async_event_id,
3557                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3558                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3559
3560         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3561         if (aeq_info & NES_AEQE_QP) {
3562                 if ((!nes_is_resource_allocated(nesadapter, nesadapter->allocated_qps,
3563                                 aeqe_cq_id)) ||
3564                                 (atomic_read(&nesqp->close_timer_started)))
3565                         return;
3566         }
3567
3568         switch (async_event_id) {
3569                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3570                         if (nesqp->term_flags)
3571                                 return; /* Ignore it, wait for close complete */
3572
3573                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3574                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3575                                         (nesqp->ibqp_state == IB_QPS_RTS) &&
3576                                         ((nesadapter->eeprom_version >> 16) != NES_A0)) {
3577                                         spin_lock_irqsave(&nesqp->lock, flags);
3578                                         nesqp->hw_iwarp_state = iwarp_state;
3579                                         nesqp->hw_tcp_state = tcp_state;
3580                                         nesqp->last_aeq = async_event_id;
3581                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3582                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3583                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3584                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3585                                         nes_cm_disconn(nesqp);
3586                                 }
3587                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3588                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3589                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3590                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3591                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3592                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3593                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3594                                                 async_event_id, nesqp->last_aeq, tcp_state);
3595                         }
3596                         break;
3597                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3598                         if (nesqp->term_flags) {
3599                                 nes_terminate_done(nesqp, 0);
3600                                 return;
3601                         }
3602                         spin_lock_irqsave(&nesqp->lock, flags);
3603                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3604                         spin_unlock_irqrestore(&nesqp->lock, flags);
3605                         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_CLOSING, 0, 0);
3606                         nes_cm_disconn(nesqp);
3607                         break;
3608
3609                 case NES_AEQE_AEID_RESET_SENT:
3610                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3611                         spin_lock_irqsave(&nesqp->lock, flags);
3612                         nesqp->hw_iwarp_state = iwarp_state;
3613                         nesqp->hw_tcp_state = tcp_state;
3614                         nesqp->last_aeq = async_event_id;
3615                         nesqp->hte_added = 0;
3616                         spin_unlock_irqrestore(&nesqp->lock, flags);
3617                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3618                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3619                         nes_cm_disconn(nesqp);
3620                         break;
3621
3622                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3623                         if (atomic_read(&nesqp->close_timer_started))
3624                                 return;
3625                         spin_lock_irqsave(&nesqp->lock, flags);
3626                         nesqp->hw_iwarp_state = iwarp_state;
3627                         nesqp->hw_tcp_state = tcp_state;
3628                         nesqp->last_aeq = async_event_id;
3629                         spin_unlock_irqrestore(&nesqp->lock, flags);
3630                         nes_cm_disconn(nesqp);
3631                         break;
3632
3633                 case NES_AEQE_AEID_TERMINATE_SENT:
3634                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3635                         break;
3636
3637                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3638                         nes_terminate_received(nesdev, nesqp, aeqe);
3639                         break;
3640
3641                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3642                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3643                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3644                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3645                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3646                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3647                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3648                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3649                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3650                 case NES_AEQE_AEID_AMP_TO_WRAP:
3651                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3652                                         nesqp->hwqp.qp_id, async_event_id);
3653                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3654                         break;
3655
3656                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3657                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3658                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3659                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3660                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3661                                 aeq_info &= 0xffff0000;
3662                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3663                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3664                         }
3665
3666                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3667                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3668                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3669                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3670                 case NES_AEQE_AEID_AMP_BAD_QP:
3671                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3672                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3673                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3674                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3675                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3676                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3677                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3678                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3679                 case NES_AEQE_AEID_AMP_BAD_PD:
3680                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3681                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3682                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3683                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3684                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3685                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3686                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3687                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3688                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3689                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3690                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3691                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3692                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3693                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3694                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3695                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3696                 case NES_AEQE_AEID_BAD_CLOSE:
3697                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3698                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3699                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3700                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3701                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3702                                         nesqp->hwqp.qp_id, async_event_id);
3703                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3704                         break;
3705
3706                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3707                         context <<= 1;
3708                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3709                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3710                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3711                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3712                         if (resource_allocated) {
3713                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3714                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3715                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3716                                 if (hw_cq) {
3717                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3718                                         if (nescq->ibcq.event_handler) {
3719                                                 ibevent.device = nescq->ibcq.device;
3720                                                 ibevent.event = IB_EVENT_CQ_ERR;
3721                                                 ibevent.element.cq = &nescq->ibcq;
3722                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3723                                         }
3724                                 }
3725                         }
3726                         break;
3727
3728                 default:
3729                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3730                                         async_event_id);
3731                         break;
3732         }
3733
3734 }
3735
3736 /**
3737  * nes_iwarp_ce_handler
3738  */
3739 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3740 {
3741         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3742
3743         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3744                         nescq->hw_cq.cq_number); */
3745         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3746
3747         if (nescq->ibcq.comp_handler)
3748                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3749
3750         return;
3751 }
3752
3753
3754 /**
3755  * nes_manage_apbvt()
3756  */
3757 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3758                 u32 nic_index, u32 add_port)
3759 {
3760         struct nes_device *nesdev = nesvnic->nesdev;
3761         struct nes_hw_cqp_wqe *cqp_wqe;
3762         struct nes_cqp_request *cqp_request;
3763         int ret = 0;
3764         u16 major_code;
3765
3766         /* Send manage APBVT request to CQP */
3767         cqp_request = nes_get_cqp_request(nesdev);
3768         if (cqp_request == NULL) {
3769                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3770                 return -ENOMEM;
3771         }
3772         cqp_request->waiting = 1;
3773         cqp_wqe = &cqp_request->cqp_wqe;
3774
3775         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3776                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3777                         accel_local_port, accel_local_port, nic_index);
3778
3779         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3780         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3781                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3782         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3783                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3784
3785         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3786
3787         atomic_set(&cqp_request->refcount, 2);
3788         nes_post_cqp_request(nesdev, cqp_request);
3789
3790         if (add_port == NES_MANAGE_APBVT_ADD)
3791                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3792                                 NES_EVENT_TIMEOUT);
3793         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3794                         ret, cqp_request->major_code, cqp_request->minor_code);
3795         major_code = cqp_request->major_code;
3796
3797         nes_put_cqp_request(nesdev, cqp_request);
3798
3799         if (!ret)
3800                 return -ETIME;
3801         else if (major_code)
3802                 return -EIO;
3803         else
3804                 return 0;
3805 }
3806
3807
3808 /**
3809  * nes_manage_arp_cache
3810  */
3811 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3812                 u32 ip_addr, u32 action)
3813 {
3814         struct nes_hw_cqp_wqe *cqp_wqe;
3815         struct nes_vnic *nesvnic = netdev_priv(netdev);
3816         struct nes_device *nesdev;
3817         struct nes_cqp_request *cqp_request;
3818         int arp_index;
3819
3820         nesdev = nesvnic->nesdev;
3821         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3822         if (arp_index == -1) {
3823                 return;
3824         }
3825
3826         /* update the ARP entry */
3827         cqp_request = nes_get_cqp_request(nesdev);
3828         if (cqp_request == NULL) {
3829                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3830                 return;
3831         }
3832         cqp_request->waiting = 0;
3833         cqp_wqe = &cqp_request->cqp_wqe;
3834         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3835
3836         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3837                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3838         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3839                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3840         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3841
3842         if (action == NES_ARP_ADD) {
3843                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3844                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3845                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3846                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3847                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3848                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3849         } else {
3850                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3851                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3852         }
3853
3854         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3855                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3856
3857         atomic_set(&cqp_request->refcount, 1);
3858         nes_post_cqp_request(nesdev, cqp_request);
3859 }
3860
3861
3862 /**
3863  * flush_wqes
3864  */
3865 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3866                 u32 which_wq, u32 wait_completion)
3867 {
3868         struct nes_cqp_request *cqp_request;
3869         struct nes_hw_cqp_wqe *cqp_wqe;
3870         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3871         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3872         int ret;
3873
3874         cqp_request = nes_get_cqp_request(nesdev);
3875         if (cqp_request == NULL) {
3876                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3877                 return;
3878         }
3879         if (wait_completion) {
3880                 cqp_request->waiting = 1;
3881                 atomic_set(&cqp_request->refcount, 2);
3882         } else {
3883                 cqp_request->waiting = 0;
3884         }
3885         cqp_wqe = &cqp_request->cqp_wqe;
3886         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3887
3888         /* If wqe in error was identified, set code to be put into cqe */
3889         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3890                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3891                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3892                 nesqp->term_sq_flush_code = 0;
3893         }
3894
3895         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3896                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3897                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3898                 nesqp->term_rq_flush_code = 0;
3899         }
3900
3901         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3902                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3903                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3904         }
3905
3906         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3907                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3908         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3909
3910         nes_post_cqp_request(nesdev, cqp_request);
3911
3912         if (wait_completion) {
3913                 /* Wait for CQP */
3914                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3915                                 NES_EVENT_TIMEOUT);
3916                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3917                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3918                                 ret, cqp_request->major_code, cqp_request->minor_code);
3919                 nes_put_cqp_request(nesdev, cqp_request);
3920         }
3921 }