drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42 #include <linux/slab.h>
  43
  44 #include "nes.h"
  45
  46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  47 module_param(nes_lro_max_aggr, uint, 0444);
  48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  49
  50 static int wide_ppm_offset;
  51 module_param(wide_ppm_offset, int, 0644);
  52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  53
  54 static u32 crit_err_count;
  55 u32 int_mod_timer_init;
  56 u32 int_mod_cq_depth_256;
  57 u32 int_mod_cq_depth_128;
  58 u32 int_mod_cq_depth_32;
  59 u32 int_mod_cq_depth_24;
  60 u32 int_mod_cq_depth_16;
  61 u32 int_mod_cq_depth_4;
  62 u32 int_mod_cq_depth_1;
  63 static const u8 nes_max_critical_error_count = 100;
  64 #include "nes_cm.h"
  65
  66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  69                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  74                                    struct nes_hw_aeqe *aeqe);
  75 static void process_critical_error(struct nes_device *nesdev);
  76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existent",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existent",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112 static inline void print_ip(struct nes_cm_node *cm_node)
 113 {
 114         unsigned char *rem_addr;
 115         if (cm_node) {
 116                 rem_addr = (unsigned char *)&cm_node->rem_addr;
 117                 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
 118         }
 119 }
 120
 121 /**
 122  * nes_nic_init_timer_defaults
 123  */
 124 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 125 {
 126         unsigned long flags;
 127         struct nes_adapter *nesadapter = nesdev->nesadapter;
 128         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 129
 130         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 131
 132         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 133         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 134         if (jumbomode) {
 135                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 136                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 137                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 138         } else {
 139                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 140                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 141                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 142         }
 143
 144         /* todo use netdev->mtu to set thresholds */
 145         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 146 }
 147
 148
 149 /**
 150  * nes_nic_init_timer
 151  */
 152 static void  nes_nic_init_timer(struct nes_device *nesdev)
 153 {
 154         unsigned long flags;
 155         struct nes_adapter *nesadapter = nesdev->nesadapter;
 156         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 157
 158         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 159
 160         if (shared_timer->timer_in_use_old == 0) {
 161                 nesdev->deepcq_count = 0;
 162                 shared_timer->timer_direction_upward = 0;
 163                 shared_timer->timer_direction_downward = 0;
 164                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 165                 shared_timer->timer_in_use_old = 0;
 166
 167         }
 168         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 169                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 170                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 171                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 172         }
 173         /* todo use netdev->mtu to set thresholds */
 174         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 175 }
 176
 177
 178 /**
 179  * nes_nic_tune_timer
 180  */
 181 static void nes_nic_tune_timer(struct nes_device *nesdev)
 182 {
 183         unsigned long flags;
 184         struct nes_adapter *nesadapter = nesdev->nesadapter;
 185         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 186         u16 cq_count = nesdev->currcq_count;
 187
 188         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 189
 190         if (shared_timer->cq_count_old <= cq_count)
 191                 shared_timer->cq_direction_downward = 0;
 192         else
 193                 shared_timer->cq_direction_downward++;
 194         shared_timer->cq_count_old = cq_count;
 195         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 196                 if (cq_count <= shared_timer->threshold_low &&
 197                     shared_timer->threshold_low > 4) {
 198                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 199                         shared_timer->cq_direction_downward=0;
 200                         nesdev->currcq_count = 0;
 201                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 202                         return;
 203                 }
 204         }
 205
 206         if (cq_count > 1) {
 207                 nesdev->deepcq_count += cq_count;
 208                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 209                         shared_timer->timer_direction_upward++;
 210                         shared_timer->timer_direction_downward = 0;
 211                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 212                         shared_timer->timer_direction_upward = 0;
 213                         shared_timer->timer_direction_downward = 0;
 214                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 215                         shared_timer->timer_direction_downward++;
 216                         shared_timer->timer_direction_upward = 0;
 217                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 218                         shared_timer->timer_in_use -= 2;
 219                         shared_timer->timer_direction_upward = 0;
 220                         shared_timer->timer_direction_downward++;
 221                 } else {
 222                         shared_timer->timer_in_use -= 4;
 223                         shared_timer->timer_direction_upward = 0;
 224                         shared_timer->timer_direction_downward++;
 225                 }
 226
 227                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 228                         shared_timer->timer_in_use += 3;
 229                         shared_timer->timer_direction_upward = 0;
 230                         shared_timer->timer_direction_downward = 0;
 231                 }
 232                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 233                         shared_timer->timer_in_use -= 4 ;
 234                         shared_timer->timer_direction_downward = 0;
 235                         shared_timer->timer_direction_upward = 0;
 236                 }
 237         }
 238
 239         /* boundary checking */
 240         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 241                 shared_timer->timer_in_use = shared_timer->threshold_high;
 242         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 243                 shared_timer->timer_in_use = shared_timer->threshold_low;
 244
 245         nesdev->currcq_count = 0;
 246
 247         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 248 }
 249
 250
 251 /**
 252  * nes_init_adapter - initialize adapter
 253  */
 254 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 255         struct nes_adapter *nesadapter = NULL;
 256         unsigned long num_pds;
 257         u32 u32temp;
 258         u32 port_count;
 259         u16 max_rq_wrs;
 260         u16 max_sq_wrs;
 261         u32 max_mr;
 262         u32 max_256pbl;
 263         u32 max_4kpbl;
 264         u32 max_qp;
 265         u32 max_irrq;
 266         u32 max_cq;
 267         u32 hte_index_mask;
 268         u32 adapter_size;
 269         u32 arp_table_size;
 270         u16 vendor_id;
 271         u16 device_id;
 272         u8  OneG_Mode;
 273         u8  func_index;
 274
 275         /* search the list of existing adapters */
 276         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 277                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 278                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 279                                 nesdev->pcidev->devfn,
 280                                 PCI_SLOT(nesadapter->devfn),
 281                                 nesadapter->bus_number,
 282                                 PCI_SLOT(nesdev->pcidev->devfn),
 283                                 nesdev->pcidev->bus->number );
 284                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 285                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 286                         nesadapter->ref_count++;
 287                         return nesadapter;
 288                 }
 289         }
 290
 291         /* no adapter found */
 292         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 293         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 294                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 295                                 hw_rev);
 296                 return NULL;
 297         }
 298
 299         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 300                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 301                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 302                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 303                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 304
 305         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 306
 307
 308         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 309                 return NULL;
 310
 311         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 312         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 313
 314         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 315         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 316                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 317                                 max_qp, u32temp);
 318                 max_qp = (u32)1 << (u32temp & 0x001f);
 319         }
 320
 321         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 322         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 323                         max_qp, hte_index_mask);
 324
 325         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 326
 327         max_irrq = 1 << (u32temp & 0x001f);
 328
 329         if (max_qp > max_irrq) {
 330                 max_qp = max_irrq;
 331                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 332                                 max_qp);
 333         }
 334
 335         /* there should be no reason to allocate more pds than qps */
 336         if (num_pds > max_qp)
 337                 num_pds = max_qp;
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 340         max_mr = (u32)8192 << (u32temp & 0x7);
 341
 342         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 343         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 344         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 345         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 346
 347         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 348         arp_table_size = 1 << u32temp;
 349
 350         adapter_size = (sizeof(struct nes_adapter) +
 351                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 352         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 353         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 354         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 355         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 356         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 357         adapter_size += sizeof(struct nes_qp **) * max_qp;
 358
 359         /* allocate a new adapter struct */
 360         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 361         if (nesadapter == NULL) {
 362                 return NULL;
 363         }
 364
 365         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 366                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 367
 368         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 369                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 370                 kfree(nesadapter);
 371                 return NULL;
 372         }
 373
 374         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 375                                 (nesadapter->mac_addr_low >> 24);
 376
 377         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 378                                  PCI_DEVICE_ID, &device_id);
 379         nesadapter->vendor_part_id = device_id;
 380
 381         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 382                                                         OneG_Mode)) {
 383                 kfree(nesadapter);
 384                 return NULL;
 385         }
 386         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 387
 388         memset(nesadapter->pft_mcast_map, 255,
 389                sizeof nesadapter->pft_mcast_map);
 390
 391         /* populate the new nesadapter */
 392         nesadapter->devfn = nesdev->pcidev->devfn;
 393         nesadapter->bus_number = nesdev->pcidev->bus->number;
 394         nesadapter->ref_count = 1;
 395         nesadapter->timer_int_req = 0xffff0000;
 396         nesadapter->OneG_Mode = OneG_Mode;
 397         nesadapter->doorbell_start = nesdev->doorbell_region;
 398
 399         /* nesadapter->tick_delta = clk_divisor; */
 400         nesadapter->hw_rev = hw_rev;
 401         nesadapter->port_count = port_count;
 402
 403         nesadapter->max_qp = max_qp;
 404         nesadapter->hte_index_mask = hte_index_mask;
 405         nesadapter->max_irrq = max_irrq;
 406         nesadapter->max_mr = max_mr;
 407         nesadapter->max_256pbl = max_256pbl - 1;
 408         nesadapter->max_4kpbl = max_4kpbl - 1;
 409         nesadapter->max_cq = max_cq;
 410         nesadapter->free_256pbl = max_256pbl - 1;
 411         nesadapter->free_4kpbl = max_4kpbl - 1;
 412         nesadapter->max_pd = num_pds;
 413         nesadapter->arp_table_size = arp_table_size;
 414
 415         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 416         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 417                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 418                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 419                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 420         } else {
 421                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 422                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 423                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 424                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 425         }
 426         /* Setup and enable the periodic timer */
 427         if (nesadapter->et_rx_coalesce_usecs_irq)
 428                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 429                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 430         else
 431                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 432
 433         nesadapter->base_pd = 1;
 434
 435         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 436                                        IB_DEVICE_MEM_WINDOW |
 437                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 438
 439         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 440                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 441         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 442         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 443         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 444         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 445         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 446
 447
 448         /* mark the usual suspect QPs, MR and CQs as in use */
 449         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 450                 set_bit(u32temp, nesadapter->allocated_qps);
 451                 set_bit(u32temp, nesadapter->allocated_cqs);
 452         }
 453         set_bit(0, nesadapter->allocated_mrs);
 454
 455         for (u32temp = 0; u32temp < 20; u32temp++)
 456                 set_bit(u32temp, nesadapter->allocated_pds);
 457         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 458
 459         max_rq_wrs = ((u32temp >> 8) & 3);
 460         switch (max_rq_wrs) {
 461                 case 0:
 462                         max_rq_wrs = 4;
 463                         break;
 464                 case 1:
 465                         max_rq_wrs = 16;
 466                         break;
 467                 case 2:
 468                         max_rq_wrs = 32;
 469                         break;
 470                 case 3:
 471                         max_rq_wrs = 512;
 472                         break;
 473         }
 474
 475         max_sq_wrs = (u32temp & 3);
 476         switch (max_sq_wrs) {
 477                 case 0:
 478                         max_sq_wrs = 4;
 479                         break;
 480                 case 1:
 481                         max_sq_wrs = 16;
 482                         break;
 483                 case 2:
 484                         max_sq_wrs = 32;
 485                         break;
 486                 case 3:
 487                         max_sq_wrs = 512;
 488                         break;
 489         }
 490         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 491         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 492
 493         nesadapter->max_sge = 4;
 494         nesadapter->max_cqe = 32766;
 495
 496         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 497                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 498                 kfree(nesadapter);
 499                 return NULL;
 500         }
 501
 502         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 503         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 504                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 505
 506         /* setup port configuration */
 507         if (nesadapter->port_count == 1) {
 508                 nesadapter->log_port = 0x00000000;
 509                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 510                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 511                 else
 512                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 513         } else {
 514                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 515                         nesadapter->log_port = 0x000000D8;
 516                 } else {
 517                         if (nesadapter->port_count == 2)
 518                                 nesadapter->log_port = 0x00000044;
 519                         else
 520                                 nesadapter->log_port = 0x000000e4;
 521                 }
 522                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 523         }
 524
 525         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 526                                                 nesadapter->log_port);
 527         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 528                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 529
 530         spin_lock_init(&nesadapter->resource_lock);
 531         spin_lock_init(&nesadapter->phy_lock);
 532         spin_lock_init(&nesadapter->pbl_lock);
 533         spin_lock_init(&nesadapter->periodic_timer_lock);
 534
 535         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 536         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 537         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 538         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 539
 540         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 541                 u32 pcs_control_status0, pcs_control_status1;
 542                 u32 reset_value;
 543                 u32 i = 0;
 544                 u32 int_cnt = 0;
 545                 u32 ext_cnt = 0;
 546                 unsigned long flags;
 547                 u32 j = 0;
 548
 549                 pcs_control_status0 = nes_read_indexed(nesdev,
 550                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 551                 pcs_control_status1 = nes_read_indexed(nesdev,
 552                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 553
 554                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 555                         pcs_control_status0 = nes_read_indexed(nesdev,
 556                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 557                         pcs_control_status1 = nes_read_indexed(nesdev,
 558                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 559                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 560                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 561                                 int_cnt++;
 562                         msleep(1);
 563                 }
 564                 if (int_cnt > 1) {
 565                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 566                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 567                         mh_detected++;
 568                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 569                         reset_value |= 0x0000003d;
 570                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 571
 572                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 573                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 574                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 575
 576                         pcs_control_status0 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 578                         pcs_control_status1 = nes_read_indexed(nesdev,
 579                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 580
 581                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 582                                 pcs_control_status0 = nes_read_indexed(nesdev,
 583                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 584                                 pcs_control_status1 = nes_read_indexed(nesdev,
 585                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 586                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 587                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 588                                         if (++ext_cnt > int_cnt) {
 589                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 590                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 591                                                                 0x0000F088);
 592                                                 mh_detected++;
 593                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 594                                                 reset_value |= 0x0000003d;
 595                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 596
 597                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 598                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 599                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 600                                                 break;
 601                                         }
 602                                 }
 603                                 msleep(1);
 604                         }
 605                 }
 606         }
 607
 608         if (nesadapter->hw_rev == NE020_REV) {
 609                 init_timer(&nesadapter->mh_timer);
 610                 nesadapter->mh_timer.function = nes_mh_fix;
 611                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 612                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 613                 add_timer(&nesadapter->mh_timer);
 614         } else {
 615                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 616         }
 617
 618         init_timer(&nesadapter->lc_timer);
 619         nesadapter->lc_timer.function = nes_clc;
 620         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 621         nesadapter->lc_timer.data = (unsigned long)nesdev;
 622         add_timer(&nesadapter->lc_timer);
 623
 624         list_add_tail(&nesadapter->list, &nes_adapter_list);
 625
 626         for (func_index = 0; func_index < 8; func_index++) {
 627                 pci_bus_read_config_word(nesdev->pcidev->bus,
 628                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 629                                         func_index), 0, &vendor_id);
 630                 if (vendor_id == 0xffff)
 631                         break;
 632         }
 633         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 634                 func_index, pci_name(nesdev->pcidev));
 635         nesadapter->adapter_fcn_count = func_index;
 636
 637         return nesadapter;
 638 }
 639
 640
 641 /**
 642  * nes_reset_adapter_ne020
 643  */
 644 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 645 {
 646         u32 port_count;
 647         u32 u32temp;
 648         u32 i;
 649
 650         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 651         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 652         /* TODO: assuming that both SERDES are set the same for now */
 653         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 654         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 655                         u32temp, port_count);
 656         if (*OneG_Mode)
 657                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 658         u32temp &= 0xff00ffc0;
 659         switch (port_count) {
 660                 case 1:
 661                         u32temp |= 0x00ee0000;
 662                         break;
 663                 case 2:
 664                         u32temp |= 0x00cc0000;
 665                         break;
 666                 case 4:
 667                         u32temp |= 0x00000000;
 668                         break;
 669                 default:
 670                         return 0;
 671                         break;
 672         }
 673
 674         /* check and do full reset if needed */
 675         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 676                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 677                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 678
 679                 i = 0;
 680                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 684                         return 0;
 685                 }
 686
 687                 i = 0;
 688                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 689                         mdelay(1);
 690                 if (i > 10000) {
 691                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 692                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 693                         return 0;
 694                 }
 695         }
 696
 697         /* port reset */
 698         switch (port_count) {
 699                 case 1:
 700                         u32temp |= 0x00ee0010;
 701                         break;
 702                 case 2:
 703                         u32temp |= 0x00cc0030;
 704                         break;
 705                 case 4:
 706                         u32temp |= 0x00000030;
 707                         break;
 708         }
 709
 710         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 711         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 712
 713         i = 0;
 714         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 715                 mdelay(1);
 716         if (i > 10000) {
 717                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 718                 return 0;
 719         }
 720
 721         /* serdes 0 */
 722         i = 0;
 723         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 724                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 725                 mdelay(1);
 726         if (i > 5000) {
 727                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 728                 return 0;
 729         }
 730
 731         /* serdes 1 */
 732         if (port_count > 1) {
 733                 i = 0;
 734                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 735                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 736                         mdelay(1);
 737                 if (i > 5000) {
 738                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 739                         return 0;
 740                 }
 741         }
 742
 743         return port_count;
 744 }
 745
 746
 747 /**
 748  * nes_init_serdes
 749  */
 750 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 751                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 752 {
 753         int i;
 754         u32 u32temp;
 755         u32 sds;
 756
 757         if (hw_rev != NE020_REV) {
 758                 /* init serdes 0 */
 759                 switch (nesadapter->phy_type[0]) {
 760                 case NES_PHY_TYPE_CX4:
 761                         if (wide_ppm_offset)
 762                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 763                         else
 764                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 765                         break;
 766                 case NES_PHY_TYPE_KR:
 767                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 768                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 769                         break;
 770                 case NES_PHY_TYPE_PUMA_1G:
 771                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 772                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 773                         sds |= 0x00000100;
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 775                         break;
 776                 default:
 777                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 778                         break;
 779                 }
 780
 781                 if (!OneG_Mode)
 782                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 783
 784                 if (port_count < 2)
 785                         return 0;
 786
 787                 /* init serdes 1 */
 788                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 789                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 790
 791                 switch (nesadapter->phy_type[1]) {
 792                 case NES_PHY_TYPE_ARGUS:
 793                 case NES_PHY_TYPE_SFP_D:
 794                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 795                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 796                         break;
 797                 case NES_PHY_TYPE_CX4:
 798                         if (wide_ppm_offset)
 799                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 800                         break;
 801                 case NES_PHY_TYPE_KR:
 802                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 803                         break;
 804                 case NES_PHY_TYPE_PUMA_1G:
 805                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 806                         sds |= 0x000000100;
 807                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 808                 }
 809                 if (!OneG_Mode) {
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 811                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 812                         sds &= 0xFFFFFFBF;
 813                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 814                 }
 815         } else {
 816                 /* init serdes 0 */
 817                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 818                 i = 0;
 819                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 820                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 821                         mdelay(1);
 822                 if (i > 5000) {
 823                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 824                         return 1;
 825                 }
 826                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 827                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 828                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 829                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 830                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 831                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 832                 if (OneG_Mode)
 833                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 834                 else
 835                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 836
 837                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 838                 if (port_count > 1) {
 839                         /* init serdes 1 */
 840                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 841                         i = 0;
 842                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 843                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 844                                 mdelay(1);
 845                         if (i > 5000) {
 846                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 847                                 /* return 1; */
 848                         }
 849                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 850                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 851                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 852                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 853                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 854                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 855                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 856                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 857                 }
 858         }
 859         return 0;
 860 }
 861
 862
 863 /**
 864  * nes_init_csr_ne020
 865  * Initialize registers for ne020 hardware
 866  */
 867 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 868 {
 869         u32 u32temp;
 870
 871         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 872
 873         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 874         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 875         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 876         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 877         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 878         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 879         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 880         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 881
 882         /* TODO: move these MAC register settings to NIC bringup */
 883         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 884         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 885         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 886         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 887         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 888         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 889         if (port_count > 1) {
 890                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 891                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 892                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 893                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 894                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 895                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 896                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 897         }
 898         if (port_count > 2) {
 899                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 900                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 901                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 902                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 903                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 904                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 905                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 906
 907                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 908                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 909                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 910                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 911                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 912                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 913                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 914         }
 915
 916         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 917         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 918         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 919                                                          0x00000001);
 920         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 921         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 922         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 923         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 924         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 925
 926         /* TODO: move this to code, get from EEPROM */
 927         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 928         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 929         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 930
 931         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 932         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 933
 934         if (hw_rev != NE020_REV) {
 935                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 936                 u32temp |= 0x80000000;
 937                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 938                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 939                 u32temp &= 0x7fffffff;
 940                 u32temp |= 0x7fff0010;
 941                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 942                 if (port_count > 1) {
 943                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 944                         u32temp &= 0x7fffffff;
 945                         u32temp |= 0x7fff0010;
 946                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 947                 }
 948         }
 949 }
 950
 951
 952 /**
 953  * nes_destroy_adapter - destroy the adapter structure
 954  */
 955 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 956 {
 957         struct nes_adapter *tmp_adapter;
 958
 959         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 960                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 961                                 tmp_adapter);
 962         }
 963
 964         nesadapter->ref_count--;
 965         if (!nesadapter->ref_count) {
 966                 if (nesadapter->hw_rev == NE020_REV) {
 967                         del_timer(&nesadapter->mh_timer);
 968                 }
 969                 del_timer(&nesadapter->lc_timer);
 970
 971                 list_del(&nesadapter->list);
 972                 kfree(nesadapter);
 973         }
 974 }
 975
 976
 977 /**
 978  * nes_init_cqp
 979  */
 980 int nes_init_cqp(struct nes_device *nesdev)
 981 {
 982         struct nes_adapter *nesadapter = nesdev->nesadapter;
 983         struct nes_hw_cqp_qp_context *cqp_qp_context;
 984         struct nes_hw_cqp_wqe *cqp_wqe;
 985         struct nes_hw_ceq *ceq;
 986         struct nes_hw_ceq *nic_ceq;
 987         struct nes_hw_aeq *aeq;
 988         void *vmem;
 989         dma_addr_t pmem;
 990         u32 count=0;
 991         u32 cqp_head;
 992         u64 u64temp;
 993         u32 u32temp;
 994
 995         /* allocate CQP memory */
 996         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 997         /* SQ is 512 byte aligned, others are 256 byte aligned */
 998         nesdev->cqp_mem_size = 512 +
 999                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1000                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1001                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1002                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1003                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1004                         sizeof(struct nes_hw_cqp_qp_context);
1005
1006         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1007                         &nesdev->cqp_pbase);
1008         if (!nesdev->cqp_vbase) {
1009                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1010                 return -ENOMEM;
1011         }
1012         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1013
1014         /* Allocate a twice the number of CQP requests as the SQ size */
1015         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1016                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1017         if (nesdev->nes_cqp_requests == NULL) {
1018                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1019                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1020                                 nesdev->cqp.sq_pbase);
1021                 return -ENOMEM;
1022         }
1023
1024         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1025                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1026
1027         spin_lock_init(&nesdev->cqp.lock);
1028         init_waitqueue_head(&nesdev->cqp.waitq);
1029
1030         /* Setup Various Structures */
1031         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1032                         ~(unsigned long)(512 - 1));
1033         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1034                         ~(unsigned long long)(512 - 1));
1035
1036         nesdev->cqp.sq_vbase = vmem;
1037         nesdev->cqp.sq_pbase = pmem;
1038         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1039         nesdev->cqp.sq_head = 0;
1040         nesdev->cqp.sq_tail = 0;
1041         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1042
1043         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1044         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045
1046         nesdev->ccq.cq_vbase = vmem;
1047         nesdev->ccq.cq_pbase = pmem;
1048         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1049         nesdev->ccq.cq_head = 0;
1050         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1051         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1052
1053         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1054         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055
1056         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1057         ceq = &nesadapter->ceq[nesdev->ceq_index];
1058         ceq->ceq_vbase = vmem;
1059         ceq->ceq_pbase = pmem;
1060         ceq->ceq_size = NES_CCEQ_SIZE;
1061         ceq->ceq_head = 0;
1062
1063         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1064         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065
1066         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1067         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1068         nic_ceq->ceq_vbase = vmem;
1069         nic_ceq->ceq_pbase = pmem;
1070         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1071         nic_ceq->ceq_head = 0;
1072
1073         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1074         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075
1076         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1077         aeq->aeq_vbase = vmem;
1078         aeq->aeq_pbase = pmem;
1079         aeq->aeq_size = nesadapter->max_qp;
1080         aeq->aeq_head = 0;
1081
1082         /* Setup QP Context */
1083         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1084         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085
1086         cqp_qp_context = vmem;
1087         cqp_qp_context->context_words[0] =
1088                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1089         cqp_qp_context->context_words[1] = 0;
1090         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1091         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1092
1093
1094         /* Write the address to Create CQP */
1095         if ((sizeof(dma_addr_t) > 4)) {
1096                 nes_write_indexed(nesdev,
1097                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1098                                 ((u64)pmem) >> 32);
1099         } else {
1100                 nes_write_indexed(nesdev,
1101                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1102         }
1103         nes_write_indexed(nesdev,
1104                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1105                         (u32)pmem);
1106
1107         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1108         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1109
1110         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1111                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1112                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1113         }
1114
1115         /* Write Create CCQ WQE */
1116         cqp_head = nesdev->cqp.sq_head++;
1117         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1118         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1119         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1120                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1121                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1122         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1123                             (nesdev->ccq.cq_number |
1124                              ((u32)nesdev->ceq_index << 16)));
1125         u64temp = (u64)nesdev->ccq.cq_pbase;
1126         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1127         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1128         u64temp = (unsigned long)&nesdev->ccq;
1129         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1130                         cpu_to_le32((u32)(u64temp >> 1));
1131         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1132                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1133         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1134
1135         /* Write Create CEQ WQE */
1136         cqp_head = nesdev->cqp.sq_head++;
1137         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1138         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1139         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1140                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1141         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1142         u64temp = (u64)ceq->ceq_pbase;
1143         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1144
1145         /* Write Create AEQ WQE */
1146         cqp_head = nesdev->cqp.sq_head++;
1147         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1148         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1149         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1150                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1151         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1152         u64temp = (u64)aeq->aeq_pbase;
1153         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1154
1155         /* Write Create NIC CEQ WQE */
1156         cqp_head = nesdev->cqp.sq_head++;
1157         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1158         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1159         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1160                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1161         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1162         u64temp = (u64)nic_ceq->ceq_pbase;
1163         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1164
1165         /* Poll until CCQP done */
1166         count = 0;
1167         do {
1168                 if (count++ > 1000) {
1169                         printk(KERN_ERR PFX "Error creating CQP\n");
1170                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1171                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1172                         return -1;
1173                 }
1174                 udelay(10);
1175         } while (!(nes_read_indexed(nesdev,
1176                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1177
1178         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1179                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1180
1181         u32temp = 0x04800000;
1182         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1183
1184         /* wait for the CCQ, CEQ, and AEQ to get created */
1185         count = 0;
1186         do {
1187                 if (count++ > 1000) {
1188                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1189                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1190                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1191                         return -1;
1192                 }
1193                 udelay(10);
1194         } while (((nes_read_indexed(nesdev,
1195                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1196
1197         /* dump the QP status value */
1198         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1199                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1200
1201         nesdev->cqp.sq_tail++;
1202
1203         return 0;
1204 }
1205
1206
1207 /**
1208  * nes_destroy_cqp
1209  */
1210 int nes_destroy_cqp(struct nes_device *nesdev)
1211 {
1212         struct nes_hw_cqp_wqe *cqp_wqe;
1213         u32 count = 0;
1214         u32 cqp_head;
1215         unsigned long flags;
1216
1217         do {
1218                 if (count++ > 1000)
1219                         break;
1220                 udelay(10);
1221         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1222
1223         /* Reset CCQ */
1224         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1225                         nesdev->ccq.cq_number);
1226
1227         /* Disable device interrupts */
1228         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1229
1230         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1231
1232         /* Destroy the AEQ */
1233         cqp_head = nesdev->cqp.sq_head++;
1234         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1235         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1236         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1237                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1238         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1239
1240         /* Destroy the NIC CEQ */
1241         cqp_head = nesdev->cqp.sq_head++;
1242         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1243         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1244         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1245                         ((u32)nesdev->nic_ceq_index << 8));
1246
1247         /* Destroy the CEQ */
1248         cqp_head = nesdev->cqp.sq_head++;
1249         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1250         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1251         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1252                         (nesdev->ceq_index << 8));
1253
1254         /* Destroy the CCQ */
1255         cqp_head = nesdev->cqp.sq_head++;
1256         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1257         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1258         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1259         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1260                         ((u32)nesdev->ceq_index << 16));
1261
1262         /* Destroy CQP */
1263         cqp_head = nesdev->cqp.sq_head++;
1264         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1265         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1266         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1267                         NES_CQP_QP_TYPE_CQP);
1268         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1269
1270         barrier();
1271         /* Ring doorbell (5 WQEs) */
1272         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1273
1274         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1275
1276         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1277         count = 0;
1278         do {
1279                 if (count++ > 1000) {
1280                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1281                                         PCI_FUNC(nesdev->pcidev->devfn));
1282                         break;
1283                 }
1284                 udelay(10);
1285         } while (((nes_read_indexed(nesdev,
1286                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1287
1288         /* dump the QP status value */
1289         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1290                         PCI_FUNC(nesdev->pcidev->devfn),
1291                         nes_read_indexed(nesdev,
1292                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1293
1294         kfree(nesdev->nes_cqp_requests);
1295
1296         /* Free the control structures */
1297         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1298                         nesdev->cqp.sq_pbase);
1299
1300         return 0;
1301 }
1302
1303
1304 /**
1305  * nes_init_1g_phy
1306  */
1307 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1308 {
1309         u32 counter = 0;
1310         u16 phy_data;
1311         int ret = 0;
1312
1313         nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1314         nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1315
1316         /* Reset the PHY */
1317         nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1318         udelay(100);
1319         counter = 0;
1320         do {
1321                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1322                 if (counter++ > 100) {
1323                         ret = -1;
1324                         break;
1325                 }
1326         } while (phy_data & 0x8000);
1327
1328         /* Setting no phy loopback */
1329         phy_data &= 0xbfff;
1330         phy_data |= 0x1140;
1331         nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1332         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1333         nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1334         nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1335
1336         /* Setting the interrupt mask */
1337         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1338         nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1339         nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1340
1341         /* turning on flow control */
1342         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1343         nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1344         nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1345
1346         /* Clear Half duplex */
1347         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1348         nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1349         nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1350
1351         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1352         nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1353
1354         return ret;
1355 }
1356
1357
1358 /**
1359  * nes_init_2025_phy
1360  */
1361 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1362 {
1363         u32 temp_phy_data = 0;
1364         u32 temp_phy_data2 = 0;
1365         u32 counter = 0;
1366         u32 sds;
1367         u32 mac_index = nesdev->mac_index;
1368         int ret = 0;
1369         unsigned int first_attempt = 1;
1370
1371         /* Check firmware heartbeat */
1372         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1373         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1374         udelay(1500);
1375         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1376         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1377
1378         if (temp_phy_data != temp_phy_data2) {
1379                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1380                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1381                 if ((temp_phy_data & 0xff) > 0x20)
1382                         return 0;
1383                 printk(PFX "Reinitialize external PHY\n");
1384         }
1385
1386         /* no heartbeat, configure the PHY */
1387         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1388         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1389         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1390         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1391
1392         switch (phy_type) {
1393         case NES_PHY_TYPE_ARGUS:
1394                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1398                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1400                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1401
1402                 /* setup LEDs */
1403                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1404                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1405                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1406                 break;
1407
1408         case NES_PHY_TYPE_SFP_D:
1409                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1410                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1411                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1412                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1413                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1414                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1415                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1416
1417                 /* setup LEDs */
1418                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1419                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1420                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1421                 break;
1422
1423         case NES_PHY_TYPE_KR:
1424                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1425                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1426                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1427                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1428                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1429                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1430                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1431
1432                 /* setup LEDs */
1433                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1434                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1435                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1436
1437                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1438                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1439                 break;
1440         }
1441
1442         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1443
1444         /* Bring PHY out of reset */
1445         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1446
1447         /* Check for heartbeat */
1448         counter = 0;
1449         mdelay(690);
1450         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1451         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1452         do {
1453                 if (counter++ > 150) {
1454                         printk(PFX "No PHY heartbeat\n");
1455                         break;
1456                 }
1457                 mdelay(1);
1458                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1459                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1460         } while ((temp_phy_data2 == temp_phy_data));
1461
1462         /* wait for tracking */
1463         counter = 0;
1464         do {
1465                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1466                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1467                 if (counter++ > 300) {
1468                         if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1469                                 first_attempt = 0;
1470                                 counter = 0;
1471                                 /* reset AMCC PHY and try again */
1472                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1473                                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1474                                 continue;
1475                         } else {
1476                                 ret = 1;
1477                                 break;
1478                         }
1479                 }
1480                 mdelay(10);
1481         } while ((temp_phy_data & 0xff) < 0x30);
1482
1483         /* setup signal integrity */
1484         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1485         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1486         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1487         if (phy_type == NES_PHY_TYPE_KR) {
1488                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1489         } else {
1490                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1491                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1492         }
1493
1494         /* reset serdes */
1495         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1496         sds |= 0x1;
1497         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1498         sds &= 0xfffffffe;
1499         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1500
1501         counter = 0;
1502         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1503                         && (counter++ < 5000))
1504                 ;
1505
1506         return ret;
1507 }
1508
1509
1510 /**
1511  * nes_init_phy
1512  */
1513 int nes_init_phy(struct nes_device *nesdev)
1514 {
1515         struct nes_adapter *nesadapter = nesdev->nesadapter;
1516         u32 mac_index = nesdev->mac_index;
1517         u32 tx_config = 0;
1518         unsigned long flags;
1519         u8  phy_type = nesadapter->phy_type[mac_index];
1520         u8  phy_index = nesadapter->phy_index[mac_index];
1521         int ret = 0;
1522
1523         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1524         if (phy_type == NES_PHY_TYPE_1G) {
1525                 /* setup 1G MDIO operation */
1526                 tx_config &= 0xFFFFFFE3;
1527                 tx_config |= 0x04;
1528         } else {
1529                 /* setup 10G MDIO operation */
1530                 tx_config &= 0xFFFFFFE3;
1531                 tx_config |= 0x15;
1532         }
1533         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1534
1535         spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1536
1537         switch (phy_type) {
1538         case NES_PHY_TYPE_1G:
1539                 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1540                 break;
1541         case NES_PHY_TYPE_ARGUS:
1542         case NES_PHY_TYPE_SFP_D:
1543         case NES_PHY_TYPE_KR:
1544                 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1545                 break;
1546         }
1547
1548         spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1549
1550         return ret;
1551 }
1552
1553
1554 /**
1555  * nes_replenish_nic_rq
1556  */
1557 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1558 {
1559         unsigned long flags;
1560         dma_addr_t bus_address;
1561         struct sk_buff *skb;
1562         struct nes_hw_nic_rq_wqe *nic_rqe;
1563         struct nes_hw_nic *nesnic;
1564         struct nes_device *nesdev;
1565         struct nes_rskb_cb *cb;
1566         u32 rx_wqes_posted = 0;
1567
1568         nesnic = &nesvnic->nic;
1569         nesdev = nesvnic->nesdev;
1570         spin_lock_irqsave(&nesnic->rq_lock, flags);
1571         if (nesnic->replenishing_rq !=0) {
1572                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1573                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1574                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1575                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1576                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1577                         add_timer(&nesvnic->rq_wqes_timer);
1578                 } else
1579                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1580                 return;
1581         }
1582         nesnic->replenishing_rq = 1;
1583         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1584         do {
1585                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1586                 if (skb) {
1587                         skb->dev = nesvnic->netdev;
1588
1589                         bus_address = pci_map_single(nesdev->pcidev,
1590                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1591                         cb = (struct nes_rskb_cb *)&skb->cb[0];
1592                         cb->busaddr = bus_address;
1593                         cb->maplen = nesvnic->max_frame_size;
1594
1595                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1596                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1597                                         cpu_to_le32(nesvnic->max_frame_size);
1598                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1599                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1600                                         cpu_to_le32((u32)bus_address);
1601                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1602                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1603                         nesnic->rx_skb[nesnic->rq_head] = skb;
1604                         nesnic->rq_head++;
1605                         nesnic->rq_head &= nesnic->rq_size - 1;
1606                         atomic_dec(&nesvnic->rx_skbs_needed);
1607                         barrier();
1608                         if (++rx_wqes_posted == 255) {
1609                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1610                                 rx_wqes_posted = 0;
1611                         }
1612                 } else {
1613                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1614                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1615                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1616                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1617                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1618                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1619                                 add_timer(&nesvnic->rq_wqes_timer);
1620                         } else
1621                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1622                         break;
1623                 }
1624         } while (atomic_read(&nesvnic->rx_skbs_needed));
1625         barrier();
1626         if (rx_wqes_posted)
1627                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1628         nesnic->replenishing_rq = 0;
1629 }
1630
1631
1632 /**
1633  * nes_rq_wqes_timeout
1634  */
1635 static void nes_rq_wqes_timeout(unsigned long parm)
1636 {
1637         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1638         printk("%s: Timer fired.\n", __func__);
1639         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1640         if (atomic_read(&nesvnic->rx_skbs_needed))
1641                 nes_replenish_nic_rq(nesvnic);
1642 }
1643
1644
1645 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1646                                void **tcph, u64 *hdr_flags, void *priv)
1647 {
1648         unsigned int ip_len;
1649         struct iphdr *iph;
1650         skb_reset_network_header(skb);
1651         iph = ip_hdr(skb);
1652         if (iph->protocol != IPPROTO_TCP)
1653                 return -1;
1654         ip_len = ip_hdrlen(skb);
1655         skb_set_transport_header(skb, ip_len);
1656         *tcph = tcp_hdr(skb);
1657
1658         *hdr_flags = LRO_IPV4 | LRO_TCP;
1659         *iphdr = iph;
1660         return 0;
1661 }
1662
1663
1664 /**
1665  * nes_init_nic_qp
1666  */
1667 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1668 {
1669         struct nes_hw_cqp_wqe *cqp_wqe;
1670         struct nes_hw_nic_sq_wqe *nic_sqe;
1671         struct nes_hw_nic_qp_context *nic_context;
1672         struct sk_buff *skb;
1673         struct nes_hw_nic_rq_wqe *nic_rqe;
1674         struct nes_vnic *nesvnic = netdev_priv(netdev);
1675         unsigned long flags;
1676         void *vmem;
1677         dma_addr_t pmem;
1678         u64 u64temp;
1679         int ret;
1680         u32 cqp_head;
1681         u32 counter;
1682         u32 wqe_count;
1683         struct nes_rskb_cb *cb;
1684         u8 jumbomode=0;
1685
1686         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1687         nesvnic->nic_mem_size = 256 +
1688                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1689                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1690                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1691                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1692                         sizeof(struct nes_hw_nic_qp_context);
1693
1694         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1695                         &nesvnic->nic_pbase);
1696         if (!nesvnic->nic_vbase) {
1697                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1698                 return -ENOMEM;
1699         }
1700         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1701         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1702                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1703
1704         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1705                         ~(unsigned long)(256 - 1));
1706         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1707                         ~(unsigned long long)(256 - 1));
1708
1709         /* Setup the first Fragment buffers */
1710         nesvnic->nic.first_frag_vbase = vmem;
1711
1712         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1713                 nesvnic->nic.frag_paddr[counter] = pmem;
1714                 pmem += sizeof(struct nes_first_frag);
1715         }
1716
1717         /* setup the SQ */
1718         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1719
1720         nesvnic->nic.sq_vbase = (void *)vmem;
1721         nesvnic->nic.sq_pbase = pmem;
1722         nesvnic->nic.sq_head = 0;
1723         nesvnic->nic.sq_tail = 0;
1724         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1725         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1726                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1727                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1728                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1729                                 NES_NIC_SQ_WQE_COMPLETION);
1730                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1731                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1732                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1733                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1734                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1735                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1736         }
1737
1738         nesvnic->get_cqp_request = nes_get_cqp_request;
1739         nesvnic->post_cqp_request = nes_post_cqp_request;
1740         nesvnic->mcrq_mcast_filter = NULL;
1741
1742         spin_lock_init(&nesvnic->nic.rq_lock);
1743
1744         /* setup the RQ */
1745         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1746         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747
1748
1749         nesvnic->nic.rq_vbase = vmem;
1750         nesvnic->nic.rq_pbase = pmem;
1751         nesvnic->nic.rq_head = 0;
1752         nesvnic->nic.rq_tail = 0;
1753         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1754
1755         /* setup the CQ */
1756         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1757         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758
1759         if (nesdev->nesadapter->netdev_count > 2)
1760                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1761         else
1762                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1763
1764         nesvnic->nic_cq.cq_vbase = vmem;
1765         nesvnic->nic_cq.cq_pbase = pmem;
1766         nesvnic->nic_cq.cq_head = 0;
1767         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1768
1769         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1770
1771         /* Send CreateCQ request to CQP */
1772         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1773         cqp_head = nesdev->cqp.sq_head;
1774
1775         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1776         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1777
1778         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1779                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1780                         ((u32)nesvnic->nic_cq.cq_size << 16));
1781         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1782                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1783         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1784         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1785         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1786         u64temp = (unsigned long)&nesvnic->nic_cq;
1787         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1788         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1789                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1790         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1791         if (++cqp_head >= nesdev->cqp.sq_size)
1792                 cqp_head = 0;
1793         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1794         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1795
1796         /* Send CreateQP request to CQP */
1797         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1798         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1799                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1800                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1801         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1802                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1803                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1804         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1805                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1806         }
1807
1808         u64temp = (u64)nesvnic->nic.sq_pbase;
1809         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1810         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1811         u64temp = (u64)nesvnic->nic.rq_pbase;
1812         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1813         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1814
1815         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1816                         NES_CQP_QP_TYPE_NIC);
1817         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1818         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1819                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1820         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1821
1822         if (++cqp_head >= nesdev->cqp.sq_size)
1823                 cqp_head = 0;
1824         nesdev->cqp.sq_head = cqp_head;
1825
1826         barrier();
1827
1828         /* Ring doorbell (2 WQEs) */
1829         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1830
1831         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1832         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1833                         nesvnic->nic.qp_id);
1834
1835         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1836                         NES_EVENT_TIMEOUT);
1837         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1838                         nesvnic->nic.qp_id, ret);
1839         if (!ret) {
1840                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1841                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1842                                 nesvnic->nic_pbase);
1843                 return -EIO;
1844         }
1845
1846         /* Populate the RQ */
1847         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1848                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1849                 if (!skb) {
1850                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1851
1852                         nes_destroy_nic_qp(nesvnic);
1853                         return -ENOMEM;
1854                 }
1855
1856                 skb->dev = netdev;
1857
1858                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1859                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1860                 cb = (struct nes_rskb_cb *)&skb->cb[0];
1861                 cb->busaddr = pmem;
1862                 cb->maplen = nesvnic->max_frame_size;
1863
1864                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1865                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1866                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1867                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1868                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1869                 nesvnic->nic.rx_skb[counter] = skb;
1870         }
1871
1872         wqe_count = NES_NIC_WQ_SIZE - 1;
1873         nesvnic->nic.rq_head = wqe_count;
1874         barrier();
1875         do {
1876                 counter = min(wqe_count, ((u32)255));
1877                 wqe_count -= counter;
1878                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1879         } while (wqe_count);
1880         init_timer(&nesvnic->rq_wqes_timer);
1881         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1882         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1883         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1884         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1885         {
1886                 nes_nic_init_timer(nesdev);
1887                 if (netdev->mtu > 1500)
1888                         jumbomode = 1;
1889                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1890         }
1891         if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1892                 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1893                         nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1894                         nes_destroy_nic_qp(nesvnic);
1895                 return -ENOMEM;
1896         }
1897
1898         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1899         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1900         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1901         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1902         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1903         nesvnic->lro_mgr.dev            = netdev;
1904         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1905         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1906         return 0;
1907 }
1908
1909
1910 /**
1911  * nes_destroy_nic_qp
1912  */
1913 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1914 {
1915         u64 u64temp;
1916         dma_addr_t bus_address;
1917         struct nes_device *nesdev = nesvnic->nesdev;
1918         struct nes_hw_cqp_wqe *cqp_wqe;
1919         struct nes_hw_nic_sq_wqe *nic_sqe;
1920         __le16 *wqe_fragment_length;
1921         u16  wqe_fragment_index;
1922         u32 cqp_head;
1923         u32 wqm_cfg0;
1924         unsigned long flags;
1925         struct sk_buff *rx_skb;
1926         struct nes_rskb_cb *cb;
1927         int ret;
1928
1929         if (nesdev->nesadapter->allow_unaligned_fpdus)
1930                 nes_destroy_mgt(nesvnic);
1931
1932         /* clear wqe stall before destroying NIC QP */
1933         wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1934         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1935
1936         /* Free remaining NIC receive buffers */
1937         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1938                 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1939                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1940                 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1941                         PCI_DMA_FROMDEVICE);
1942
1943                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1944                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1945         }
1946
1947         /* Free remaining NIC transmit buffers */
1948         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1949                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1950                 wqe_fragment_index = 1;
1951                 wqe_fragment_length = (__le16 *)
1952                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1953                 /* bump past the vlan tag */
1954                 wqe_fragment_length++;
1955                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1956                         u64temp = (u64)le32_to_cpu(
1957                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1958                                 wqe_fragment_index*2]);
1959                         u64temp += ((u64)le32_to_cpu(
1960                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1961                                 + wqe_fragment_index*2]))<<32;
1962                         bus_address = (dma_addr_t)u64temp;
1963                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1964                                         nesvnic->nic.first_frag_overflow)) {
1965                                 pci_unmap_single(nesdev->pcidev,
1966                                                 bus_address,
1967                                                 le16_to_cpu(wqe_fragment_length[
1968                                                         wqe_fragment_index++]),
1969                                                 PCI_DMA_TODEVICE);
1970                         }
1971                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1972                                 if (wqe_fragment_length[wqe_fragment_index]) {
1973                                         u64temp = le32_to_cpu(
1974                                                 nic_sqe->wqe_words[
1975                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1976                                                 wqe_fragment_index*2]);
1977                                         u64temp += ((u64)le32_to_cpu(
1978                                                 nic_sqe->wqe_words[
1979                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1980                                                 wqe_fragment_index*2]))<<32;
1981                                         bus_address = (dma_addr_t)u64temp;
1982                                         pci_unmap_page(nesdev->pcidev,
1983                                                         bus_address,
1984                                                         le16_to_cpu(
1985                                                         wqe_fragment_length[
1986                                                         wqe_fragment_index]),
1987                                                         PCI_DMA_TODEVICE);
1988                                 } else
1989                                         break;
1990                         }
1991                 }
1992                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1993                         dev_kfree_skb(
1994                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1995
1996                 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1997                                         & (nesvnic->nic.sq_size - 1);
1998         }
1999
2000         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2001
2002         /* Destroy NIC QP */
2003         cqp_head = nesdev->cqp.sq_head;
2004         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2005         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2006
2007         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2008                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2009         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2010                 nesvnic->nic.qp_id);
2011
2012         if (++cqp_head >= nesdev->cqp.sq_size)
2013                 cqp_head = 0;
2014
2015         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2016
2017         /* Destroy NIC CQ */
2018         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2019         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2020                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2021         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2022                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2023
2024         if (++cqp_head >= nesdev->cqp.sq_size)
2025                 cqp_head = 0;
2026
2027         nesdev->cqp.sq_head = cqp_head;
2028         barrier();
2029
2030         /* Ring doorbell (2 WQEs) */
2031         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2032
2033         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2034         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2035                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2036                         cqp_head, nesdev->cqp.sq_head,
2037                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2038
2039         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2040                         NES_EVENT_TIMEOUT);
2041
2042         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2043                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2044                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2045         if (!ret) {
2046                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2047                                 nesvnic->nic.qp_id);
2048         }
2049
2050         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2051                         nesvnic->nic_pbase);
2052
2053         /* restore old wqm_cfg0 value */
2054         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2055 }
2056
2057 /**
2058  * nes_napi_isr
2059  */
2060 int nes_napi_isr(struct nes_device *nesdev)
2061 {
2062         struct nes_adapter *nesadapter = nesdev->nesadapter;
2063         u32 int_stat;
2064
2065         if (nesdev->napi_isr_ran) {
2066                 /* interrupt status has already been read in ISR */
2067                 int_stat = nesdev->int_stat;
2068         } else {
2069                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2070                 nesdev->int_stat = int_stat;
2071                 nesdev->napi_isr_ran = 1;
2072         }
2073
2074         int_stat &= nesdev->int_req;
2075         /* iff NIC, process here, else wait for DPC */
2076         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2077                 nesdev->napi_isr_ran = 0;
2078                 nes_write32(nesdev->regs + NES_INT_STAT,
2079                         (int_stat &
2080                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2081
2082                 /* Process the CEQs */
2083                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2084
2085                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2086                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2087                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
2088                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2089                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2090                                 /* Enable Periodic timer interrupts */
2091                                 nesdev->int_req |= NES_INT_TIMER;
2092                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2093                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2094                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
2095                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
2096                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2097                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2098                         }
2099
2100                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2101                         {
2102                                 nes_nic_init_timer(nesdev);
2103                         }
2104                         /* Enable interrupts, except CEQs */
2105                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2106                 } else {
2107                         /* Enable interrupts, make sure timer is off */
2108                         nesdev->int_req &= ~NES_INT_TIMER;
2109                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2110                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2111                 }
2112                 nesdev->deepcq_count = 0;
2113                 return 1;
2114         } else {
2115                 return 0;
2116         }
2117 }
2118
2119 static void process_critical_error(struct nes_device *nesdev)
2120 {
2121         u32 debug_error;
2122         u32 nes_idx_debug_error_masks0 = 0;
2123         u16 error_module = 0;
2124
2125         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2126         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2127                         (u16)debug_error);
2128         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2129                         0x01010000 | (debug_error & 0x0000ffff));
2130         if (crit_err_count++ > 10)
2131                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2132         error_module = (u16) (debug_error & 0x1F00) >> 8;
2133         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2134                         nes_max_critical_error_count) {
2135                 printk(KERN_ERR PFX "Masking off critical error for module "
2136                         "0x%02X\n", (u16)error_module);
2137                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2138                         NES_IDX_DEBUG_ERROR_MASKS0);
2139                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2140                         nes_idx_debug_error_masks0 | (1 << error_module));
2141         }
2142 }
2143 /**
2144  * nes_dpc
2145  */
2146 void nes_dpc(unsigned long param)
2147 {
2148         struct nes_device *nesdev = (struct nes_device *)param;
2149         struct nes_adapter *nesadapter = nesdev->nesadapter;
2150         u32 counter;
2151         u32 loop_counter = 0;
2152         u32 int_status_bit;
2153         u32 int_stat;
2154         u32 timer_stat;
2155         u32 temp_int_stat;
2156         u32 intf_int_stat;
2157         u32 processed_intf_int = 0;
2158         u16 processed_timer_int = 0;
2159         u16 completion_ints = 0;
2160         u16 timer_ints = 0;
2161
2162         /* nes_debug(NES_DBG_ISR, "\n"); */
2163
2164         do {
2165                 timer_stat = 0;
2166                 if (nesdev->napi_isr_ran) {
2167                         nesdev->napi_isr_ran = 0;
2168                         int_stat = nesdev->int_stat;
2169                 } else
2170                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2171                 if (processed_intf_int != 0)
2172                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2173                 else
2174                         int_stat &= nesdev->int_req;
2175                 if (processed_timer_int == 0) {
2176                         processed_timer_int = 1;
2177                         if (int_stat & NES_INT_TIMER) {
2178                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2179                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2180                                         int_stat &= ~NES_INT_TIMER;
2181                                 }
2182                         }
2183                 } else {
2184                         int_stat &= ~NES_INT_TIMER;
2185                 }
2186
2187                 if (int_stat) {
2188                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2189                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2190                                 /* Ack the interrupts */
2191                                 nes_write32(nesdev->regs+NES_INT_STAT,
2192                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2193                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2194                         }
2195
2196                         temp_int_stat = int_stat;
2197                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2198                                 if (int_stat & int_status_bit) {
2199                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2200                                         temp_int_stat &= ~int_status_bit;
2201                                         completion_ints = 1;
2202                                 }
2203                                 if (!(temp_int_stat & 0x0000ffff))
2204                                         break;
2205                                 int_status_bit <<= 1;
2206                         }
2207
2208                         /* Process the AEQ for this pci function */
2209                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2210                         if (int_stat & int_status_bit) {
2211                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2212                         }
2213
2214                         /* Process the MAC interrupt for this pci function */
2215                         int_status_bit = 1 << (24 + nesdev->mac_index);
2216                         if (int_stat & int_status_bit) {
2217                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2218                         }
2219
2220                         if (int_stat & NES_INT_TIMER) {
2221                                 if (timer_stat & nesdev->timer_int_req) {
2222                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2223                                                         (timer_stat & nesdev->timer_int_req) |
2224                                                         ~(nesdev->nesadapter->timer_int_req));
2225                                         timer_ints = 1;
2226                                 }
2227                         }
2228
2229                         if (int_stat & NES_INT_INTF) {
2230                                 processed_intf_int = 1;
2231                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2232                                 intf_int_stat &= nesdev->intf_int_req;
2233                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2234                                         process_critical_error(nesdev);
2235                                 }
2236                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2237                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2238                                         BUG();
2239                                 }
2240                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2241                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2242                                         BUG();
2243                                 }
2244                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2245                         }
2246
2247                         if (int_stat & NES_INT_TSW) {
2248                         }
2249                 }
2250                 /* Don't use the interface interrupt bit stay in loop */
2251                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2252                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2253         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2254
2255         if (timer_ints == 1) {
2256                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2257                         if (completion_ints == 0) {
2258                                 nesdev->timer_only_int_count++;
2259                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2260                                         nesdev->timer_only_int_count = 0;
2261                                         nesdev->int_req &= ~NES_INT_TIMER;
2262                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2263                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2264                                 } else {
2265                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2266                                 }
2267                         } else {
2268                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2269                                 {
2270                                         nes_nic_init_timer(nesdev);
2271                                 }
2272                                 nesdev->timer_only_int_count = 0;
2273                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2274                         }
2275                 } else {
2276                         nesdev->timer_only_int_count = 0;
2277                         nesdev->int_req &= ~NES_INT_TIMER;
2278                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2279                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2280                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2281                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2282                 }
2283         } else {
2284                 if ( (completion_ints == 1) &&
2285                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2286                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2287                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2288                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2289                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2290                         nesdev->timer_only_int_count = 0;
2291                         nesdev->int_req |= NES_INT_TIMER;
2292                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2293                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2294                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2295                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2296                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2297                 } else {
2298                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2299                 }
2300         }
2301         nesdev->deepcq_count = 0;
2302 }
2303
2304
2305 /**
2306  * nes_process_ceq
2307  */
2308 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2309 {
2310         u64 u64temp;
2311         struct nes_hw_cq *cq;
2312         u32 head;
2313         u32 ceq_size;
2314
2315         /* nes_debug(NES_DBG_CQ, "\n"); */
2316         head = ceq->ceq_head;
2317         ceq_size = ceq->ceq_size;
2318
2319         do {
2320                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2321                                 NES_CEQE_VALID) {
2322                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2323                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2324                         u64temp <<= 1;
2325                         cq = *((struct nes_hw_cq **)&u64temp);
2326                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2327                         barrier();
2328                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2329
2330                         /* call the event handler */
2331                         cq->ce_handler(nesdev, cq);
2332
2333                         if (++head >= ceq_size)
2334                                 head = 0;
2335                 } else {
2336                         break;
2337                 }
2338
2339         } while (1);
2340
2341         ceq->ceq_head = head;
2342 }
2343
2344
2345 /**
2346  * nes_process_aeq
2347  */
2348 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2349 {
2350         /* u64 u64temp; */
2351         u32 head;
2352         u32 aeq_size;
2353         u32 aeqe_misc;
2354         u32 aeqe_cq_id;
2355         struct nes_hw_aeqe volatile *aeqe;
2356
2357         head = aeq->aeq_head;
2358         aeq_size = aeq->aeq_size;
2359
2360         do {
2361                 aeqe = &aeq->aeq_vbase[head];
2362                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2363                         break;
2364                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2365                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2366                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2367                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2368                                 /* dealing with an accelerated QP related AE */
2369                                 /*
2370                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2371                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2372                                  */
2373                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2374                         } else {
2375                                 /* TODO: dealing with a CQP related AE */
2376                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2377                                                 (u16)(aeqe_misc >> 16));
2378                         }
2379                 }
2380
2381                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2382
2383                 if (++head >= aeq_size)
2384                         head = 0;
2385
2386                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2387         }
2388         while (1);
2389         aeq->aeq_head = head;
2390 }
2391
2392 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2393 {
2394         struct nes_adapter *nesadapter = nesdev->nesadapter;
2395         u32 reset_value;
2396         u32 i=0;
2397         u32 u32temp;
2398
2399         if (nesadapter->hw_rev == NE020_REV) {
2400                 return;
2401         }
2402         mh_detected++;
2403
2404         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2405
2406         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2407                 reset_value |= 0x0000001d;
2408         else
2409                 reset_value |= 0x0000002d;
2410
2411         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2412                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2413                         nesadapter->link_interrupt_count[0] = 0;
2414                         nesadapter->link_interrupt_count[1] = 0;
2415                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2416                         if (0x00000040 & u32temp)
2417                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2418                         else
2419                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2420
2421                         reset_value |= 0x0000003d;
2422                 }
2423                 nesadapter->link_interrupt_count[mac_index] = 0;
2424         }
2425
2426         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2427
2428         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2429                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2430
2431         if (0x0000003d == (reset_value & 0x0000003d)) {
2432                 u32 pcs_control_status0, pcs_control_status1;
2433
2434                 for (i = 0; i < 10; i++) {
2435                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2436                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2437                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2438                              && (pcs_control_status0 & 0x00100000))
2439                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2440                                 && (pcs_control_status1 & 0x00100000)))
2441                                 continue;
2442                         else
2443                                 break;
2444                 }
2445                 if (10 == i) {
2446                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2447                         if (0x00000040 & u32temp)
2448                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2449                         else
2450                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2451
2452                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2453
2454                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2455                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2456                 }
2457         }
2458 }
2459
2460 /**
2461  * nes_process_mac_intr
2462  */
2463 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2464 {
2465         unsigned long flags;
2466         u32 pcs_control_status;
2467         struct nes_adapter *nesadapter = nesdev->nesadapter;
2468         struct nes_vnic *nesvnic;
2469         u32 mac_status;
2470         u32 mac_index = nesdev->mac_index;
2471         u32 u32temp;
2472         u16 phy_data;
2473         u16 temp_phy_data;
2474         u32 pcs_val  = 0x0f0f0000;
2475         u32 pcs_mask = 0x0f1f0000;
2476         u32 cdr_ctrl;
2477
2478         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2479         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2480                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2481                 return;
2482         }
2483         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2484
2485         /* ack the MAC interrupt */
2486         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2487         /* Clear the interrupt */
2488         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2489
2490         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2491
2492         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2493                 nesdev->link_status_interrupts++;
2494                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2495                         nes_reset_link(nesdev, mac_index);
2496
2497                 /* read the PHY interrupt status register */
2498                 if ((nesadapter->OneG_Mode) &&
2499                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2500                         do {
2501                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2502                                                 nesadapter->phy_index[mac_index], &phy_data);
2503                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2504                                                 nesadapter->phy_index[mac_index], phy_data);
2505                         } while (phy_data&0x8000);
2506
2507                         temp_phy_data = 0;
2508                         do {
2509                                 nes_read_1G_phy_reg(nesdev, 0x11,
2510                                                 nesadapter->phy_index[mac_index], &phy_data);
2511                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2512                                                 nesadapter->phy_index[mac_index], phy_data);
2513                                 if (temp_phy_data == phy_data)
2514                                         break;
2515                                 temp_phy_data = phy_data;
2516                         } while (1);
2517
2518                         nes_read_1G_phy_reg(nesdev, 0x1e,
2519                                         nesadapter->phy_index[mac_index], &phy_data);
2520                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2521                                         nesadapter->phy_index[mac_index], phy_data);
2522
2523                         nes_read_1G_phy_reg(nesdev, 1,
2524                                         nesadapter->phy_index[mac_index], &phy_data);
2525                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2526                                         nesadapter->phy_index[mac_index], phy_data);
2527
2528                         if (temp_phy_data & 0x1000) {
2529                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2530                                 phy_data = 4;
2531                         } else {
2532                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2533                         }
2534                 }
2535                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2536                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2537                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2538
2539                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2540                         switch (mac_index) {
2541                         case 1:
2542                         case 3:
2543                                 pcs_control_status = nes_read_indexed(nesdev,
2544                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2545                                 break;
2546                         default:
2547                                 pcs_control_status = nes_read_indexed(nesdev,
2548                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2549                                 break;
2550                         }
2551                 } else {
2552                         pcs_control_status = nes_read_indexed(nesdev,
2553                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2554                         pcs_control_status = nes_read_indexed(nesdev,
2555                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2556                 }
2557
2558                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2559                                 mac_index, pcs_control_status);
2560                 if ((nesadapter->OneG_Mode) &&
2561                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2562                         u32temp = 0x01010000;
2563                         if (nesadapter->port_count > 2) {
2564                                 u32temp |= 0x02020000;
2565                         }
2566                         if ((pcs_control_status & u32temp)!= u32temp) {
2567                                 phy_data = 0;
2568                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2569                         }
2570                 } else {
2571                         switch (nesadapter->phy_type[mac_index]) {
2572                         case NES_PHY_TYPE_ARGUS:
2573                         case NES_PHY_TYPE_SFP_D:
2574                         case NES_PHY_TYPE_KR:
2575                                 /* clear the alarms */
2576                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2577                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2578                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2579                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2580                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2581                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2582                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2583                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2584                                 /* check link status */
2585                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2586                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2587
2588                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2589                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2590                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2591                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2592
2593                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2594
2595                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2596                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2597                                 break;
2598
2599                         case NES_PHY_TYPE_PUMA_1G:
2600                                 if (mac_index < 2)
2601                                         pcs_val = pcs_mask = 0x01010000;
2602                                 else
2603                                         pcs_val = pcs_mask = 0x02020000;
2604                                 /* fall through */
2605                         default:
2606                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2607                                 break;
2608                         }
2609                 }
2610
2611                 if (phy_data & 0x0004) {
2612                         if (wide_ppm_offset &&
2613                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2614                             (nesadapter->hw_rev != NE020_REV)) {
2615                                 cdr_ctrl = nes_read_indexed(nesdev,
2616                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2617                                                             mac_index * 0x200);
2618                                 nes_write_indexed(nesdev,
2619                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2620                                                   mac_index * 0x200,
2621                                                   cdr_ctrl | 0x000F0000);
2622                         }
2623                         nesadapter->mac_link_down[mac_index] = 0;
2624                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2625                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2626                                                 nesvnic->linkup);
2627                                 if (nesvnic->linkup == 0) {
2628                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2629                                                         nesvnic->netdev->name, nesvnic->netdev);
2630                                         if (netif_queue_stopped(nesvnic->netdev))
2631                                                 netif_start_queue(nesvnic->netdev);
2632                                         nesvnic->linkup = 1;
2633                                         netif_carrier_on(nesvnic->netdev);
2634
2635                                         spin_lock(&nesvnic->port_ibevent_lock);
2636                                         if (nesvnic->of_device_registered) {
2637                                                 if (nesdev->iw_status == 0) {
2638                                                         nesdev->iw_status = 1;
2639                                                         nes_port_ibevent(nesvnic);
2640                                                 }
2641                                         }
2642                                         spin_unlock(&nesvnic->port_ibevent_lock);
2643                                 }
2644                         }
2645                 } else {
2646                         if (wide_ppm_offset &&
2647                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2648                             (nesadapter->hw_rev != NE020_REV)) {
2649                                 cdr_ctrl = nes_read_indexed(nesdev,
2650                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2651                                                             mac_index * 0x200);
2652                                 nes_write_indexed(nesdev,
2653                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2654                                                   mac_index * 0x200,
2655                                                   cdr_ctrl & 0xFFF0FFFF);
2656                         }
2657                         nesadapter->mac_link_down[mac_index] = 1;
2658                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2659                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2660                                                 nesvnic->linkup);
2661                                 if (nesvnic->linkup == 1) {
2662                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2663                                                         nesvnic->netdev->name, nesvnic->netdev);
2664                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2665                                                 netif_stop_queue(nesvnic->netdev);
2666                                         nesvnic->linkup = 0;
2667                                         netif_carrier_off(nesvnic->netdev);
2668
2669                                         spin_lock(&nesvnic->port_ibevent_lock);
2670                                         if (nesvnic->of_device_registered) {
2671                                                 if (nesdev->iw_status == 1) {
2672                                                         nesdev->iw_status = 0;
2673                                                         nes_port_ibevent(nesvnic);
2674                                                 }
2675                                         }
2676                                         spin_unlock(&nesvnic->port_ibevent_lock);
2677                                 }
2678                         }
2679                 }
2680                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2681                         if (nesdev->link_recheck)
2682                                 cancel_delayed_work(&nesdev->work);
2683                         nesdev->link_recheck = 1;
2684                         schedule_delayed_work(&nesdev->work,
2685                                               NES_LINK_RECHECK_DELAY);
2686                 }
2687         }
2688
2689         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2690
2691         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2692 }
2693
2694 void nes_recheck_link_status(struct work_struct *work)
2695 {
2696         unsigned long flags;
2697         struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2698         struct nes_adapter *nesadapter = nesdev->nesadapter;
2699         struct nes_vnic *nesvnic;
2700         u32 mac_index = nesdev->mac_index;
2701         u16 phy_data;
2702         u16 temp_phy_data;
2703
2704         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2705
2706         /* check link status */
2707         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2708         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2709
2710         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2711         nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2712         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2713         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2714
2715         phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2716
2717         nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2718                 __func__, phy_data,
2719                 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2720
2721         if (phy_data & 0x0004) {
2722                 nesadapter->mac_link_down[mac_index] = 0;
2723                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2724                         if (nesvnic->linkup == 0) {
2725                                 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2726                                                 nesvnic->netdev->name, nesvnic->netdev);
2727                                 if (netif_queue_stopped(nesvnic->netdev))
2728                                         netif_start_queue(nesvnic->netdev);
2729                                 nesvnic->linkup = 1;
2730                                 netif_carrier_on(nesvnic->netdev);
2731
2732                                 spin_lock(&nesvnic->port_ibevent_lock);
2733                                 if (nesvnic->of_device_registered) {
2734                                         if (nesdev->iw_status == 0) {
2735                                                 nesdev->iw_status = 1;
2736                                                 nes_port_ibevent(nesvnic);
2737                                         }
2738                                 }
2739                                 spin_unlock(&nesvnic->port_ibevent_lock);
2740                         }
2741                 }
2742
2743         } else {
2744                 nesadapter->mac_link_down[mac_index] = 1;
2745                 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2746                         if (nesvnic->linkup == 1) {
2747                                 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2748                                                 nesvnic->netdev->name, nesvnic->netdev);
2749                                 if (!(netif_queue_stopped(nesvnic->netdev)))
2750                                         netif_stop_queue(nesvnic->netdev);
2751                                 nesvnic->linkup = 0;
2752                                 netif_carrier_off(nesvnic->netdev);
2753
2754                                 spin_lock(&nesvnic->port_ibevent_lock);
2755                                 if (nesvnic->of_device_registered) {
2756                                         if (nesdev->iw_status == 1) {
2757                                                 nesdev->iw_status = 0;
2758                                                 nes_port_ibevent(nesvnic);
2759                                         }
2760                                 }
2761                                 spin_unlock(&nesvnic->port_ibevent_lock);
2762                         }
2763                 }
2764         }
2765         if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2766                 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2767         else
2768                 nesdev->link_recheck = 0;
2769
2770         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2771 }
2772
2773
2774 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2775 {
2776         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2777
2778         napi_schedule(&nesvnic->napi);
2779 }
2780
2781
2782 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2783 * getting out of nic_ce_handler
2784 */
2785 #define MAX_RQES_TO_PROCESS     384
2786
2787 /**
2788  * nes_nic_ce_handler
2789  */
2790 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2791 {
2792         u64 u64temp;
2793         dma_addr_t bus_address;
2794         struct nes_hw_nic *nesnic;
2795         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2796         struct nes_adapter *nesadapter = nesdev->nesadapter;
2797         struct nes_hw_nic_rq_wqe *nic_rqe;
2798         struct nes_hw_nic_sq_wqe *nic_sqe;
2799         struct sk_buff *skb;
2800         struct sk_buff *rx_skb;
2801         struct nes_rskb_cb *cb;
2802         __le16 *wqe_fragment_length;
2803         u32 head;
2804         u32 cq_size;
2805         u32 rx_pkt_size;
2806         u32 cqe_count=0;
2807         u32 cqe_errv;
2808         u32 cqe_misc;
2809         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2810         u16 vlan_tag;
2811         u16 pkt_type;
2812         u16 rqes_processed = 0;
2813         u8 sq_cqes = 0;
2814         u8 nes_use_lro = 0;
2815
2816         head = cq->cq_head;
2817         cq_size = cq->cq_size;
2818         cq->cqes_pending = 1;
2819         if (nesvnic->netdev->features & NETIF_F_LRO)
2820                 nes_use_lro = 1;
2821         do {
2822                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2823                                 NES_NIC_CQE_VALID) {
2824                         nesnic = &nesvnic->nic;
2825                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2826                         if (cqe_misc & NES_NIC_CQE_SQ) {
2827                                 sq_cqes++;
2828                                 wqe_fragment_index = 1;
2829                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2830                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2831                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2832                                 /* bump past the vlan tag */
2833                                 wqe_fragment_length++;
2834                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2835                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2836                                                         wqe_fragment_index * 2]);
2837                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2838                                                         wqe_fragment_index * 2])) << 32;
2839                                         bus_address = (dma_addr_t)u64temp;
2840                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2841                                                 pci_unmap_single(nesdev->pcidev,
2842                                                                 bus_address,
2843                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2844                                                                 PCI_DMA_TODEVICE);
2845                                         }
2846                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2847                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2848                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2849                                                                                 wqe_fragment_index * 2]);
2850                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2851                                                                                 + wqe_fragment_index * 2])) <<32;
2852                                                         bus_address = (dma_addr_t)u64temp;
2853                                                         pci_unmap_page(nesdev->pcidev,
2854                                                                         bus_address,
2855                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2856                                                                         PCI_DMA_TODEVICE);
2857                                                 } else
2858                                                         break;
2859                                         }
2860                                 }
2861                                 if (skb)
2862                                         dev_kfree_skb_any(skb);
2863                                 nesnic->sq_tail++;
2864                                 nesnic->sq_tail &= nesnic->sq_size-1;
2865                                 if (sq_cqes > 128) {
2866                                         barrier();
2867                                         /* restart the queue if it had been stopped */
2868                                         if (netif_queue_stopped(nesvnic->netdev))
2869                                                 netif_wake_queue(nesvnic->netdev);
2870                                         sq_cqes = 0;
2871                                 }
2872                         } else {
2873                                 rqes_processed ++;
2874
2875                                 cq->rx_cqes_completed++;
2876                                 cq->rx_pkts_indicated++;
2877                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2878                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2879                                 /* Get the skb */
2880                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2881                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2882                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2883                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2884                                 pci_unmap_single(nesdev->pcidev, bus_address,
2885                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2886                                 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2887                                 cb->busaddr = 0;
2888                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2889                                 /* rx_skb->len = rx_pkt_size; */
2890                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2891                                 skb_put(rx_skb, rx_pkt_size);
2892                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2893                                 nesnic->rq_tail++;
2894                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2895
2896                                 atomic_inc(&nesvnic->rx_skbs_needed);
2897                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2898                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2899                                                         cq->cq_number | (cqe_count << 16));
2900                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2901                                         nesdev->currcq_count += cqe_count;
2902                                         cqe_count = 0;
2903                                         nes_replenish_nic_rq(nesvnic);
2904                                 }
2905                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2906                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2907                                 rx_skb->ip_summed = CHECKSUM_NONE;
2908
2909                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2910                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2911                                         if ((cqe_errv &
2912                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2913                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2914                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2915                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2916                                         } else
2917                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2918                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2919                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2920
2921                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2922                                         if ((cqe_errv &
2923                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2924                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2925                                                 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2926                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2927                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2928                                                                   nesvnic->netdev->name); */
2929                                                 }
2930                                         } else
2931                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2932                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2933                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2934                                         }
2935                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2936                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2937
2938                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2939                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2940                                                 rx_skb = NULL;
2941                                 }
2942                                 if (rx_skb == NULL)
2943                                         goto skip_rx_indicate0;
2944
2945
2946                                 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2947                                         vlan_tag = (u16)(le32_to_cpu(
2948                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2949                                                         >> 16);
2950                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2951                                                         nesvnic->netdev->name, vlan_tag);
2952
2953                                         __vlan_hwaccel_put_tag(rx_skb, vlan_tag);
2954                                 }
2955                                 if (nes_use_lro)
2956                                         lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2957                                 else
2958                                         netif_receive_skb(rx_skb);
2959
2960 skip_rx_indicate0:
2961                                 ;
2962                                 /* nesvnic->netstats.rx_packets++; */
2963                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2964                         }
2965
2966                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2967                         /* Accounting... */
2968                         cqe_count++;
2969                         if (++head >= cq_size)
2970                                 head = 0;
2971                         if (cqe_count == 255) {
2972                                 /* Replenish Nic CQ */
2973                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2974                                                 cq->cq_number | (cqe_count << 16));
2975                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2976                                 nesdev->currcq_count += cqe_count;
2977                                 cqe_count = 0;
2978                         }
2979
2980                         if (cq->rx_cqes_completed >= nesvnic->budget)
2981                                 break;
2982                 } else {
2983                         cq->cqes_pending = 0;
2984                         break;
2985                 }
2986
2987         } while (1);
2988
2989         if (nes_use_lro)
2990                 lro_flush_all(&nesvnic->lro_mgr);
2991         if (sq_cqes) {
2992                 barrier();
2993                 /* restart the queue if it had been stopped */
2994                 if (netif_queue_stopped(nesvnic->netdev))
2995                         netif_wake_queue(nesvnic->netdev);
2996         }
2997         cq->cq_head = head;
2998         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2999                         cq->cq_number, cqe_count, cq->cq_head); */
3000         cq->cqe_allocs_pending = cqe_count;
3001         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3002         {
3003                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3004                 nesdev->currcq_count += cqe_count;
3005                 nes_nic_tune_timer(nesdev);
3006         }
3007         if (atomic_read(&nesvnic->rx_skbs_needed))
3008                 nes_replenish_nic_rq(nesvnic);
3009 }
3010
3011
3012
3013 /**
3014  * nes_cqp_ce_handler
3015  */
3016 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3017 {
3018         u64 u64temp;
3019         unsigned long flags;
3020         struct nes_hw_cqp *cqp = NULL;
3021         struct nes_cqp_request *cqp_request;
3022         struct nes_hw_cqp_wqe *cqp_wqe;
3023         u32 head;
3024         u32 cq_size;
3025         u32 cqe_count=0;
3026         u32 error_code;
3027         u32 opcode;
3028         u32 ctx_index;
3029         /* u32 counter; */
3030
3031         head = cq->cq_head;
3032         cq_size = cq->cq_size;
3033
3034         do {
3035                 /* process the CQE */
3036                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3037                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3038
3039                 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3040                 if (opcode & NES_CQE_VALID) {
3041                         cqp = &nesdev->cqp;
3042
3043                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3044                         if (error_code) {
3045                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3046                                                 " Major/Minor codes = 0x%04X:%04X.\n",
3047                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3048                                                 (u16)(error_code >> 16),
3049                                                 (u16)error_code);
3050                         }
3051
3052                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3053                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3054                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
3055                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3056
3057                         cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3058                         if (cqp_request) {
3059                                 if (cqp_request->waiting) {
3060                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3061                                         cqp_request->major_code = (u16)(error_code >> 16);
3062                                         cqp_request->minor_code = (u16)error_code;
3063                                         barrier();
3064                                         cqp_request->request_done = 1;
3065                                         wake_up(&cqp_request->waitq);
3066                                         nes_put_cqp_request(nesdev, cqp_request);
3067                                 } else {
3068                                         if (cqp_request->callback)
3069                                                 cqp_request->cqp_callback(nesdev, cqp_request);
3070                                         nes_free_cqp_request(nesdev, cqp_request);
3071                                 }
3072                         } else {
3073                                 wake_up(&nesdev->cqp.waitq);
3074                         }
3075
3076                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3077                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3078                         if (++cqp->sq_tail >= cqp->sq_size)
3079                                 cqp->sq_tail = 0;
3080
3081                         /* Accounting... */
3082                         cqe_count++;
3083                         if (++head >= cq_size)
3084                                 head = 0;
3085                 } else {
3086                         break;
3087                 }
3088         } while (1);
3089         cq->cq_head = head;
3090
3091         spin_lock_irqsave(&nesdev->cqp.lock, flags);
3092         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3093                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3094                         (nesdev->cqp.sq_size - 1)) != 1)) {
3095                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3096                                 struct nes_cqp_request, list);
3097                 list_del_init(&cqp_request->list);
3098                 head = nesdev->cqp.sq_head++;
3099                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3100                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3101                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3102                 barrier();
3103
3104                 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3105                 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3106                         ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3107                 else
3108                         ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3109                 cqp_wqe->wqe_words[ctx_index] =
3110                         cpu_to_le32((u32)((unsigned long)cqp_request));
3111                 cqp_wqe->wqe_words[ctx_index + 1] =
3112                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3113                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3114                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3115                 /* Ring doorbell (1 WQEs) */
3116                 barrier();
3117                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3118         }
3119         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3120
3121         /* Arm the CCQ */
3122         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3123                         cq->cq_number);
3124         nes_read32(nesdev->regs+NES_CQE_ALLOC);
3125 }
3126
3127 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3128 {
3129         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3130                 /* skip over ethernet header */
3131                 pkt += ETH_HLEN;
3132
3133                 /* Skip over IP and TCP headers */
3134                 pkt += 4 * (pkt[0] & 0x0f);
3135                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3136         }
3137         return pkt;
3138 }
3139
3140 /* Determine if incoming error pkt is rdma layer */
3141 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3142 {
3143         u8 *pkt;
3144         u16 *mpa;
3145         u32 opcode = 0xffffffff;
3146
3147         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3148                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3149                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3150                 opcode = be16_to_cpu(mpa[1]) & 0xf;
3151         }
3152
3153         return opcode;
3154 }
3155
3156 /* Build iWARP terminate header */
3157 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3158 {
3159         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3160         u16 ddp_seg_len;
3161         int copy_len = 0;
3162         u8 is_tagged = 0;
3163         u8 flush_code = 0;
3164         struct nes_terminate_hdr *termhdr;
3165
3166         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3167         memset(termhdr, 0, 64);
3168
3169         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3170
3171                 /* Use data from offending packet to fill in ddp & rdma hdrs */
3172                 pkt = locate_mpa(pkt, aeq_info);
3173                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3174                 if (ddp_seg_len) {
3175                         copy_len = 2;
3176                         termhdr->hdrct = DDP_LEN_FLAG;
3177                         if (pkt[2] & 0x80) {
3178                                 is_tagged = 1;
3179                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3180                                         copy_len += TERM_DDP_LEN_TAGGED;
3181                                         termhdr->hdrct |= DDP_HDR_FLAG;
3182                                 }
3183                         } else {
3184                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3185                                         copy_len += TERM_DDP_LEN_UNTAGGED;
3186                                         termhdr->hdrct |= DDP_HDR_FLAG;
3187                                 }
3188
3189                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3190                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3191                                                 copy_len += TERM_RDMA_LEN;
3192                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
3193                                         }
3194                                 }
3195                         }
3196                 }
3197         }
3198
3199         switch (async_event_id) {
3200         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3201                 switch (iwarp_opcode(nesqp, aeq_info)) {
3202                 case IWARP_OPCODE_WRITE:
3203                         flush_code = IB_WC_LOC_PROT_ERR;
3204                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3205                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3206                         break;
3207                 default:
3208                         flush_code = IB_WC_REM_ACCESS_ERR;
3209                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3210                         termhdr->error_code = RDMAP_INV_STAG;
3211                 }
3212                 break;
3213         case NES_AEQE_AEID_AMP_INVALID_STAG:
3214                 flush_code = IB_WC_REM_ACCESS_ERR;
3215                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3216                 termhdr->error_code = RDMAP_INV_STAG;
3217                 break;
3218         case NES_AEQE_AEID_AMP_BAD_QP:
3219                 flush_code = IB_WC_LOC_QP_OP_ERR;
3220                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3221                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3222                 break;
3223         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3224         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3225                 switch (iwarp_opcode(nesqp, aeq_info)) {
3226                 case IWARP_OPCODE_SEND_INV:
3227                 case IWARP_OPCODE_SEND_SE_INV:
3228                         flush_code = IB_WC_REM_OP_ERR;
3229                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3230                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3231                         break;
3232                 default:
3233                         flush_code = IB_WC_REM_ACCESS_ERR;
3234                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3235                         termhdr->error_code = RDMAP_INV_STAG;
3236                 }
3237                 break;
3238         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3239                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3240                         flush_code = IB_WC_LOC_PROT_ERR;
3241                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3242                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3243                 } else {
3244                         flush_code = IB_WC_REM_ACCESS_ERR;
3245                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3246                         termhdr->error_code = RDMAP_INV_BOUNDS;
3247                 }
3248                 break;
3249         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3250         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3251         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3252                 flush_code = IB_WC_REM_ACCESS_ERR;
3253                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3254                 termhdr->error_code = RDMAP_ACCESS;
3255                 break;
3256         case NES_AEQE_AEID_AMP_TO_WRAP:
3257                 flush_code = IB_WC_REM_ACCESS_ERR;
3258                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3259                 termhdr->error_code = RDMAP_TO_WRAP;
3260                 break;
3261         case NES_AEQE_AEID_AMP_BAD_PD:
3262                 switch (iwarp_opcode(nesqp, aeq_info)) {
3263                 case IWARP_OPCODE_WRITE:
3264                         flush_code = IB_WC_LOC_PROT_ERR;
3265                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3266                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3267                         break;
3268                 case IWARP_OPCODE_SEND_INV:
3269                 case IWARP_OPCODE_SEND_SE_INV:
3270                         flush_code = IB_WC_REM_ACCESS_ERR;
3271                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3272                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3273                         break;
3274                 default:
3275                         flush_code = IB_WC_REM_ACCESS_ERR;
3276                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3277                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3278                 }
3279                 break;
3280         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3281                 flush_code = IB_WC_LOC_LEN_ERR;
3282                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3283                 termhdr->error_code = MPA_MARKER;
3284                 break;
3285         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3286                 flush_code = IB_WC_GENERAL_ERR;
3287                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3288                 termhdr->error_code = MPA_CRC;
3289                 break;
3290         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3291         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3292                 flush_code = IB_WC_LOC_LEN_ERR;
3293                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3294                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3295                 break;
3296         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3297         case NES_AEQE_AEID_DDP_NO_L_BIT:
3298                 flush_code = IB_WC_FATAL_ERR;
3299                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3300                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3301                 break;
3302         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3303         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3304                 flush_code = IB_WC_GENERAL_ERR;
3305                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3306                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3307                 break;
3308         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3309                 flush_code = IB_WC_LOC_LEN_ERR;
3310                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3311                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3312                 break;
3313         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3314                 flush_code = IB_WC_GENERAL_ERR;
3315                 if (is_tagged) {
3316                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3317                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3318                 } else {
3319                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3320                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3321                 }
3322                 break;
3323         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3324                 flush_code = IB_WC_GENERAL_ERR;
3325                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3326                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3327                 break;
3328         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3329                 flush_code = IB_WC_REM_OP_ERR;
3330                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3331                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3332                 break;
3333         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3334                 flush_code = IB_WC_GENERAL_ERR;
3335                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3336                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3337                 break;
3338         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3339                 flush_code = IB_WC_GENERAL_ERR;
3340                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3341                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3342                 break;
3343         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3344                 flush_code = IB_WC_LOC_QP_OP_ERR;
3345                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3346                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3347                 break;
3348         default:
3349                 flush_code = IB_WC_FATAL_ERR;
3350                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3351                 termhdr->error_code = RDMAP_UNSPECIFIED;
3352                 break;
3353         }
3354
3355         if (copy_len)
3356                 memcpy(termhdr + 1, pkt, copy_len);
3357
3358         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3359                 if (aeq_info & NES_AEQE_SQ)
3360                         nesqp->term_sq_flush_code = flush_code;
3361                 else
3362                         nesqp->term_rq_flush_code = flush_code;
3363         }
3364
3365         return sizeof(struct nes_terminate_hdr) + copy_len;
3366 }
3367
3368 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3369                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3370 {
3371         u64 context;
3372         unsigned long flags;
3373         u32 aeq_info;
3374         u16 async_event_id;
3375         u8 tcp_state;
3376         u8 iwarp_state;
3377         u32 termlen = 0;
3378         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3379                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3380         struct nes_adapter *nesadapter = nesdev->nesadapter;
3381
3382         if (nesqp->term_flags & NES_TERM_SENT)
3383                 return; /* Sanity check */
3384
3385         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3386         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3387         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3388         async_event_id = (u16)aeq_info;
3389
3390         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3391                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3392         if (!context) {
3393                 WARN_ON(!context);
3394                 return;
3395         }
3396
3397         nesqp = (struct nes_qp *)(unsigned long)context;
3398         spin_lock_irqsave(&nesqp->lock, flags);
3399         nesqp->hw_iwarp_state = iwarp_state;
3400         nesqp->hw_tcp_state = tcp_state;
3401         nesqp->last_aeq = async_event_id;
3402         nesqp->terminate_eventtype = eventtype;
3403         spin_unlock_irqrestore(&nesqp->lock, flags);
3404
3405         if (nesadapter->send_term_ok)
3406                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3407         else
3408                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3409
3410         if (!nesdev->iw_status)  {
3411                 nesqp->term_flags = NES_TERM_DONE;
3412                 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3413                 nes_cm_disconn(nesqp);
3414         } else {
3415                 nes_terminate_start_timer(nesqp);
3416                 nesqp->term_flags |= NES_TERM_SENT;
3417                 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3418         }
3419 }
3420
3421 static void nes_terminate_send_fin(struct nes_device *nesdev,
3422                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3423 {
3424         u32 aeq_info;
3425         u16 async_event_id;
3426         u8 tcp_state;
3427         u8 iwarp_state;
3428         unsigned long flags;
3429
3430         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3431         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3432         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3433         async_event_id = (u16)aeq_info;
3434
3435         spin_lock_irqsave(&nesqp->lock, flags);
3436         nesqp->hw_iwarp_state = iwarp_state;
3437         nesqp->hw_tcp_state = tcp_state;
3438         nesqp->last_aeq = async_event_id;
3439         spin_unlock_irqrestore(&nesqp->lock, flags);
3440
3441         /* Send the fin only */
3442         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3443                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3444 }
3445
3446 /* Cleanup after a terminate sent or received */
3447 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3448 {
3449         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3450         unsigned long flags;
3451         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3452         struct nes_device *nesdev = nesvnic->nesdev;
3453         u8 first_time = 0;
3454
3455         spin_lock_irqsave(&nesqp->lock, flags);
3456         if (nesqp->hte_added) {
3457                 nesqp->hte_added = 0;
3458                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3459         }
3460
3461         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3462         nesqp->term_flags |= NES_TERM_DONE;
3463         spin_unlock_irqrestore(&nesqp->lock, flags);
3464
3465         /* Make sure we go through this only once */
3466         if (first_time) {
3467                 if (timeout_occurred == 0)
3468                         del_timer(&nesqp->terminate_timer);
3469                 else
3470                         next_iwarp_state |= NES_CQP_QP_RESET;
3471
3472                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3473                 nes_cm_disconn(nesqp);
3474         }
3475 }
3476
3477 static void nes_terminate_received(struct nes_device *nesdev,
3478                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3479 {
3480         u32 aeq_info;
3481         u8 *pkt;
3482         u32 *mpa;
3483         u8 ddp_ctl;
3484         u8 rdma_ctl;
3485         u16 aeq_id = 0;
3486
3487         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3488         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3489                 /* Terminate is not a performance path so the silicon */
3490                 /* did not validate the frame - do it now */
3491                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3492                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3493                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3494                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3495                 if ((ddp_ctl & 0xc0) != 0x40)
3496                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3497                 else if ((ddp_ctl & 0x03) != 1)
3498                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3499                 else if (be32_to_cpu(mpa[2]) != 2)
3500                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3501                 else if (be32_to_cpu(mpa[3]) != 1)
3502                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3503                 else if (be32_to_cpu(mpa[4]) != 0)
3504                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3505                 else if ((rdma_ctl & 0xc0) != 0x40)
3506                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3507
3508                 if (aeq_id) {
3509                         /* Bad terminate recvd - send back a terminate */
3510                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3511                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3512                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3513                         return;
3514                 }
3515         }
3516
3517         nesqp->term_flags |= NES_TERM_RCVD;
3518         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3519         nes_terminate_start_timer(nesqp);
3520         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3521 }
3522
3523 /* Timeout routine in case terminate fails to complete */
3524 void nes_terminate_timeout(unsigned long context)
3525 {
3526         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3527
3528         nes_terminate_done(nesqp, 1);
3529 }
3530
3531 /* Set a timer in case hw cannot complete the terminate sequence */
3532 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3533 {
3534         mod_timer(&nesqp->terminate_timer, (jiffies + HZ));
3535 }
3536
3537 /**
3538  * nes_process_iwarp_aeqe
3539  */
3540 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3541                                    struct nes_hw_aeqe *aeqe)
3542 {
3543         u64 context;
3544         unsigned long flags;
3545         struct nes_qp *nesqp;
3546         struct nes_hw_cq *hw_cq;
3547         struct nes_cq *nescq;
3548         int resource_allocated;
3549         struct nes_adapter *nesadapter = nesdev->nesadapter;
3550         u32 aeq_info;
3551         u32 next_iwarp_state = 0;
3552         u32 aeqe_cq_id;
3553         u16 async_event_id;
3554         u8 tcp_state;
3555         u8 iwarp_state;
3556         struct ib_event ibevent;
3557
3558         nes_debug(NES_DBG_AEQ, "\n");
3559         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3560         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3561                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3562                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3563         } else {
3564                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3565                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3566                 BUG_ON(!context);
3567         }
3568
3569         /* context is nesqp unless async_event_id == CQ ERROR */
3570         nesqp = (struct nes_qp *)(unsigned long)context;
3571         async_event_id = (u16)aeq_info;
3572         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3573         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3574         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3575                         " Tcp state = %s, iWARP state = %s\n",
3576                         async_event_id,
3577                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3578                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3579
3580         aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3581         if (aeq_info & NES_AEQE_QP) {
3582                 if (!nes_is_resource_allocated(nesadapter,
3583                                 nesadapter->allocated_qps,
3584                                 aeqe_cq_id))
3585                         return;
3586         }
3587
3588         switch (async_event_id) {
3589                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3590                         if (nesqp->term_flags)
3591                                 return; /* Ignore it, wait for close complete */
3592
3593                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3594                                 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3595                                         (nesqp->ibqp_state == IB_QPS_RTS)) {
3596                                         spin_lock_irqsave(&nesqp->lock, flags);
3597                                         nesqp->hw_iwarp_state = iwarp_state;
3598                                         nesqp->hw_tcp_state = tcp_state;
3599                                         nesqp->last_aeq = async_event_id;
3600                                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3601                                         nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3602                                         spin_unlock_irqrestore(&nesqp->lock, flags);
3603                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3604                                         nes_cm_disconn(nesqp);
3605                                 }
3606                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3607                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3608                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3609                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3610                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3611                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3612                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3613                                                 async_event_id, nesqp->last_aeq, tcp_state);
3614                         }
3615                         break;
3616                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3617                         if (nesqp->term_flags) {
3618                                 nes_terminate_done(nesqp, 0);
3619                                 return;
3620                         }
3621                         spin_lock_irqsave(&nesqp->lock, flags);
3622                         nesqp->hw_iwarp_state = iwarp_state;
3623                         nesqp->hw_tcp_state = tcp_state;
3624                         nesqp->last_aeq = async_event_id;
3625                         spin_unlock_irqrestore(&nesqp->lock, flags);
3626                         nes_cm_disconn(nesqp);
3627                         break;
3628
3629                 case NES_AEQE_AEID_RESET_SENT:
3630                         tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3631                         spin_lock_irqsave(&nesqp->lock, flags);
3632                         nesqp->hw_iwarp_state = iwarp_state;
3633                         nesqp->hw_tcp_state = tcp_state;
3634                         nesqp->last_aeq = async_event_id;
3635                         nesqp->hte_added = 0;
3636                         spin_unlock_irqrestore(&nesqp->lock, flags);
3637                         next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3638                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3639                         nes_cm_disconn(nesqp);
3640                         break;
3641
3642                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3643                         if (atomic_read(&nesqp->close_timer_started))
3644                                 return;
3645                         spin_lock_irqsave(&nesqp->lock, flags);
3646                         nesqp->hw_iwarp_state = iwarp_state;
3647                         nesqp->hw_tcp_state = tcp_state;
3648                         nesqp->last_aeq = async_event_id;
3649                         spin_unlock_irqrestore(&nesqp->lock, flags);
3650                         nes_cm_disconn(nesqp);
3651                         break;
3652
3653                 case NES_AEQE_AEID_TERMINATE_SENT:
3654                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3655                         break;
3656
3657                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3658                         nes_terminate_received(nesdev, nesqp, aeqe);
3659                         break;
3660
3661                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3662                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3663                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3664                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3665                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3666                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3667                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3668                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3669                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3670                 case NES_AEQE_AEID_AMP_TO_WRAP:
3671                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3672                                         nesqp->hwqp.qp_id, async_event_id);
3673                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3674                         break;
3675
3676                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3677                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3678                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3679                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3680                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3681                                 aeq_info &= 0xffff0000;
3682                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3683                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3684                         }
3685
3686                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3687                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3688                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3689                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3690                 case NES_AEQE_AEID_AMP_BAD_QP:
3691                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3692                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3693                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3694                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3695                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3696                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3697                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3698                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3699                 case NES_AEQE_AEID_AMP_BAD_PD:
3700                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3701                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3702                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3703                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3704                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3705                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3706                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3707                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3708                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3709                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3710                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3711                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3712                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3713                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3714                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3715                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3716                 case NES_AEQE_AEID_BAD_CLOSE:
3717                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3718                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3719                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3720                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3721                         printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3722                                         nesqp->hwqp.qp_id, async_event_id);
3723                         print_ip(nesqp->cm_node);
3724                         if (!atomic_read(&nesqp->close_timer_started))
3725                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3726                         break;
3727
3728                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3729                         context <<= 1;
3730                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3731                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3732                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3733                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3734                         if (resource_allocated) {
3735                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3736                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3737                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3738                                 if (hw_cq) {
3739                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3740                                         if (nescq->ibcq.event_handler) {
3741                                                 ibevent.device = nescq->ibcq.device;
3742                                                 ibevent.event = IB_EVENT_CQ_ERR;
3743                                                 ibevent.element.cq = &nescq->ibcq;
3744                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3745                                         }
3746                                 }
3747                         }
3748                         break;
3749
3750                 default:
3751                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3752                                         async_event_id);
3753                         break;
3754         }
3755
3756 }
3757
3758 /**
3759  * nes_iwarp_ce_handler
3760  */
3761 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3762 {
3763         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3764
3765         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3766                         nescq->hw_cq.cq_number); */
3767         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3768
3769         if (nescq->ibcq.comp_handler)
3770                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3771
3772         return;
3773 }
3774
3775
3776 /**
3777  * nes_manage_apbvt()
3778  */
3779 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3780                 u32 nic_index, u32 add_port)
3781 {
3782         struct nes_device *nesdev = nesvnic->nesdev;
3783         struct nes_hw_cqp_wqe *cqp_wqe;
3784         struct nes_cqp_request *cqp_request;
3785         int ret = 0;
3786         u16 major_code;
3787
3788         /* Send manage APBVT request to CQP */
3789         cqp_request = nes_get_cqp_request(nesdev);
3790         if (cqp_request == NULL) {
3791                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3792                 return -ENOMEM;
3793         }
3794         cqp_request->waiting = 1;
3795         cqp_wqe = &cqp_request->cqp_wqe;
3796
3797         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3798                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3799                         accel_local_port, accel_local_port, nic_index);
3800
3801         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3802         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3803                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3804         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3805                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3806
3807         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3808
3809         atomic_set(&cqp_request->refcount, 2);
3810         nes_post_cqp_request(nesdev, cqp_request);
3811
3812         if (add_port == NES_MANAGE_APBVT_ADD)
3813                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3814                                 NES_EVENT_TIMEOUT);
3815         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3816                         ret, cqp_request->major_code, cqp_request->minor_code);
3817         major_code = cqp_request->major_code;
3818
3819         nes_put_cqp_request(nesdev, cqp_request);
3820
3821         if (!ret)
3822                 return -ETIME;
3823         else if (major_code)
3824                 return -EIO;
3825         else
3826                 return 0;
3827 }
3828
3829
3830 /**
3831  * nes_manage_arp_cache
3832  */
3833 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3834                 u32 ip_addr, u32 action)
3835 {
3836         struct nes_hw_cqp_wqe *cqp_wqe;
3837         struct nes_vnic *nesvnic = netdev_priv(netdev);
3838         struct nes_device *nesdev;
3839         struct nes_cqp_request *cqp_request;
3840         int arp_index;
3841
3842         nesdev = nesvnic->nesdev;
3843         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3844         if (arp_index == -1) {
3845                 return;
3846         }
3847
3848         /* update the ARP entry */
3849         cqp_request = nes_get_cqp_request(nesdev);
3850         if (cqp_request == NULL) {
3851                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3852                 return;
3853         }
3854         cqp_request->waiting = 0;
3855         cqp_wqe = &cqp_request->cqp_wqe;
3856         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3857
3858         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3859                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3860         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3861                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3862         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3863
3864         if (action == NES_ARP_ADD) {
3865                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3866                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3867                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3868                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3869                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3870                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3871         } else {
3872                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3873                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3874         }
3875
3876         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3877                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3878
3879         atomic_set(&cqp_request->refcount, 1);
3880         nes_post_cqp_request(nesdev, cqp_request);
3881 }
3882
3883
3884 /**
3885  * flush_wqes
3886  */
3887 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3888                 u32 which_wq, u32 wait_completion)
3889 {
3890         struct nes_cqp_request *cqp_request;
3891         struct nes_hw_cqp_wqe *cqp_wqe;
3892         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3893         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3894         int ret;
3895
3896         cqp_request = nes_get_cqp_request(nesdev);
3897         if (cqp_request == NULL) {
3898                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3899                 return;
3900         }
3901         if (wait_completion) {
3902                 cqp_request->waiting = 1;
3903                 atomic_set(&cqp_request->refcount, 2);
3904         } else {
3905                 cqp_request->waiting = 0;
3906         }
3907         cqp_wqe = &cqp_request->cqp_wqe;
3908         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3909
3910         /* If wqe in error was identified, set code to be put into cqe */
3911         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3912                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3913                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3914                 nesqp->term_sq_flush_code = 0;
3915         }
3916
3917         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3918                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3919                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3920                 nesqp->term_rq_flush_code = 0;
3921         }
3922
3923         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3924                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3925                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3926         }
3927
3928         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3929                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3930         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3931
3932         nes_post_cqp_request(nesdev, cqp_request);
3933
3934         if (wait_completion) {
3935                 /* Wait for CQP */
3936                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3937                                 NES_EVENT_TIMEOUT);
3938                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3939                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3940                                 ret, cqp_request->major_code, cqp_request->minor_code);
3941                 nes_put_cqp_request(nesdev, cqp_request);
3942         }
3943 }