drivers/infiniband/hw/nes/nes_hw.c

   1 /*
   2  * Copyright (c) 2006 - 2009 Intel Corporation.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  *
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/moduleparam.h>
  36 #include <linux/netdevice.h>
  37 #include <linux/etherdevice.h>
  38 #include <linux/ip.h>
  39 #include <linux/tcp.h>
  40 #include <linux/if_vlan.h>
  41 #include <linux/inet_lro.h>
  42
  43 #include "nes.h"
  44
  45 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
  46 module_param(nes_lro_max_aggr, uint, 0444);
  47 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
  48
  49 static int wide_ppm_offset;
  50 module_param(wide_ppm_offset, int, 0644);
  51 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
  52
  53 static u32 crit_err_count;
  54 u32 int_mod_timer_init;
  55 u32 int_mod_cq_depth_256;
  56 u32 int_mod_cq_depth_128;
  57 u32 int_mod_cq_depth_32;
  58 u32 int_mod_cq_depth_24;
  59 u32 int_mod_cq_depth_16;
  60 u32 int_mod_cq_depth_4;
  61 u32 int_mod_cq_depth_1;
  62 static const u8 nes_max_critical_error_count = 100;
  63 #include "nes_cm.h"
  64
  65 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
  66 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
  67 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
  68                                 struct nes_adapter *nesadapter, u8  OneG_Mode);
  69 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
  70 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
  71 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
  72 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
  73                                    struct nes_hw_aeqe *aeqe);
  74 static void process_critical_error(struct nes_device *nesdev);
  75 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
  76 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
  77 static void nes_terminate_timeout(unsigned long context);
  78 static void nes_terminate_start_timer(struct nes_qp *nesqp);
  79
  80 #ifdef CONFIG_INFINIBAND_NES_DEBUG
  81 static unsigned char *nes_iwarp_state_str[] = {
  82         "Non-Existant",
  83         "Idle",
  84         "RTS",
  85         "Closing",
  86         "RSVD1",
  87         "Terminate",
  88         "Error",
  89         "RSVD2",
  90 };
  91
  92 static unsigned char *nes_tcp_state_str[] = {
  93         "Non-Existant",
  94         "Closed",
  95         "Listen",
  96         "SYN Sent",
  97         "SYN Rcvd",
  98         "Established",
  99         "Close Wait",
 100         "FIN Wait 1",
 101         "Closing",
 102         "Last Ack",
 103         "FIN Wait 2",
 104         "Time Wait",
 105         "RSVD1",
 106         "RSVD2",
 107         "RSVD3",
 108         "RSVD4",
 109 };
 110 #endif
 111
 112
 113 /**
 114  * nes_nic_init_timer_defaults
 115  */
 116 void  nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
 117 {
 118         unsigned long flags;
 119         struct nes_adapter *nesadapter = nesdev->nesadapter;
 120         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 121
 122         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 123
 124         shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
 125         shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
 126         if (jumbomode) {
 127                 shared_timer->threshold_low    = DEFAULT_JUMBO_NES_QL_LOW;
 128                 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
 129                 shared_timer->threshold_high   = DEFAULT_JUMBO_NES_QL_HIGH;
 130         } else {
 131                 shared_timer->threshold_low    = DEFAULT_NES_QL_LOW;
 132                 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
 133                 shared_timer->threshold_high   = DEFAULT_NES_QL_HIGH;
 134         }
 135
 136         /* todo use netdev->mtu to set thresholds */
 137         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 138 }
 139
 140
 141 /**
 142  * nes_nic_init_timer
 143  */
 144 static void  nes_nic_init_timer(struct nes_device *nesdev)
 145 {
 146         unsigned long flags;
 147         struct nes_adapter *nesadapter = nesdev->nesadapter;
 148         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 149
 150         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 151
 152         if (shared_timer->timer_in_use_old == 0) {
 153                 nesdev->deepcq_count = 0;
 154                 shared_timer->timer_direction_upward = 0;
 155                 shared_timer->timer_direction_downward = 0;
 156                 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
 157                 shared_timer->timer_in_use_old = 0;
 158
 159         }
 160         if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
 161                 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
 162                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
 163                         0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
 164         }
 165         /* todo use netdev->mtu to set thresholds */
 166         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 167 }
 168
 169
 170 /**
 171  * nes_nic_tune_timer
 172  */
 173 static void nes_nic_tune_timer(struct nes_device *nesdev)
 174 {
 175         unsigned long flags;
 176         struct nes_adapter *nesadapter = nesdev->nesadapter;
 177         struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
 178         u16 cq_count = nesdev->currcq_count;
 179
 180         spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
 181
 182         if (shared_timer->cq_count_old <= cq_count)
 183                 shared_timer->cq_direction_downward = 0;
 184         else
 185                 shared_timer->cq_direction_downward++;
 186         shared_timer->cq_count_old = cq_count;
 187         if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
 188                 if (cq_count <= shared_timer->threshold_low &&
 189                     shared_timer->threshold_low > 4) {
 190                         shared_timer->threshold_low = shared_timer->threshold_low/2;
 191                         shared_timer->cq_direction_downward=0;
 192                         nesdev->currcq_count = 0;
 193                         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 194                         return;
 195                 }
 196         }
 197
 198         if (cq_count > 1) {
 199                 nesdev->deepcq_count += cq_count;
 200                 if (cq_count <= shared_timer->threshold_low) {       /* increase timer gently */
 201                         shared_timer->timer_direction_upward++;
 202                         shared_timer->timer_direction_downward = 0;
 203                 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
 204                         shared_timer->timer_direction_upward = 0;
 205                         shared_timer->timer_direction_downward = 0;
 206                 } else if (cq_count <= shared_timer->threshold_high) {  /* decrease timer gently */
 207                         shared_timer->timer_direction_downward++;
 208                         shared_timer->timer_direction_upward = 0;
 209                 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
 210                         shared_timer->timer_in_use -= 2;
 211                         shared_timer->timer_direction_upward = 0;
 212                         shared_timer->timer_direction_downward++;
 213                 } else {
 214                         shared_timer->timer_in_use -= 4;
 215                         shared_timer->timer_direction_upward = 0;
 216                         shared_timer->timer_direction_downward++;
 217                 }
 218
 219                 if (shared_timer->timer_direction_upward > 3 ) {  /* using history */
 220                         shared_timer->timer_in_use += 3;
 221                         shared_timer->timer_direction_upward = 0;
 222                         shared_timer->timer_direction_downward = 0;
 223                 }
 224                 if (shared_timer->timer_direction_downward > 5) { /* using history */
 225                         shared_timer->timer_in_use -= 4 ;
 226                         shared_timer->timer_direction_downward = 0;
 227                         shared_timer->timer_direction_upward = 0;
 228                 }
 229         }
 230
 231         /* boundary checking */
 232         if (shared_timer->timer_in_use > shared_timer->threshold_high)
 233                 shared_timer->timer_in_use = shared_timer->threshold_high;
 234         else if (shared_timer->timer_in_use < shared_timer->threshold_low)
 235                 shared_timer->timer_in_use = shared_timer->threshold_low;
 236
 237         nesdev->currcq_count = 0;
 238
 239         spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
 240 }
 241
 242
 243 /**
 244  * nes_init_adapter - initialize adapter
 245  */
 246 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
 247         struct nes_adapter *nesadapter = NULL;
 248         unsigned long num_pds;
 249         u32 u32temp;
 250         u32 port_count;
 251         u16 max_rq_wrs;
 252         u16 max_sq_wrs;
 253         u32 max_mr;
 254         u32 max_256pbl;
 255         u32 max_4kpbl;
 256         u32 max_qp;
 257         u32 max_irrq;
 258         u32 max_cq;
 259         u32 hte_index_mask;
 260         u32 adapter_size;
 261         u32 arp_table_size;
 262         u16 vendor_id;
 263         u16 device_id;
 264         u8  OneG_Mode;
 265         u8  func_index;
 266
 267         /* search the list of existing adapters */
 268         list_for_each_entry(nesadapter, &nes_adapter_list, list) {
 269                 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
 270                                 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
 271                                 nesdev->pcidev->devfn,
 272                                 PCI_SLOT(nesadapter->devfn),
 273                                 nesadapter->bus_number,
 274                                 PCI_SLOT(nesdev->pcidev->devfn),
 275                                 nesdev->pcidev->bus->number );
 276                 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
 277                                 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
 278                         nesadapter->ref_count++;
 279                         return nesadapter;
 280                 }
 281         }
 282
 283         /* no adapter found */
 284         num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
 285         if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
 286                 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
 287                                 hw_rev);
 288                 return NULL;
 289         }
 290
 291         nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
 292                         nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
 293                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
 294                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
 295                         nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
 296
 297         nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
 298
 299
 300         if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
 301                 return NULL;
 302
 303         max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
 304         nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
 305
 306         u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
 307         if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
 308                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
 309                                 max_qp, u32temp);
 310                 max_qp = (u32)1 << (u32temp & 0x001f);
 311         }
 312
 313         hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
 314         nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
 315                         max_qp, hte_index_mask);
 316
 317         u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
 318
 319         max_irrq = 1 << (u32temp & 0x001f);
 320
 321         if (max_qp > max_irrq) {
 322                 max_qp = max_irrq;
 323                 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
 324                                 max_qp);
 325         }
 326
 327         /* there should be no reason to allocate more pds than qps */
 328         if (num_pds > max_qp)
 329                 num_pds = max_qp;
 330
 331         u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
 332         max_mr = (u32)8192 << (u32temp & 0x7);
 333
 334         u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
 335         max_256pbl = (u32)1 << (u32temp & 0x0000001f);
 336         max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
 337         max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
 338
 339         u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
 340         arp_table_size = 1 << u32temp;
 341
 342         adapter_size = (sizeof(struct nes_adapter) +
 343                         (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
 344         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
 345         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
 346         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
 347         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
 348         adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
 349         adapter_size += sizeof(struct nes_qp **) * max_qp;
 350
 351         /* allocate a new adapter struct */
 352         nesadapter = kzalloc(adapter_size, GFP_KERNEL);
 353         if (nesadapter == NULL) {
 354                 return NULL;
 355         }
 356
 357         nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
 358                         nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
 359
 360         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 361                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 362                 kfree(nesadapter);
 363                 return NULL;
 364         }
 365
 366         nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
 367                                 (nesadapter->mac_addr_low >> 24);
 368
 369         pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
 370                                  PCI_DEVICE_ID, &device_id);
 371         nesadapter->vendor_part_id = device_id;
 372
 373         if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
 374                                                         OneG_Mode)) {
 375                 kfree(nesadapter);
 376                 return NULL;
 377         }
 378         nes_init_csr_ne020(nesdev, hw_rev, port_count);
 379
 380         memset(nesadapter->pft_mcast_map, 255,
 381                sizeof nesadapter->pft_mcast_map);
 382
 383         /* populate the new nesadapter */
 384         nesadapter->devfn = nesdev->pcidev->devfn;
 385         nesadapter->bus_number = nesdev->pcidev->bus->number;
 386         nesadapter->ref_count = 1;
 387         nesadapter->timer_int_req = 0xffff0000;
 388         nesadapter->OneG_Mode = OneG_Mode;
 389         nesadapter->doorbell_start = nesdev->doorbell_region;
 390
 391         /* nesadapter->tick_delta = clk_divisor; */
 392         nesadapter->hw_rev = hw_rev;
 393         nesadapter->port_count = port_count;
 394
 395         nesadapter->max_qp = max_qp;
 396         nesadapter->hte_index_mask = hte_index_mask;
 397         nesadapter->max_irrq = max_irrq;
 398         nesadapter->max_mr = max_mr;
 399         nesadapter->max_256pbl = max_256pbl - 1;
 400         nesadapter->max_4kpbl = max_4kpbl - 1;
 401         nesadapter->max_cq = max_cq;
 402         nesadapter->free_256pbl = max_256pbl - 1;
 403         nesadapter->free_4kpbl = max_4kpbl - 1;
 404         nesadapter->max_pd = num_pds;
 405         nesadapter->arp_table_size = arp_table_size;
 406
 407         nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
 408         if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
 409                 nesadapter->et_use_adaptive_rx_coalesce = 0;
 410                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
 411                 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
 412         } else {
 413                 nesadapter->et_use_adaptive_rx_coalesce = 1;
 414                 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
 415                 nesadapter->et_rx_coalesce_usecs_irq = 0;
 416                 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
 417         }
 418         /* Setup and enable the periodic timer */
 419         if (nesadapter->et_rx_coalesce_usecs_irq)
 420                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
 421                                 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
 422         else
 423                 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
 424
 425         nesadapter->base_pd = 1;
 426
 427         nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
 428                                        IB_DEVICE_MEM_WINDOW |
 429                                        IB_DEVICE_MEM_MGT_EXTENSIONS;
 430
 431         nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
 432                         [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
 433         nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
 434         nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
 435         nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
 436         nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
 437         nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
 438
 439
 440         /* mark the usual suspect QPs, MR and CQs as in use */
 441         for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
 442                 set_bit(u32temp, nesadapter->allocated_qps);
 443                 set_bit(u32temp, nesadapter->allocated_cqs);
 444         }
 445         set_bit(0, nesadapter->allocated_mrs);
 446
 447         for (u32temp = 0; u32temp < 20; u32temp++)
 448                 set_bit(u32temp, nesadapter->allocated_pds);
 449         u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
 450
 451         max_rq_wrs = ((u32temp >> 8) & 3);
 452         switch (max_rq_wrs) {
 453                 case 0:
 454                         max_rq_wrs = 4;
 455                         break;
 456                 case 1:
 457                         max_rq_wrs = 16;
 458                         break;
 459                 case 2:
 460                         max_rq_wrs = 32;
 461                         break;
 462                 case 3:
 463                         max_rq_wrs = 512;
 464                         break;
 465         }
 466
 467         max_sq_wrs = (u32temp & 3);
 468         switch (max_sq_wrs) {
 469                 case 0:
 470                         max_sq_wrs = 4;
 471                         break;
 472                 case 1:
 473                         max_sq_wrs = 16;
 474                         break;
 475                 case 2:
 476                         max_sq_wrs = 32;
 477                         break;
 478                 case 3:
 479                         max_sq_wrs = 512;
 480                         break;
 481         }
 482         nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
 483         nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
 484
 485         nesadapter->max_sge = 4;
 486         nesadapter->max_cqe = 32766;
 487
 488         if (nes_read_eeprom_values(nesdev, nesadapter)) {
 489                 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
 490                 kfree(nesadapter);
 491                 return NULL;
 492         }
 493
 494         u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
 495         nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
 496                         (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
 497
 498         /* setup port configuration */
 499         if (nesadapter->port_count == 1) {
 500                 nesadapter->log_port = 0x00000000;
 501                 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
 502                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
 503                 else
 504                         nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 505         } else {
 506                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 507                         nesadapter->log_port = 0x000000D8;
 508                 } else {
 509                         if (nesadapter->port_count == 2)
 510                                 nesadapter->log_port = 0x00000044;
 511                         else
 512                                 nesadapter->log_port = 0x000000e4;
 513                 }
 514                 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
 515         }
 516
 517         nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
 518                                                 nesadapter->log_port);
 519         nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
 520                         nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
 521
 522         spin_lock_init(&nesadapter->resource_lock);
 523         spin_lock_init(&nesadapter->phy_lock);
 524         spin_lock_init(&nesadapter->pbl_lock);
 525         spin_lock_init(&nesadapter->periodic_timer_lock);
 526
 527         INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
 528         INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
 529         INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
 530         INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
 531
 532         if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
 533                 u32 pcs_control_status0, pcs_control_status1;
 534                 u32 reset_value;
 535                 u32 i = 0;
 536                 u32 int_cnt = 0;
 537                 u32 ext_cnt = 0;
 538                 unsigned long flags;
 539                 u32 j = 0;
 540
 541                 pcs_control_status0 = nes_read_indexed(nesdev,
 542                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 543                 pcs_control_status1 = nes_read_indexed(nesdev,
 544                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 545
 546                 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 547                         pcs_control_status0 = nes_read_indexed(nesdev,
 548                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 549                         pcs_control_status1 = nes_read_indexed(nesdev,
 550                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 551                         if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 552                             || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
 553                                 int_cnt++;
 554                         msleep(1);
 555                 }
 556                 if (int_cnt > 1) {
 557                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
 558                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
 559                         mh_detected++;
 560                         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 561                         reset_value |= 0x0000003d;
 562                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 563
 564                         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 565                                 & 0x00000040) != 0x00000040) && (j++ < 5000));
 566                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 567
 568                         pcs_control_status0 = nes_read_indexed(nesdev,
 569                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 570                         pcs_control_status1 = nes_read_indexed(nesdev,
 571                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 572
 573                         for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
 574                                 pcs_control_status0 = nes_read_indexed(nesdev,
 575                                         NES_IDX_PHY_PCS_CONTROL_STATUS0);
 576                                 pcs_control_status1 = nes_read_indexed(nesdev,
 577                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
 578                                 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
 579                                         || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
 580                                         if (++ext_cnt > int_cnt) {
 581                                                 spin_lock_irqsave(&nesadapter->phy_lock, flags);
 582                                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
 583                                                                 0x0000F088);
 584                                                 mh_detected++;
 585                                                 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 586                                                 reset_value |= 0x0000003d;
 587                                                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
 588
 589                                                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
 590                                                         & 0x00000040) != 0x00000040) && (j++ < 5000));
 591                                                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
 592                                                 break;
 593                                         }
 594                                 }
 595                                 msleep(1);
 596                         }
 597                 }
 598         }
 599
 600         if (nesadapter->hw_rev == NE020_REV) {
 601                 init_timer(&nesadapter->mh_timer);
 602                 nesadapter->mh_timer.function = nes_mh_fix;
 603                 nesadapter->mh_timer.expires = jiffies + (HZ/5);  /* 1 second */
 604                 nesadapter->mh_timer.data = (unsigned long)nesdev;
 605                 add_timer(&nesadapter->mh_timer);
 606         } else {
 607                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
 608         }
 609
 610         init_timer(&nesadapter->lc_timer);
 611         nesadapter->lc_timer.function = nes_clc;
 612         nesadapter->lc_timer.expires = jiffies + 3600 * HZ;  /* 1 hour */
 613         nesadapter->lc_timer.data = (unsigned long)nesdev;
 614         add_timer(&nesadapter->lc_timer);
 615
 616         list_add_tail(&nesadapter->list, &nes_adapter_list);
 617
 618         for (func_index = 0; func_index < 8; func_index++) {
 619                 pci_bus_read_config_word(nesdev->pcidev->bus,
 620                                         PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
 621                                         func_index), 0, &vendor_id);
 622                 if (vendor_id == 0xffff)
 623                         break;
 624         }
 625         nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
 626                 func_index, pci_name(nesdev->pcidev));
 627         nesadapter->adapter_fcn_count = func_index;
 628
 629         return nesadapter;
 630 }
 631
 632
 633 /**
 634  * nes_reset_adapter_ne020
 635  */
 636 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
 637 {
 638         u32 port_count;
 639         u32 u32temp;
 640         u32 i;
 641
 642         u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
 643         port_count = ((u32temp & 0x00000300) >> 8) + 1;
 644         /* TODO: assuming that both SERDES are set the same for now */
 645         *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
 646         nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
 647                         u32temp, port_count);
 648         if (*OneG_Mode)
 649                 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
 650         u32temp &= 0xff00ffc0;
 651         switch (port_count) {
 652                 case 1:
 653                         u32temp |= 0x00ee0000;
 654                         break;
 655                 case 2:
 656                         u32temp |= 0x00cc0000;
 657                         break;
 658                 case 4:
 659                         u32temp |= 0x00000000;
 660                         break;
 661                 default:
 662                         return 0;
 663                         break;
 664         }
 665
 666         /* check and do full reset if needed */
 667         if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
 668                 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
 669                 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 670
 671                 i = 0;
 672                 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 673                         mdelay(1);
 674                 if (i > 10000) {
 675                         nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
 676                         return 0;
 677                 }
 678
 679                 i = 0;
 680                 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
 681                         mdelay(1);
 682                 if (i > 10000) {
 683                         printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
 684                                nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
 685                         return 0;
 686                 }
 687         }
 688
 689         /* port reset */
 690         switch (port_count) {
 691                 case 1:
 692                         u32temp |= 0x00ee0010;
 693                         break;
 694                 case 2:
 695                         u32temp |= 0x00cc0030;
 696                         break;
 697                 case 4:
 698                         u32temp |= 0x00000030;
 699                         break;
 700         }
 701
 702         nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
 703         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
 704
 705         i = 0;
 706         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
 707                 mdelay(1);
 708         if (i > 10000) {
 709                 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
 710                 return 0;
 711         }
 712
 713         /* serdes 0 */
 714         i = 0;
 715         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 716                         & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 717                 mdelay(1);
 718         if (i > 5000) {
 719                 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
 720                 return 0;
 721         }
 722
 723         /* serdes 1 */
 724         if (port_count > 1) {
 725                 i = 0;
 726                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 727                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 728                         mdelay(1);
 729                 if (i > 5000) {
 730                         nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
 731                         return 0;
 732                 }
 733         }
 734
 735         return port_count;
 736 }
 737
 738
 739 /**
 740  * nes_init_serdes
 741  */
 742 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
 743                                 struct nes_adapter *nesadapter, u8  OneG_Mode)
 744 {
 745         int i;
 746         u32 u32temp;
 747         u32 sds;
 748
 749         if (hw_rev != NE020_REV) {
 750                 /* init serdes 0 */
 751                 if (wide_ppm_offset && (nesadapter->phy_type[0] == NES_PHY_TYPE_CX4))
 752                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
 753                 else
 754                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
 755
 756                 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
 757                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
 758                         sds |= 0x00000100;
 759                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
 760                 }
 761                 if (!OneG_Mode)
 762                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
 763
 764                 if (port_count < 2)
 765                         return 0;
 766
 767                 /* init serdes 1 */
 768                 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
 769                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
 770
 771                 switch (nesadapter->phy_type[1]) {
 772                 case NES_PHY_TYPE_ARGUS:
 773                 case NES_PHY_TYPE_SFP_D:
 774                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
 775                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
 776                         break;
 777                 case NES_PHY_TYPE_CX4:
 778                         if (wide_ppm_offset)
 779                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
 780                         break;
 781                 case NES_PHY_TYPE_PUMA_1G:
 782                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 783                         sds |= 0x000000100;
 784                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 785                 }
 786                 if (!OneG_Mode) {
 787                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
 788                         sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
 789                         sds &= 0xFFFFFFBF;
 790                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
 791                 }
 792         } else {
 793                 /* init serdes 0 */
 794                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
 795                 i = 0;
 796                 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
 797                                 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
 798                         mdelay(1);
 799                 if (i > 5000) {
 800                         nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
 801                         return 1;
 802                 }
 803                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
 804                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
 805                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
 806                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
 807                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
 808                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
 809                 if (OneG_Mode)
 810                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
 811                 else
 812                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
 813
 814                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
 815                 if (port_count > 1) {
 816                         /* init serdes 1 */
 817                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
 818                         i = 0;
 819                         while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
 820                                 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
 821                                 mdelay(1);
 822                         if (i > 5000) {
 823                                 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
 824                                 /* return 1; */
 825                         }
 826                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
 827                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
 828                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
 829                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
 830                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
 831                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
 832                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
 833                         nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
 834                 }
 835         }
 836         return 0;
 837 }
 838
 839
 840 /**
 841  * nes_init_csr_ne020
 842  * Initialize registers for ne020 hardware
 843  */
 844 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
 845 {
 846         u32 u32temp;
 847
 848         nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
 849
 850         nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
 851         /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
 852         nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
 853         nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
 854         /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
 855         nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
 856         nes_write_indexed(nesdev, 0x00000600, 0x55555555);
 857         nes_write_indexed(nesdev, 0x00000604, 0x55555555);
 858
 859         /* TODO: move these MAC register settings to NIC bringup */
 860         nes_write_indexed(nesdev, 0x00002000, 0x00000001);
 861         nes_write_indexed(nesdev, 0x00002004, 0x00000001);
 862         nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
 863         nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
 864         nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
 865         nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
 866         if (port_count > 1) {
 867                 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
 868                 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
 869                 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
 870                 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
 871                 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
 872                 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
 873                 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
 874         }
 875         if (port_count > 2) {
 876                 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
 877                 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
 878                 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
 879                 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
 880                 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
 881                 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
 882                 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
 883
 884                 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
 885                 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
 886                 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
 887                 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
 888                 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
 889                 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
 890                 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
 891         }
 892
 893         nes_write_indexed(nesdev, 0x00005000, 0x00018000);
 894         /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
 895         nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
 896                                                          0x00000001);
 897         nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
 898         nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
 899         nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
 900         nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
 901         nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
 902
 903         /* TODO: move this to code, get from EEPROM */
 904         nes_write_indexed(nesdev, 0x00000900, 0x20000001);
 905         nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
 906         nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
 907
 908         nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
 909         /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
 910
 911         if (hw_rev != NE020_REV) {
 912                 u32temp = nes_read_indexed(nesdev, 0x000008e8);
 913                 u32temp |= 0x80000000;
 914                 nes_write_indexed(nesdev, 0x000008e8, u32temp);
 915                 u32temp = nes_read_indexed(nesdev, 0x000021f8);
 916                 u32temp &= 0x7fffffff;
 917                 u32temp |= 0x7fff0010;
 918                 nes_write_indexed(nesdev, 0x000021f8, u32temp);
 919                 if (port_count > 1) {
 920                         u32temp = nes_read_indexed(nesdev, 0x000023f8);
 921                         u32temp &= 0x7fffffff;
 922                         u32temp |= 0x7fff0010;
 923                         nes_write_indexed(nesdev, 0x000023f8, u32temp);
 924                 }
 925         }
 926 }
 927
 928
 929 /**
 930  * nes_destroy_adapter - destroy the adapter structure
 931  */
 932 void nes_destroy_adapter(struct nes_adapter *nesadapter)
 933 {
 934         struct nes_adapter *tmp_adapter;
 935
 936         list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
 937                 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
 938                                 tmp_adapter);
 939         }
 940
 941         nesadapter->ref_count--;
 942         if (!nesadapter->ref_count) {
 943                 if (nesadapter->hw_rev == NE020_REV) {
 944                         del_timer(&nesadapter->mh_timer);
 945                 }
 946                 del_timer(&nesadapter->lc_timer);
 947
 948                 list_del(&nesadapter->list);
 949                 kfree(nesadapter);
 950         }
 951 }
 952
 953
 954 /**
 955  * nes_init_cqp
 956  */
 957 int nes_init_cqp(struct nes_device *nesdev)
 958 {
 959         struct nes_adapter *nesadapter = nesdev->nesadapter;
 960         struct nes_hw_cqp_qp_context *cqp_qp_context;
 961         struct nes_hw_cqp_wqe *cqp_wqe;
 962         struct nes_hw_ceq *ceq;
 963         struct nes_hw_ceq *nic_ceq;
 964         struct nes_hw_aeq *aeq;
 965         void *vmem;
 966         dma_addr_t pmem;
 967         u32 count=0;
 968         u32 cqp_head;
 969         u64 u64temp;
 970         u32 u32temp;
 971
 972         /* allocate CQP memory */
 973         /* Need to add max_cq to the aeq size once cq overflow checking is added back */
 974         /* SQ is 512 byte aligned, others are 256 byte aligned */
 975         nesdev->cqp_mem_size = 512 +
 976                         (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
 977                         (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
 978                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
 979                         max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
 980                         (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
 981                         sizeof(struct nes_hw_cqp_qp_context);
 982
 983         nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
 984                         &nesdev->cqp_pbase);
 985         if (!nesdev->cqp_vbase) {
 986                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
 987                 return -ENOMEM;
 988         }
 989         memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
 990
 991         /* Allocate a twice the number of CQP requests as the SQ size */
 992         nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
 993                         2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
 994         if (nesdev->nes_cqp_requests == NULL) {
 995                 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
 996                 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
 997                                 nesdev->cqp.sq_pbase);
 998                 return -ENOMEM;
 999         }
1000
1001         nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1002                         nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1003
1004         spin_lock_init(&nesdev->cqp.lock);
1005         init_waitqueue_head(&nesdev->cqp.waitq);
1006
1007         /* Setup Various Structures */
1008         vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1009                         ~(unsigned long)(512 - 1));
1010         pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1011                         ~(unsigned long long)(512 - 1));
1012
1013         nesdev->cqp.sq_vbase = vmem;
1014         nesdev->cqp.sq_pbase = pmem;
1015         nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1016         nesdev->cqp.sq_head = 0;
1017         nesdev->cqp.sq_tail = 0;
1018         nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1019
1020         vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1021         pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1022
1023         nesdev->ccq.cq_vbase = vmem;
1024         nesdev->ccq.cq_pbase = pmem;
1025         nesdev->ccq.cq_size = NES_CCQ_SIZE;
1026         nesdev->ccq.cq_head = 0;
1027         nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1028         nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1029
1030         vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1031         pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1032
1033         nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1034         ceq = &nesadapter->ceq[nesdev->ceq_index];
1035         ceq->ceq_vbase = vmem;
1036         ceq->ceq_pbase = pmem;
1037         ceq->ceq_size = NES_CCEQ_SIZE;
1038         ceq->ceq_head = 0;
1039
1040         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1041         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1042
1043         nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1044         nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1045         nic_ceq->ceq_vbase = vmem;
1046         nic_ceq->ceq_pbase = pmem;
1047         nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1048         nic_ceq->ceq_head = 0;
1049
1050         vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1051         pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1052
1053         aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1054         aeq->aeq_vbase = vmem;
1055         aeq->aeq_pbase = pmem;
1056         aeq->aeq_size = nesadapter->max_qp;
1057         aeq->aeq_head = 0;
1058
1059         /* Setup QP Context */
1060         vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1061         pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1062
1063         cqp_qp_context = vmem;
1064         cqp_qp_context->context_words[0] =
1065                         cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1066         cqp_qp_context->context_words[1] = 0;
1067         cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1068         cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1069
1070
1071         /* Write the address to Create CQP */
1072         if ((sizeof(dma_addr_t) > 4)) {
1073                 nes_write_indexed(nesdev,
1074                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1075                                 ((u64)pmem) >> 32);
1076         } else {
1077                 nes_write_indexed(nesdev,
1078                                 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1079         }
1080         nes_write_indexed(nesdev,
1081                         NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1082                         (u32)pmem);
1083
1084         INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1085         INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1086
1087         for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1088                 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1089                 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1090         }
1091
1092         /* Write Create CCQ WQE */
1093         cqp_head = nesdev->cqp.sq_head++;
1094         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1095         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1096         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1097                         (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1098                         NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1099         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1100                             (nesdev->ccq.cq_number |
1101                              ((u32)nesdev->ceq_index << 16)));
1102         u64temp = (u64)nesdev->ccq.cq_pbase;
1103         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1104         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1105         u64temp = (unsigned long)&nesdev->ccq;
1106         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1107                         cpu_to_le32((u32)(u64temp >> 1));
1108         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1109                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1110         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1111
1112         /* Write Create CEQ WQE */
1113         cqp_head = nesdev->cqp.sq_head++;
1114         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1115         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1116         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1117                             (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1118         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1119         u64temp = (u64)ceq->ceq_pbase;
1120         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1121
1122         /* Write Create AEQ WQE */
1123         cqp_head = nesdev->cqp.sq_head++;
1124         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1125         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1126         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1127                         (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1128         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1129         u64temp = (u64)aeq->aeq_pbase;
1130         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1131
1132         /* Write Create NIC CEQ WQE */
1133         cqp_head = nesdev->cqp.sq_head++;
1134         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1135         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1136         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1137                         (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1138         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1139         u64temp = (u64)nic_ceq->ceq_pbase;
1140         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1141
1142         /* Poll until CCQP done */
1143         count = 0;
1144         do {
1145                 if (count++ > 1000) {
1146                         printk(KERN_ERR PFX "Error creating CQP\n");
1147                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1148                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1149                         return -1;
1150                 }
1151                 udelay(10);
1152         } while (!(nes_read_indexed(nesdev,
1153                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1154
1155         nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1156                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1157
1158         u32temp = 0x04800000;
1159         nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1160
1161         /* wait for the CCQ, CEQ, and AEQ to get created */
1162         count = 0;
1163         do {
1164                 if (count++ > 1000) {
1165                         printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1166                         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1167                                         nesdev->cqp_vbase, nesdev->cqp_pbase);
1168                         return -1;
1169                 }
1170                 udelay(10);
1171         } while (((nes_read_indexed(nesdev,
1172                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1173
1174         /* dump the QP status value */
1175         nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1176                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1177
1178         nesdev->cqp.sq_tail++;
1179
1180         return 0;
1181 }
1182
1183
1184 /**
1185  * nes_destroy_cqp
1186  */
1187 int nes_destroy_cqp(struct nes_device *nesdev)
1188 {
1189         struct nes_hw_cqp_wqe *cqp_wqe;
1190         u32 count = 0;
1191         u32 cqp_head;
1192         unsigned long flags;
1193
1194         do {
1195                 if (count++ > 1000)
1196                         break;
1197                 udelay(10);
1198         } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1199
1200         /* Reset CCQ */
1201         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1202                         nesdev->ccq.cq_number);
1203
1204         /* Disable device interrupts */
1205         nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1206
1207         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1208
1209         /* Destroy the AEQ */
1210         cqp_head = nesdev->cqp.sq_head++;
1211         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1212         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1213         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1214                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1215         cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1216
1217         /* Destroy the NIC CEQ */
1218         cqp_head = nesdev->cqp.sq_head++;
1219         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1220         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1221         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1222                         ((u32)nesdev->nic_ceq_index << 8));
1223
1224         /* Destroy the CEQ */
1225         cqp_head = nesdev->cqp.sq_head++;
1226         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1227         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1228         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1229                         (nesdev->ceq_index << 8));
1230
1231         /* Destroy the CCQ */
1232         cqp_head = nesdev->cqp.sq_head++;
1233         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1234         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1235         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1236         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1237                         ((u32)nesdev->ceq_index << 16));
1238
1239         /* Destroy CQP */
1240         cqp_head = nesdev->cqp.sq_head++;
1241         nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1242         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1243         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1244                         NES_CQP_QP_TYPE_CQP);
1245         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1246
1247         barrier();
1248         /* Ring doorbell (5 WQEs) */
1249         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1250
1251         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1252
1253         /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1254         count = 0;
1255         do {
1256                 if (count++ > 1000) {
1257                         printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1258                                         PCI_FUNC(nesdev->pcidev->devfn));
1259                         break;
1260                 }
1261                 udelay(10);
1262         } while (((nes_read_indexed(nesdev,
1263                         NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1264
1265         /* dump the QP status value */
1266         nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1267                         PCI_FUNC(nesdev->pcidev->devfn),
1268                         nes_read_indexed(nesdev,
1269                         NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1270
1271         kfree(nesdev->nes_cqp_requests);
1272
1273         /* Free the control structures */
1274         pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1275                         nesdev->cqp.sq_pbase);
1276
1277         return 0;
1278 }
1279
1280
1281 /**
1282  * nes_init_phy
1283  */
1284 int nes_init_phy(struct nes_device *nesdev)
1285 {
1286         struct nes_adapter *nesadapter = nesdev->nesadapter;
1287         u32 counter = 0;
1288         u32 sds;
1289         u32 mac_index = nesdev->mac_index;
1290         u32 tx_config = 0;
1291         u16 phy_data;
1292         u32 temp_phy_data = 0;
1293         u32 temp_phy_data2 = 0;
1294         u8  phy_type = nesadapter->phy_type[mac_index];
1295         u8  phy_index = nesadapter->phy_index[mac_index];
1296
1297         if ((nesadapter->OneG_Mode) &&
1298             (phy_type != NES_PHY_TYPE_PUMA_1G)) {
1299                 nes_debug(NES_DBG_PHY, "1G PHY, mac_index = %d.\n", mac_index);
1300                 if (phy_type == NES_PHY_TYPE_1G) {
1301                         tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1302                         tx_config &= 0xFFFFFFE3;
1303                         tx_config |= 0x04;
1304                         nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1305                 }
1306
1307                 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1308                 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1309
1310                 /* Reset the PHY */
1311                 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1312                 udelay(100);
1313                 counter = 0;
1314                 do {
1315                         nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1316                         if (counter++ > 100)
1317                                 break;
1318                 } while (phy_data & 0x8000);
1319
1320                 /* Setting no phy loopback */
1321                 phy_data &= 0xbfff;
1322                 phy_data |= 0x1140;
1323                 nes_write_1G_phy_reg(nesdev, 0, phy_index,  phy_data);
1324                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1325                 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1326                 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1327
1328                 /* Setting the interrupt mask */
1329                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1330                 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1331                 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1332
1333                 /* turning on flow control */
1334                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1335                 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1336                 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1337
1338                 /* Clear Half duplex */
1339                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1340                 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1341                 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1342
1343                 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1344                 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1345
1346                 return 0;
1347         }
1348
1349         if ((phy_type == NES_PHY_TYPE_IRIS) ||
1350             (phy_type == NES_PHY_TYPE_ARGUS) ||
1351             (phy_type == NES_PHY_TYPE_SFP_D)) {
1352                 /* setup 10G MDIO operation */
1353                 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1354                 tx_config &= 0xFFFFFFE3;
1355                 tx_config |= 0x15;
1356                 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1357         }
1358         if ((phy_type == NES_PHY_TYPE_ARGUS) ||
1359             (phy_type == NES_PHY_TYPE_SFP_D)) {
1360                 u32 first_time = 1;
1361
1362                 /* Check firmware heartbeat */
1363                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1364                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1365                 udelay(1500);
1366                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1367                 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1368
1369                 if (temp_phy_data != temp_phy_data2) {
1370                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1371                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1372                         if ((temp_phy_data & 0xff) > 0x20)
1373                                 return 0;
1374                         printk(PFX "Reinitializing PHY\n");
1375                 }
1376
1377                 /* no heartbeat, configure the PHY */
1378                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1379                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1380                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1381                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1382                 if (phy_type == NES_PHY_TYPE_ARGUS) {
1383                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1384                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1385                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1386                 } else {
1387                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1388                         nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1389                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1390                 }
1391                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1392                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1393
1394                 /* setup LEDs */
1395                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1396                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1397                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1398
1399                 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1400
1401                 /* Bring PHY out of reset */
1402                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1403
1404                 /* Check for heartbeat */
1405                 counter = 0;
1406                 mdelay(690);
1407                 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1408                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1409                 do {
1410                         if (counter++ > 150) {
1411                                 printk(PFX "No PHY heartbeat\n");
1412                                 break;
1413                         }
1414                         mdelay(1);
1415                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1416                         temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1417                 } while ((temp_phy_data2 == temp_phy_data));
1418
1419                 /* wait for tracking */
1420                 counter = 0;
1421                 do {
1422                         nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1423                         temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1424                         if (counter++ > 300) {
1425                                 if (((temp_phy_data & 0xff) == 0x0) && first_time) {
1426                                         first_time = 0;
1427                                         counter = 0;
1428                                         /* reset AMCC PHY and try again */
1429                                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1430                                         nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1431                                         continue;
1432                                 } else {
1433                                         printk(PFX "PHY did not track\n");
1434                                         break;
1435                                 }
1436                         }
1437                         mdelay(10);
1438                 } while ((temp_phy_data & 0xff) < 0x30);
1439
1440                 /* setup signal integrity */
1441                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1442                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1443                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1444                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1445                 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1446
1447                 /* reset serdes */
1448                 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1449                                        mac_index * 0x200);
1450                 sds |= 0x1;
1451                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1452                                   mac_index * 0x200, sds);
1453                 sds &= 0xfffffffe;
1454                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 +
1455                                   mac_index * 0x200, sds);
1456
1457                 counter = 0;
1458                 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1459                                 && (counter++ < 5000))
1460                         ;
1461         }
1462         return 0;
1463 }
1464
1465
1466 /**
1467  * nes_replenish_nic_rq
1468  */
1469 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1470 {
1471         unsigned long flags;
1472         dma_addr_t bus_address;
1473         struct sk_buff *skb;
1474         struct nes_hw_nic_rq_wqe *nic_rqe;
1475         struct nes_hw_nic *nesnic;
1476         struct nes_device *nesdev;
1477         u32 rx_wqes_posted = 0;
1478
1479         nesnic = &nesvnic->nic;
1480         nesdev = nesvnic->nesdev;
1481         spin_lock_irqsave(&nesnic->rq_lock, flags);
1482         if (nesnic->replenishing_rq !=0) {
1483                 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1484                                 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1485                         atomic_set(&nesvnic->rx_skb_timer_running, 1);
1486                         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1487                         nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1488                         add_timer(&nesvnic->rq_wqes_timer);
1489                 } else
1490                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1491                 return;
1492         }
1493         nesnic->replenishing_rq = 1;
1494         spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1495         do {
1496                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1497                 if (skb) {
1498                         skb->dev = nesvnic->netdev;
1499
1500                         bus_address = pci_map_single(nesdev->pcidev,
1501                                         skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1502
1503                         nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1504                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1505                                         cpu_to_le32(nesvnic->max_frame_size);
1506                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1507                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1508                                         cpu_to_le32((u32)bus_address);
1509                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1510                                         cpu_to_le32((u32)((u64)bus_address >> 32));
1511                         nesnic->rx_skb[nesnic->rq_head] = skb;
1512                         nesnic->rq_head++;
1513                         nesnic->rq_head &= nesnic->rq_size - 1;
1514                         atomic_dec(&nesvnic->rx_skbs_needed);
1515                         barrier();
1516                         if (++rx_wqes_posted == 255) {
1517                                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1518                                 rx_wqes_posted = 0;
1519                         }
1520                 } else {
1521                         spin_lock_irqsave(&nesnic->rq_lock, flags);
1522                         if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1523                                         (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1524                                 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1525                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1526                                 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2);      /* 1/2 second */
1527                                 add_timer(&nesvnic->rq_wqes_timer);
1528                         } else
1529                                 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1530                         break;
1531                 }
1532         } while (atomic_read(&nesvnic->rx_skbs_needed));
1533         barrier();
1534         if (rx_wqes_posted)
1535                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1536         nesnic->replenishing_rq = 0;
1537 }
1538
1539
1540 /**
1541  * nes_rq_wqes_timeout
1542  */
1543 static void nes_rq_wqes_timeout(unsigned long parm)
1544 {
1545         struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1546         printk("%s: Timer fired.\n", __func__);
1547         atomic_set(&nesvnic->rx_skb_timer_running, 0);
1548         if (atomic_read(&nesvnic->rx_skbs_needed))
1549                 nes_replenish_nic_rq(nesvnic);
1550 }
1551
1552
1553 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1554                                void **tcph, u64 *hdr_flags, void *priv)
1555 {
1556         unsigned int ip_len;
1557         struct iphdr *iph;
1558         skb_reset_network_header(skb);
1559         iph = ip_hdr(skb);
1560         if (iph->protocol != IPPROTO_TCP)
1561                 return -1;
1562         ip_len = ip_hdrlen(skb);
1563         skb_set_transport_header(skb, ip_len);
1564         *tcph = tcp_hdr(skb);
1565
1566         *hdr_flags = LRO_IPV4 | LRO_TCP;
1567         *iphdr = iph;
1568         return 0;
1569 }
1570
1571
1572 /**
1573  * nes_init_nic_qp
1574  */
1575 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1576 {
1577         struct nes_hw_cqp_wqe *cqp_wqe;
1578         struct nes_hw_nic_sq_wqe *nic_sqe;
1579         struct nes_hw_nic_qp_context *nic_context;
1580         struct sk_buff *skb;
1581         struct nes_hw_nic_rq_wqe *nic_rqe;
1582         struct nes_vnic *nesvnic = netdev_priv(netdev);
1583         unsigned long flags;
1584         void *vmem;
1585         dma_addr_t pmem;
1586         u64 u64temp;
1587         int ret;
1588         u32 cqp_head;
1589         u32 counter;
1590         u32 wqe_count;
1591         u8 jumbomode=0;
1592
1593         /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1594         nesvnic->nic_mem_size = 256 +
1595                         (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1596                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1597                         (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1598                         (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1599                         sizeof(struct nes_hw_nic_qp_context);
1600
1601         nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1602                         &nesvnic->nic_pbase);
1603         if (!nesvnic->nic_vbase) {
1604                 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1605                 return -ENOMEM;
1606         }
1607         memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1608         nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1609                         nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1610
1611         vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1612                         ~(unsigned long)(256 - 1));
1613         pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1614                         ~(unsigned long long)(256 - 1));
1615
1616         /* Setup the first Fragment buffers */
1617         nesvnic->nic.first_frag_vbase = vmem;
1618
1619         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1620                 nesvnic->nic.frag_paddr[counter] = pmem;
1621                 pmem += sizeof(struct nes_first_frag);
1622         }
1623
1624         /* setup the SQ */
1625         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1626
1627         nesvnic->nic.sq_vbase = (void *)vmem;
1628         nesvnic->nic.sq_pbase = pmem;
1629         nesvnic->nic.sq_head = 0;
1630         nesvnic->nic.sq_tail = 0;
1631         nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1632         for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1633                 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1634                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1635                                 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1636                                 NES_NIC_SQ_WQE_COMPLETION);
1637                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1638                                 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1639                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1640                                 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1641                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1642                                 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1643         }
1644
1645         nesvnic->get_cqp_request = nes_get_cqp_request;
1646         nesvnic->post_cqp_request = nes_post_cqp_request;
1647         nesvnic->mcrq_mcast_filter = NULL;
1648
1649         spin_lock_init(&nesvnic->nic.rq_lock);
1650
1651         /* setup the RQ */
1652         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1653         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1654
1655
1656         nesvnic->nic.rq_vbase = vmem;
1657         nesvnic->nic.rq_pbase = pmem;
1658         nesvnic->nic.rq_head = 0;
1659         nesvnic->nic.rq_tail = 0;
1660         nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1661
1662         /* setup the CQ */
1663         vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1664         pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1665
1666         if (nesdev->nesadapter->netdev_count > 2)
1667                 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1668         else
1669                 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1670
1671         nesvnic->nic_cq.cq_vbase = vmem;
1672         nesvnic->nic_cq.cq_pbase = pmem;
1673         nesvnic->nic_cq.cq_head = 0;
1674         nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1675
1676         nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1677
1678         /* Send CreateCQ request to CQP */
1679         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1680         cqp_head = nesdev->cqp.sq_head;
1681
1682         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1683         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1684
1685         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1686                         NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1687                         ((u32)nesvnic->nic_cq.cq_size << 16));
1688         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1689                         nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1690         u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1691         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1692         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =  0;
1693         u64temp = (unsigned long)&nesvnic->nic_cq;
1694         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =  cpu_to_le32((u32)(u64temp >> 1));
1695         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1696                         cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1697         cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1698         if (++cqp_head >= nesdev->cqp.sq_size)
1699                 cqp_head = 0;
1700         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1701         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1702
1703         /* Send CreateQP request to CQP */
1704         nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1705         nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1706                         cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1707                         ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1708         nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1709                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1710                         nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1711         if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1712                 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1713         }
1714
1715         u64temp = (u64)nesvnic->nic.sq_pbase;
1716         nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1717         nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1718         u64temp = (u64)nesvnic->nic.rq_pbase;
1719         nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX]  = cpu_to_le32((u32)u64temp);
1720         nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1721
1722         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1723                         NES_CQP_QP_TYPE_NIC);
1724         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1725         u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1726                         (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1727         set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1728
1729         if (++cqp_head >= nesdev->cqp.sq_size)
1730                 cqp_head = 0;
1731         nesdev->cqp.sq_head = cqp_head;
1732
1733         barrier();
1734
1735         /* Ring doorbell (2 WQEs) */
1736         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1737
1738         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1739         nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1740                         nesvnic->nic.qp_id);
1741
1742         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1743                         NES_EVENT_TIMEOUT);
1744         nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1745                         nesvnic->nic.qp_id, ret);
1746         if (!ret) {
1747                 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1748                 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1749                                 nesvnic->nic_pbase);
1750                 return -EIO;
1751         }
1752
1753         /* Populate the RQ */
1754         for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1755                 skb = dev_alloc_skb(nesvnic->max_frame_size);
1756                 if (!skb) {
1757                         nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1758
1759                         nes_destroy_nic_qp(nesvnic);
1760                         return -ENOMEM;
1761                 }
1762
1763                 skb->dev = netdev;
1764
1765                 pmem = pci_map_single(nesdev->pcidev, skb->data,
1766                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1767
1768                 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1769                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1770                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1771                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]  = cpu_to_le32((u32)pmem);
1772                 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1773                 nesvnic->nic.rx_skb[counter] = skb;
1774         }
1775
1776         wqe_count = NES_NIC_WQ_SIZE - 1;
1777         nesvnic->nic.rq_head = wqe_count;
1778         barrier();
1779         do {
1780                 counter = min(wqe_count, ((u32)255));
1781                 wqe_count -= counter;
1782                 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1783         } while (wqe_count);
1784         init_timer(&nesvnic->rq_wqes_timer);
1785         nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1786         nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1787         nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1788         if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1789         {
1790                 nes_nic_init_timer(nesdev);
1791                 if (netdev->mtu > 1500)
1792                         jumbomode = 1;
1793                 nes_nic_init_timer_defaults(nesdev, jumbomode);
1794         }
1795         nesvnic->lro_mgr.max_aggr       = nes_lro_max_aggr;
1796         nesvnic->lro_mgr.max_desc       = NES_MAX_LRO_DESCRIPTORS;
1797         nesvnic->lro_mgr.lro_arr        = nesvnic->lro_desc;
1798         nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1799         nesvnic->lro_mgr.features       = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1800         nesvnic->lro_mgr.dev            = netdev;
1801         nesvnic->lro_mgr.ip_summed      = CHECKSUM_UNNECESSARY;
1802         nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1803         return 0;
1804 }
1805
1806
1807 /**
1808  * nes_destroy_nic_qp
1809  */
1810 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1811 {
1812         u64 u64temp;
1813         dma_addr_t bus_address;
1814         struct nes_device *nesdev = nesvnic->nesdev;
1815         struct nes_hw_cqp_wqe *cqp_wqe;
1816         struct nes_hw_nic_sq_wqe *nic_sqe;
1817         struct nes_hw_nic_rq_wqe *nic_rqe;
1818         __le16 *wqe_fragment_length;
1819         u16  wqe_fragment_index;
1820         u64 wqe_frag;
1821         u32 cqp_head;
1822         unsigned long flags;
1823         int ret;
1824
1825         /* Free remaining NIC receive buffers */
1826         while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1827                 nic_rqe   = &nesvnic->nic.rq_vbase[nesvnic->nic.rq_tail];
1828                 wqe_frag  = (u64)le32_to_cpu(
1829                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
1830                 wqe_frag |= ((u64)le32_to_cpu(
1831                         nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX]))<<32;
1832                 pci_unmap_single(nesdev->pcidev, (dma_addr_t)wqe_frag,
1833                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1834                 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1835                 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1836         }
1837
1838         /* Free remaining NIC transmit buffers */
1839         while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1840                 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1841                 wqe_fragment_index = 1;
1842                 wqe_fragment_length = (__le16 *)
1843                         &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1844                 /* bump past the vlan tag */
1845                 wqe_fragment_length++;
1846                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1847                         u64temp = (u64)le32_to_cpu(
1848                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1849                                 wqe_fragment_index*2]);
1850                         u64temp += ((u64)le32_to_cpu(
1851                                 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1852                                 + wqe_fragment_index*2]))<<32;
1853                         bus_address = (dma_addr_t)u64temp;
1854                         if (test_and_clear_bit(nesvnic->nic.sq_tail,
1855                                         nesvnic->nic.first_frag_overflow)) {
1856                                 pci_unmap_single(nesdev->pcidev,
1857                                                 bus_address,
1858                                                 le16_to_cpu(wqe_fragment_length[
1859                                                         wqe_fragment_index++]),
1860                                                 PCI_DMA_TODEVICE);
1861                         }
1862                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1863                                 if (wqe_fragment_length[wqe_fragment_index]) {
1864                                         u64temp = le32_to_cpu(
1865                                                 nic_sqe->wqe_words[
1866                                                 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1867                                                 wqe_fragment_index*2]);
1868                                         u64temp += ((u64)le32_to_cpu(
1869                                                 nic_sqe->wqe_words[
1870                                                 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1871                                                 wqe_fragment_index*2]))<<32;
1872                                         bus_address = (dma_addr_t)u64temp;
1873                                         pci_unmap_page(nesdev->pcidev,
1874                                                         bus_address,
1875                                                         le16_to_cpu(
1876                                                         wqe_fragment_length[
1877                                                         wqe_fragment_index]),
1878                                                         PCI_DMA_TODEVICE);
1879                                 } else
1880                                         break;
1881                         }
1882                 }
1883                 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1884                         dev_kfree_skb(
1885                                 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1886
1887                 nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail)
1888                                         & (nesvnic->nic.sq_size - 1);
1889         }
1890
1891         spin_lock_irqsave(&nesdev->cqp.lock, flags);
1892
1893         /* Destroy NIC QP */
1894         cqp_head = nesdev->cqp.sq_head;
1895         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1896         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1897
1898         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1899                 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
1900         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1901                 nesvnic->nic.qp_id);
1902
1903         if (++cqp_head >= nesdev->cqp.sq_size)
1904                 cqp_head = 0;
1905
1906         cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1907
1908         /* Destroy NIC CQ */
1909         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1910         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1911                 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
1912         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1913                 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
1914
1915         if (++cqp_head >= nesdev->cqp.sq_size)
1916                 cqp_head = 0;
1917
1918         nesdev->cqp.sq_head = cqp_head;
1919         barrier();
1920
1921         /* Ring doorbell (2 WQEs) */
1922         nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1923
1924         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1925         nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
1926                         " cqp.sq_tail=%u, cqp.sq_size=%u\n",
1927                         cqp_head, nesdev->cqp.sq_head,
1928                         nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
1929
1930         ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1931                         NES_EVENT_TIMEOUT);
1932
1933         nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
1934                         " cqp.sq_head=%u, cqp.sq_tail=%u\n",
1935                         ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
1936         if (!ret) {
1937                 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
1938                                 nesvnic->nic.qp_id);
1939         }
1940
1941         pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1942                         nesvnic->nic_pbase);
1943 }
1944
1945 /**
1946  * nes_napi_isr
1947  */
1948 int nes_napi_isr(struct nes_device *nesdev)
1949 {
1950         struct nes_adapter *nesadapter = nesdev->nesadapter;
1951         u32 int_stat;
1952
1953         if (nesdev->napi_isr_ran) {
1954                 /* interrupt status has already been read in ISR */
1955                 int_stat = nesdev->int_stat;
1956         } else {
1957                 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
1958                 nesdev->int_stat = int_stat;
1959                 nesdev->napi_isr_ran = 1;
1960         }
1961
1962         int_stat &= nesdev->int_req;
1963         /* iff NIC, process here, else wait for DPC */
1964         if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
1965                 nesdev->napi_isr_ran = 0;
1966                 nes_write32(nesdev->regs + NES_INT_STAT,
1967                         (int_stat &
1968                         ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
1969
1970                 /* Process the CEQs */
1971                 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
1972
1973                 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
1974                                         (!nesadapter->et_use_adaptive_rx_coalesce)) ||
1975                                         ((nesadapter->et_use_adaptive_rx_coalesce) &&
1976                                          (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
1977                         if ((nesdev->int_req & NES_INT_TIMER) == 0) {
1978                                 /* Enable Periodic timer interrupts */
1979                                 nesdev->int_req |= NES_INT_TIMER;
1980                                 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
1981                                 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
1982                                 nes_write32(nesdev->regs+NES_TIMER_STAT,
1983                                                 nesdev->timer_int_req  | ~(nesdev->nesadapter->timer_int_req));
1984                                 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
1985                                                 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
1986                         }
1987
1988                         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
1989                         {
1990                                 nes_nic_init_timer(nesdev);
1991                         }
1992                         /* Enable interrupts, except CEQs */
1993                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
1994                 } else {
1995                         /* Enable interrupts, make sure timer is off */
1996                         nesdev->int_req &= ~NES_INT_TIMER;
1997                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
1998                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
1999                 }
2000                 nesdev->deepcq_count = 0;
2001                 return 1;
2002         } else {
2003                 return 0;
2004         }
2005 }
2006
2007 static void process_critical_error(struct nes_device *nesdev)
2008 {
2009         u32 debug_error;
2010         u32 nes_idx_debug_error_masks0 = 0;
2011         u16 error_module = 0;
2012
2013         debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2014         printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2015                         (u16)debug_error);
2016         nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2017                         0x01010000 | (debug_error & 0x0000ffff));
2018         if (crit_err_count++ > 10)
2019                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2020         error_module = (u16) (debug_error & 0x1F00) >> 8;
2021         if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2022                         nes_max_critical_error_count) {
2023                 printk(KERN_ERR PFX "Masking off critical error for module "
2024                         "0x%02X\n", (u16)error_module);
2025                 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2026                         NES_IDX_DEBUG_ERROR_MASKS0);
2027                 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2028                         nes_idx_debug_error_masks0 | (1 << error_module));
2029         }
2030 }
2031 /**
2032  * nes_dpc
2033  */
2034 void nes_dpc(unsigned long param)
2035 {
2036         struct nes_device *nesdev = (struct nes_device *)param;
2037         struct nes_adapter *nesadapter = nesdev->nesadapter;
2038         u32 counter;
2039         u32 loop_counter = 0;
2040         u32 int_status_bit;
2041         u32 int_stat;
2042         u32 timer_stat;
2043         u32 temp_int_stat;
2044         u32 intf_int_stat;
2045         u32 processed_intf_int = 0;
2046         u16 processed_timer_int = 0;
2047         u16 completion_ints = 0;
2048         u16 timer_ints = 0;
2049
2050         /* nes_debug(NES_DBG_ISR, "\n"); */
2051
2052         do {
2053                 timer_stat = 0;
2054                 if (nesdev->napi_isr_ran) {
2055                         nesdev->napi_isr_ran = 0;
2056                         int_stat = nesdev->int_stat;
2057                 } else
2058                         int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2059                 if (processed_intf_int != 0)
2060                         int_stat &= nesdev->int_req & ~NES_INT_INTF;
2061                 else
2062                         int_stat &= nesdev->int_req;
2063                 if (processed_timer_int == 0) {
2064                         processed_timer_int = 1;
2065                         if (int_stat & NES_INT_TIMER) {
2066                                 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2067                                 if ((timer_stat & nesdev->timer_int_req) == 0) {
2068                                         int_stat &= ~NES_INT_TIMER;
2069                                 }
2070                         }
2071                 } else {
2072                         int_stat &= ~NES_INT_TIMER;
2073                 }
2074
2075                 if (int_stat) {
2076                         if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2077                                         NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2078                                 /* Ack the interrupts */
2079                                 nes_write32(nesdev->regs+NES_INT_STAT,
2080                                         (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2081                                         NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2082                         }
2083
2084                         temp_int_stat = int_stat;
2085                         for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2086                                 if (int_stat & int_status_bit) {
2087                                         nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2088                                         temp_int_stat &= ~int_status_bit;
2089                                         completion_ints = 1;
2090                                 }
2091                                 if (!(temp_int_stat & 0x0000ffff))
2092                                         break;
2093                                 int_status_bit <<= 1;
2094                         }
2095
2096                         /* Process the AEQ for this pci function */
2097                         int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2098                         if (int_stat & int_status_bit) {
2099                                 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2100                         }
2101
2102                         /* Process the MAC interrupt for this pci function */
2103                         int_status_bit = 1 << (24 + nesdev->mac_index);
2104                         if (int_stat & int_status_bit) {
2105                                 nes_process_mac_intr(nesdev, nesdev->mac_index);
2106                         }
2107
2108                         if (int_stat & NES_INT_TIMER) {
2109                                 if (timer_stat & nesdev->timer_int_req) {
2110                                         nes_write32(nesdev->regs + NES_TIMER_STAT,
2111                                                         (timer_stat & nesdev->timer_int_req) |
2112                                                         ~(nesdev->nesadapter->timer_int_req));
2113                                         timer_ints = 1;
2114                                 }
2115                         }
2116
2117                         if (int_stat & NES_INT_INTF) {
2118                                 processed_intf_int = 1;
2119                                 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2120                                 intf_int_stat &= nesdev->intf_int_req;
2121                                 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2122                                         process_critical_error(nesdev);
2123                                 }
2124                                 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2125                                         printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2126                                         BUG();
2127                                 }
2128                                 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2129                                         printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2130                                         BUG();
2131                                 }
2132                                 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2133                         }
2134
2135                         if (int_stat & NES_INT_TSW) {
2136                         }
2137                 }
2138                 /* Don't use the interface interrupt bit stay in loop */
2139                 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2140                                 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2141         } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2142
2143         if (timer_ints == 1) {
2144                 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2145                         if (completion_ints == 0) {
2146                                 nesdev->timer_only_int_count++;
2147                                 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2148                                         nesdev->timer_only_int_count = 0;
2149                                         nesdev->int_req &= ~NES_INT_TIMER;
2150                                         nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2151                                         nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2152                                 } else {
2153                                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2154                                 }
2155                         } else {
2156                                 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2157                                 {
2158                                         nes_nic_init_timer(nesdev);
2159                                 }
2160                                 nesdev->timer_only_int_count = 0;
2161                                 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2162                         }
2163                 } else {
2164                         nesdev->timer_only_int_count = 0;
2165                         nesdev->int_req &= ~NES_INT_TIMER;
2166                         nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2167                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2168                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2169                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2170                 }
2171         } else {
2172                 if ( (completion_ints == 1) &&
2173                          (((nesadapter->et_rx_coalesce_usecs_irq) &&
2174                            (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2175                           ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2176                            (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2177                         /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2178                         nesdev->timer_only_int_count = 0;
2179                         nesdev->int_req |= NES_INT_TIMER;
2180                         nes_write32(nesdev->regs+NES_TIMER_STAT,
2181                                         nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2182                         nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2183                                         ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2184                         nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2185                 } else {
2186                         nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2187                 }
2188         }
2189         nesdev->deepcq_count = 0;
2190 }
2191
2192
2193 /**
2194  * nes_process_ceq
2195  */
2196 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2197 {
2198         u64 u64temp;
2199         struct nes_hw_cq *cq;
2200         u32 head;
2201         u32 ceq_size;
2202
2203         /* nes_debug(NES_DBG_CQ, "\n"); */
2204         head = ceq->ceq_head;
2205         ceq_size = ceq->ceq_size;
2206
2207         do {
2208                 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2209                                 NES_CEQE_VALID) {
2210                         u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2211                                                 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2212                         u64temp <<= 1;
2213                         cq = *((struct nes_hw_cq **)&u64temp);
2214                         /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2215                         barrier();
2216                         ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2217
2218                         /* call the event handler */
2219                         cq->ce_handler(nesdev, cq);
2220
2221                         if (++head >= ceq_size)
2222                                 head = 0;
2223                 } else {
2224                         break;
2225                 }
2226
2227         } while (1);
2228
2229         ceq->ceq_head = head;
2230 }
2231
2232
2233 /**
2234  * nes_process_aeq
2235  */
2236 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2237 {
2238         /* u64 u64temp; */
2239         u32 head;
2240         u32 aeq_size;
2241         u32 aeqe_misc;
2242         u32 aeqe_cq_id;
2243         struct nes_hw_aeqe volatile *aeqe;
2244
2245         head = aeq->aeq_head;
2246         aeq_size = aeq->aeq_size;
2247
2248         do {
2249                 aeqe = &aeq->aeq_vbase[head];
2250                 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2251                         break;
2252                 aeqe_misc  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2253                 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2254                 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2255                         if (aeqe_cq_id >= NES_FIRST_QPN) {
2256                                 /* dealing with an accelerated QP related AE */
2257                                 /*
2258                                  * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2259                                  *           ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2260                                  */
2261                                 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2262                         } else {
2263                                 /* TODO: dealing with a CQP related AE */
2264                                 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2265                                                 (u16)(aeqe_misc >> 16));
2266                         }
2267                 }
2268
2269                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2270
2271                 if (++head >= aeq_size)
2272                         head = 0;
2273
2274                 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2275         }
2276         while (1);
2277         aeq->aeq_head = head;
2278 }
2279
2280 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2281 {
2282         struct nes_adapter *nesadapter = nesdev->nesadapter;
2283         u32 reset_value;
2284         u32 i=0;
2285         u32 u32temp;
2286
2287         if (nesadapter->hw_rev == NE020_REV) {
2288                 return;
2289         }
2290         mh_detected++;
2291
2292         reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2293
2294         if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2295                 reset_value |= 0x0000001d;
2296         else
2297                 reset_value |= 0x0000002d;
2298
2299         if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2300                 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2301                         nesadapter->link_interrupt_count[0] = 0;
2302                         nesadapter->link_interrupt_count[1] = 0;
2303                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2304                         if (0x00000040 & u32temp)
2305                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2306                         else
2307                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2308
2309                         reset_value |= 0x0000003d;
2310                 }
2311                 nesadapter->link_interrupt_count[mac_index] = 0;
2312         }
2313
2314         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2315
2316         while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2317                         & 0x00000040) != 0x00000040) && (i++ < 5000));
2318
2319         if (0x0000003d == (reset_value & 0x0000003d)) {
2320                 u32 pcs_control_status0, pcs_control_status1;
2321
2322                 for (i = 0; i < 10; i++) {
2323                         pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2324                         pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2325                         if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2326                              && (pcs_control_status0 & 0x00100000))
2327                             || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2328                                 && (pcs_control_status1 & 0x00100000)))
2329                                 continue;
2330                         else
2331                                 break;
2332                 }
2333                 if (10 == i) {
2334                         u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2335                         if (0x00000040 & u32temp)
2336                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2337                         else
2338                                 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2339
2340                         nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2341
2342                         while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2343                                  & 0x00000040) != 0x00000040) && (i++ < 5000));
2344                 }
2345         }
2346 }
2347
2348 /**
2349  * nes_process_mac_intr
2350  */
2351 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2352 {
2353         unsigned long flags;
2354         u32 pcs_control_status;
2355         struct nes_adapter *nesadapter = nesdev->nesadapter;
2356         struct nes_vnic *nesvnic;
2357         u32 mac_status;
2358         u32 mac_index = nesdev->mac_index;
2359         u32 u32temp;
2360         u16 phy_data;
2361         u16 temp_phy_data;
2362         u32 pcs_val  = 0x0f0f0000;
2363         u32 pcs_mask = 0x0f1f0000;
2364         u32 cdr_ctrl;
2365
2366         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2367         if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2368                 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2369                 return;
2370         }
2371         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2372         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2373
2374         /* ack the MAC interrupt */
2375         mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2376         /* Clear the interrupt */
2377         nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2378
2379         nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2380
2381         if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2382                 nesdev->link_status_interrupts++;
2383                 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) {
2384                         spin_lock_irqsave(&nesadapter->phy_lock, flags);
2385                         nes_reset_link(nesdev, mac_index);
2386                         spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2387                 }
2388                 /* read the PHY interrupt status register */
2389                 if ((nesadapter->OneG_Mode) &&
2390                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2391                         do {
2392                                 nes_read_1G_phy_reg(nesdev, 0x1a,
2393                                                 nesadapter->phy_index[mac_index], &phy_data);
2394                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2395                                                 nesadapter->phy_index[mac_index], phy_data);
2396                         } while (phy_data&0x8000);
2397
2398                         temp_phy_data = 0;
2399                         do {
2400                                 nes_read_1G_phy_reg(nesdev, 0x11,
2401                                                 nesadapter->phy_index[mac_index], &phy_data);
2402                                 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2403                                                 nesadapter->phy_index[mac_index], phy_data);
2404                                 if (temp_phy_data == phy_data)
2405                                         break;
2406                                 temp_phy_data = phy_data;
2407                         } while (1);
2408
2409                         nes_read_1G_phy_reg(nesdev, 0x1e,
2410                                         nesadapter->phy_index[mac_index], &phy_data);
2411                         nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2412                                         nesadapter->phy_index[mac_index], phy_data);
2413
2414                         nes_read_1G_phy_reg(nesdev, 1,
2415                                         nesadapter->phy_index[mac_index], &phy_data);
2416                         nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2417                                         nesadapter->phy_index[mac_index], phy_data);
2418
2419                         if (temp_phy_data & 0x1000) {
2420                                 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2421                                 phy_data = 4;
2422                         } else {
2423                                 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2424                         }
2425                 }
2426                 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2427                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2428                                 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2429
2430                 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2431                         switch (mac_index) {
2432                         case 1:
2433                         case 3:
2434                                 pcs_control_status = nes_read_indexed(nesdev,
2435                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2436                                 break;
2437                         default:
2438                                 pcs_control_status = nes_read_indexed(nesdev,
2439                                                 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2440                                 break;
2441                         }
2442                 } else {
2443                         pcs_control_status = nes_read_indexed(nesdev,
2444                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2445                         pcs_control_status = nes_read_indexed(nesdev,
2446                                         NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2447                 }
2448
2449                 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2450                                 mac_index, pcs_control_status);
2451                 if ((nesadapter->OneG_Mode) &&
2452                                 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2453                         u32temp = 0x01010000;
2454                         if (nesadapter->port_count > 2) {
2455                                 u32temp |= 0x02020000;
2456                         }
2457                         if ((pcs_control_status & u32temp)!= u32temp) {
2458                                 phy_data = 0;
2459                                 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2460                         }
2461                 } else {
2462                         switch (nesadapter->phy_type[mac_index]) {
2463                         case NES_PHY_TYPE_IRIS:
2464                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2465                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2466                                 u32temp = 20;
2467                                 do {
2468                                         nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 1);
2469                                         phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2470                                         if ((phy_data == temp_phy_data) || (!(--u32temp)))
2471                                                 break;
2472                                         temp_phy_data = phy_data;
2473                                 } while (1);
2474                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2475                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2476                                 break;
2477
2478                         case NES_PHY_TYPE_ARGUS:
2479                         case NES_PHY_TYPE_SFP_D:
2480                                 /* clear the alarms */
2481                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2482                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2483                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2484                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2485                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2486                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2487                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2488                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2489                                 /* check link status */
2490                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2491                                 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2492
2493                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2494                                 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2495                                 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2496                                 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2497
2498                                 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2499
2500                                 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2501                                         __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2502                                 break;
2503
2504                         case NES_PHY_TYPE_PUMA_1G:
2505                                 if (mac_index < 2)
2506                                         pcs_val = pcs_mask = 0x01010000;
2507                                 else
2508                                         pcs_val = pcs_mask = 0x02020000;
2509                                 /* fall through */
2510                         default:
2511                                 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2512                                 break;
2513                         }
2514                 }
2515
2516                 if (phy_data & 0x0004) {
2517                         if (wide_ppm_offset &&
2518                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2519                             (nesadapter->hw_rev != NE020_REV)) {
2520                                 cdr_ctrl = nes_read_indexed(nesdev,
2521                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2522                                                             mac_index * 0x200);
2523                                 nes_write_indexed(nesdev,
2524                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2525                                                   mac_index * 0x200,
2526                                                   cdr_ctrl | 0x000F0000);
2527                         }
2528                         nesadapter->mac_link_down[mac_index] = 0;
2529                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2530                                 nes_debug(NES_DBG_PHY, "The Link is UP!!.  linkup was %d\n",
2531                                                 nesvnic->linkup);
2532                                 if (nesvnic->linkup == 0) {
2533                                         printk(PFX "The Link is now up for port %s, netdev %p.\n",
2534                                                         nesvnic->netdev->name, nesvnic->netdev);
2535                                         if (netif_queue_stopped(nesvnic->netdev))
2536                                                 netif_start_queue(nesvnic->netdev);
2537                                         nesvnic->linkup = 1;
2538                                         netif_carrier_on(nesvnic->netdev);
2539                                 }
2540                         }
2541                 } else {
2542                         if (wide_ppm_offset &&
2543                             (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2544                             (nesadapter->hw_rev != NE020_REV)) {
2545                                 cdr_ctrl = nes_read_indexed(nesdev,
2546                                                             NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2547                                                             mac_index * 0x200);
2548                                 nes_write_indexed(nesdev,
2549                                                   NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2550                                                   mac_index * 0x200,
2551                                                   cdr_ctrl & 0xFFF0FFFF);
2552                         }
2553                         nesadapter->mac_link_down[mac_index] = 1;
2554                         list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2555                                 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2556                                                 nesvnic->linkup);
2557                                 if (nesvnic->linkup == 1) {
2558                                         printk(PFX "The Link is now down for port %s, netdev %p.\n",
2559                                                         nesvnic->netdev->name, nesvnic->netdev);
2560                                         if (!(netif_queue_stopped(nesvnic->netdev)))
2561                                                 netif_stop_queue(nesvnic->netdev);
2562                                         nesvnic->linkup = 0;
2563                                         netif_carrier_off(nesvnic->netdev);
2564                                 }
2565                         }
2566                 }
2567         }
2568
2569         nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2570 }
2571
2572
2573
2574 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2575 {
2576         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2577
2578         napi_schedule(&nesvnic->napi);
2579 }
2580
2581
2582 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2583 * getting out of nic_ce_handler
2584 */
2585 #define MAX_RQES_TO_PROCESS     384
2586
2587 /**
2588  * nes_nic_ce_handler
2589  */
2590 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2591 {
2592         u64 u64temp;
2593         dma_addr_t bus_address;
2594         struct nes_hw_nic *nesnic;
2595         struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2596         struct nes_adapter *nesadapter = nesdev->nesadapter;
2597         struct nes_hw_nic_rq_wqe *nic_rqe;
2598         struct nes_hw_nic_sq_wqe *nic_sqe;
2599         struct sk_buff *skb;
2600         struct sk_buff *rx_skb;
2601         __le16 *wqe_fragment_length;
2602         u32 head;
2603         u32 cq_size;
2604         u32 rx_pkt_size;
2605         u32 cqe_count=0;
2606         u32 cqe_errv;
2607         u32 cqe_misc;
2608         u16 wqe_fragment_index = 1;     /* first fragment (0) is used by copy buffer */
2609         u16 vlan_tag;
2610         u16 pkt_type;
2611         u16 rqes_processed = 0;
2612         u8 sq_cqes = 0;
2613         u8 nes_use_lro = 0;
2614
2615         head = cq->cq_head;
2616         cq_size = cq->cq_size;
2617         cq->cqes_pending = 1;
2618         if (nesvnic->netdev->features & NETIF_F_LRO)
2619                 nes_use_lro = 1;
2620         do {
2621                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2622                                 NES_NIC_CQE_VALID) {
2623                         nesnic = &nesvnic->nic;
2624                         cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2625                         if (cqe_misc & NES_NIC_CQE_SQ) {
2626                                 sq_cqes++;
2627                                 wqe_fragment_index = 1;
2628                                 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2629                                 skb = nesnic->tx_skb[nesnic->sq_tail];
2630                                 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2631                                 /* bump past the vlan tag */
2632                                 wqe_fragment_length++;
2633                                 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2634                                         u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2635                                                         wqe_fragment_index * 2]);
2636                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2637                                                         wqe_fragment_index * 2])) << 32;
2638                                         bus_address = (dma_addr_t)u64temp;
2639                                         if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2640                                                 pci_unmap_single(nesdev->pcidev,
2641                                                                 bus_address,
2642                                                                 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2643                                                                 PCI_DMA_TODEVICE);
2644                                         }
2645                                         for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2646                                                 if (wqe_fragment_length[wqe_fragment_index]) {
2647                                                         u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2648                                                                                 wqe_fragment_index * 2]);
2649                                                         u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2650                                                                                 + wqe_fragment_index * 2])) <<32;
2651                                                         bus_address = (dma_addr_t)u64temp;
2652                                                         pci_unmap_page(nesdev->pcidev,
2653                                                                         bus_address,
2654                                                                         le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2655                                                                         PCI_DMA_TODEVICE);
2656                                                 } else
2657                                                         break;
2658                                         }
2659                                 }
2660                                 if (skb)
2661                                         dev_kfree_skb_any(skb);
2662                                 nesnic->sq_tail++;
2663                                 nesnic->sq_tail &= nesnic->sq_size-1;
2664                                 if (sq_cqes > 128) {
2665                                         barrier();
2666                                 /* restart the queue if it had been stopped */
2667                                 if (netif_queue_stopped(nesvnic->netdev))
2668                                         netif_wake_queue(nesvnic->netdev);
2669                                         sq_cqes = 0;
2670                                 }
2671                         } else {
2672                                 rqes_processed ++;
2673
2674                                 cq->rx_cqes_completed++;
2675                                 cq->rx_pkts_indicated++;
2676                                 rx_pkt_size = cqe_misc & 0x0000ffff;
2677                                 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2678                                 /* Get the skb */
2679                                 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2680                                 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2681                                 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2682                                 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2683                                 pci_unmap_single(nesdev->pcidev, bus_address,
2684                                                 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2685                                 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2686                                 /* rx_skb->len = rx_pkt_size; */
2687                                 rx_skb->len = 0;  /* TODO: see if this is necessary */
2688                                 skb_put(rx_skb, rx_pkt_size);
2689                                 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2690                                 nesnic->rq_tail++;
2691                                 nesnic->rq_tail &= nesnic->rq_size - 1;
2692
2693                                 atomic_inc(&nesvnic->rx_skbs_needed);
2694                                 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2695                                         nes_write32(nesdev->regs+NES_CQE_ALLOC,
2696                                                         cq->cq_number | (cqe_count << 16));
2697                                         /* nesadapter->tune_timer.cq_count += cqe_count; */
2698                                         nesdev->currcq_count += cqe_count;
2699                                         cqe_count = 0;
2700                                         nes_replenish_nic_rq(nesvnic);
2701                                 }
2702                                 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2703                                 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2704                                 rx_skb->ip_summed = CHECKSUM_NONE;
2705
2706                                 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2707                                                 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2708                                         if ((cqe_errv &
2709                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2710                                                         NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2711                                                 if (nesvnic->rx_checksum_disabled == 0) {
2712                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2713                                                 }
2714                                         } else
2715                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2716                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2717                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2718
2719                                 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2720                                         if ((cqe_errv &
2721                                                         (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2722                                                         NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2723                                                 if (nesvnic->rx_checksum_disabled == 0) {
2724                                                         rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2725                                                         /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2726                                                                   nesvnic->netdev->name); */
2727                                                 }
2728                                         } else
2729                                                 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2730                                                                 " errv = 0x%X, pkt_type = 0x%X.\n",
2731                                                                 nesvnic->netdev->name, cqe_errv, pkt_type);
2732                                         }
2733                                 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2734                                                         pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2735
2736                                 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2737                                         if (nes_cm_recv(rx_skb, nesvnic->netdev))
2738                                                 rx_skb = NULL;
2739                                 }
2740                                 if (rx_skb == NULL)
2741                                         goto skip_rx_indicate0;
2742
2743
2744                                 if ((cqe_misc & NES_NIC_CQE_TAG_VALID) &&
2745                                     (nesvnic->vlan_grp != NULL)) {
2746                                         vlan_tag = (u16)(le32_to_cpu(
2747                                                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2748                                                         >> 16);
2749                                         nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2750                                                         nesvnic->netdev->name, vlan_tag);
2751                                         if (nes_use_lro)
2752                                                 lro_vlan_hwaccel_receive_skb(&nesvnic->lro_mgr, rx_skb,
2753                                                                 nesvnic->vlan_grp, vlan_tag, NULL);
2754                                         else
2755                                                 nes_vlan_rx(rx_skb, nesvnic->vlan_grp, vlan_tag);
2756                                 } else {
2757                                         if (nes_use_lro)
2758                                                 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2759                                         else
2760                                                 nes_netif_rx(rx_skb);
2761                                 }
2762
2763 skip_rx_indicate0:
2764                                 ;
2765                                 /* nesvnic->netstats.rx_packets++; */
2766                                 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2767                         }
2768
2769                         cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2770                         /* Accounting... */
2771                         cqe_count++;
2772                         if (++head >= cq_size)
2773                                 head = 0;
2774                         if (cqe_count == 255) {
2775                                 /* Replenish Nic CQ */
2776                                 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2777                                                 cq->cq_number | (cqe_count << 16));
2778                                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2779                                 nesdev->currcq_count += cqe_count;
2780                                 cqe_count = 0;
2781                         }
2782
2783                         if (cq->rx_cqes_completed >= nesvnic->budget)
2784                                 break;
2785                 } else {
2786                         cq->cqes_pending = 0;
2787                         break;
2788                 }
2789
2790         } while (1);
2791
2792         if (nes_use_lro)
2793                 lro_flush_all(&nesvnic->lro_mgr);
2794         if (sq_cqes) {
2795                 barrier();
2796                 /* restart the queue if it had been stopped */
2797                 if (netif_queue_stopped(nesvnic->netdev))
2798                         netif_wake_queue(nesvnic->netdev);
2799         }
2800         cq->cq_head = head;
2801         /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
2802                         cq->cq_number, cqe_count, cq->cq_head); */
2803         cq->cqe_allocs_pending = cqe_count;
2804         if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2805         {
2806                 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2807                 nesdev->currcq_count += cqe_count;
2808                 nes_nic_tune_timer(nesdev);
2809         }
2810         if (atomic_read(&nesvnic->rx_skbs_needed))
2811                 nes_replenish_nic_rq(nesvnic);
2812 }
2813
2814
2815 /**
2816  * nes_cqp_ce_handler
2817  */
2818 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
2819 {
2820         u64 u64temp;
2821         unsigned long flags;
2822         struct nes_hw_cqp *cqp = NULL;
2823         struct nes_cqp_request *cqp_request;
2824         struct nes_hw_cqp_wqe *cqp_wqe;
2825         u32 head;
2826         u32 cq_size;
2827         u32 cqe_count=0;
2828         u32 error_code;
2829         /* u32 counter; */
2830
2831         head = cq->cq_head;
2832         cq_size = cq->cq_size;
2833
2834         do {
2835                 /* process the CQE */
2836                 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
2837                           le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
2838
2839                 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) {
2840                         u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
2841                                         cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
2842                                         ((u64)(le32_to_cpu(cq->cq_vbase[head].
2843                                         cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
2844                         cqp = *((struct nes_hw_cqp **)&u64temp);
2845
2846                         error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
2847                         if (error_code) {
2848                                 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
2849                                                 " Major/Minor codes = 0x%04X:%04X.\n",
2850                                                 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
2851                                                 (u16)(error_code >> 16),
2852                                                 (u16)error_code);
2853                                 nes_debug(NES_DBG_CQP, "cqp: qp_id=%u, sq_head=%u, sq_tail=%u\n",
2854                                                 cqp->qp_id, cqp->sq_head, cqp->sq_tail);
2855                         }
2856
2857                         u64temp = (((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2858                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) |
2859                                         ((u64)(le32_to_cpu(nesdev->cqp.sq_vbase[cqp->sq_tail].
2860                                         wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX])));
2861                         cqp_request = *((struct nes_cqp_request **)&u64temp);
2862                         if (cqp_request) {
2863                                 if (cqp_request->waiting) {
2864                                         /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
2865                                         cqp_request->major_code = (u16)(error_code >> 16);
2866                                         cqp_request->minor_code = (u16)error_code;
2867                                         barrier();
2868                                         cqp_request->request_done = 1;
2869                                         wake_up(&cqp_request->waitq);
2870                                         nes_put_cqp_request(nesdev, cqp_request);
2871                                 } else {
2872                                         if (cqp_request->callback)
2873                                                 cqp_request->cqp_callback(nesdev, cqp_request);
2874                                         nes_free_cqp_request(nesdev, cqp_request);
2875                                 }
2876                         } else {
2877                                 wake_up(&nesdev->cqp.waitq);
2878                         }
2879
2880                         cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
2881                         nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
2882                         if (++cqp->sq_tail >= cqp->sq_size)
2883                                 cqp->sq_tail = 0;
2884
2885                         /* Accounting... */
2886                         cqe_count++;
2887                         if (++head >= cq_size)
2888                                 head = 0;
2889                 } else {
2890                         break;
2891                 }
2892         } while (1);
2893         cq->cq_head = head;
2894
2895         spin_lock_irqsave(&nesdev->cqp.lock, flags);
2896         while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
2897                         ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
2898                         (nesdev->cqp.sq_size - 1)) != 1)) {
2899                 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
2900                                 struct nes_cqp_request, list);
2901                 list_del_init(&cqp_request->list);
2902                 head = nesdev->cqp.sq_head++;
2903                 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
2904                 cqp_wqe = &nesdev->cqp.sq_vbase[head];
2905                 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
2906                 barrier();
2907                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] =
2908                         cpu_to_le32((u32)((unsigned long)cqp_request));
2909                 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] =
2910                         cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
2911                 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
2912                                 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
2913                 /* Ring doorbell (1 WQEs) */
2914                 barrier();
2915                 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
2916         }
2917         spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2918
2919         /* Arm the CCQ */
2920         nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
2921                         cq->cq_number);
2922         nes_read32(nesdev->regs+NES_CQE_ALLOC);
2923 }
2924
2925
2926 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
2927 {
2928         u16 pkt_len;
2929
2930         if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
2931                 /* skip over ethernet header */
2932                 pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2));
2933                 pkt += ETH_HLEN;
2934
2935                 /* Skip over IP and TCP headers */
2936                 pkt += 4 * (pkt[0] & 0x0f);
2937                 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
2938         }
2939         return pkt;
2940 }
2941
2942 /* Determine if incoming error pkt is rdma layer */
2943 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
2944 {
2945         u8 *pkt;
2946         u16 *mpa;
2947         u32 opcode = 0xffffffff;
2948
2949         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2950                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2951                 mpa = (u16 *)locate_mpa(pkt, aeq_info);
2952                 opcode = be16_to_cpu(mpa[1]) & 0xf;
2953         }
2954
2955         return opcode;
2956 }
2957
2958 /* Build iWARP terminate header */
2959 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
2960 {
2961         u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
2962         u16 ddp_seg_len;
2963         int copy_len = 0;
2964         u8 is_tagged = 0;
2965         u8 flush_code = 0;
2966         struct nes_terminate_hdr *termhdr;
2967
2968         termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
2969         memset(termhdr, 0, 64);
2970
2971         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
2972
2973                 /* Use data from offending packet to fill in ddp & rdma hdrs */
2974                 pkt = locate_mpa(pkt, aeq_info);
2975                 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
2976                 if (ddp_seg_len) {
2977                         copy_len = 2;
2978                         termhdr->hdrct = DDP_LEN_FLAG;
2979                         if (pkt[2] & 0x80) {
2980                                 is_tagged = 1;
2981                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
2982                                         copy_len += TERM_DDP_LEN_TAGGED;
2983                                         termhdr->hdrct |= DDP_HDR_FLAG;
2984                                 }
2985                         } else {
2986                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
2987                                         copy_len += TERM_DDP_LEN_UNTAGGED;
2988                                         termhdr->hdrct |= DDP_HDR_FLAG;
2989                                 }
2990
2991                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
2992                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
2993                                                 copy_len += TERM_RDMA_LEN;
2994                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
2995                                         }
2996                                 }
2997                         }
2998                 }
2999         }
3000
3001         switch (async_event_id) {
3002         case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3003                 switch (iwarp_opcode(nesqp, aeq_info)) {
3004                 case IWARP_OPCODE_WRITE:
3005                         flush_code = IB_WC_LOC_PROT_ERR;
3006                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3007                         termhdr->error_code = DDP_TAGGED_INV_STAG;
3008                         break;
3009                 default:
3010                         flush_code = IB_WC_REM_ACCESS_ERR;
3011                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3012                         termhdr->error_code = RDMAP_INV_STAG;
3013                 }
3014                 break;
3015         case NES_AEQE_AEID_AMP_INVALID_STAG:
3016                 flush_code = IB_WC_REM_ACCESS_ERR;
3017                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3018                 termhdr->error_code = RDMAP_INV_STAG;
3019                 break;
3020         case NES_AEQE_AEID_AMP_BAD_QP:
3021                 flush_code = IB_WC_LOC_QP_OP_ERR;
3022                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3023                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3024                 break;
3025         case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3026         case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3027                 switch (iwarp_opcode(nesqp, aeq_info)) {
3028                 case IWARP_OPCODE_SEND_INV:
3029                 case IWARP_OPCODE_SEND_SE_INV:
3030                         flush_code = IB_WC_REM_OP_ERR;
3031                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3032                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3033                         break;
3034                 default:
3035                         flush_code = IB_WC_REM_ACCESS_ERR;
3036                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3037                         termhdr->error_code = RDMAP_INV_STAG;
3038                 }
3039                 break;
3040         case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3041                 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3042                         flush_code = IB_WC_LOC_PROT_ERR;
3043                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3044                         termhdr->error_code = DDP_TAGGED_BOUNDS;
3045                 } else {
3046                         flush_code = IB_WC_REM_ACCESS_ERR;
3047                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3048                         termhdr->error_code = RDMAP_INV_BOUNDS;
3049                 }
3050                 break;
3051         case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3052         case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3053         case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3054                 flush_code = IB_WC_REM_ACCESS_ERR;
3055                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3056                 termhdr->error_code = RDMAP_ACCESS;
3057                 break;
3058         case NES_AEQE_AEID_AMP_TO_WRAP:
3059                 flush_code = IB_WC_REM_ACCESS_ERR;
3060                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3061                 termhdr->error_code = RDMAP_TO_WRAP;
3062                 break;
3063         case NES_AEQE_AEID_AMP_BAD_PD:
3064                 switch (iwarp_opcode(nesqp, aeq_info)) {
3065                 case IWARP_OPCODE_WRITE:
3066                         flush_code = IB_WC_LOC_PROT_ERR;
3067                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3068                         termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3069                         break;
3070                 case IWARP_OPCODE_SEND_INV:
3071                 case IWARP_OPCODE_SEND_SE_INV:
3072                         flush_code = IB_WC_REM_ACCESS_ERR;
3073                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3074                         termhdr->error_code = RDMAP_CANT_INV_STAG;
3075                         break;
3076                 default:
3077                         flush_code = IB_WC_REM_ACCESS_ERR;
3078                         termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3079                         termhdr->error_code = RDMAP_UNASSOC_STAG;
3080                 }
3081                 break;
3082         case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3083                 flush_code = IB_WC_LOC_LEN_ERR;
3084                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3085                 termhdr->error_code = MPA_MARKER;
3086                 break;
3087         case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3088                 flush_code = IB_WC_GENERAL_ERR;
3089                 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3090                 termhdr->error_code = MPA_CRC;
3091                 break;
3092         case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3093         case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3094                 flush_code = IB_WC_LOC_LEN_ERR;
3095                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3096                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3097                 break;
3098         case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3099         case NES_AEQE_AEID_DDP_NO_L_BIT:
3100                 flush_code = IB_WC_FATAL_ERR;
3101                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3102                 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3103                 break;
3104         case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3105         case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3106                 flush_code = IB_WC_GENERAL_ERR;
3107                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3108                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3109                 break;
3110         case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3111                 flush_code = IB_WC_LOC_LEN_ERR;
3112                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3113                 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3114                 break;
3115         case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3116                 flush_code = IB_WC_GENERAL_ERR;
3117                 if (is_tagged) {
3118                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3119                         termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3120                 } else {
3121                         termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3122                         termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3123                 }
3124                 break;
3125         case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3126                 flush_code = IB_WC_GENERAL_ERR;
3127                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3128                 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3129                 break;
3130         case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3131                 flush_code = IB_WC_REM_OP_ERR;
3132                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3133                 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3134                 break;
3135         case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3136                 flush_code = IB_WC_GENERAL_ERR;
3137                 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3138                 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3139                 break;
3140         case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3141                 flush_code = IB_WC_GENERAL_ERR;
3142                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3143                 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3144                 break;
3145         case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3146                 flush_code = IB_WC_LOC_QP_OP_ERR;
3147                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3148                 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3149                 break;
3150         default:
3151                 flush_code = IB_WC_FATAL_ERR;
3152                 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3153                 termhdr->error_code = RDMAP_UNSPECIFIED;
3154                 break;
3155         }
3156
3157         if (copy_len)
3158                 memcpy(termhdr + 1, pkt, copy_len);
3159
3160         if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3161                 if (aeq_info & NES_AEQE_SQ)
3162                         nesqp->term_sq_flush_code = flush_code;
3163                 else
3164                         nesqp->term_rq_flush_code = flush_code;
3165         }
3166
3167         return sizeof(struct nes_terminate_hdr) + copy_len;
3168 }
3169
3170 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3171                  struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3172 {
3173         u64 context;
3174         unsigned long flags;
3175         u32 aeq_info;
3176         u16 async_event_id;
3177         u8 tcp_state;
3178         u8 iwarp_state;
3179         u32 termlen = 0;
3180         u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3181                            NES_CQP_QP_TERM_DONT_SEND_FIN;
3182         struct nes_adapter *nesadapter = nesdev->nesadapter;
3183
3184         if (nesqp->term_flags & NES_TERM_SENT)
3185                 return; /* Sanity check */
3186
3187         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3188         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3189         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3190         async_event_id = (u16)aeq_info;
3191
3192         context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3193                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3194         if (!context) {
3195                 WARN_ON(!context);
3196                 return;
3197         }
3198
3199         nesqp = (struct nes_qp *)(unsigned long)context;
3200         spin_lock_irqsave(&nesqp->lock, flags);
3201         nesqp->hw_iwarp_state = iwarp_state;
3202         nesqp->hw_tcp_state = tcp_state;
3203         nesqp->last_aeq = async_event_id;
3204         nesqp->terminate_eventtype = eventtype;
3205         spin_unlock_irqrestore(&nesqp->lock, flags);
3206
3207         if (nesadapter->send_term_ok)
3208                 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3209         else
3210                 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3211
3212         nes_terminate_start_timer(nesqp);
3213         nesqp->term_flags |= NES_TERM_SENT;
3214         nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3215 }
3216
3217 static void nes_terminate_send_fin(struct nes_device *nesdev,
3218                           struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3219 {
3220         u32 aeq_info;
3221         u16 async_event_id;
3222         u8 tcp_state;
3223         u8 iwarp_state;
3224         unsigned long flags;
3225
3226         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3227         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3228         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3229         async_event_id = (u16)aeq_info;
3230
3231         spin_lock_irqsave(&nesqp->lock, flags);
3232         nesqp->hw_iwarp_state = iwarp_state;
3233         nesqp->hw_tcp_state = tcp_state;
3234         nesqp->last_aeq = async_event_id;
3235         spin_unlock_irqrestore(&nesqp->lock, flags);
3236
3237         /* Send the fin only */
3238         nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3239                 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3240 }
3241
3242 /* Cleanup after a terminate sent or received */
3243 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3244 {
3245         u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3246         unsigned long flags;
3247         struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3248         struct nes_device *nesdev = nesvnic->nesdev;
3249         u8 first_time = 0;
3250
3251         spin_lock_irqsave(&nesqp->lock, flags);
3252         if (nesqp->hte_added) {
3253                 nesqp->hte_added = 0;
3254                 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3255         }
3256
3257         first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3258         nesqp->term_flags |= NES_TERM_DONE;
3259         spin_unlock_irqrestore(&nesqp->lock, flags);
3260
3261         /* Make sure we go through this only once */
3262         if (first_time) {
3263                 if (timeout_occurred == 0)
3264                         del_timer(&nesqp->terminate_timer);
3265                 else
3266                         next_iwarp_state |= NES_CQP_QP_RESET;
3267
3268                 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3269                 nes_cm_disconn(nesqp);
3270         }
3271 }
3272
3273 static void nes_terminate_received(struct nes_device *nesdev,
3274                                 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3275 {
3276         u32 aeq_info;
3277         u8 *pkt;
3278         u32 *mpa;
3279         u8 ddp_ctl;
3280         u8 rdma_ctl;
3281         u16 aeq_id = 0;
3282
3283         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3284         if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3285                 /* Terminate is not a performance path so the silicon */
3286                 /* did not validate the frame - do it now */
3287                 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3288                 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3289                 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3290                 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3291                 if ((ddp_ctl & 0xc0) != 0x40)
3292                         aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3293                 else if ((ddp_ctl & 0x03) != 1)
3294                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3295                 else if (be32_to_cpu(mpa[2]) != 2)
3296                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3297                 else if (be32_to_cpu(mpa[3]) != 1)
3298                         aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3299                 else if (be32_to_cpu(mpa[4]) != 0)
3300                         aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3301                 else if ((rdma_ctl & 0xc0) != 0x40)
3302                         aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3303
3304                 if (aeq_id) {
3305                         /* Bad terminate recvd - send back a terminate */
3306                         aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3307                         aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3308                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3309                         return;
3310                 }
3311         }
3312
3313         nesqp->term_flags |= NES_TERM_RCVD;
3314         nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3315         nes_terminate_start_timer(nesqp);
3316         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3317 }
3318
3319 /* Timeout routine in case terminate fails to complete */
3320 static void nes_terminate_timeout(unsigned long context)
3321 {
3322         struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3323
3324         nes_terminate_done(nesqp, 1);
3325 }
3326
3327 /* Set a timer in case hw cannot complete the terminate sequence */
3328 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3329 {
3330         init_timer(&nesqp->terminate_timer);
3331         nesqp->terminate_timer.function = nes_terminate_timeout;
3332         nesqp->terminate_timer.expires = jiffies + HZ;
3333         nesqp->terminate_timer.data = (unsigned long)nesqp;
3334         add_timer(&nesqp->terminate_timer);
3335 }
3336
3337 /**
3338  * nes_process_iwarp_aeqe
3339  */
3340 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3341                                    struct nes_hw_aeqe *aeqe)
3342 {
3343         u64 context;
3344         unsigned long flags;
3345         struct nes_qp *nesqp;
3346         struct nes_hw_cq *hw_cq;
3347         struct nes_cq *nescq;
3348         int resource_allocated;
3349         struct nes_adapter *nesadapter = nesdev->nesadapter;
3350         u32 aeq_info;
3351         u32 next_iwarp_state = 0;
3352         u16 async_event_id;
3353         u8 tcp_state;
3354         u8 iwarp_state;
3355         int must_disconn = 1;
3356         int must_terminate = 0;
3357         struct ib_event ibevent;
3358
3359         nes_debug(NES_DBG_AEQ, "\n");
3360         aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3361         if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3362                 context  = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3363                 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3364         } else {
3365                 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3366                                                 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3367                 BUG_ON(!context);
3368         }
3369
3370         async_event_id = (u16)aeq_info;
3371         tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3372         iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3373         nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3374                         " Tcp state = %s, iWARP state = %s\n",
3375                         async_event_id,
3376                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3377                         nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3378
3379         switch (async_event_id) {
3380                 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3381                         nesqp = (struct nes_qp *)(unsigned long)context;
3382
3383                         if (nesqp->term_flags)
3384                                 return; /* Ignore it, wait for close complete */
3385
3386                         if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3387                                 nesqp->cm_id->add_ref(nesqp->cm_id);
3388                                 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3389                                                 NES_TIMER_TYPE_CLOSE, 1, 0);
3390                                 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3391                                                 " need ae to finish up, original_last_aeq = 0x%04X."
3392                                                 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3393                                                 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3394                                                 async_event_id, nesqp->last_aeq, tcp_state);
3395                         }
3396
3397                         if ((tcp_state != NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3398                                         (nesqp->ibqp_state != IB_QPS_RTS)) {
3399                                 /* FIN Received but tcp state or IB state moved on,
3400                                                 should expect a close complete */
3401                                 return;
3402                         }
3403
3404                 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3405                         nesqp = (struct nes_qp *)(unsigned long)context;
3406                         if (nesqp->term_flags) {
3407                                 nes_terminate_done(nesqp, 0);
3408                                 return;
3409                         }
3410
3411                 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3412                 case NES_AEQE_AEID_RESET_SENT:
3413                         nesqp = (struct nes_qp *)(unsigned long)context;
3414                         if (async_event_id == NES_AEQE_AEID_RESET_SENT) {
3415                                 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3416                         }
3417                         spin_lock_irqsave(&nesqp->lock, flags);
3418                         nesqp->hw_iwarp_state = iwarp_state;
3419                         nesqp->hw_tcp_state = tcp_state;
3420                         nesqp->last_aeq = async_event_id;
3421
3422                         if ((tcp_state == NES_AEQE_TCP_STATE_CLOSED) ||
3423                                         (tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT)) {
3424                                 nesqp->hte_added = 0;
3425                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3426                         }
3427
3428                         if ((nesqp->ibqp_state == IB_QPS_RTS) &&
3429                                         ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) ||
3430                                         (async_event_id == NES_AEQE_AEID_LLP_CONNECTION_RESET))) {
3431                                 switch (nesqp->hw_iwarp_state) {
3432                                         case NES_AEQE_IWARP_STATE_RTS:
3433                                                 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3434                                                 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3435                                                 break;
3436                                         case NES_AEQE_IWARP_STATE_TERMINATE:
3437                                                 must_disconn = 0; /* terminate path takes care of disconn */
3438                                                 if (nesqp->term_flags == 0)
3439                                                         must_terminate = 1;
3440                                                 break;
3441                                 }
3442                         } else {
3443                                 if (async_event_id ==  NES_AEQE_AEID_LLP_FIN_RECEIVED) {
3444                                         /* FIN Received but ib state not RTS,
3445                                                         close complete will be on its way */
3446                                         must_disconn = 0;
3447                                 }
3448                         }
3449                         spin_unlock_irqrestore(&nesqp->lock, flags);
3450
3451                         if (must_terminate)
3452                                 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3453                         else if (must_disconn) {
3454                                 if (next_iwarp_state) {
3455                                         nes_debug(NES_DBG_AEQ, "issuing hw modifyqp for QP%u. next state = 0x%08X\n",
3456                                                   nesqp->hwqp.qp_id, next_iwarp_state);
3457                                         nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3458                                 }
3459                                 nes_cm_disconn(nesqp);
3460                         }
3461                         break;
3462
3463                 case NES_AEQE_AEID_TERMINATE_SENT:
3464                         nesqp = (struct nes_qp *)(unsigned long)context;
3465                         nes_terminate_send_fin(nesdev, nesqp, aeqe);
3466                         break;
3467
3468                 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3469                         nesqp = (struct nes_qp *)(unsigned long)context;
3470                         nes_terminate_received(nesdev, nesqp, aeqe);
3471                         break;
3472
3473                 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3474                 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3475                 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3476                 case NES_AEQE_AEID_AMP_INVALID_STAG:
3477                 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3478                 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3479                 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3480                 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3481                 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3482                 case NES_AEQE_AEID_AMP_TO_WRAP:
3483                         nesqp = (struct nes_qp *)(unsigned long)context;
3484                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3485                         break;
3486
3487                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3488                 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3489                 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3490                 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3491                         nesqp = (struct nes_qp *)(unsigned long)context;
3492                         if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3493                                 aeq_info &= 0xffff0000;
3494                                 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3495                                 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3496                         }
3497
3498                 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3499                 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3500                 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3501                 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3502                 case NES_AEQE_AEID_AMP_BAD_QP:
3503                 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3504                 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3505                 case NES_AEQE_AEID_DDP_NO_L_BIT:
3506                 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3507                 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3508                 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3509                 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3510                 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3511                 case NES_AEQE_AEID_AMP_BAD_PD:
3512                 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3513                 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3514                 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3515                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3516                 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3517                 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3518                 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3519                 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3520                 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3521                 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3522                 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3523                 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3524                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3525                 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3526                 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3527                 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3528                 case NES_AEQE_AEID_BAD_CLOSE:
3529                 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3530                 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3531                 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3532                 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3533                         nesqp = (struct nes_qp *)(unsigned long)context;
3534                         nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3535                         break;
3536
3537                 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3538                         context <<= 1;
3539                         nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3540                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3541                         resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3542                                         le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3543                         if (resource_allocated) {
3544                                 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3545                                                 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3546                                 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3547                                 if (hw_cq) {
3548                                         nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3549                                         if (nescq->ibcq.event_handler) {
3550                                                 ibevent.device = nescq->ibcq.device;
3551                                                 ibevent.event = IB_EVENT_CQ_ERR;
3552                                                 ibevent.element.cq = &nescq->ibcq;
3553                                                 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3554                                         }
3555                                 }
3556                         }
3557                         break;
3558
3559                 default:
3560                         nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3561                                         async_event_id);
3562                         break;
3563         }
3564
3565 }
3566
3567 /**
3568  * nes_iwarp_ce_handler
3569  */
3570 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3571 {
3572         struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3573
3574         /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3575                         nescq->hw_cq.cq_number); */
3576         nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3577
3578         if (nescq->ibcq.comp_handler)
3579                 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3580
3581         return;
3582 }
3583
3584
3585 /**
3586  * nes_manage_apbvt()
3587  */
3588 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3589                 u32 nic_index, u32 add_port)
3590 {
3591         struct nes_device *nesdev = nesvnic->nesdev;
3592         struct nes_hw_cqp_wqe *cqp_wqe;
3593         struct nes_cqp_request *cqp_request;
3594         int ret = 0;
3595         u16 major_code;
3596
3597         /* Send manage APBVT request to CQP */
3598         cqp_request = nes_get_cqp_request(nesdev);
3599         if (cqp_request == NULL) {
3600                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3601                 return -ENOMEM;
3602         }
3603         cqp_request->waiting = 1;
3604         cqp_wqe = &cqp_request->cqp_wqe;
3605
3606         nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3607                         (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3608                         accel_local_port, accel_local_port, nic_index);
3609
3610         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3611         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3612                         ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3613         set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3614                         ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3615
3616         nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3617
3618         atomic_set(&cqp_request->refcount, 2);
3619         nes_post_cqp_request(nesdev, cqp_request);
3620
3621         if (add_port == NES_MANAGE_APBVT_ADD)
3622                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3623                                 NES_EVENT_TIMEOUT);
3624         nes_debug(NES_DBG_QP, "Completed, ret=%u,  CQP Major:Minor codes = 0x%04X:0x%04X\n",
3625                         ret, cqp_request->major_code, cqp_request->minor_code);
3626         major_code = cqp_request->major_code;
3627
3628         nes_put_cqp_request(nesdev, cqp_request);
3629
3630         if (!ret)
3631                 return -ETIME;
3632         else if (major_code)
3633                 return -EIO;
3634         else
3635                 return 0;
3636 }
3637
3638
3639 /**
3640  * nes_manage_arp_cache
3641  */
3642 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3643                 u32 ip_addr, u32 action)
3644 {
3645         struct nes_hw_cqp_wqe *cqp_wqe;
3646         struct nes_vnic *nesvnic = netdev_priv(netdev);
3647         struct nes_device *nesdev;
3648         struct nes_cqp_request *cqp_request;
3649         int arp_index;
3650
3651         nesdev = nesvnic->nesdev;
3652         arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3653         if (arp_index == -1) {
3654                 return;
3655         }
3656
3657         /* update the ARP entry */
3658         cqp_request = nes_get_cqp_request(nesdev);
3659         if (cqp_request == NULL) {
3660                 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3661                 return;
3662         }
3663         cqp_request->waiting = 0;
3664         cqp_wqe = &cqp_request->cqp_wqe;
3665         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3666
3667         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3668                         NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3669         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3670                         (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3671         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3672
3673         if (action == NES_ARP_ADD) {
3674                 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3675                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3676                                 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3677                                 (((u32)mac_addr[4]) << 8)  | (u32)mac_addr[5]);
3678                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3679                                 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3680         } else {
3681                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3682                 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3683         }
3684
3685         nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3686                         nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3687
3688         atomic_set(&cqp_request->refcount, 1);
3689         nes_post_cqp_request(nesdev, cqp_request);
3690 }
3691
3692
3693 /**
3694  * flush_wqes
3695  */
3696 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3697                 u32 which_wq, u32 wait_completion)
3698 {
3699         struct nes_cqp_request *cqp_request;
3700         struct nes_hw_cqp_wqe *cqp_wqe;
3701         u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3702         u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3703         int ret;
3704
3705         cqp_request = nes_get_cqp_request(nesdev);
3706         if (cqp_request == NULL) {
3707                 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3708                 return;
3709         }
3710         if (wait_completion) {
3711                 cqp_request->waiting = 1;
3712                 atomic_set(&cqp_request->refcount, 2);
3713         } else {
3714                 cqp_request->waiting = 0;
3715         }
3716         cqp_wqe = &cqp_request->cqp_wqe;
3717         nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3718
3719         /* If wqe in error was identified, set code to be put into cqe */
3720         if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3721                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3722                 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3723                 nesqp->term_sq_flush_code = 0;
3724         }
3725
3726         if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3727                 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3728                 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3729                 nesqp->term_rq_flush_code = 0;
3730         }
3731
3732         if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3733                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3734                 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3735         }
3736
3737         cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3738                         cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3739         cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3740
3741         nes_post_cqp_request(nesdev, cqp_request);
3742
3743         if (wait_completion) {
3744                 /* Wait for CQP */
3745                 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3746                                 NES_EVENT_TIMEOUT);
3747                 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3748                                 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3749                                 ret, cqp_request->major_code, cqp_request->minor_code);
3750                 nes_put_cqp_request(nesdev, cqp_request);
3751         }
3752 }