drivers/net/vxge/vxge-config.c

   1 /******************************************************************************
   2  * This software may be used and distributed according to the terms of
   3  * the GNU General Public License (GPL), incorporated herein by reference.
   4  * Drivers based on or derived from this code fall under the GPL and must
   5  * retain the authorship, copyright and license notice.  This file is not
   6  * a complete program and may only be used when the entire operating
   7  * system is licensed under the GPL.
   8  * See the file COPYING in this distribution for more information.
   9  *
  10  * vxge-config.c: Driver for Neterion Inc's X3100 Series 10GbE PCIe I/O
  11  *                Virtualized Server Adapter.
  12  * Copyright(c) 2002-2009 Neterion Inc.
  13  ******************************************************************************/
  14 #include <linux/vmalloc.h>
  15 #include <linux/etherdevice.h>
  16 #include <linux/pci.h>
  17 #include <linux/pci_hotplug.h>
  18
  19 #include "vxge-traffic.h"
  20 #include "vxge-config.h"
  21
  22 /*
  23  * __vxge_hw_channel_allocate - Allocate memory for channel
  24  * This function allocates required memory for the channel and various arrays
  25  * in the channel
  26  */
  27 struct __vxge_hw_channel*
  28 __vxge_hw_channel_allocate(struct __vxge_hw_vpath_handle *vph,
  29                            enum __vxge_hw_channel_type type,
  30         u32 length, u32 per_dtr_space, void *userdata)
  31 {
  32         struct __vxge_hw_channel *channel;
  33         struct __vxge_hw_device *hldev;
  34         int size = 0;
  35         u32 vp_id;
  36
  37         hldev = vph->vpath->hldev;
  38         vp_id = vph->vpath->vp_id;
  39
  40         switch (type) {
  41         case VXGE_HW_CHANNEL_TYPE_FIFO:
  42                 size = sizeof(struct __vxge_hw_fifo);
  43                 break;
  44         case VXGE_HW_CHANNEL_TYPE_RING:
  45                 size = sizeof(struct __vxge_hw_ring);
  46                 break;
  47         default:
  48                 break;
  49         }
  50
  51         channel = kzalloc(size, GFP_KERNEL);
  52         if (channel == NULL)
  53                 goto exit0;
  54         INIT_LIST_HEAD(&channel->item);
  55
  56         channel->common_reg = hldev->common_reg;
  57         channel->first_vp_id = hldev->first_vp_id;
  58         channel->type = type;
  59         channel->devh = hldev;
  60         channel->vph = vph;
  61         channel->userdata = userdata;
  62         channel->per_dtr_space = per_dtr_space;
  63         channel->length = length;
  64         channel->vp_id = vp_id;
  65
  66         channel->work_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  67         if (channel->work_arr == NULL)
  68                 goto exit1;
  69
  70         channel->free_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  71         if (channel->free_arr == NULL)
  72                 goto exit1;
  73         channel->free_ptr = length;
  74
  75         channel->reserve_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  76         if (channel->reserve_arr == NULL)
  77                 goto exit1;
  78         channel->reserve_ptr = length;
  79         channel->reserve_top = 0;
  80
  81         channel->orig_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  82         if (channel->orig_arr == NULL)
  83                 goto exit1;
  84
  85         return channel;
  86 exit1:
  87         __vxge_hw_channel_free(channel);
  88
  89 exit0:
  90         return NULL;
  91 }
  92
  93 /*
  94  * __vxge_hw_channel_free - Free memory allocated for channel
  95  * This function deallocates memory from the channel and various arrays
  96  * in the channel
  97  */
  98 void __vxge_hw_channel_free(struct __vxge_hw_channel *channel)
  99 {
 100         kfree(channel->work_arr);
 101         kfree(channel->free_arr);
 102         kfree(channel->reserve_arr);
 103         kfree(channel->orig_arr);
 104         kfree(channel);
 105 }
 106
 107 /*
 108  * __vxge_hw_channel_initialize - Initialize a channel
 109  * This function initializes a channel by properly setting the
 110  * various references
 111  */
 112 enum vxge_hw_status
 113 __vxge_hw_channel_initialize(struct __vxge_hw_channel *channel)
 114 {
 115         u32 i;
 116         struct __vxge_hw_virtualpath *vpath;
 117
 118         vpath = channel->vph->vpath;
 119
 120         if ((channel->reserve_arr != NULL) && (channel->orig_arr != NULL)) {
 121                 for (i = 0; i < channel->length; i++)
 122                         channel->orig_arr[i] = channel->reserve_arr[i];
 123         }
 124
 125         switch (channel->type) {
 126         case VXGE_HW_CHANNEL_TYPE_FIFO:
 127                 vpath->fifoh = (struct __vxge_hw_fifo *)channel;
 128                 channel->stats = &((struct __vxge_hw_fifo *)
 129                                 channel)->stats->common_stats;
 130                 break;
 131         case VXGE_HW_CHANNEL_TYPE_RING:
 132                 vpath->ringh = (struct __vxge_hw_ring *)channel;
 133                 channel->stats = &((struct __vxge_hw_ring *)
 134                                 channel)->stats->common_stats;
 135                 break;
 136         default:
 137                 break;
 138         }
 139
 140         return VXGE_HW_OK;
 141 }
 142
 143 /*
 144  * __vxge_hw_channel_reset - Resets a channel
 145  * This function resets a channel by properly setting the various references
 146  */
 147 enum vxge_hw_status
 148 __vxge_hw_channel_reset(struct __vxge_hw_channel *channel)
 149 {
 150         u32 i;
 151
 152         for (i = 0; i < channel->length; i++) {
 153                 if (channel->reserve_arr != NULL)
 154                         channel->reserve_arr[i] = channel->orig_arr[i];
 155                 if (channel->free_arr != NULL)
 156                         channel->free_arr[i] = NULL;
 157                 if (channel->work_arr != NULL)
 158                         channel->work_arr[i] = NULL;
 159         }
 160         channel->free_ptr = channel->length;
 161         channel->reserve_ptr = channel->length;
 162         channel->reserve_top = 0;
 163         channel->post_index = 0;
 164         channel->compl_index = 0;
 165
 166         return VXGE_HW_OK;
 167 }
 168
 169 /*
 170  * __vxge_hw_device_pci_e_init
 171  * Initialize certain PCI/PCI-X configuration registers
 172  * with recommended values. Save config space for future hw resets.
 173  */
 174 void
 175 __vxge_hw_device_pci_e_init(struct __vxge_hw_device *hldev)
 176 {
 177         u16 cmd = 0;
 178
 179         /* Set the PErr Repconse bit and SERR in PCI command register. */
 180         pci_read_config_word(hldev->pdev, PCI_COMMAND, &cmd);
 181         cmd |= 0x140;
 182         pci_write_config_word(hldev->pdev, PCI_COMMAND, cmd);
 183
 184         pci_save_state(hldev->pdev);
 185
 186         return;
 187 }
 188
 189 /*
 190  * __vxge_hw_device_register_poll
 191  * Will poll certain register for specified amount of time.
 192  * Will poll until masked bit is not cleared.
 193  */
 194 enum vxge_hw_status
 195 __vxge_hw_device_register_poll(void __iomem *reg, u64 mask, u32 max_millis)
 196 {
 197         u64 val64;
 198         u32 i = 0;
 199         enum vxge_hw_status ret = VXGE_HW_FAIL;
 200
 201         udelay(10);
 202
 203         do {
 204                 val64 = readq(reg);
 205                 if (!(val64 & mask))
 206                         return VXGE_HW_OK;
 207                 udelay(100);
 208         } while (++i <= 9);
 209
 210         i = 0;
 211         do {
 212                 val64 = readq(reg);
 213                 if (!(val64 & mask))
 214                         return VXGE_HW_OK;
 215                 mdelay(1);
 216         } while (++i <= max_millis);
 217
 218         return ret;
 219 }
 220
 221  /* __vxge_hw_device_vpath_reset_in_prog_check - Check if vpath reset
 222  * in progress
 223  * This routine checks the vpath reset in progress register is turned zero
 224  */
 225 enum vxge_hw_status
 226 __vxge_hw_device_vpath_reset_in_prog_check(u64 __iomem *vpath_rst_in_prog)
 227 {
 228         enum vxge_hw_status status;
 229         status = __vxge_hw_device_register_poll(vpath_rst_in_prog,
 230                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(0x1ffff),
 231                         VXGE_HW_DEF_DEVICE_POLL_MILLIS);
 232         return status;
 233 }
 234
 235 /*
 236  * __vxge_hw_device_toc_get
 237  * This routine sets the swapper and reads the toc pointer and returns the
 238  * memory mapped address of the toc
 239  */
 240 struct vxge_hw_toc_reg __iomem *
 241 __vxge_hw_device_toc_get(void __iomem *bar0)
 242 {
 243         u64 val64;
 244         struct vxge_hw_toc_reg __iomem *toc = NULL;
 245         enum vxge_hw_status status;
 246
 247         struct vxge_hw_legacy_reg __iomem *legacy_reg =
 248                 (struct vxge_hw_legacy_reg __iomem *)bar0;
 249
 250         status = __vxge_hw_legacy_swapper_set(legacy_reg);
 251         if (status != VXGE_HW_OK)
 252                 goto exit;
 253
 254         val64 = readq(&legacy_reg->toc_first_pointer);
 255         toc = (struct vxge_hw_toc_reg __iomem *)(bar0+val64);
 256 exit:
 257         return toc;
 258 }
 259
 260 /*
 261  * __vxge_hw_device_reg_addr_get
 262  * This routine sets the swapper and reads the toc pointer and initializes the
 263  * register location pointers in the device object. It waits until the ric is
 264  * completed initializing registers.
 265  */
 266 enum vxge_hw_status
 267 __vxge_hw_device_reg_addr_get(struct __vxge_hw_device *hldev)
 268 {
 269         u64 val64;
 270         u32 i;
 271         enum vxge_hw_status status = VXGE_HW_OK;
 272
 273         hldev->legacy_reg = (struct vxge_hw_legacy_reg __iomem *)hldev->bar0;
 274
 275         hldev->toc_reg = __vxge_hw_device_toc_get(hldev->bar0);
 276         if (hldev->toc_reg  == NULL) {
 277                 status = VXGE_HW_FAIL;
 278                 goto exit;
 279         }
 280
 281         val64 = readq(&hldev->toc_reg->toc_common_pointer);
 282         hldev->common_reg =
 283         (struct vxge_hw_common_reg __iomem *)(hldev->bar0 + val64);
 284
 285         val64 = readq(&hldev->toc_reg->toc_mrpcim_pointer);
 286         hldev->mrpcim_reg =
 287                 (struct vxge_hw_mrpcim_reg __iomem *)(hldev->bar0 + val64);
 288
 289         for (i = 0; i < VXGE_HW_TITAN_SRPCIM_REG_SPACES; i++) {
 290                 val64 = readq(&hldev->toc_reg->toc_srpcim_pointer[i]);
 291                 hldev->srpcim_reg[i] =
 292                         (struct vxge_hw_srpcim_reg __iomem *)
 293                                 (hldev->bar0 + val64);
 294         }
 295
 296         for (i = 0; i < VXGE_HW_TITAN_VPMGMT_REG_SPACES; i++) {
 297                 val64 = readq(&hldev->toc_reg->toc_vpmgmt_pointer[i]);
 298                 hldev->vpmgmt_reg[i] =
 299                 (struct vxge_hw_vpmgmt_reg __iomem *)(hldev->bar0 + val64);
 300         }
 301
 302         for (i = 0; i < VXGE_HW_TITAN_VPATH_REG_SPACES; i++) {
 303                 val64 = readq(&hldev->toc_reg->toc_vpath_pointer[i]);
 304                 hldev->vpath_reg[i] =
 305                         (struct vxge_hw_vpath_reg __iomem *)
 306                                 (hldev->bar0 + val64);
 307         }
 308
 309         val64 = readq(&hldev->toc_reg->toc_kdfc);
 310
 311         switch (VXGE_HW_TOC_GET_KDFC_INITIAL_BIR(val64)) {
 312         case 0:
 313                 hldev->kdfc = (u8 __iomem *)(hldev->bar0 +
 314                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 315                 break;
 316         case 2:
 317                 hldev->kdfc = (u8 __iomem *)(hldev->bar1 +
 318                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 319                 break;
 320         case 4:
 321                 hldev->kdfc = (u8 __iomem *)(hldev->bar2 +
 322                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 323                 break;
 324         default:
 325                 break;
 326         }
 327
 328         status = __vxge_hw_device_vpath_reset_in_prog_check(
 329                         (u64 __iomem *)&hldev->common_reg->vpath_rst_in_prog);
 330 exit:
 331         return status;
 332 }
 333
 334 /*
 335  * __vxge_hw_device_id_get
 336  * This routine returns sets the device id and revision numbers into the device
 337  * structure
 338  */
 339 void __vxge_hw_device_id_get(struct __vxge_hw_device *hldev)
 340 {
 341         u64 val64;
 342
 343         val64 = readq(&hldev->common_reg->titan_asic_id);
 344         hldev->device_id =
 345                 (u16)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_DEVICE_ID(val64);
 346
 347         hldev->major_revision =
 348                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MAJOR_REVISION(val64);
 349
 350         hldev->minor_revision =
 351                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MINOR_REVISION(val64);
 352
 353         return;
 354 }
 355
 356 /*
 357  * __vxge_hw_device_access_rights_get: Get Access Rights of the driver
 358  * This routine returns the Access Rights of the driver
 359  */
 360 static u32
 361 __vxge_hw_device_access_rights_get(u32 host_type, u32 func_id)
 362 {
 363         u32 access_rights = VXGE_HW_DEVICE_ACCESS_RIGHT_VPATH;
 364
 365         switch (host_type) {
 366         case VXGE_HW_NO_MR_NO_SR_NORMAL_FUNCTION:
 367                 if (func_id == 0) {
 368                         access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 369                                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 370                 }
 371                 break;
 372         case VXGE_HW_MR_NO_SR_VH0_BASE_FUNCTION:
 373                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 374                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 375                 break;
 376         case VXGE_HW_NO_MR_SR_VH0_FUNCTION0:
 377                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 378                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 379                 break;
 380         case VXGE_HW_NO_MR_SR_VH0_VIRTUAL_FUNCTION:
 381         case VXGE_HW_SR_VH_VIRTUAL_FUNCTION:
 382         case VXGE_HW_MR_SR_VH0_INVALID_CONFIG:
 383                 break;
 384         case VXGE_HW_SR_VH_FUNCTION0:
 385         case VXGE_HW_VH_NORMAL_FUNCTION:
 386                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 387                 break;
 388         }
 389
 390         return access_rights;
 391 }
 392 /*
 393  * __vxge_hw_device_host_info_get
 394  * This routine returns the host type assignments
 395  */
 396 void __vxge_hw_device_host_info_get(struct __vxge_hw_device *hldev)
 397 {
 398         u64 val64;
 399         u32 i;
 400
 401         val64 = readq(&hldev->common_reg->host_type_assignments);
 402
 403         hldev->host_type =
 404            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 405
 406         hldev->vpath_assignments = readq(&hldev->common_reg->vpath_assignments);
 407
 408         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 409
 410                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 411                         continue;
 412
 413                 hldev->func_id =
 414                         __vxge_hw_vpath_func_id_get(i, hldev->vpmgmt_reg[i]);
 415
 416                 hldev->access_rights = __vxge_hw_device_access_rights_get(
 417                         hldev->host_type, hldev->func_id);
 418
 419                 hldev->first_vp_id = i;
 420                 break;
 421         }
 422
 423         return;
 424 }
 425
 426 /*
 427  * __vxge_hw_verify_pci_e_info - Validate the pci-e link parameters such as
 428  * link width and signalling rate.
 429  */
 430 static enum vxge_hw_status
 431 __vxge_hw_verify_pci_e_info(struct __vxge_hw_device *hldev)
 432 {
 433         int exp_cap;
 434         u16 lnk;
 435
 436         /* Get the negotiated link width and speed from PCI config space */
 437         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
 438         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
 439
 440         if ((lnk & PCI_EXP_LNKSTA_CLS) != 1)
 441                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 442
 443         switch ((lnk & PCI_EXP_LNKSTA_NLW) >> 4) {
 444         case PCIE_LNK_WIDTH_RESRV:
 445         case PCIE_LNK_X1:
 446         case PCIE_LNK_X2:
 447         case PCIE_LNK_X4:
 448         case PCIE_LNK_X8:
 449                 break;
 450         default:
 451                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 452         }
 453
 454         return VXGE_HW_OK;
 455 }
 456
 457 static enum vxge_hw_status
 458 __vxge_hw_device_is_privilaged(struct __vxge_hw_device *hldev)
 459 {
 460         if ((hldev->host_type == VXGE_HW_NO_MR_NO_SR_NORMAL_FUNCTION ||
 461         hldev->host_type == VXGE_HW_MR_NO_SR_VH0_BASE_FUNCTION ||
 462         hldev->host_type == VXGE_HW_NO_MR_SR_VH0_FUNCTION0) &&
 463         (hldev->func_id == 0))
 464                 return VXGE_HW_OK;
 465         else
 466                 return VXGE_HW_ERR_PRIVILAGED_OPEARATION;
 467 }
 468
 469 /*
 470  * vxge_hw_wrr_rebalance - Rebalance the RX_WRR and KDFC_WRR calandars.
 471  * Rebalance the RX_WRR and KDFC_WRR calandars.
 472  */
 473 static enum
 474 vxge_hw_status vxge_hw_wrr_rebalance(struct __vxge_hw_device *hldev)
 475 {
 476         u64 val64;
 477         u32 wrr_states[VXGE_HW_WEIGHTED_RR_SERVICE_STATES];
 478         u32 i, j, how_often = 1;
 479         enum vxge_hw_status status = VXGE_HW_OK;
 480
 481         status = __vxge_hw_device_is_privilaged(hldev);
 482         if (status != VXGE_HW_OK)
 483                 goto exit;
 484
 485         /* Reset the priorities assigned to the WRR arbitration
 486         phases for the receive traffic */
 487         for (i = 0; i < VXGE_HW_WRR_RING_COUNT; i++)
 488                 writeq(0, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 489
 490         /* Reset the transmit FIFO servicing calendar for FIFOs */
 491         for (i = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 492                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_0) + i));
 493                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_20) + i));
 494         }
 495
 496         /* Assign WRR priority  0 for all FIFOs */
 497         for (i = 1; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 498                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(0),
 499                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl)  + i));
 500
 501                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(0),
 502                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 503         }
 504
 505         /* Reset to service non-offload doorbells */
 506         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 507         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 508
 509         /* Set priority 0 to all receive queues */
 510         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_0);
 511         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_1);
 512         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_2);
 513
 514         /* Initialize all the slots as unused */
 515         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 516                 wrr_states[i] = -1;
 517
 518         /* Prepare the Fifo service states */
 519         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 520
 521                 if (!hldev->config.vp_config[i].min_bandwidth)
 522                         continue;
 523
 524                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 525                                 hldev->config.vp_config[i].min_bandwidth;
 526                 if (how_often) {
 527
 528                         for (j = 0; j < VXGE_HW_WRR_FIFO_SERVICE_STATES;) {
 529                                 if (wrr_states[j] == -1) {
 530                                         wrr_states[j] = i;
 531                                         /* Make sure each fifo is serviced
 532                                          * atleast once */
 533                                         if (i == j)
 534                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 535                                         else
 536                                                 j += how_often;
 537                                 } else
 538                                         j++;
 539                         }
 540                 }
 541         }
 542
 543         /* Fill the unused slots with 0 */
 544         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 545                 if (wrr_states[j] == -1)
 546                         wrr_states[j] = 0;
 547         }
 548
 549         /* Assign WRR priority number for FIFOs */
 550         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 551                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(i),
 552                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl) + i));
 553
 554                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(i),
 555                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 556         }
 557
 558         /* Modify the servicing algorithm applied to the 3 types of doorbells.
 559         i.e, none-offload, message and offload */
 560         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_0(0) |
 561                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_1(0) |
 562                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_2(0) |
 563                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_3(0) |
 564                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_4(1) |
 565                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_5(0) |
 566                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_6(0) |
 567                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_7(0),
 568                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 569
 570         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_1_NUMBER_8(1),
 571                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 572
 573         for (i = 0, j = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 574
 575                 val64 = VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_0(wrr_states[j++]);
 576                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_1(wrr_states[j++]);
 577                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_2(wrr_states[j++]);
 578                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_3(wrr_states[j++]);
 579                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_4(wrr_states[j++]);
 580                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_5(wrr_states[j++]);
 581                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_6(wrr_states[j++]);
 582                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_7(wrr_states[j++]);
 583
 584                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_0 + i));
 585                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_20 + i));
 586         }
 587
 588         /* Set up the priorities assigned to receive queues */
 589         writeq(VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_0(0) |
 590                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_1(1) |
 591                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_2(2) |
 592                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_3(3) |
 593                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_4(4) |
 594                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_5(5) |
 595                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_6(6) |
 596                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_7(7),
 597                         &hldev->mrpcim_reg->rx_queue_priority_0);
 598
 599         writeq(VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_8(8) |
 600                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_9(9) |
 601                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_10(10) |
 602                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_11(11) |
 603                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_12(12) |
 604                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_13(13) |
 605                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_14(14) |
 606                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_15(15),
 607                         &hldev->mrpcim_reg->rx_queue_priority_1);
 608
 609         writeq(VXGE_HW_RX_QUEUE_PRIORITY_2_RX_Q_NUMBER_16(16),
 610                                 &hldev->mrpcim_reg->rx_queue_priority_2);
 611
 612         /* Initialize all the slots as unused */
 613         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 614                 wrr_states[i] = -1;
 615
 616         /* Prepare the Ring service states */
 617         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 618
 619                 if (!hldev->config.vp_config[i].min_bandwidth)
 620                         continue;
 621
 622                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 623                                 hldev->config.vp_config[i].min_bandwidth;
 624
 625                 if (how_often) {
 626                         for (j = 0; j < VXGE_HW_WRR_RING_SERVICE_STATES;) {
 627                                 if (wrr_states[j] == -1) {
 628                                         wrr_states[j] = i;
 629                                         /* Make sure each ring is
 630                                          * serviced atleast once */
 631                                         if (i == j)
 632                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 633                                         else
 634                                                 j += how_often;
 635                                 } else
 636                                         j++;
 637                         }
 638                 }
 639         }
 640
 641         /* Fill the unused slots with 0 */
 642         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 643                 if (wrr_states[j] == -1)
 644                         wrr_states[j] = 0;
 645         }
 646
 647         for (i = 0, j = 0; i < VXGE_HW_WRR_RING_COUNT; i++) {
 648                 val64 =  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_0(
 649                                 wrr_states[j++]);
 650                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_1(
 651                                 wrr_states[j++]);
 652                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_2(
 653                                 wrr_states[j++]);
 654                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_3(
 655                                 wrr_states[j++]);
 656                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_4(
 657                                 wrr_states[j++]);
 658                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_5(
 659                                 wrr_states[j++]);
 660                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_6(
 661                                 wrr_states[j++]);
 662                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_7(
 663                                 wrr_states[j++]);
 664
 665                 writeq(val64, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 666         }
 667 exit:
 668         return status;
 669 }
 670
 671 /*
 672  * __vxge_hw_device_initialize
 673  * Initialize Titan-V hardware.
 674  */
 675 enum vxge_hw_status __vxge_hw_device_initialize(struct __vxge_hw_device *hldev)
 676 {
 677         enum vxge_hw_status status = VXGE_HW_OK;
 678
 679         /* Validate the pci-e link width and speed */
 680         status = __vxge_hw_verify_pci_e_info(hldev);
 681         if (status != VXGE_HW_OK)
 682                 goto exit;
 683
 684         vxge_hw_wrr_rebalance(hldev);
 685 exit:
 686         return status;
 687 }
 688
 689 /**
 690  * vxge_hw_device_hw_info_get - Get the hw information
 691  * Returns the vpath mask that has the bits set for each vpath allocated
 692  * for the driver, FW version information and the first mac addresse for
 693  * each vpath
 694  */
 695 enum vxge_hw_status __devinit
 696 vxge_hw_device_hw_info_get(void __iomem *bar0,
 697                            struct vxge_hw_device_hw_info *hw_info)
 698 {
 699         u32 i;
 700         u64 val64;
 701         struct vxge_hw_toc_reg __iomem *toc;
 702         struct vxge_hw_mrpcim_reg __iomem *mrpcim_reg;
 703         struct vxge_hw_common_reg __iomem *common_reg;
 704         struct vxge_hw_vpath_reg __iomem *vpath_reg;
 705         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg;
 706         enum vxge_hw_status status;
 707
 708         memset(hw_info, 0, sizeof(struct vxge_hw_device_hw_info));
 709
 710         toc = __vxge_hw_device_toc_get(bar0);
 711         if (toc == NULL) {
 712                 status = VXGE_HW_ERR_CRITICAL;
 713                 goto exit;
 714         }
 715
 716         val64 = readq(&toc->toc_common_pointer);
 717         common_reg = (struct vxge_hw_common_reg __iomem *)(bar0 + val64);
 718
 719         status = __vxge_hw_device_vpath_reset_in_prog_check(
 720                 (u64 __iomem *)&common_reg->vpath_rst_in_prog);
 721         if (status != VXGE_HW_OK)
 722                 goto exit;
 723
 724         hw_info->vpath_mask = readq(&common_reg->vpath_assignments);
 725
 726         val64 = readq(&common_reg->host_type_assignments);
 727
 728         hw_info->host_type =
 729            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 730
 731         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 732
 733                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 734                         continue;
 735
 736                 val64 = readq(&toc->toc_vpmgmt_pointer[i]);
 737
 738                 vpmgmt_reg = (struct vxge_hw_vpmgmt_reg __iomem *)
 739                                 (bar0 + val64);
 740
 741                 hw_info->func_id = __vxge_hw_vpath_func_id_get(i, vpmgmt_reg);
 742                 if (__vxge_hw_device_access_rights_get(hw_info->host_type,
 743                         hw_info->func_id) &
 744                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM) {
 745
 746                         val64 = readq(&toc->toc_mrpcim_pointer);
 747
 748                         mrpcim_reg = (struct vxge_hw_mrpcim_reg __iomem *)
 749                                         (bar0 + val64);
 750
 751                         writeq(0, &mrpcim_reg->xgmac_gen_fw_memo_mask);
 752                         wmb();
 753                 }
 754
 755                 val64 = readq(&toc->toc_vpath_pointer[i]);
 756
 757                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 758
 759                 hw_info->function_mode =
 760                         __vxge_hw_vpath_pci_func_mode_get(i, vpath_reg);
 761
 762                 status = __vxge_hw_vpath_fw_ver_get(i, vpath_reg, hw_info);
 763                 if (status != VXGE_HW_OK)
 764                         goto exit;
 765
 766                 status = __vxge_hw_vpath_card_info_get(i, vpath_reg, hw_info);
 767                 if (status != VXGE_HW_OK)
 768                         goto exit;
 769
 770                 break;
 771         }
 772
 773         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 774
 775                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 776                         continue;
 777
 778                 val64 = readq(&toc->toc_vpath_pointer[i]);
 779                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 780
 781                 status =  __vxge_hw_vpath_addr_get(i, vpath_reg,
 782                                 hw_info->mac_addrs[i],
 783                                 hw_info->mac_addr_masks[i]);
 784                 if (status != VXGE_HW_OK)
 785                         goto exit;
 786         }
 787 exit:
 788         return status;
 789 }
 790
 791 /*
 792  * vxge_hw_device_initialize - Initialize Titan device.
 793  * Initialize Titan device. Note that all the arguments of this public API
 794  * are 'IN', including @hldev. Driver cooperates with
 795  * OS to find new Titan device, locate its PCI and memory spaces.
 796  *
 797  * When done, the driver allocates sizeof(struct __vxge_hw_device) bytes for HW
 798  * to enable the latter to perform Titan hardware initialization.
 799  */
 800 enum vxge_hw_status __devinit
 801 vxge_hw_device_initialize(
 802         struct __vxge_hw_device **devh,
 803         struct vxge_hw_device_attr *attr,
 804         struct vxge_hw_device_config *device_config)
 805 {
 806         u32 i;
 807         u32 nblocks = 0;
 808         struct __vxge_hw_device *hldev = NULL;
 809         enum vxge_hw_status status = VXGE_HW_OK;
 810
 811         status = __vxge_hw_device_config_check(device_config);
 812         if (status != VXGE_HW_OK)
 813                 goto exit;
 814
 815         hldev = (struct __vxge_hw_device *)
 816                         vmalloc(sizeof(struct __vxge_hw_device));
 817         if (hldev == NULL) {
 818                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 819                 goto exit;
 820         }
 821
 822         memset(hldev, 0, sizeof(struct __vxge_hw_device));
 823         hldev->magic = VXGE_HW_DEVICE_MAGIC;
 824
 825         vxge_hw_device_debug_set(hldev, VXGE_ERR, VXGE_COMPONENT_ALL);
 826
 827         /* apply config */
 828         memcpy(&hldev->config, device_config,
 829                 sizeof(struct vxge_hw_device_config));
 830
 831         hldev->bar0 = attr->bar0;
 832         hldev->bar1 = attr->bar1;
 833         hldev->bar2 = attr->bar2;
 834         hldev->pdev = attr->pdev;
 835
 836         hldev->uld_callbacks.link_up = attr->uld_callbacks.link_up;
 837         hldev->uld_callbacks.link_down = attr->uld_callbacks.link_down;
 838         hldev->uld_callbacks.crit_err = attr->uld_callbacks.crit_err;
 839
 840         __vxge_hw_device_pci_e_init(hldev);
 841
 842         status = __vxge_hw_device_reg_addr_get(hldev);
 843         if (status != VXGE_HW_OK)
 844                 goto exit;
 845         __vxge_hw_device_id_get(hldev);
 846
 847         __vxge_hw_device_host_info_get(hldev);
 848
 849         /* Incrementing for stats blocks */
 850         nblocks++;
 851
 852         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 853
 854                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 855                         continue;
 856
 857                 if (device_config->vp_config[i].ring.enable ==
 858                         VXGE_HW_RING_ENABLE)
 859                         nblocks += device_config->vp_config[i].ring.ring_blocks;
 860
 861                 if (device_config->vp_config[i].fifo.enable ==
 862                         VXGE_HW_FIFO_ENABLE)
 863                         nblocks += device_config->vp_config[i].fifo.fifo_blocks;
 864                 nblocks++;
 865         }
 866
 867         if (__vxge_hw_blockpool_create(hldev,
 868                 &hldev->block_pool,
 869                 device_config->dma_blockpool_initial + nblocks,
 870                 device_config->dma_blockpool_max + nblocks) != VXGE_HW_OK) {
 871
 872                 vxge_hw_device_terminate(hldev);
 873                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 874                 goto exit;
 875         }
 876
 877         status = __vxge_hw_device_initialize(hldev);
 878
 879         if (status != VXGE_HW_OK) {
 880                 vxge_hw_device_terminate(hldev);
 881                 goto exit;
 882         }
 883
 884         *devh = hldev;
 885 exit:
 886         return status;
 887 }
 888
 889 /*
 890  * vxge_hw_device_terminate - Terminate Titan device.
 891  * Terminate HW device.
 892  */
 893 void
 894 vxge_hw_device_terminate(struct __vxge_hw_device *hldev)
 895 {
 896         vxge_assert(hldev->magic == VXGE_HW_DEVICE_MAGIC);
 897
 898         hldev->magic = VXGE_HW_DEVICE_DEAD;
 899         __vxge_hw_blockpool_destroy(&hldev->block_pool);
 900         vfree(hldev);
 901 }
 902
 903 /*
 904  * vxge_hw_device_stats_get - Get the device hw statistics.
 905  * Returns the vpath h/w stats for the device.
 906  */
 907 enum vxge_hw_status
 908 vxge_hw_device_stats_get(struct __vxge_hw_device *hldev,
 909                         struct vxge_hw_device_stats_hw_info *hw_stats)
 910 {
 911         u32 i;
 912         enum vxge_hw_status status = VXGE_HW_OK;
 913
 914         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 915
 916                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)) ||
 917                         (hldev->virtual_paths[i].vp_open ==
 918                                 VXGE_HW_VP_NOT_OPEN))
 919                         continue;
 920
 921                 memcpy(hldev->virtual_paths[i].hw_stats_sav,
 922                                 hldev->virtual_paths[i].hw_stats,
 923                                 sizeof(struct vxge_hw_vpath_stats_hw_info));
 924
 925                 status = __vxge_hw_vpath_stats_get(
 926                         &hldev->virtual_paths[i],
 927                         hldev->virtual_paths[i].hw_stats);
 928         }
 929
 930         memcpy(hw_stats, &hldev->stats.hw_dev_info_stats,
 931                         sizeof(struct vxge_hw_device_stats_hw_info));
 932
 933         return status;
 934 }
 935
 936 /*
 937  * vxge_hw_driver_stats_get - Get the device sw statistics.
 938  * Returns the vpath s/w stats for the device.
 939  */
 940 enum vxge_hw_status vxge_hw_driver_stats_get(
 941                         struct __vxge_hw_device *hldev,
 942                         struct vxge_hw_device_stats_sw_info *sw_stats)
 943 {
 944         enum vxge_hw_status status = VXGE_HW_OK;
 945
 946         memcpy(sw_stats, &hldev->stats.sw_dev_info_stats,
 947                 sizeof(struct vxge_hw_device_stats_sw_info));
 948
 949         return status;
 950 }
 951
 952 /*
 953  * vxge_hw_mrpcim_stats_access - Access the statistics from the given location
 954  *                           and offset and perform an operation
 955  * Get the statistics from the given location and offset.
 956  */
 957 enum vxge_hw_status
 958 vxge_hw_mrpcim_stats_access(struct __vxge_hw_device *hldev,
 959                             u32 operation, u32 location, u32 offset, u64 *stat)
 960 {
 961         u64 val64;
 962         enum vxge_hw_status status = VXGE_HW_OK;
 963
 964         status = __vxge_hw_device_is_privilaged(hldev);
 965         if (status != VXGE_HW_OK)
 966                 goto exit;
 967
 968         val64 = VXGE_HW_XMAC_STATS_SYS_CMD_OP(operation) |
 969                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE |
 970                 VXGE_HW_XMAC_STATS_SYS_CMD_LOC_SEL(location) |
 971                 VXGE_HW_XMAC_STATS_SYS_CMD_OFFSET_SEL(offset);
 972
 973         status = __vxge_hw_pio_mem_write64(val64,
 974                                 &hldev->mrpcim_reg->xmac_stats_sys_cmd,
 975                                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE,
 976                                 hldev->config.device_poll_millis);
 977
 978         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
 979                 *stat = readq(&hldev->mrpcim_reg->xmac_stats_sys_data);
 980         else
 981                 *stat = 0;
 982 exit:
 983         return status;
 984 }
 985
 986 /*
 987  * vxge_hw_device_xmac_aggr_stats_get - Get the Statistics on aggregate port
 988  * Get the Statistics on aggregate port
 989  */
 990 enum vxge_hw_status
 991 vxge_hw_device_xmac_aggr_stats_get(struct __vxge_hw_device *hldev, u32 port,
 992                                    struct vxge_hw_xmac_aggr_stats *aggr_stats)
 993 {
 994         u64 *val64;
 995         int i;
 996         u32 offset = VXGE_HW_STATS_AGGRn_OFFSET;
 997         enum vxge_hw_status status = VXGE_HW_OK;
 998
 999         val64 = (u64 *)aggr_stats;
1000
1001         status = __vxge_hw_device_is_privilaged(hldev);
1002         if (status != VXGE_HW_OK)
1003                 goto exit;
1004
1005         for (i = 0; i < sizeof(struct vxge_hw_xmac_aggr_stats) / 8; i++) {
1006                 status = vxge_hw_mrpcim_stats_access(hldev,
1007                                         VXGE_HW_STATS_OP_READ,
1008                                         VXGE_HW_STATS_LOC_AGGR,
1009                                         ((offset + (104 * port)) >> 3), val64);
1010                 if (status != VXGE_HW_OK)
1011                         goto exit;
1012
1013                 offset += 8;
1014                 val64++;
1015         }
1016 exit:
1017         return status;
1018 }
1019
1020 /*
1021  * vxge_hw_device_xmac_port_stats_get - Get the Statistics on a port
1022  * Get the Statistics on port
1023  */
1024 enum vxge_hw_status
1025 vxge_hw_device_xmac_port_stats_get(struct __vxge_hw_device *hldev, u32 port,
1026                                    struct vxge_hw_xmac_port_stats *port_stats)
1027 {
1028         u64 *val64;
1029         enum vxge_hw_status status = VXGE_HW_OK;
1030         int i;
1031         u32 offset = 0x0;
1032         val64 = (u64 *) port_stats;
1033
1034         status = __vxge_hw_device_is_privilaged(hldev);
1035         if (status != VXGE_HW_OK)
1036                 goto exit;
1037
1038         for (i = 0; i < sizeof(struct vxge_hw_xmac_port_stats) / 8; i++) {
1039                 status = vxge_hw_mrpcim_stats_access(hldev,
1040                                         VXGE_HW_STATS_OP_READ,
1041                                         VXGE_HW_STATS_LOC_AGGR,
1042                                         ((offset + (608 * port)) >> 3), val64);
1043                 if (status != VXGE_HW_OK)
1044                         goto exit;
1045
1046                 offset += 8;
1047                 val64++;
1048         }
1049
1050 exit:
1051         return status;
1052 }
1053
1054 /*
1055  * vxge_hw_device_xmac_stats_get - Get the XMAC Statistics
1056  * Get the XMAC Statistics
1057  */
1058 enum vxge_hw_status
1059 vxge_hw_device_xmac_stats_get(struct __vxge_hw_device *hldev,
1060                               struct vxge_hw_xmac_stats *xmac_stats)
1061 {
1062         enum vxge_hw_status status = VXGE_HW_OK;
1063         u32 i;
1064
1065         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1066                                         0, &xmac_stats->aggr_stats[0]);
1067
1068         if (status != VXGE_HW_OK)
1069                 goto exit;
1070
1071         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1072                                 1, &xmac_stats->aggr_stats[1]);
1073         if (status != VXGE_HW_OK)
1074                 goto exit;
1075
1076         for (i = 0; i <= VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
1077
1078                 status = vxge_hw_device_xmac_port_stats_get(hldev,
1079                                         i, &xmac_stats->port_stats[i]);
1080                 if (status != VXGE_HW_OK)
1081                         goto exit;
1082         }
1083
1084         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1085
1086                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
1087                         continue;
1088
1089                 status = __vxge_hw_vpath_xmac_tx_stats_get(
1090                                         &hldev->virtual_paths[i],
1091                                         &xmac_stats->vpath_tx_stats[i]);
1092                 if (status != VXGE_HW_OK)
1093                         goto exit;
1094
1095                 status = __vxge_hw_vpath_xmac_rx_stats_get(
1096                                         &hldev->virtual_paths[i],
1097                                         &xmac_stats->vpath_rx_stats[i]);
1098                 if (status != VXGE_HW_OK)
1099                         goto exit;
1100         }
1101 exit:
1102         return status;
1103 }
1104
1105 /*
1106  * vxge_hw_device_debug_set - Set the debug module, level and timestamp
1107  * This routine is used to dynamically change the debug output
1108  */
1109 void vxge_hw_device_debug_set(struct __vxge_hw_device *hldev,
1110                               enum vxge_debug_level level, u32 mask)
1111 {
1112         if (hldev == NULL)
1113                 return;
1114
1115 #if defined(VXGE_DEBUG_TRACE_MASK) || \
1116         defined(VXGE_DEBUG_ERR_MASK)
1117         hldev->debug_module_mask = mask;
1118         hldev->debug_level = level;
1119 #endif
1120
1121 #if defined(VXGE_DEBUG_ERR_MASK)
1122         hldev->level_err = level & VXGE_ERR;
1123 #endif
1124
1125 #if defined(VXGE_DEBUG_TRACE_MASK)
1126         hldev->level_trace = level & VXGE_TRACE;
1127 #endif
1128 }
1129
1130 /*
1131  * vxge_hw_device_error_level_get - Get the error level
1132  * This routine returns the current error level set
1133  */
1134 u32 vxge_hw_device_error_level_get(struct __vxge_hw_device *hldev)
1135 {
1136 #if defined(VXGE_DEBUG_ERR_MASK)
1137         if (hldev == NULL)
1138                 return VXGE_ERR;
1139         else
1140                 return hldev->level_err;
1141 #else
1142         return 0;
1143 #endif
1144 }
1145
1146 /*
1147  * vxge_hw_device_trace_level_get - Get the trace level
1148  * This routine returns the current trace level set
1149  */
1150 u32 vxge_hw_device_trace_level_get(struct __vxge_hw_device *hldev)
1151 {
1152 #if defined(VXGE_DEBUG_TRACE_MASK)
1153         if (hldev == NULL)
1154                 return VXGE_TRACE;
1155         else
1156                 return hldev->level_trace;
1157 #else
1158         return 0;
1159 #endif
1160 }
1161 /*
1162  * vxge_hw_device_debug_mask_get - Get the debug mask
1163  * This routine returns the current debug mask set
1164  */
1165 u32 vxge_hw_device_debug_mask_get(struct __vxge_hw_device *hldev)
1166 {
1167 #if defined(VXGE_DEBUG_TRACE_MASK) || defined(VXGE_DEBUG_ERR_MASK)
1168         if (hldev == NULL)
1169                 return 0;
1170         return hldev->debug_module_mask;
1171 #else
1172         return 0;
1173 #endif
1174 }
1175
1176 /*
1177  * vxge_hw_getpause_data -Pause frame frame generation and reception.
1178  * Returns the Pause frame generation and reception capability of the NIC.
1179  */
1180 enum vxge_hw_status vxge_hw_device_getpause_data(struct __vxge_hw_device *hldev,
1181                                                  u32 port, u32 *tx, u32 *rx)
1182 {
1183         u64 val64;
1184         enum vxge_hw_status status = VXGE_HW_OK;
1185
1186         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1187                 status = VXGE_HW_ERR_INVALID_DEVICE;
1188                 goto exit;
1189         }
1190
1191         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1192                 status = VXGE_HW_ERR_INVALID_PORT;
1193                 goto exit;
1194         }
1195
1196         if (!(hldev->access_rights & VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
1197                 status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
1198                 goto exit;
1199         }
1200
1201         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1202         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN)
1203                 *tx = 1;
1204         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN)
1205                 *rx = 1;
1206 exit:
1207         return status;
1208 }
1209
1210 /*
1211  * vxge_hw_device_setpause_data -  set/reset pause frame generation.
1212  * It can be used to set or reset Pause frame generation or reception
1213  * support of the NIC.
1214  */
1215
1216 enum vxge_hw_status vxge_hw_device_setpause_data(struct __vxge_hw_device *hldev,
1217                                                  u32 port, u32 tx, u32 rx)
1218 {
1219         u64 val64;
1220         enum vxge_hw_status status = VXGE_HW_OK;
1221
1222         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1223                 status = VXGE_HW_ERR_INVALID_DEVICE;
1224                 goto exit;
1225         }
1226
1227         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1228                 status = VXGE_HW_ERR_INVALID_PORT;
1229                 goto exit;
1230         }
1231
1232         status = __vxge_hw_device_is_privilaged(hldev);
1233         if (status != VXGE_HW_OK)
1234                 goto exit;
1235
1236         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1237         if (tx)
1238                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1239         else
1240                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1241         if (rx)
1242                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1243         else
1244                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1245
1246         writeq(val64, &hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1247 exit:
1248         return status;
1249 }
1250
1251 u16 vxge_hw_device_link_width_get(struct __vxge_hw_device *hldev)
1252 {
1253         int link_width, exp_cap;
1254         u16 lnk;
1255
1256         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
1257         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
1258         link_width = (lnk & VXGE_HW_PCI_EXP_LNKCAP_LNK_WIDTH) >> 4;
1259         return link_width;
1260 }
1261
1262 /*
1263  * __vxge_hw_ring_block_memblock_idx - Return the memblock index
1264  * This function returns the index of memory block
1265  */
1266 static inline u32
1267 __vxge_hw_ring_block_memblock_idx(u8 *block)
1268 {
1269         return (u32)*((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET));
1270 }
1271
1272 /*
1273  * __vxge_hw_ring_block_memblock_idx_set - Sets the memblock index
1274  * This function sets index to a memory block
1275  */
1276 static inline void
1277 __vxge_hw_ring_block_memblock_idx_set(u8 *block, u32 memblock_idx)
1278 {
1279         *((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET)) = memblock_idx;
1280 }
1281
1282 /*
1283  * __vxge_hw_ring_block_next_pointer_set - Sets the next block pointer
1284  * in RxD block
1285  * Sets the next block pointer in RxD block
1286  */
1287 static inline void
1288 __vxge_hw_ring_block_next_pointer_set(u8 *block, dma_addr_t dma_next)
1289 {
1290         *((u64 *)(block + VXGE_HW_RING_NEXT_BLOCK_POINTER_OFFSET)) = dma_next;
1291 }
1292
1293 /*
1294  * __vxge_hw_ring_first_block_address_get - Returns the dma address of the
1295  *             first block
1296  * Returns the dma address of the first RxD block
1297  */
1298 u64 __vxge_hw_ring_first_block_address_get(struct __vxge_hw_ring *ring)
1299 {
1300         struct vxge_hw_mempool_dma *dma_object;
1301
1302         dma_object = ring->mempool->memblocks_dma_arr;
1303         vxge_assert(dma_object != NULL);
1304
1305         return dma_object->addr;
1306 }
1307
1308 /*
1309  * __vxge_hw_ring_item_dma_addr - Return the dma address of an item
1310  * This function returns the dma address of a given item
1311  */
1312 static dma_addr_t __vxge_hw_ring_item_dma_addr(struct vxge_hw_mempool *mempoolh,
1313                                                void *item)
1314 {
1315         u32 memblock_idx;
1316         void *memblock;
1317         struct vxge_hw_mempool_dma *memblock_dma_object;
1318         ptrdiff_t dma_item_offset;
1319
1320         /* get owner memblock index */
1321         memblock_idx = __vxge_hw_ring_block_memblock_idx(item);
1322
1323         /* get owner memblock by memblock index */
1324         memblock = mempoolh->memblocks_arr[memblock_idx];
1325
1326         /* get memblock DMA object by memblock index */
1327         memblock_dma_object = mempoolh->memblocks_dma_arr + memblock_idx;
1328
1329         /* calculate offset in the memblock of this item */
1330         dma_item_offset = (u8 *)item - (u8 *)memblock;
1331
1332         return memblock_dma_object->addr + dma_item_offset;
1333 }
1334
1335 /*
1336  * __vxge_hw_ring_rxdblock_link - Link the RxD blocks
1337  * This function returns the dma address of a given item
1338  */
1339 static void __vxge_hw_ring_rxdblock_link(struct vxge_hw_mempool *mempoolh,
1340                                          struct __vxge_hw_ring *ring, u32 from,
1341                                          u32 to)
1342 {
1343         u8 *to_item , *from_item;
1344         dma_addr_t to_dma;
1345
1346         /* get "from" RxD block */
1347         from_item = mempoolh->items_arr[from];
1348         vxge_assert(from_item);
1349
1350         /* get "to" RxD block */
1351         to_item = mempoolh->items_arr[to];
1352         vxge_assert(to_item);
1353
1354         /* return address of the beginning of previous RxD block */
1355         to_dma = __vxge_hw_ring_item_dma_addr(mempoolh, to_item);
1356
1357         /* set next pointer for this RxD block to point on
1358          * previous item's DMA start address */
1359         __vxge_hw_ring_block_next_pointer_set(from_item, to_dma);
1360 }
1361
1362 /*
1363  * __vxge_hw_ring_mempool_item_alloc - Allocate List blocks for RxD
1364  * block callback
1365  * This function is callback passed to __vxge_hw_mempool_create to create memory
1366  * pool for RxD block
1367  */
1368 static void
1369 __vxge_hw_ring_mempool_item_alloc(struct vxge_hw_mempool *mempoolh,
1370                                   u32 memblock_index,
1371                                   struct vxge_hw_mempool_dma *dma_object,
1372                                   u32 index, u32 is_last)
1373 {
1374         u32 i;
1375         void *item = mempoolh->items_arr[index];
1376         struct __vxge_hw_ring *ring =
1377                 (struct __vxge_hw_ring *)mempoolh->userdata;
1378
1379         /* format rxds array */
1380         for (i = 0; i < ring->rxds_per_block; i++) {
1381                 void *rxdblock_priv;
1382                 void *uld_priv;
1383                 struct vxge_hw_ring_rxd_1 *rxdp;
1384
1385                 u32 reserve_index = ring->channel.reserve_ptr -
1386                                 (index * ring->rxds_per_block + i + 1);
1387                 u32 memblock_item_idx;
1388
1389                 ring->channel.reserve_arr[reserve_index] = ((u8 *)item) +
1390                                                 i * ring->rxd_size;
1391
1392                 /* Note: memblock_item_idx is index of the item within
1393                  *       the memblock. For instance, in case of three RxD-blocks
1394                  *       per memblock this value can be 0, 1 or 2. */
1395                 rxdblock_priv = __vxge_hw_mempool_item_priv(mempoolh,
1396                                         memblock_index, item,
1397                                         &memblock_item_idx);
1398
1399                 rxdp = (struct vxge_hw_ring_rxd_1 *)
1400                                 ring->channel.reserve_arr[reserve_index];
1401
1402                 uld_priv = ((u8 *)rxdblock_priv + ring->rxd_priv_size * i);
1403
1404                 /* pre-format Host_Control */
1405                 rxdp->host_control = (u64)(size_t)uld_priv;
1406         }
1407
1408         __vxge_hw_ring_block_memblock_idx_set(item, memblock_index);
1409
1410         if (is_last) {
1411                 /* link last one with first one */
1412                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index, 0);
1413         }
1414
1415         if (index > 0) {
1416                 /* link this RxD block with previous one */
1417                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index - 1, index);
1418         }
1419
1420         return;
1421 }
1422
1423 /*
1424  * __vxge_hw_ring_initial_replenish - Initial replenish of RxDs
1425  * This function replenishes the RxDs from reserve array to work array
1426  */
1427 enum vxge_hw_status
1428 vxge_hw_ring_replenish(struct __vxge_hw_ring *ring, u16 min_flag)
1429 {
1430         void *rxd;
1431         int i = 0;
1432         struct __vxge_hw_channel *channel;
1433         enum vxge_hw_status status = VXGE_HW_OK;
1434
1435         channel = &ring->channel;
1436
1437         while (vxge_hw_channel_dtr_count(channel) > 0) {
1438
1439                 status = vxge_hw_ring_rxd_reserve(ring, &rxd);
1440
1441                 vxge_assert(status == VXGE_HW_OK);
1442
1443                 if (ring->rxd_init) {
1444                         status = ring->rxd_init(rxd, channel->userdata);
1445                         if (status != VXGE_HW_OK) {
1446                                 vxge_hw_ring_rxd_free(ring, rxd);
1447                                 goto exit;
1448                         }
1449                 }
1450
1451                 vxge_hw_ring_rxd_post(ring, rxd);
1452                 if (min_flag) {
1453                         i++;
1454                         if (i == VXGE_HW_RING_MIN_BUFF_ALLOCATION)
1455                                 break;
1456                 }
1457         }
1458         status = VXGE_HW_OK;
1459 exit:
1460         return status;
1461 }
1462
1463 /*
1464  * __vxge_hw_ring_create - Create a Ring
1465  * This function creates Ring and initializes it.
1466  *
1467  */
1468 enum vxge_hw_status
1469 __vxge_hw_ring_create(struct __vxge_hw_vpath_handle *vp,
1470                       struct vxge_hw_ring_attr *attr)
1471 {
1472         enum vxge_hw_status status = VXGE_HW_OK;
1473         struct __vxge_hw_ring *ring;
1474         u32 ring_length;
1475         struct vxge_hw_ring_config *config;
1476         struct __vxge_hw_device *hldev;
1477         u32 vp_id;
1478         struct vxge_hw_mempool_cbs ring_mp_callback;
1479
1480         if ((vp == NULL) || (attr == NULL)) {
1481                 status = VXGE_HW_FAIL;
1482                 goto exit;
1483         }
1484
1485         hldev = vp->vpath->hldev;
1486         vp_id = vp->vpath->vp_id;
1487
1488         config = &hldev->config.vp_config[vp_id].ring;
1489
1490         ring_length = config->ring_blocks *
1491                         vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1492
1493         ring = (struct __vxge_hw_ring *)__vxge_hw_channel_allocate(vp,
1494                                                 VXGE_HW_CHANNEL_TYPE_RING,
1495                                                 ring_length,
1496                                                 attr->per_rxd_space,
1497                                                 attr->userdata);
1498
1499         if (ring == NULL) {
1500                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1501                 goto exit;
1502         }
1503
1504         vp->vpath->ringh = ring;
1505         ring->vp_id = vp_id;
1506         ring->vp_reg = vp->vpath->vp_reg;
1507         ring->common_reg = hldev->common_reg;
1508         ring->stats = &vp->vpath->sw_stats->ring_stats;
1509         ring->config = config;
1510         ring->callback = attr->callback;
1511         ring->rxd_init = attr->rxd_init;
1512         ring->rxd_term = attr->rxd_term;
1513         ring->buffer_mode = config->buffer_mode;
1514         ring->rxds_limit = config->rxds_limit;
1515
1516         ring->rxd_size = vxge_hw_ring_rxd_size_get(config->buffer_mode);
1517         ring->rxd_priv_size =
1518                 sizeof(struct __vxge_hw_ring_rxd_priv) + attr->per_rxd_space;
1519         ring->per_rxd_space = attr->per_rxd_space;
1520
1521         ring->rxd_priv_size =
1522                 ((ring->rxd_priv_size + VXGE_CACHE_LINE_SIZE - 1) /
1523                 VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
1524
1525         /* how many RxDs can fit into one block. Depends on configured
1526          * buffer_mode. */
1527         ring->rxds_per_block =
1528                 vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1529
1530         /* calculate actual RxD block private size */
1531         ring->rxdblock_priv_size = ring->rxd_priv_size * ring->rxds_per_block;
1532         ring_mp_callback.item_func_alloc = __vxge_hw_ring_mempool_item_alloc;
1533         ring->mempool = __vxge_hw_mempool_create(hldev,
1534                                 VXGE_HW_BLOCK_SIZE,
1535                                 VXGE_HW_BLOCK_SIZE,
1536                                 ring->rxdblock_priv_size,
1537                                 ring->config->ring_blocks,
1538                                 ring->config->ring_blocks,
1539                                 &ring_mp_callback,
1540                                 ring);
1541
1542         if (ring->mempool == NULL) {
1543                 __vxge_hw_ring_delete(vp);
1544                 return VXGE_HW_ERR_OUT_OF_MEMORY;
1545         }
1546
1547         status = __vxge_hw_channel_initialize(&ring->channel);
1548         if (status != VXGE_HW_OK) {
1549                 __vxge_hw_ring_delete(vp);
1550                 goto exit;
1551         }
1552
1553         /* Note:
1554          * Specifying rxd_init callback means two things:
1555          * 1) rxds need to be initialized by driver at channel-open time;
1556          * 2) rxds need to be posted at channel-open time
1557          *    (that's what the initial_replenish() below does)
1558          * Currently we don't have a case when the 1) is done without the 2).
1559          */
1560         if (ring->rxd_init) {
1561                 status = vxge_hw_ring_replenish(ring, 1);
1562                 if (status != VXGE_HW_OK) {
1563                         __vxge_hw_ring_delete(vp);
1564                         goto exit;
1565                 }
1566         }
1567
1568         /* initial replenish will increment the counter in its post() routine,
1569          * we have to reset it */
1570         ring->stats->common_stats.usage_cnt = 0;
1571 exit:
1572         return status;
1573 }
1574
1575 /*
1576  * __vxge_hw_ring_abort - Returns the RxD
1577  * This function terminates the RxDs of ring
1578  */
1579 enum vxge_hw_status __vxge_hw_ring_abort(struct __vxge_hw_ring *ring)
1580 {
1581         void *rxdh;
1582         struct __vxge_hw_channel *channel;
1583
1584         channel = &ring->channel;
1585
1586         for (;;) {
1587                 vxge_hw_channel_dtr_try_complete(channel, &rxdh);
1588
1589                 if (rxdh == NULL)
1590                         break;
1591
1592                 vxge_hw_channel_dtr_complete(channel);
1593
1594                 if (ring->rxd_term)
1595                         ring->rxd_term(rxdh, VXGE_HW_RXD_STATE_POSTED,
1596                                 channel->userdata);
1597
1598                 vxge_hw_channel_dtr_free(channel, rxdh);
1599         }
1600
1601         return VXGE_HW_OK;
1602 }
1603
1604 /*
1605  * __vxge_hw_ring_reset - Resets the ring
1606  * This function resets the ring during vpath reset operation
1607  */
1608 enum vxge_hw_status __vxge_hw_ring_reset(struct __vxge_hw_ring *ring)
1609 {
1610         enum vxge_hw_status status = VXGE_HW_OK;
1611         struct __vxge_hw_channel *channel;
1612
1613         channel = &ring->channel;
1614
1615         __vxge_hw_ring_abort(ring);
1616
1617         status = __vxge_hw_channel_reset(channel);
1618
1619         if (status != VXGE_HW_OK)
1620                 goto exit;
1621
1622         if (ring->rxd_init) {
1623                 status = vxge_hw_ring_replenish(ring, 1);
1624                 if (status != VXGE_HW_OK)
1625                         goto exit;
1626         }
1627 exit:
1628         return status;
1629 }
1630
1631 /*
1632  * __vxge_hw_ring_delete - Removes the ring
1633  * This function freeup the memory pool and removes the ring
1634  */
1635 enum vxge_hw_status __vxge_hw_ring_delete(struct __vxge_hw_vpath_handle *vp)
1636 {
1637         struct __vxge_hw_ring *ring = vp->vpath->ringh;
1638
1639         __vxge_hw_ring_abort(ring);
1640
1641         if (ring->mempool)
1642                 __vxge_hw_mempool_destroy(ring->mempool);
1643
1644         vp->vpath->ringh = NULL;
1645         __vxge_hw_channel_free(&ring->channel);
1646
1647         return VXGE_HW_OK;
1648 }
1649
1650 /*
1651  * __vxge_hw_mempool_grow
1652  * Will resize mempool up to %num_allocate value.
1653  */
1654 enum vxge_hw_status
1655 __vxge_hw_mempool_grow(struct vxge_hw_mempool *mempool, u32 num_allocate,
1656                        u32 *num_allocated)
1657 {
1658         u32 i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
1659         u32 n_items = mempool->items_per_memblock;
1660         u32 start_block_idx = mempool->memblocks_allocated;
1661         u32 end_block_idx = mempool->memblocks_allocated + num_allocate;
1662         enum vxge_hw_status status = VXGE_HW_OK;
1663
1664         *num_allocated = 0;
1665
1666         if (end_block_idx > mempool->memblocks_max) {
1667                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1668                 goto exit;
1669         }
1670
1671         for (i = start_block_idx; i < end_block_idx; i++) {
1672                 u32 j;
1673                 u32 is_last = ((end_block_idx - 1) == i);
1674                 struct vxge_hw_mempool_dma *dma_object =
1675                         mempool->memblocks_dma_arr + i;
1676                 void *the_memblock;
1677
1678                 /* allocate memblock's private part. Each DMA memblock
1679                  * has a space allocated for item's private usage upon
1680                  * mempool's user request. Each time mempool grows, it will
1681                  * allocate new memblock and its private part at once.
1682                  * This helps to minimize memory usage a lot. */
1683                 mempool->memblocks_priv_arr[i] =
1684                                 vmalloc(mempool->items_priv_size * n_items);
1685                 if (mempool->memblocks_priv_arr[i] == NULL) {
1686                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1687                         goto exit;
1688                 }
1689
1690                 memset(mempool->memblocks_priv_arr[i], 0,
1691                              mempool->items_priv_size * n_items);
1692
1693                 /* allocate DMA-capable memblock */
1694                 mempool->memblocks_arr[i] =
1695                         __vxge_hw_blockpool_malloc(mempool->devh,
1696                                 mempool->memblock_size, dma_object);
1697                 if (mempool->memblocks_arr[i] == NULL) {
1698                         vfree(mempool->memblocks_priv_arr[i]);
1699                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1700                         goto exit;
1701                 }
1702
1703                 (*num_allocated)++;
1704                 mempool->memblocks_allocated++;
1705
1706                 memset(mempool->memblocks_arr[i], 0, mempool->memblock_size);
1707
1708                 the_memblock = mempool->memblocks_arr[i];
1709
1710                 /* fill the items hash array */
1711                 for (j = 0; j < n_items; j++) {
1712                         u32 index = i * n_items + j;
1713
1714                         if (first_time && index >= mempool->items_initial)
1715                                 break;
1716
1717                         mempool->items_arr[index] =
1718                                 ((char *)the_memblock + j*mempool->item_size);
1719
1720                         /* let caller to do more job on each item */
1721                         if (mempool->item_func_alloc != NULL)
1722                                 mempool->item_func_alloc(mempool, i,
1723                                         dma_object, index, is_last);
1724
1725                         mempool->items_current = index + 1;
1726                 }
1727
1728                 if (first_time && mempool->items_current ==
1729                                         mempool->items_initial)
1730                         break;
1731         }
1732 exit:
1733         return status;
1734 }
1735
1736 /*
1737  * vxge_hw_mempool_create
1738  * This function will create memory pool object. Pool may grow but will
1739  * never shrink. Pool consists of number of dynamically allocated blocks
1740  * with size enough to hold %items_initial number of items. Memory is
1741  * DMA-able but client must map/unmap before interoperating with the device.
1742  */
1743 struct vxge_hw_mempool*
1744 __vxge_hw_mempool_create(
1745         struct __vxge_hw_device *devh,
1746         u32 memblock_size,
1747         u32 item_size,
1748         u32 items_priv_size,
1749         u32 items_initial,
1750         u32 items_max,
1751         struct vxge_hw_mempool_cbs *mp_callback,
1752         void *userdata)
1753 {
1754         enum vxge_hw_status status = VXGE_HW_OK;
1755         u32 memblocks_to_allocate;
1756         struct vxge_hw_mempool *mempool = NULL;
1757         u32 allocated;
1758
1759         if (memblock_size < item_size) {
1760                 status = VXGE_HW_FAIL;
1761                 goto exit;
1762         }
1763
1764         mempool = (struct vxge_hw_mempool *)
1765                         vmalloc(sizeof(struct vxge_hw_mempool));
1766         if (mempool == NULL) {
1767                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1768                 goto exit;
1769         }
1770         memset(mempool, 0, sizeof(struct vxge_hw_mempool));
1771
1772         mempool->devh                   = devh;
1773         mempool->memblock_size          = memblock_size;
1774         mempool->items_max              = items_max;
1775         mempool->items_initial          = items_initial;
1776         mempool->item_size              = item_size;
1777         mempool->items_priv_size        = items_priv_size;
1778         mempool->item_func_alloc        = mp_callback->item_func_alloc;
1779         mempool->userdata               = userdata;
1780
1781         mempool->memblocks_allocated = 0;
1782
1783         mempool->items_per_memblock = memblock_size / item_size;
1784
1785         mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
1786                                         mempool->items_per_memblock;
1787
1788         /* allocate array of memblocks */
1789         mempool->memblocks_arr =
1790                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1791         if (mempool->memblocks_arr == NULL) {
1792                 __vxge_hw_mempool_destroy(mempool);
1793                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1794                 mempool = NULL;
1795                 goto exit;
1796         }
1797         memset(mempool->memblocks_arr, 0,
1798                 sizeof(void *) * mempool->memblocks_max);
1799
1800         /* allocate array of private parts of items per memblocks */
1801         mempool->memblocks_priv_arr =
1802                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1803         if (mempool->memblocks_priv_arr == NULL) {
1804                 __vxge_hw_mempool_destroy(mempool);
1805                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1806                 mempool = NULL;
1807                 goto exit;
1808         }
1809         memset(mempool->memblocks_priv_arr, 0,
1810                     sizeof(void *) * mempool->memblocks_max);
1811
1812         /* allocate array of memblocks DMA objects */
1813         mempool->memblocks_dma_arr = (struct vxge_hw_mempool_dma *)
1814                 vmalloc(sizeof(struct vxge_hw_mempool_dma) *
1815                         mempool->memblocks_max);
1816
1817         if (mempool->memblocks_dma_arr == NULL) {
1818                 __vxge_hw_mempool_destroy(mempool);
1819                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1820                 mempool = NULL;
1821                 goto exit;
1822         }
1823         memset(mempool->memblocks_dma_arr, 0,
1824                         sizeof(struct vxge_hw_mempool_dma) *
1825                         mempool->memblocks_max);
1826
1827         /* allocate hash array of items */
1828         mempool->items_arr =
1829                 (void **) vmalloc(sizeof(void *) * mempool->items_max);
1830         if (mempool->items_arr == NULL) {
1831                 __vxge_hw_mempool_destroy(mempool);
1832                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1833                 mempool = NULL;
1834                 goto exit;
1835         }
1836         memset(mempool->items_arr, 0, sizeof(void *) * mempool->items_max);
1837
1838         /* calculate initial number of memblocks */
1839         memblocks_to_allocate = (mempool->items_initial +
1840                                  mempool->items_per_memblock - 1) /
1841                                                 mempool->items_per_memblock;
1842
1843         /* pre-allocate the mempool */
1844         status = __vxge_hw_mempool_grow(mempool, memblocks_to_allocate,
1845                                         &allocated);
1846         if (status != VXGE_HW_OK) {
1847                 __vxge_hw_mempool_destroy(mempool);
1848                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1849                 mempool = NULL;
1850                 goto exit;
1851         }
1852
1853 exit:
1854         return mempool;
1855 }
1856
1857 /*
1858  * vxge_hw_mempool_destroy
1859  */
1860 void __vxge_hw_mempool_destroy(struct vxge_hw_mempool *mempool)
1861 {
1862         u32 i, j;
1863         struct __vxge_hw_device *devh = mempool->devh;
1864
1865         for (i = 0; i < mempool->memblocks_allocated; i++) {
1866                 struct vxge_hw_mempool_dma *dma_object;
1867
1868                 vxge_assert(mempool->memblocks_arr[i]);
1869                 vxge_assert(mempool->memblocks_dma_arr + i);
1870
1871                 dma_object = mempool->memblocks_dma_arr + i;
1872
1873                 for (j = 0; j < mempool->items_per_memblock; j++) {
1874                         u32 index = i * mempool->items_per_memblock + j;
1875
1876                         /* to skip last partially filled(if any) memblock */
1877                         if (index >= mempool->items_current)
1878                                 break;
1879                 }
1880
1881                 vfree(mempool->memblocks_priv_arr[i]);
1882
1883                 __vxge_hw_blockpool_free(devh, mempool->memblocks_arr[i],
1884                                 mempool->memblock_size, dma_object);
1885         }
1886
1887         vfree(mempool->items_arr);
1888
1889         vfree(mempool->memblocks_dma_arr);
1890
1891         vfree(mempool->memblocks_priv_arr);
1892
1893         vfree(mempool->memblocks_arr);
1894
1895         vfree(mempool);
1896 }
1897
1898 /*
1899  * __vxge_hw_device_fifo_config_check - Check fifo configuration.
1900  * Check the fifo configuration
1901  */
1902 enum vxge_hw_status
1903 __vxge_hw_device_fifo_config_check(struct vxge_hw_fifo_config *fifo_config)
1904 {
1905         if ((fifo_config->fifo_blocks < VXGE_HW_MIN_FIFO_BLOCKS) ||
1906              (fifo_config->fifo_blocks > VXGE_HW_MAX_FIFO_BLOCKS))
1907                 return VXGE_HW_BADCFG_FIFO_BLOCKS;
1908
1909         return VXGE_HW_OK;
1910 }
1911
1912 /*
1913  * __vxge_hw_device_vpath_config_check - Check vpath configuration.
1914  * Check the vpath configuration
1915  */
1916 enum vxge_hw_status
1917 __vxge_hw_device_vpath_config_check(struct vxge_hw_vp_config *vp_config)
1918 {
1919         enum vxge_hw_status status;
1920
1921         if ((vp_config->min_bandwidth < VXGE_HW_VPATH_BANDWIDTH_MIN) ||
1922                 (vp_config->min_bandwidth >
1923                                         VXGE_HW_VPATH_BANDWIDTH_MAX))
1924                 return VXGE_HW_BADCFG_VPATH_MIN_BANDWIDTH;
1925
1926         status = __vxge_hw_device_fifo_config_check(&vp_config->fifo);
1927         if (status != VXGE_HW_OK)
1928                 return status;
1929
1930         if ((vp_config->mtu != VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) &&
1931                 ((vp_config->mtu < VXGE_HW_VPATH_MIN_INITIAL_MTU) ||
1932                 (vp_config->mtu > VXGE_HW_VPATH_MAX_INITIAL_MTU)))
1933                 return VXGE_HW_BADCFG_VPATH_MTU;
1934
1935         if ((vp_config->rpa_strip_vlan_tag !=
1936                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) &&
1937                 (vp_config->rpa_strip_vlan_tag !=
1938                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_ENABLE) &&
1939                 (vp_config->rpa_strip_vlan_tag !=
1940                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_DISABLE))
1941                 return VXGE_HW_BADCFG_VPATH_RPA_STRIP_VLAN_TAG;
1942
1943         return VXGE_HW_OK;
1944 }
1945
1946 /*
1947  * __vxge_hw_device_config_check - Check device configuration.
1948  * Check the device configuration
1949  */
1950 enum vxge_hw_status
1951 __vxge_hw_device_config_check(struct vxge_hw_device_config *new_config)
1952 {
1953         u32 i;
1954         enum vxge_hw_status status;
1955
1956         if ((new_config->intr_mode != VXGE_HW_INTR_MODE_IRQLINE) &&
1957            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX) &&
1958            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX_ONE_SHOT) &&
1959            (new_config->intr_mode != VXGE_HW_INTR_MODE_DEF))
1960                 return VXGE_HW_BADCFG_INTR_MODE;
1961
1962         if ((new_config->rts_mac_en != VXGE_HW_RTS_MAC_DISABLE) &&
1963            (new_config->rts_mac_en != VXGE_HW_RTS_MAC_ENABLE))
1964                 return VXGE_HW_BADCFG_RTS_MAC_EN;
1965
1966         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1967                 status = __vxge_hw_device_vpath_config_check(
1968                                 &new_config->vp_config[i]);
1969                 if (status != VXGE_HW_OK)
1970                         return status;
1971         }
1972
1973         return VXGE_HW_OK;
1974 }
1975
1976 /*
1977  * vxge_hw_device_config_default_get - Initialize device config with defaults.
1978  * Initialize Titan device config with default values.
1979  */
1980 enum vxge_hw_status __devinit
1981 vxge_hw_device_config_default_get(struct vxge_hw_device_config *device_config)
1982 {
1983         u32 i;
1984
1985         device_config->dma_blockpool_initial =
1986                                         VXGE_HW_INITIAL_DMA_BLOCK_POOL_SIZE;
1987         device_config->dma_blockpool_max = VXGE_HW_MAX_DMA_BLOCK_POOL_SIZE;
1988         device_config->intr_mode = VXGE_HW_INTR_MODE_DEF;
1989         device_config->rth_en = VXGE_HW_RTH_DEFAULT;
1990         device_config->rth_it_type = VXGE_HW_RTH_IT_TYPE_DEFAULT;
1991         device_config->device_poll_millis =  VXGE_HW_DEF_DEVICE_POLL_MILLIS;
1992         device_config->rts_mac_en =  VXGE_HW_RTS_MAC_DEFAULT;
1993
1994         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1995
1996                 device_config->vp_config[i].vp_id = i;
1997
1998                 device_config->vp_config[i].min_bandwidth =
1999                                 VXGE_HW_VPATH_BANDWIDTH_DEFAULT;
2000
2001                 device_config->vp_config[i].ring.enable = VXGE_HW_RING_DEFAULT;
2002
2003                 device_config->vp_config[i].ring.ring_blocks =
2004                                 VXGE_HW_DEF_RING_BLOCKS;
2005
2006                 device_config->vp_config[i].ring.buffer_mode =
2007                                 VXGE_HW_RING_RXD_BUFFER_MODE_DEFAULT;
2008
2009                 device_config->vp_config[i].ring.scatter_mode =
2010                                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT;
2011
2012                 device_config->vp_config[i].ring.rxds_limit =
2013                                 VXGE_HW_DEF_RING_RXDS_LIMIT;
2014
2015                 device_config->vp_config[i].fifo.enable = VXGE_HW_FIFO_ENABLE;
2016
2017                 device_config->vp_config[i].fifo.fifo_blocks =
2018                                 VXGE_HW_MIN_FIFO_BLOCKS;
2019
2020                 device_config->vp_config[i].fifo.max_frags =
2021                                 VXGE_HW_MAX_FIFO_FRAGS;
2022
2023                 device_config->vp_config[i].fifo.memblock_size =
2024                                 VXGE_HW_DEF_FIFO_MEMBLOCK_SIZE;
2025
2026                 device_config->vp_config[i].fifo.alignment_size =
2027                                 VXGE_HW_DEF_FIFO_ALIGNMENT_SIZE;
2028
2029                 device_config->vp_config[i].fifo.intr =
2030                                 VXGE_HW_FIFO_QUEUE_INTR_DEFAULT;
2031
2032                 device_config->vp_config[i].fifo.no_snoop_bits =
2033                                 VXGE_HW_FIFO_NO_SNOOP_DEFAULT;
2034                 device_config->vp_config[i].tti.intr_enable =
2035                                 VXGE_HW_TIM_INTR_DEFAULT;
2036
2037                 device_config->vp_config[i].tti.btimer_val =
2038                                 VXGE_HW_USE_FLASH_DEFAULT;
2039
2040                 device_config->vp_config[i].tti.timer_ac_en =
2041                                 VXGE_HW_USE_FLASH_DEFAULT;
2042
2043                 device_config->vp_config[i].tti.timer_ci_en =
2044                                 VXGE_HW_USE_FLASH_DEFAULT;
2045
2046                 device_config->vp_config[i].tti.timer_ri_en =
2047                                 VXGE_HW_USE_FLASH_DEFAULT;
2048
2049                 device_config->vp_config[i].tti.rtimer_val =
2050                                 VXGE_HW_USE_FLASH_DEFAULT;
2051
2052                 device_config->vp_config[i].tti.util_sel =
2053                                 VXGE_HW_USE_FLASH_DEFAULT;
2054
2055                 device_config->vp_config[i].tti.ltimer_val =
2056                                 VXGE_HW_USE_FLASH_DEFAULT;
2057
2058                 device_config->vp_config[i].tti.urange_a =
2059                                 VXGE_HW_USE_FLASH_DEFAULT;
2060
2061                 device_config->vp_config[i].tti.uec_a =
2062                                 VXGE_HW_USE_FLASH_DEFAULT;
2063
2064                 device_config->vp_config[i].tti.urange_b =
2065                                 VXGE_HW_USE_FLASH_DEFAULT;
2066
2067                 device_config->vp_config[i].tti.uec_b =
2068                                 VXGE_HW_USE_FLASH_DEFAULT;
2069
2070                 device_config->vp_config[i].tti.urange_c =
2071                                 VXGE_HW_USE_FLASH_DEFAULT;
2072
2073                 device_config->vp_config[i].tti.uec_c =
2074                                 VXGE_HW_USE_FLASH_DEFAULT;
2075
2076                 device_config->vp_config[i].tti.uec_d =
2077                                 VXGE_HW_USE_FLASH_DEFAULT;
2078
2079                 device_config->vp_config[i].rti.intr_enable =
2080                                 VXGE_HW_TIM_INTR_DEFAULT;
2081
2082                 device_config->vp_config[i].rti.btimer_val =
2083                                 VXGE_HW_USE_FLASH_DEFAULT;
2084
2085                 device_config->vp_config[i].rti.timer_ac_en =
2086                                 VXGE_HW_USE_FLASH_DEFAULT;
2087
2088                 device_config->vp_config[i].rti.timer_ci_en =
2089                                 VXGE_HW_USE_FLASH_DEFAULT;
2090
2091                 device_config->vp_config[i].rti.timer_ri_en =
2092                                 VXGE_HW_USE_FLASH_DEFAULT;
2093
2094                 device_config->vp_config[i].rti.rtimer_val =
2095                                 VXGE_HW_USE_FLASH_DEFAULT;
2096
2097                 device_config->vp_config[i].rti.util_sel =
2098                                 VXGE_HW_USE_FLASH_DEFAULT;
2099
2100                 device_config->vp_config[i].rti.ltimer_val =
2101                                 VXGE_HW_USE_FLASH_DEFAULT;
2102
2103                 device_config->vp_config[i].rti.urange_a =
2104                                 VXGE_HW_USE_FLASH_DEFAULT;
2105
2106                 device_config->vp_config[i].rti.uec_a =
2107                                 VXGE_HW_USE_FLASH_DEFAULT;
2108
2109                 device_config->vp_config[i].rti.urange_b =
2110                                 VXGE_HW_USE_FLASH_DEFAULT;
2111
2112                 device_config->vp_config[i].rti.uec_b =
2113                                 VXGE_HW_USE_FLASH_DEFAULT;
2114
2115                 device_config->vp_config[i].rti.urange_c =
2116                                 VXGE_HW_USE_FLASH_DEFAULT;
2117
2118                 device_config->vp_config[i].rti.uec_c =
2119                                 VXGE_HW_USE_FLASH_DEFAULT;
2120
2121                 device_config->vp_config[i].rti.uec_d =
2122                                 VXGE_HW_USE_FLASH_DEFAULT;
2123
2124                 device_config->vp_config[i].mtu =
2125                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU;
2126
2127                 device_config->vp_config[i].rpa_strip_vlan_tag =
2128                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT;
2129         }
2130
2131         return VXGE_HW_OK;
2132 }
2133
2134 /*
2135  * _hw_legacy_swapper_set - Set the swapper bits for the legacy secion.
2136  * Set the swapper bits appropriately for the lagacy section.
2137  */
2138 enum vxge_hw_status
2139 __vxge_hw_legacy_swapper_set(struct vxge_hw_legacy_reg __iomem *legacy_reg)
2140 {
2141         u64 val64;
2142         enum vxge_hw_status status = VXGE_HW_OK;
2143
2144         val64 = readq(&legacy_reg->toc_swapper_fb);
2145
2146         wmb();
2147
2148         switch (val64) {
2149
2150         case VXGE_HW_SWAPPER_INITIAL_VALUE:
2151                 return status;
2152
2153         case VXGE_HW_SWAPPER_BYTE_SWAPPED_BIT_FLIPPED:
2154                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2155                         &legacy_reg->pifm_rd_swap_en);
2156                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2157                         &legacy_reg->pifm_rd_flip_en);
2158                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2159                         &legacy_reg->pifm_wr_swap_en);
2160                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2161                         &legacy_reg->pifm_wr_flip_en);
2162                 break;
2163
2164         case VXGE_HW_SWAPPER_BYTE_SWAPPED:
2165                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2166                         &legacy_reg->pifm_rd_swap_en);
2167                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2168                         &legacy_reg->pifm_wr_swap_en);
2169                 break;
2170
2171         case VXGE_HW_SWAPPER_BIT_FLIPPED:
2172                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2173                         &legacy_reg->pifm_rd_flip_en);
2174                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2175                         &legacy_reg->pifm_wr_flip_en);
2176                 break;
2177         }
2178
2179         wmb();
2180
2181         val64 = readq(&legacy_reg->toc_swapper_fb);
2182
2183         if (val64 != VXGE_HW_SWAPPER_INITIAL_VALUE)
2184                 status = VXGE_HW_ERR_SWAPPER_CTRL;
2185
2186         return status;
2187 }
2188
2189 /*
2190  * __vxge_hw_vpath_swapper_set - Set the swapper bits for the vpath.
2191  * Set the swapper bits appropriately for the vpath.
2192  */
2193 enum vxge_hw_status
2194 __vxge_hw_vpath_swapper_set(struct vxge_hw_vpath_reg __iomem *vpath_reg)
2195 {
2196 #ifndef __BIG_ENDIAN
2197         u64 val64;
2198
2199         val64 = readq(&vpath_reg->vpath_general_cfg1);
2200         wmb();
2201         val64 |= VXGE_HW_VPATH_GENERAL_CFG1_CTL_BYTE_SWAPEN;
2202         writeq(val64, &vpath_reg->vpath_general_cfg1);
2203         wmb();
2204 #endif
2205         return VXGE_HW_OK;
2206 }
2207
2208 /*
2209  * __vxge_hw_kdfc_swapper_set - Set the swapper bits for the kdfc.
2210  * Set the swapper bits appropriately for the vpath.
2211  */
2212 enum vxge_hw_status
2213 __vxge_hw_kdfc_swapper_set(
2214         struct vxge_hw_legacy_reg __iomem *legacy_reg,
2215         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2216 {
2217         u64 val64;
2218
2219         val64 = readq(&legacy_reg->pifm_wr_swap_en);
2220
2221         if (val64 == VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE) {
2222                 val64 = readq(&vpath_reg->kdfcctl_cfg0);
2223                 wmb();
2224
2225                 val64 |= VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO0 |
2226                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO1  |
2227                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO2;
2228
2229                 writeq(val64, &vpath_reg->kdfcctl_cfg0);
2230                 wmb();
2231         }
2232
2233         return VXGE_HW_OK;
2234 }
2235
2236 /*
2237  * vxge_hw_mgmt_device_config - Retrieve device configuration.
2238  * Get device configuration. Permits to retrieve at run-time configuration
2239  * values that were used to initialize and configure the device.
2240  */
2241 enum vxge_hw_status
2242 vxge_hw_mgmt_device_config(struct __vxge_hw_device *hldev,
2243                            struct vxge_hw_device_config *dev_config, int size)
2244 {
2245
2246         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC))
2247                 return VXGE_HW_ERR_INVALID_DEVICE;
2248
2249         if (size != sizeof(struct vxge_hw_device_config))
2250                 return VXGE_HW_ERR_VERSION_CONFLICT;
2251
2252         memcpy(dev_config, &hldev->config,
2253                 sizeof(struct vxge_hw_device_config));
2254
2255         return VXGE_HW_OK;
2256 }
2257
2258 /*
2259  * vxge_hw_mgmt_reg_read - Read Titan register.
2260  */
2261 enum vxge_hw_status
2262 vxge_hw_mgmt_reg_read(struct __vxge_hw_device *hldev,
2263                       enum vxge_hw_mgmt_reg_type type,
2264                       u32 index, u32 offset, u64 *value)
2265 {
2266         enum vxge_hw_status status = VXGE_HW_OK;
2267
2268         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2269                 status = VXGE_HW_ERR_INVALID_DEVICE;
2270                 goto exit;
2271         }
2272
2273         switch (type) {
2274         case vxge_hw_mgmt_reg_type_legacy:
2275                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2276                         status = VXGE_HW_ERR_INVALID_OFFSET;
2277                         break;
2278                 }
2279                 *value = readq((void __iomem *)hldev->legacy_reg + offset);
2280                 break;
2281         case vxge_hw_mgmt_reg_type_toc:
2282                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2283                         status = VXGE_HW_ERR_INVALID_OFFSET;
2284                         break;
2285                 }
2286                 *value = readq((void __iomem *)hldev->toc_reg + offset);
2287                 break;
2288         case vxge_hw_mgmt_reg_type_common:
2289                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2290                         status = VXGE_HW_ERR_INVALID_OFFSET;
2291                         break;
2292                 }
2293                 *value = readq((void __iomem *)hldev->common_reg + offset);
2294                 break;
2295         case vxge_hw_mgmt_reg_type_mrpcim:
2296                 if (!(hldev->access_rights &
2297                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2298                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2299                         break;
2300                 }
2301                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2302                         status = VXGE_HW_ERR_INVALID_OFFSET;
2303                         break;
2304                 }
2305                 *value = readq((void __iomem *)hldev->mrpcim_reg + offset);
2306                 break;
2307         case vxge_hw_mgmt_reg_type_srpcim:
2308                 if (!(hldev->access_rights &
2309                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2310                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2311                         break;
2312                 }
2313                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2314                         status = VXGE_HW_ERR_INVALID_INDEX;
2315                         break;
2316                 }
2317                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2318                         status = VXGE_HW_ERR_INVALID_OFFSET;
2319                         break;
2320                 }
2321                 *value = readq((void __iomem *)hldev->srpcim_reg[index] +
2322                                 offset);
2323                 break;
2324         case vxge_hw_mgmt_reg_type_vpmgmt:
2325                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2326                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2327                         status = VXGE_HW_ERR_INVALID_INDEX;
2328                         break;
2329                 }
2330                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2331                         status = VXGE_HW_ERR_INVALID_OFFSET;
2332                         break;
2333                 }
2334                 *value = readq((void __iomem *)hldev->vpmgmt_reg[index] +
2335                                 offset);
2336                 break;
2337         case vxge_hw_mgmt_reg_type_vpath:
2338                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) ||
2339                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2340                         status = VXGE_HW_ERR_INVALID_INDEX;
2341                         break;
2342                 }
2343                 if (index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) {
2344                         status = VXGE_HW_ERR_INVALID_INDEX;
2345                         break;
2346                 }
2347                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2348                         status = VXGE_HW_ERR_INVALID_OFFSET;
2349                         break;
2350                 }
2351                 *value = readq((void __iomem *)hldev->vpath_reg[index] +
2352                                 offset);
2353                 break;
2354         default:
2355                 status = VXGE_HW_ERR_INVALID_TYPE;
2356                 break;
2357         }
2358
2359 exit:
2360         return status;
2361 }
2362
2363 /*
2364  * vxge_hw_mgmt_reg_Write - Write Titan register.
2365  */
2366 enum vxge_hw_status
2367 vxge_hw_mgmt_reg_write(struct __vxge_hw_device *hldev,
2368                       enum vxge_hw_mgmt_reg_type type,
2369                       u32 index, u32 offset, u64 value)
2370 {
2371         enum vxge_hw_status status = VXGE_HW_OK;
2372
2373         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2374                 status = VXGE_HW_ERR_INVALID_DEVICE;
2375                 goto exit;
2376         }
2377
2378         switch (type) {
2379         case vxge_hw_mgmt_reg_type_legacy:
2380                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2381                         status = VXGE_HW_ERR_INVALID_OFFSET;
2382                         break;
2383                 }
2384                 writeq(value, (void __iomem *)hldev->legacy_reg + offset);
2385                 break;
2386         case vxge_hw_mgmt_reg_type_toc:
2387                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2388                         status = VXGE_HW_ERR_INVALID_OFFSET;
2389                         break;
2390                 }
2391                 writeq(value, (void __iomem *)hldev->toc_reg + offset);
2392                 break;
2393         case vxge_hw_mgmt_reg_type_common:
2394                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2395                         status = VXGE_HW_ERR_INVALID_OFFSET;
2396                         break;
2397                 }
2398                 writeq(value, (void __iomem *)hldev->common_reg + offset);
2399                 break;
2400         case vxge_hw_mgmt_reg_type_mrpcim:
2401                 if (!(hldev->access_rights &
2402                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2403                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2404                         break;
2405                 }
2406                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2407                         status = VXGE_HW_ERR_INVALID_OFFSET;
2408                         break;
2409                 }
2410                 writeq(value, (void __iomem *)hldev->mrpcim_reg + offset);
2411                 break;
2412         case vxge_hw_mgmt_reg_type_srpcim:
2413                 if (!(hldev->access_rights &
2414                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2415                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2416                         break;
2417                 }
2418                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2419                         status = VXGE_HW_ERR_INVALID_INDEX;
2420                         break;
2421                 }
2422                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2423                         status = VXGE_HW_ERR_INVALID_OFFSET;
2424                         break;
2425                 }
2426                 writeq(value, (void __iomem *)hldev->srpcim_reg[index] +
2427                         offset);
2428
2429                 break;
2430         case vxge_hw_mgmt_reg_type_vpmgmt:
2431                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2432                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2433                         status = VXGE_HW_ERR_INVALID_INDEX;
2434                         break;
2435                 }
2436                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2437                         status = VXGE_HW_ERR_INVALID_OFFSET;
2438                         break;
2439                 }
2440                 writeq(value, (void __iomem *)hldev->vpmgmt_reg[index] +
2441                         offset);
2442                 break;
2443         case vxge_hw_mgmt_reg_type_vpath:
2444                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES-1) ||
2445                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2446                         status = VXGE_HW_ERR_INVALID_INDEX;
2447                         break;
2448                 }
2449                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2450                         status = VXGE_HW_ERR_INVALID_OFFSET;
2451                         break;
2452                 }
2453                 writeq(value, (void __iomem *)hldev->vpath_reg[index] +
2454                         offset);
2455                 break;
2456         default:
2457                 status = VXGE_HW_ERR_INVALID_TYPE;
2458                 break;
2459         }
2460 exit:
2461         return status;
2462 }
2463
2464 /*
2465  * __vxge_hw_fifo_mempool_item_alloc - Allocate List blocks for TxD
2466  * list callback
2467  * This function is callback passed to __vxge_hw_mempool_create to create memory
2468  * pool for TxD list
2469  */
2470 static void
2471 __vxge_hw_fifo_mempool_item_alloc(
2472         struct vxge_hw_mempool *mempoolh,
2473         u32 memblock_index, struct vxge_hw_mempool_dma *dma_object,
2474         u32 index, u32 is_last)
2475 {
2476         u32 memblock_item_idx;
2477         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
2478         struct vxge_hw_fifo_txd *txdp =
2479                 (struct vxge_hw_fifo_txd *)mempoolh->items_arr[index];
2480         struct __vxge_hw_fifo *fifo =
2481                         (struct __vxge_hw_fifo *)mempoolh->userdata;
2482         void *memblock = mempoolh->memblocks_arr[memblock_index];
2483
2484         vxge_assert(txdp);
2485
2486         txdp->host_control = (u64) (size_t)
2487         __vxge_hw_mempool_item_priv(mempoolh, memblock_index, txdp,
2488                                         &memblock_item_idx);
2489
2490         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo, txdp);
2491
2492         vxge_assert(txdl_priv);
2493
2494         fifo->channel.reserve_arr[fifo->channel.reserve_ptr - 1 - index] = txdp;
2495
2496         /* pre-format HW's TxDL's private */
2497         txdl_priv->dma_offset = (char *)txdp - (char *)memblock;
2498         txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
2499         txdl_priv->dma_handle = dma_object->handle;
2500         txdl_priv->memblock   = memblock;
2501         txdl_priv->first_txdp = txdp;
2502         txdl_priv->next_txdl_priv = NULL;
2503         txdl_priv->alloc_frags = 0;
2504
2505         return;
2506 }
2507
2508 /*
2509  * __vxge_hw_fifo_create - Create a FIFO
2510  * This function creates FIFO and initializes it.
2511  */
2512 enum vxge_hw_status
2513 __vxge_hw_fifo_create(struct __vxge_hw_vpath_handle *vp,
2514                       struct vxge_hw_fifo_attr *attr)
2515 {
2516         enum vxge_hw_status status = VXGE_HW_OK;
2517         struct __vxge_hw_fifo *fifo;
2518         struct vxge_hw_fifo_config *config;
2519         u32 txdl_size, txdl_per_memblock;
2520         struct vxge_hw_mempool_cbs fifo_mp_callback;
2521         struct __vxge_hw_virtualpath *vpath;
2522
2523         if ((vp == NULL) || (attr == NULL)) {
2524                 status = VXGE_HW_ERR_INVALID_HANDLE;
2525                 goto exit;
2526         }
2527         vpath = vp->vpath;
2528         config = &vpath->hldev->config.vp_config[vpath->vp_id].fifo;
2529
2530         txdl_size = config->max_frags * sizeof(struct vxge_hw_fifo_txd);
2531
2532         txdl_per_memblock = config->memblock_size / txdl_size;
2533
2534         fifo = (struct __vxge_hw_fifo *)__vxge_hw_channel_allocate(vp,
2535                                         VXGE_HW_CHANNEL_TYPE_FIFO,
2536                                         config->fifo_blocks * txdl_per_memblock,
2537                                         attr->per_txdl_space, attr->userdata);
2538
2539         if (fifo == NULL) {
2540                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2541                 goto exit;
2542         }
2543
2544         vpath->fifoh = fifo;
2545         fifo->nofl_db = vpath->nofl_db;
2546
2547         fifo->vp_id = vpath->vp_id;
2548         fifo->vp_reg = vpath->vp_reg;
2549         fifo->stats = &vpath->sw_stats->fifo_stats;
2550
2551         fifo->config = config;
2552
2553         /* apply "interrupts per txdl" attribute */
2554         fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_UTILZ;
2555
2556         if (fifo->config->intr)
2557                 fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_PER_LIST;
2558
2559         fifo->no_snoop_bits = config->no_snoop_bits;
2560
2561         /*
2562          * FIFO memory management strategy:
2563          *
2564          * TxDL split into three independent parts:
2565          *      - set of TxD's
2566          *      - TxD HW private part
2567          *      - driver private part
2568          *
2569          * Adaptative memory allocation used. i.e. Memory allocated on
2570          * demand with the size which will fit into one memory block.
2571          * One memory block may contain more than one TxDL.
2572          *
2573          * During "reserve" operations more memory can be allocated on demand
2574          * for example due to FIFO full condition.
2575          *
2576          * Pool of memory memblocks never shrinks except in __vxge_hw_fifo_close
2577          * routine which will essentially stop the channel and free resources.
2578          */
2579
2580         /* TxDL common private size == TxDL private  +  driver private */
2581         fifo->priv_size =
2582                 sizeof(struct __vxge_hw_fifo_txdl_priv) + attr->per_txdl_space;
2583         fifo->priv_size = ((fifo->priv_size  +  VXGE_CACHE_LINE_SIZE - 1) /
2584                         VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
2585
2586         fifo->per_txdl_space = attr->per_txdl_space;
2587
2588         /* recompute txdl size to be cacheline aligned */
2589         fifo->txdl_size = txdl_size;
2590         fifo->txdl_per_memblock = txdl_per_memblock;
2591
2592         fifo->txdl_term = attr->txdl_term;
2593         fifo->callback = attr->callback;
2594
2595         if (fifo->txdl_per_memblock == 0) {
2596                 __vxge_hw_fifo_delete(vp);
2597                 status = VXGE_HW_ERR_INVALID_BLOCK_SIZE;
2598                 goto exit;
2599         }
2600
2601         fifo_mp_callback.item_func_alloc = __vxge_hw_fifo_mempool_item_alloc;
2602
2603         fifo->mempool =
2604                 __vxge_hw_mempool_create(vpath->hldev,
2605                         fifo->config->memblock_size,
2606                         fifo->txdl_size,
2607                         fifo->priv_size,
2608                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2609                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2610                         &fifo_mp_callback,
2611                         fifo);
2612
2613         if (fifo->mempool == NULL) {
2614                 __vxge_hw_fifo_delete(vp);
2615                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2616                 goto exit;
2617         }
2618
2619         status = __vxge_hw_channel_initialize(&fifo->channel);
2620         if (status != VXGE_HW_OK) {
2621                 __vxge_hw_fifo_delete(vp);
2622                 goto exit;
2623         }
2624
2625         vxge_assert(fifo->channel.reserve_ptr);
2626 exit:
2627         return status;
2628 }
2629
2630 /*
2631  * __vxge_hw_fifo_abort - Returns the TxD
2632  * This function terminates the TxDs of fifo
2633  */
2634 enum vxge_hw_status __vxge_hw_fifo_abort(struct __vxge_hw_fifo *fifo)
2635 {
2636         void *txdlh;
2637
2638         for (;;) {
2639                 vxge_hw_channel_dtr_try_complete(&fifo->channel, &txdlh);
2640
2641                 if (txdlh == NULL)
2642                         break;
2643
2644                 vxge_hw_channel_dtr_complete(&fifo->channel);
2645
2646                 if (fifo->txdl_term) {
2647                         fifo->txdl_term(txdlh,
2648                         VXGE_HW_TXDL_STATE_POSTED,
2649                         fifo->channel.userdata);
2650                 }
2651
2652                 vxge_hw_channel_dtr_free(&fifo->channel, txdlh);
2653         }
2654
2655         return VXGE_HW_OK;
2656 }
2657
2658 /*
2659  * __vxge_hw_fifo_reset - Resets the fifo
2660  * This function resets the fifo during vpath reset operation
2661  */
2662 enum vxge_hw_status __vxge_hw_fifo_reset(struct __vxge_hw_fifo *fifo)
2663 {
2664         enum vxge_hw_status status = VXGE_HW_OK;
2665
2666         __vxge_hw_fifo_abort(fifo);
2667         status = __vxge_hw_channel_reset(&fifo->channel);
2668
2669         return status;
2670 }
2671
2672 /*
2673  * __vxge_hw_fifo_delete - Removes the FIFO
2674  * This function freeup the memory pool and removes the FIFO
2675  */
2676 enum vxge_hw_status __vxge_hw_fifo_delete(struct __vxge_hw_vpath_handle *vp)
2677 {
2678         struct __vxge_hw_fifo *fifo = vp->vpath->fifoh;
2679
2680         __vxge_hw_fifo_abort(fifo);
2681
2682         if (fifo->mempool)
2683                 __vxge_hw_mempool_destroy(fifo->mempool);
2684
2685         vp->vpath->fifoh = NULL;
2686
2687         __vxge_hw_channel_free(&fifo->channel);
2688
2689         return VXGE_HW_OK;
2690 }
2691
2692 /*
2693  * __vxge_hw_vpath_pci_read - Read the content of given address
2694  *                          in pci config space.
2695  * Read from the vpath pci config space.
2696  */
2697 enum vxge_hw_status
2698 __vxge_hw_vpath_pci_read(struct __vxge_hw_virtualpath *vpath,
2699                          u32 phy_func_0, u32 offset, u32 *val)
2700 {
2701         u64 val64;
2702         enum vxge_hw_status status = VXGE_HW_OK;
2703         struct vxge_hw_vpath_reg __iomem *vp_reg = vpath->vp_reg;
2704
2705         val64 = VXGE_HW_PCI_CONFIG_ACCESS_CFG1_ADDRESS(offset);
2706
2707         if (phy_func_0)
2708                 val64 |= VXGE_HW_PCI_CONFIG_ACCESS_CFG1_SEL_FUNC0;
2709
2710         writeq(val64, &vp_reg->pci_config_access_cfg1);
2711         wmb();
2712         writeq(VXGE_HW_PCI_CONFIG_ACCESS_CFG2_REQ,
2713                         &vp_reg->pci_config_access_cfg2);
2714         wmb();
2715
2716         status = __vxge_hw_device_register_poll(
2717                         &vp_reg->pci_config_access_cfg2,
2718                         VXGE_HW_INTR_MASK_ALL, VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2719
2720         if (status != VXGE_HW_OK)
2721                 goto exit;
2722
2723         val64 = readq(&vp_reg->pci_config_access_status);
2724
2725         if (val64 & VXGE_HW_PCI_CONFIG_ACCESS_STATUS_ACCESS_ERR) {
2726                 status = VXGE_HW_FAIL;
2727                 *val = 0;
2728         } else
2729                 *val = (u32)vxge_bVALn(val64, 32, 32);
2730 exit:
2731         return status;
2732 }
2733
2734 /*
2735  * __vxge_hw_vpath_func_id_get - Get the function id of the vpath.
2736  * Returns the function number of the vpath.
2737  */
2738 u32
2739 __vxge_hw_vpath_func_id_get(u32 vp_id,
2740         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg)
2741 {
2742         u64 val64;
2743
2744         val64 = readq(&vpmgmt_reg->vpath_to_func_map_cfg1);
2745
2746         return
2747          (u32)VXGE_HW_VPATH_TO_FUNC_MAP_CFG1_GET_VPATH_TO_FUNC_MAP_CFG1(val64);
2748 }
2749
2750 /*
2751  * __vxge_hw_read_rts_ds - Program RTS steering critieria
2752  */
2753 static inline void
2754 __vxge_hw_read_rts_ds(struct vxge_hw_vpath_reg __iomem *vpath_reg,
2755                       u64 dta_struct_sel)
2756 {
2757         writeq(0, &vpath_reg->rts_access_steer_ctrl);
2758         wmb();
2759         writeq(dta_struct_sel, &vpath_reg->rts_access_steer_data0);
2760         writeq(0, &vpath_reg->rts_access_steer_data1);
2761         wmb();
2762         return;
2763 }
2764
2765
2766 /*
2767  * __vxge_hw_vpath_card_info_get - Get the serial numbers,
2768  * part number and product description.
2769  */
2770 enum vxge_hw_status
2771 __vxge_hw_vpath_card_info_get(
2772         u32 vp_id,
2773         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2774         struct vxge_hw_device_hw_info *hw_info)
2775 {
2776         u32 i, j;
2777         u64 val64;
2778         u64 data1 = 0ULL;
2779         u64 data2 = 0ULL;
2780         enum vxge_hw_status status = VXGE_HW_OK;
2781         u8 *serial_number = hw_info->serial_number;
2782         u8 *part_number = hw_info->part_number;
2783         u8 *product_desc = hw_info->product_desc;
2784
2785         __vxge_hw_read_rts_ds(vpath_reg,
2786                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_SERIAL_NUMBER);
2787
2788         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2789                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2790                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2791                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2792                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2793                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2794
2795         status = __vxge_hw_pio_mem_write64(val64,
2796                                 &vpath_reg->rts_access_steer_ctrl,
2797                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2798                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2799
2800         if (status != VXGE_HW_OK)
2801                 return status;
2802
2803         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2804
2805         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2806                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2807                 ((u64 *)serial_number)[0] = be64_to_cpu(data1);
2808
2809                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2810                 ((u64 *)serial_number)[1] = be64_to_cpu(data2);
2811                 status = VXGE_HW_OK;
2812         } else
2813                 *serial_number = 0;
2814
2815         __vxge_hw_read_rts_ds(vpath_reg,
2816                         VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PART_NUMBER);
2817
2818         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2819                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2820                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2821                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2822                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2823                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2824
2825         status = __vxge_hw_pio_mem_write64(val64,
2826                                 &vpath_reg->rts_access_steer_ctrl,
2827                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2828                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2829
2830         if (status != VXGE_HW_OK)
2831                 return status;
2832
2833         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2834
2835         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2836
2837                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2838                 ((u64 *)part_number)[0] = be64_to_cpu(data1);
2839
2840                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2841                 ((u64 *)part_number)[1] = be64_to_cpu(data2);
2842
2843                 status = VXGE_HW_OK;
2844
2845         } else
2846                 *part_number = 0;
2847
2848         j = 0;
2849
2850         for (i = VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_0;
2851              i <= VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_3; i++) {
2852
2853                 __vxge_hw_read_rts_ds(vpath_reg, i);
2854
2855                 val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2856                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2857                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2858                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2859                         VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2860                         VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2861
2862                 status = __vxge_hw_pio_mem_write64(val64,
2863                                 &vpath_reg->rts_access_steer_ctrl,
2864                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2865                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2866
2867                 if (status != VXGE_HW_OK)
2868                         return status;
2869
2870                 val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2871
2872                 if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2873
2874                         data1 = readq(&vpath_reg->rts_access_steer_data0);
2875                         ((u64 *)product_desc)[j++] = be64_to_cpu(data1);
2876
2877                         data2 = readq(&vpath_reg->rts_access_steer_data1);
2878                         ((u64 *)product_desc)[j++] = be64_to_cpu(data2);
2879
2880                         status = VXGE_HW_OK;
2881                 } else
2882                         *product_desc = 0;
2883         }
2884
2885         return status;
2886 }
2887
2888 /*
2889  * __vxge_hw_vpath_fw_ver_get - Get the fw version
2890  * Returns FW Version
2891  */
2892 enum vxge_hw_status
2893 __vxge_hw_vpath_fw_ver_get(
2894         u32 vp_id,
2895         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2896         struct vxge_hw_device_hw_info *hw_info)
2897 {
2898         u64 val64;
2899         u64 data1 = 0ULL;
2900         u64 data2 = 0ULL;
2901         struct vxge_hw_device_version *fw_version = &hw_info->fw_version;
2902         struct vxge_hw_device_date *fw_date = &hw_info->fw_date;
2903         struct vxge_hw_device_version *flash_version = &hw_info->flash_version;
2904         struct vxge_hw_device_date *flash_date = &hw_info->flash_date;
2905         enum vxge_hw_status status = VXGE_HW_OK;
2906
2907         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2908                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY) |
2909                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2910                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2911                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2912                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2913
2914         status = __vxge_hw_pio_mem_write64(val64,
2915                                 &vpath_reg->rts_access_steer_ctrl,
2916                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2917                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2918
2919         if (status != VXGE_HW_OK)
2920                 goto exit;
2921
2922         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2923
2924         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2925
2926                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2927                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2928
2929                 fw_date->day =
2930                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_DAY(
2931                                                 data1);
2932                 fw_date->month =
2933                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MONTH(
2934                                                 data1);
2935                 fw_date->year =
2936                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_YEAR(
2937                                                 data1);
2938
2939                 snprintf(fw_date->date, VXGE_HW_FW_STRLEN, "%2.2d/%2.2d/%4.4d",
2940                         fw_date->month, fw_date->day, fw_date->year);
2941
2942                 fw_version->major =
2943                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MAJOR(data1);
2944                 fw_version->minor =
2945                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MINOR(data1);
2946                 fw_version->build =
2947                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_BUILD(data1);
2948
2949                 snprintf(fw_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2950                     fw_version->major, fw_version->minor, fw_version->build);
2951
2952                 flash_date->day =
2953                   (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_DAY(data2);
2954                 flash_date->month =
2955                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MONTH(data2);
2956                 flash_date->year =
2957                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_YEAR(data2);
2958
2959                 snprintf(flash_date->date, VXGE_HW_FW_STRLEN,
2960                         "%2.2d/%2.2d/%4.4d",
2961                         flash_date->month, flash_date->day, flash_date->year);
2962
2963                 flash_version->major =
2964                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MAJOR(data2);
2965                 flash_version->minor =
2966                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MINOR(data2);
2967                 flash_version->build =
2968                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_BUILD(data2);
2969
2970                 snprintf(flash_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2971                         flash_version->major, flash_version->minor,
2972                         flash_version->build);
2973
2974                 status = VXGE_HW_OK;
2975
2976         } else
2977                 status = VXGE_HW_FAIL;
2978 exit:
2979         return status;
2980 }
2981
2982 /*
2983  * __vxge_hw_vpath_pci_func_mode_get - Get the pci mode
2984  * Returns pci function mode
2985  */
2986 u64
2987 __vxge_hw_vpath_pci_func_mode_get(
2988         u32  vp_id,
2989         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2990 {
2991         u64 val64;
2992         u64 data1 = 0ULL;
2993         enum vxge_hw_status status = VXGE_HW_OK;
2994
2995         __vxge_hw_read_rts_ds(vpath_reg,
2996                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PCI_MODE);
2997
2998         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2999                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
3000                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3001                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3002                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3003                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3004
3005         status = __vxge_hw_pio_mem_write64(val64,
3006                                 &vpath_reg->rts_access_steer_ctrl,
3007                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3008                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3009
3010         if (status != VXGE_HW_OK)
3011                 goto exit;
3012
3013         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3014
3015         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3016                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3017                 status = VXGE_HW_OK;
3018         } else {
3019                 data1 = 0;
3020                 status = VXGE_HW_FAIL;
3021         }
3022 exit:
3023         return data1;
3024 }
3025
3026 /**
3027  * vxge_hw_device_flick_link_led - Flick (blink) link LED.
3028  * @hldev: HW device.
3029  * @on_off: TRUE if flickering to be on, FALSE to be off
3030  *
3031  * Flicker the link LED.
3032  */
3033 enum vxge_hw_status
3034 vxge_hw_device_flick_link_led(struct __vxge_hw_device *hldev,
3035                                u64 on_off)
3036 {
3037         u64 val64;
3038         enum vxge_hw_status status = VXGE_HW_OK;
3039         struct vxge_hw_vpath_reg __iomem *vp_reg;
3040
3041         if (hldev == NULL) {
3042                 status = VXGE_HW_ERR_INVALID_DEVICE;
3043                 goto exit;
3044         }
3045
3046         vp_reg = hldev->vpath_reg[hldev->first_vp_id];
3047
3048         writeq(0, &vp_reg->rts_access_steer_ctrl);
3049         wmb();
3050         writeq(on_off, &vp_reg->rts_access_steer_data0);
3051         writeq(0, &vp_reg->rts_access_steer_data1);
3052         wmb();
3053
3054         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3055                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LED_CONTROL) |
3056                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3057                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3058                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3059                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3060
3061         status = __vxge_hw_pio_mem_write64(val64,
3062                                 &vp_reg->rts_access_steer_ctrl,
3063                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3064                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3065 exit:
3066         return status;
3067 }
3068
3069 /*
3070  * __vxge_hw_vpath_rts_table_get - Get the entries from RTS access tables
3071  */
3072 enum vxge_hw_status
3073 __vxge_hw_vpath_rts_table_get(
3074         struct __vxge_hw_vpath_handle *vp,
3075         u32 action, u32 rts_table, u32 offset, u64 *data1, u64 *data2)
3076 {
3077         u64 val64;
3078         struct __vxge_hw_virtualpath *vpath;
3079         struct vxge_hw_vpath_reg __iomem *vp_reg;
3080
3081         enum vxge_hw_status status = VXGE_HW_OK;
3082
3083         if (vp == NULL) {
3084                 status = VXGE_HW_ERR_INVALID_HANDLE;
3085                 goto exit;
3086         }
3087
3088         vpath = vp->vpath;
3089         vp_reg = vpath->vp_reg;
3090
3091         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3092                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3093                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3094                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3095
3096         if ((rts_table ==
3097                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT) ||
3098             (rts_table ==
3099                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT) ||
3100             (rts_table ==
3101                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MASK) ||
3102             (rts_table ==
3103                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_KEY)) {
3104                 val64 = val64 | VXGE_HW_RTS_ACCESS_STEER_CTRL_TABLE_SEL;
3105         }
3106
3107         status = __vxge_hw_pio_mem_write64(val64,
3108                                 &vp_reg->rts_access_steer_ctrl,
3109                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3110                                 vpath->hldev->config.device_poll_millis);
3111
3112         if (status != VXGE_HW_OK)
3113                 goto exit;
3114
3115         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3116
3117         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3118
3119                 *data1 = readq(&vp_reg->rts_access_steer_data0);
3120
3121                 if ((rts_table ==
3122                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3123                 (rts_table ==
3124                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3125                         *data2 = readq(&vp_reg->rts_access_steer_data1);
3126                 }
3127                 status = VXGE_HW_OK;
3128         } else
3129                 status = VXGE_HW_FAIL;
3130 exit:
3131         return status;
3132 }
3133
3134 /*
3135  * __vxge_hw_vpath_rts_table_set - Set the entries of RTS access tables
3136  */
3137 enum vxge_hw_status
3138 __vxge_hw_vpath_rts_table_set(
3139         struct __vxge_hw_vpath_handle *vp, u32 action, u32 rts_table,
3140         u32 offset, u64 data1, u64 data2)
3141 {
3142         u64 val64;
3143         struct __vxge_hw_virtualpath *vpath;
3144         enum vxge_hw_status status = VXGE_HW_OK;
3145         struct vxge_hw_vpath_reg __iomem *vp_reg;
3146
3147         if (vp == NULL) {
3148                 status = VXGE_HW_ERR_INVALID_HANDLE;
3149                 goto exit;
3150         }
3151
3152         vpath = vp->vpath;
3153         vp_reg = vpath->vp_reg;
3154
3155         writeq(data1, &vp_reg->rts_access_steer_data0);
3156         wmb();
3157
3158         if ((rts_table == VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3159             (rts_table ==
3160                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3161                 writeq(data2, &vp_reg->rts_access_steer_data1);
3162                 wmb();
3163         }
3164
3165         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3166                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3167                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3168                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3169
3170         status = __vxge_hw_pio_mem_write64(val64,
3171                                 &vp_reg->rts_access_steer_ctrl,
3172                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3173                                 vpath->hldev->config.device_poll_millis);
3174
3175         if (status != VXGE_HW_OK)
3176                 goto exit;
3177
3178         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3179
3180         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS)
3181                 status = VXGE_HW_OK;
3182         else
3183                 status = VXGE_HW_FAIL;
3184 exit:
3185         return status;
3186 }
3187
3188 /*
3189  * __vxge_hw_vpath_addr_get - Get the hw address entry for this vpath
3190  *               from MAC address table.
3191  */
3192 enum vxge_hw_status
3193 __vxge_hw_vpath_addr_get(
3194         u32 vp_id, struct vxge_hw_vpath_reg __iomem *vpath_reg,
3195         u8 (macaddr)[ETH_ALEN], u8 (macaddr_mask)[ETH_ALEN])
3196 {
3197         u32 i;
3198         u64 val64;
3199         u64 data1 = 0ULL;
3200         u64 data2 = 0ULL;
3201         enum vxge_hw_status status = VXGE_HW_OK;
3202
3203         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3204                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LIST_FIRST_ENTRY) |
3205                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3206                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) |
3207                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3208                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3209
3210         status = __vxge_hw_pio_mem_write64(val64,
3211                                 &vpath_reg->rts_access_steer_ctrl,
3212                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3213                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3214
3215         if (status != VXGE_HW_OK)
3216                 goto exit;
3217
3218         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3219
3220         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3221
3222                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3223                 data2 = readq(&vpath_reg->rts_access_steer_data1);
3224
3225                 data1 = VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_DA_MAC_ADDR(data1);
3226                 data2 = VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_DA_MAC_ADDR_MASK(
3227                                                         data2);
3228
3229                 for (i = ETH_ALEN; i > 0; i--) {
3230                         macaddr[i-1] = (u8)(data1 & 0xFF);
3231                         data1 >>= 8;
3232
3233                         macaddr_mask[i-1] = (u8)(data2 & 0xFF);
3234                         data2 >>= 8;
3235                 }
3236                 status = VXGE_HW_OK;
3237         } else
3238                 status = VXGE_HW_FAIL;
3239 exit:
3240         return status;
3241 }
3242
3243 /*
3244  * vxge_hw_vpath_rts_rth_set - Set/configure RTS hashing.
3245  */
3246 enum vxge_hw_status vxge_hw_vpath_rts_rth_set(
3247                         struct __vxge_hw_vpath_handle *vp,
3248                         enum vxge_hw_rth_algoritms algorithm,
3249                         struct vxge_hw_rth_hash_types *hash_type,
3250                         u16 bucket_size)
3251 {
3252         u64 data0, data1;
3253         enum vxge_hw_status status = VXGE_HW_OK;
3254
3255         if (vp == NULL) {
3256                 status = VXGE_HW_ERR_INVALID_HANDLE;
3257                 goto exit;
3258         }
3259
3260         status = __vxge_hw_vpath_rts_table_get(vp,
3261                      VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY,
3262                      VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3263                         0, &data0, &data1);
3264
3265         data0 &= ~(VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(0xf) |
3266                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(0x3));
3267
3268         data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_EN |
3269         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(bucket_size) |
3270         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(algorithm);
3271
3272         if (hash_type->hash_type_tcpipv4_en)
3273                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV4_EN;
3274
3275         if (hash_type->hash_type_ipv4_en)
3276                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV4_EN;
3277
3278         if (hash_type->hash_type_tcpipv6_en)
3279                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EN;
3280
3281         if (hash_type->hash_type_ipv6_en)
3282                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EN;
3283
3284         if (hash_type->hash_type_tcpipv6ex_en)
3285                 data0 |=
3286                 VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EX_EN;
3287
3288         if (hash_type->hash_type_ipv6ex_en)
3289                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EX_EN;
3290
3291         if (VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_RTH_GEN_ACTIVE_TABLE(data0))
3292                 data0 &= ~VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3293         else
3294                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3295
3296         status = __vxge_hw_vpath_rts_table_set(vp,
3297                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY,
3298                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3299                 0, data0, 0);
3300 exit:
3301         return status;
3302 }
3303
3304 static void
3305 vxge_hw_rts_rth_data0_data1_get(u32 j, u64 *data0, u64 *data1,
3306                                 u16 flag, u8 *itable)
3307 {
3308         switch (flag) {
3309         case 1:
3310                 *data0 = VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_NUM(j)|
3311                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_ENTRY_EN |
3312                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_DATA(
3313                         itable[j]);
3314         case 2:
3315                 *data0 |=
3316                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_NUM(j)|
3317                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_ENTRY_EN |
3318                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_DATA(
3319                         itable[j]);
3320         case 3:
3321                 *data1 = VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_NUM(j)|
3322                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_ENTRY_EN |
3323                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_DATA(
3324                         itable[j]);
3325         case 4:
3326                 *data1 |=
3327                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_NUM(j)|
3328                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_ENTRY_EN |
3329                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_DATA(
3330                         itable[j]);
3331         default:
3332                 return;
3333         }
3334 }
3335 /*
3336  * vxge_hw_vpath_rts_rth_itable_set - Set/configure indirection table (IT).
3337  */
3338 enum vxge_hw_status vxge_hw_vpath_rts_rth_itable_set(
3339                         struct __vxge_hw_vpath_handle **vpath_handles,
3340                         u32 vpath_count,
3341                         u8 *mtable,
3342                         u8 *itable,
3343                         u32 itable_size)
3344 {
3345         u32 i, j, action, rts_table;
3346         u64 data0;
3347         u64 data1;
3348         u32 max_entries;
3349         enum vxge_hw_status status = VXGE_HW_OK;
3350         struct __vxge_hw_vpath_handle *vp = vpath_handles[0];
3351
3352         if (vp == NULL) {
3353                 status = VXGE_HW_ERR_INVALID_HANDLE;
3354                 goto exit;
3355         }
3356
3357         max_entries = (((u32)1) << itable_size);
3358
3359         if (vp->vpath->hldev->config.rth_it_type
3360                                 == VXGE_HW_RTH_IT_TYPE_SOLO_IT) {
3361                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3362                 rts_table =
3363                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT;
3364
3365                 for (j = 0; j < max_entries; j++) {
3366
3367                         data1 = 0;
3368
3369                         data0 =
3370                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3371                                 itable[j]);
3372
3373                         status = __vxge_hw_vpath_rts_table_set(vpath_handles[0],
3374                                 action, rts_table, j, data0, data1);
3375
3376                         if (status != VXGE_HW_OK)
3377                                 goto exit;
3378                 }
3379
3380                 for (j = 0; j < max_entries; j++) {
3381
3382                         data1 = 0;
3383
3384                         data0 =
3385                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_ENTRY_EN |
3386                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3387                                 itable[j]);
3388
3389                         status = __vxge_hw_vpath_rts_table_set(
3390                                 vpath_handles[mtable[itable[j]]], action,
3391                                 rts_table, j, data0, data1);
3392
3393                         if (status != VXGE_HW_OK)
3394                                 goto exit;
3395                 }
3396         } else {
3397                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3398                 rts_table =
3399                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT;
3400                 for (i = 0; i < vpath_count; i++) {
3401
3402                         for (j = 0; j < max_entries;) {
3403
3404                                 data0 = 0;
3405                                 data1 = 0;
3406
3407                                 while (j < max_entries) {
3408                                         if (mtable[itable[j]] != i) {
3409                                                 j++;
3410                                                 continue;
3411                                         }
3412                                         vxge_hw_rts_rth_data0_data1_get(j,
3413                                                 &data0, &data1, 1, itable);
3414                                         j++;
3415                                         break;
3416                                 }
3417
3418                                 while (j < max_entries) {
3419                                         if (mtable[itable[j]] != i) {
3420                                                 j++;
3421                                                 continue;
3422                                         }
3423                                         vxge_hw_rts_rth_data0_data1_get(j,
3424                                                 &data0, &data1, 2, itable);
3425                                         j++;
3426                                         break;
3427                                 }
3428
3429                                 while (j < max_entries) {
3430                                         if (mtable[itable[j]] != i) {
3431                                                 j++;
3432                                                 continue;
3433                                         }
3434                                         vxge_hw_rts_rth_data0_data1_get(j,
3435                                                 &data0, &data1, 3, itable);
3436                                         j++;
3437                                         break;
3438                                 }
3439
3440                                 while (j < max_entries) {
3441                                         if (mtable[itable[j]] != i) {
3442                                                 j++;
3443                                                 continue;
3444                                         }
3445                                         vxge_hw_rts_rth_data0_data1_get(j,
3446                                                 &data0, &data1, 4, itable);
3447                                         j++;
3448                                         break;
3449                                 }
3450
3451                                 if (data0 != 0) {
3452                                         status = __vxge_hw_vpath_rts_table_set(
3453                                                         vpath_handles[i],
3454                                                         action, rts_table,
3455                                                         0, data0, data1);
3456
3457                                         if (status != VXGE_HW_OK)
3458                                                 goto exit;
3459                                 }
3460                         }
3461                 }
3462         }
3463 exit:
3464         return status;
3465 }
3466
3467 /**
3468  * vxge_hw_vpath_check_leak - Check for memory leak
3469  * @ringh: Handle to the ring object used for receive
3470  *
3471  * If PRC_RXD_DOORBELL_VPn.NEW_QW_CNT is larger or equal to
3472  * PRC_CFG6_VPn.RXD_SPAT then a leak has occurred.
3473  * Returns: VXGE_HW_FAIL, if leak has occurred.
3474  *
3475  */
3476 enum vxge_hw_status
3477 vxge_hw_vpath_check_leak(struct __vxge_hw_ring *ring)
3478 {
3479         enum vxge_hw_status status = VXGE_HW_OK;
3480         u64 rxd_new_count, rxd_spat;
3481
3482         if (ring == NULL)
3483                 return status;
3484
3485         rxd_new_count = readl(&ring->vp_reg->prc_rxd_doorbell);
3486         rxd_spat = readq(&ring->vp_reg->prc_cfg6);
3487         rxd_spat = VXGE_HW_PRC_CFG6_RXD_SPAT(rxd_spat);
3488
3489         if (rxd_new_count >= rxd_spat)
3490                 status = VXGE_HW_FAIL;
3491
3492         return status;
3493 }
3494
3495 /*
3496  * __vxge_hw_vpath_mgmt_read
3497  * This routine reads the vpath_mgmt registers
3498  */
3499 static enum vxge_hw_status
3500 __vxge_hw_vpath_mgmt_read(
3501         struct __vxge_hw_device *hldev,
3502         struct __vxge_hw_virtualpath *vpath)
3503 {
3504         u32 i, mtu = 0, max_pyld = 0;
3505         u64 val64;
3506         enum vxge_hw_status status = VXGE_HW_OK;
3507
3508         for (i = 0; i < VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
3509
3510                 val64 = readq(&vpath->vpmgmt_reg->
3511                                 rxmac_cfg0_port_vpmgmt_clone[i]);
3512                 max_pyld =
3513                         (u32)
3514                         VXGE_HW_RXMAC_CFG0_PORT_VPMGMT_CLONE_GET_MAX_PYLD_LEN
3515                         (val64);
3516                 if (mtu < max_pyld)
3517                         mtu = max_pyld;
3518         }
3519
3520         vpath->max_mtu = mtu + VXGE_HW_MAC_HEADER_MAX_SIZE;
3521
3522         val64 = readq(&vpath->vpmgmt_reg->xmac_vsport_choices_vp);
3523
3524         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
3525                 if (val64 & vxge_mBIT(i))
3526                         vpath->vsport_number = i;
3527         }
3528
3529         val64 = readq(&vpath->vpmgmt_reg->xgmac_gen_status_vpmgmt_clone);
3530
3531         if (val64 & VXGE_HW_XGMAC_GEN_STATUS_VPMGMT_CLONE_XMACJ_NTWK_OK)
3532                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_UP);
3533         else
3534                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_DOWN);
3535
3536         return status;
3537 }
3538
3539 /*
3540  * __vxge_hw_vpath_reset_check - Check if resetting the vpath completed
3541  * This routine checks the vpath_rst_in_prog register to see if
3542  * adapter completed the reset process for the vpath
3543  */
3544 enum vxge_hw_status
3545 __vxge_hw_vpath_reset_check(struct __vxge_hw_virtualpath *vpath)
3546 {
3547         enum vxge_hw_status status;
3548
3549         status = __vxge_hw_device_register_poll(
3550                         &vpath->hldev->common_reg->vpath_rst_in_prog,
3551                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(
3552                                 1 << (16 - vpath->vp_id)),
3553                         vpath->hldev->config.device_poll_millis);
3554
3555         return status;
3556 }
3557
3558 /*
3559  * __vxge_hw_vpath_reset
3560  * This routine resets the vpath on the device
3561  */
3562 enum vxge_hw_status
3563 __vxge_hw_vpath_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3564 {
3565         u64 val64;
3566         enum vxge_hw_status status = VXGE_HW_OK;
3567
3568         val64 = VXGE_HW_CMN_RSTHDLR_CFG0_SW_RESET_VPATH(1 << (16 - vp_id));
3569
3570         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
3571                                 &hldev->common_reg->cmn_rsthdlr_cfg0);
3572
3573         return status;
3574 }
3575
3576 /*
3577  * __vxge_hw_vpath_sw_reset
3578  * This routine resets the vpath structures
3579  */
3580 enum vxge_hw_status
3581 __vxge_hw_vpath_sw_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3582 {
3583         enum vxge_hw_status status = VXGE_HW_OK;
3584         struct __vxge_hw_virtualpath *vpath;
3585
3586         vpath = (struct __vxge_hw_virtualpath *)&hldev->virtual_paths[vp_id];
3587
3588         if (vpath->ringh) {
3589                 status = __vxge_hw_ring_reset(vpath->ringh);
3590                 if (status != VXGE_HW_OK)
3591                         goto exit;
3592         }
3593
3594         if (vpath->fifoh)
3595                 status = __vxge_hw_fifo_reset(vpath->fifoh);
3596 exit:
3597         return status;
3598 }
3599
3600 /*
3601  * __vxge_hw_vpath_prc_configure
3602  * This routine configures the prc registers of virtual path using the config
3603  * passed
3604  */
3605 void
3606 __vxge_hw_vpath_prc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3607 {
3608         u64 val64;
3609         struct __vxge_hw_virtualpath *vpath;
3610         struct vxge_hw_vp_config *vp_config;
3611         struct vxge_hw_vpath_reg __iomem *vp_reg;
3612
3613         vpath = &hldev->virtual_paths[vp_id];
3614         vp_reg = vpath->vp_reg;
3615         vp_config = vpath->vp_config;
3616
3617         if (vp_config->ring.enable == VXGE_HW_RING_DISABLE)
3618                 return;
3619
3620         val64 = readq(&vp_reg->prc_cfg1);
3621         val64 |= VXGE_HW_PRC_CFG1_RTI_TINT_DISABLE;
3622         writeq(val64, &vp_reg->prc_cfg1);
3623
3624         val64 = readq(&vpath->vp_reg->prc_cfg6);
3625         val64 |= VXGE_HW_PRC_CFG6_DOORBELL_MODE_EN;
3626         writeq(val64, &vpath->vp_reg->prc_cfg6);
3627
3628         val64 = readq(&vp_reg->prc_cfg7);
3629
3630         if (vpath->vp_config->ring.scatter_mode !=
3631                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT) {
3632
3633                 val64 &= ~VXGE_HW_PRC_CFG7_SCATTER_MODE(0x3);
3634
3635                 switch (vpath->vp_config->ring.scatter_mode) {
3636                 case VXGE_HW_RING_SCATTER_MODE_A:
3637                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3638                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_A);
3639                         break;
3640                 case VXGE_HW_RING_SCATTER_MODE_B:
3641                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3642                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_B);
3643                         break;
3644                 case VXGE_HW_RING_SCATTER_MODE_C:
3645                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3646                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_C);
3647                         break;
3648                 }
3649         }
3650
3651         writeq(val64, &vp_reg->prc_cfg7);
3652
3653         writeq(VXGE_HW_PRC_CFG5_RXD0_ADD(
3654                                 __vxge_hw_ring_first_block_address_get(
3655                                         vpath->ringh) >> 3), &vp_reg->prc_cfg5);
3656
3657         val64 = readq(&vp_reg->prc_cfg4);
3658         val64 |= VXGE_HW_PRC_CFG4_IN_SVC;
3659         val64 &= ~VXGE_HW_PRC_CFG4_RING_MODE(0x3);
3660
3661         val64 |= VXGE_HW_PRC_CFG4_RING_MODE(
3662                         VXGE_HW_PRC_CFG4_RING_MODE_ONE_BUFFER);
3663
3664         if (hldev->config.rth_en == VXGE_HW_RTH_DISABLE)
3665                 val64 |= VXGE_HW_PRC_CFG4_RTH_DISABLE;
3666         else
3667                 val64 &= ~VXGE_HW_PRC_CFG4_RTH_DISABLE;
3668
3669         writeq(val64, &vp_reg->prc_cfg4);
3670         return;
3671 }
3672
3673 /*
3674  * __vxge_hw_vpath_kdfc_configure
3675  * This routine configures the kdfc registers of virtual path using the
3676  * config passed
3677  */
3678 enum vxge_hw_status
3679 __vxge_hw_vpath_kdfc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3680 {
3681         u64 val64;
3682         u64 vpath_stride;
3683         enum vxge_hw_status status = VXGE_HW_OK;
3684         struct __vxge_hw_virtualpath *vpath;
3685         struct vxge_hw_vpath_reg __iomem *vp_reg;
3686
3687         vpath = &hldev->virtual_paths[vp_id];
3688         vp_reg = vpath->vp_reg;
3689         status = __vxge_hw_kdfc_swapper_set(hldev->legacy_reg, vp_reg);
3690
3691         if (status != VXGE_HW_OK)
3692                 goto exit;
3693
3694         val64 = readq(&vp_reg->kdfc_drbl_triplet_total);
3695
3696         vpath->max_kdfc_db =
3697                 (u32)VXGE_HW_KDFC_DRBL_TRIPLET_TOTAL_GET_KDFC_MAX_SIZE(
3698                         val64+1)/2;
3699
3700         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3701
3702                 vpath->max_nofl_db = vpath->max_kdfc_db;
3703
3704                 if (vpath->max_nofl_db <
3705                         ((vpath->vp_config->fifo.memblock_size /
3706                         (vpath->vp_config->fifo.max_frags *
3707                         sizeof(struct vxge_hw_fifo_txd))) *
3708                         vpath->vp_config->fifo.fifo_blocks)) {
3709
3710                         return VXGE_HW_BADCFG_FIFO_BLOCKS;
3711                 }
3712                 val64 = VXGE_HW_KDFC_FIFO_TRPL_PARTITION_LENGTH_0(
3713                                 (vpath->max_nofl_db*2)-1);
3714         }
3715
3716         writeq(val64, &vp_reg->kdfc_fifo_trpl_partition);
3717
3718         writeq(VXGE_HW_KDFC_FIFO_TRPL_CTRL_TRIPLET_ENABLE,
3719                 &vp_reg->kdfc_fifo_trpl_ctrl);
3720
3721         val64 = readq(&vp_reg->kdfc_trpl_fifo_0_ctrl);
3722
3723         val64 &= ~(VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(0x3) |
3724                    VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0xFF));
3725
3726         val64 |= VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(
3727                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE_NON_OFFLOAD_ONLY) |
3728 #ifndef __BIG_ENDIAN
3729                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SWAP_EN |
3730 #endif
3731                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0);
3732
3733         writeq(val64, &vp_reg->kdfc_trpl_fifo_0_ctrl);
3734         writeq((u64)0, &vp_reg->kdfc_trpl_fifo_0_wb_address);
3735         wmb();
3736         vpath_stride = readq(&hldev->toc_reg->toc_kdfc_vpath_stride);
3737
3738         vpath->nofl_db =
3739                 (struct __vxge_hw_non_offload_db_wrapper __iomem *)
3740                 (hldev->kdfc + (vp_id *
3741                 VXGE_HW_TOC_KDFC_VPATH_STRIDE_GET_TOC_KDFC_VPATH_STRIDE(
3742                                         vpath_stride)));
3743 exit:
3744         return status;
3745 }
3746
3747 /*
3748  * __vxge_hw_vpath_mac_configure
3749  * This routine configures the mac of virtual path using the config passed
3750  */
3751 enum vxge_hw_status
3752 __vxge_hw_vpath_mac_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3753 {
3754         u64 val64;
3755         enum vxge_hw_status status = VXGE_HW_OK;
3756         struct __vxge_hw_virtualpath *vpath;
3757         struct vxge_hw_vp_config *vp_config;
3758         struct vxge_hw_vpath_reg __iomem *vp_reg;
3759
3760         vpath = &hldev->virtual_paths[vp_id];
3761         vp_reg = vpath->vp_reg;
3762         vp_config = vpath->vp_config;
3763
3764         writeq(VXGE_HW_XMAC_VSPORT_CHOICE_VSPORT_NUMBER(
3765                         vpath->vsport_number), &vp_reg->xmac_vsport_choice);
3766
3767         if (vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
3768
3769                 val64 = readq(&vp_reg->xmac_rpa_vcfg);
3770
3771                 if (vp_config->rpa_strip_vlan_tag !=
3772                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) {
3773                         if (vp_config->rpa_strip_vlan_tag)
3774                                 val64 |= VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3775                         else
3776                                 val64 &= ~VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3777                 }
3778
3779                 writeq(val64, &vp_reg->xmac_rpa_vcfg);
3780                 val64 = readq(&vp_reg->rxmac_vcfg0);
3781
3782                 if (vp_config->mtu !=
3783                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) {
3784                         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
3785                         if ((vp_config->mtu  +
3786                                 VXGE_HW_MAC_HEADER_MAX_SIZE) < vpath->max_mtu)
3787                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3788                                         vp_config->mtu  +
3789                                         VXGE_HW_MAC_HEADER_MAX_SIZE);
3790                         else
3791                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3792                                         vpath->max_mtu);
3793                 }
3794
3795                 writeq(val64, &vp_reg->rxmac_vcfg0);
3796
3797                 val64 = readq(&vp_reg->rxmac_vcfg1);
3798
3799                 val64 &= ~(VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(0x3) |
3800                         VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE);
3801
3802                 if (hldev->config.rth_it_type ==
3803                                 VXGE_HW_RTH_IT_TYPE_MULTI_IT) {
3804                         val64 |= VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(
3805                                 0x2) |
3806                                 VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE;
3807                 }
3808
3809                 writeq(val64, &vp_reg->rxmac_vcfg1);
3810         }
3811         return status;
3812 }
3813
3814 /*
3815  * __vxge_hw_vpath_tim_configure
3816  * This routine configures the tim registers of virtual path using the config
3817  * passed
3818  */
3819 enum vxge_hw_status
3820 __vxge_hw_vpath_tim_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3821 {
3822         u64 val64;
3823         enum vxge_hw_status status = VXGE_HW_OK;
3824         struct __vxge_hw_virtualpath *vpath;
3825         struct vxge_hw_vpath_reg __iomem *vp_reg;
3826         struct vxge_hw_vp_config *config;
3827
3828         vpath = &hldev->virtual_paths[vp_id];
3829         vp_reg = vpath->vp_reg;
3830         config = vpath->vp_config;
3831
3832         writeq((u64)0, &vp_reg->tim_dest_addr);
3833         writeq((u64)0, &vp_reg->tim_vpath_map);
3834         writeq((u64)0, &vp_reg->tim_bitmap);
3835         writeq((u64)0, &vp_reg->tim_remap);
3836
3837         if (config->ring.enable == VXGE_HW_RING_ENABLE)
3838                 writeq(VXGE_HW_TIM_RING_ASSN_INT_NUM(
3839                         (vp_id * VXGE_HW_MAX_INTR_PER_VP) +
3840                         VXGE_HW_VPATH_INTR_RX), &vp_reg->tim_ring_assn);
3841
3842         val64 = readq(&vp_reg->tim_pci_cfg);
3843         val64 |= VXGE_HW_TIM_PCI_CFG_ADD_PAD;
3844         writeq(val64, &vp_reg->tim_pci_cfg);
3845
3846         if (config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3847
3848                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3849
3850                 if (config->tti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3851                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3852                                 0x3ffffff);
3853                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3854                                         config->tti.btimer_val);
3855                 }
3856
3857                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3858
3859                 if (config->tti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3860                         if (config->tti.timer_ac_en)
3861                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3862                         else
3863                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3864                 }
3865
3866                 if (config->tti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3867                         if (config->tti.timer_ci_en)
3868                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3869                         else
3870                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3871                 }
3872
3873                 if (config->tti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3874                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3875                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3876                                         config->tti.urange_a);
3877                 }
3878
3879                 if (config->tti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3880                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3881                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3882                                         config->tti.urange_b);
3883                 }
3884
3885                 if (config->tti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3886                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3887                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3888                                         config->tti.urange_c);
3889                 }
3890
3891                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3892                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3893
3894                 if (config->tti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
3895                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
3896                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
3897                                                 config->tti.uec_a);
3898                 }
3899
3900                 if (config->tti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
3901                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
3902                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
3903                                                 config->tti.uec_b);
3904                 }
3905
3906                 if (config->tti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
3907                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
3908                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
3909                                                 config->tti.uec_c);
3910                 }
3911
3912                 if (config->tti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
3913                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
3914                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
3915                                                 config->tti.uec_d);
3916                 }
3917
3918                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3919                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3920
3921                 if (config->tti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
3922                         if (config->tti.timer_ri_en)
3923                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3924                         else
3925                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3926                 }
3927
3928                 if (config->tti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3929                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3930                                         0x3ffffff);
3931                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3932                                         config->tti.rtimer_val);
3933                 }
3934
3935                 if (config->tti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
3936                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
3937                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
3938                                         config->tti.util_sel);
3939                 }
3940
3941                 if (config->tti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3942                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3943                                         0x3ffffff);
3944                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3945                                         config->tti.ltimer_val);
3946                 }
3947
3948                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3949         }
3950
3951         if (config->ring.enable == VXGE_HW_RING_ENABLE) {
3952
3953                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3954
3955                 if (config->rti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3956                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3957                                         0x3ffffff);
3958                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3959                                         config->rti.btimer_val);
3960                 }
3961
3962                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3963
3964                 if (config->rti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3965                         if (config->rti.timer_ac_en)
3966                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3967                         else
3968                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3969                 }
3970
3971                 if (config->rti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3972                         if (config->rti.timer_ci_en)
3973                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3974                         else
3975                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3976                 }
3977
3978                 if (config->rti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3979                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3980                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3981                                         config->rti.urange_a);
3982                 }
3983
3984                 if (config->rti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3985                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3986                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3987                                         config->rti.urange_b);
3988                 }
3989
3990                 if (config->rti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3991                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3992                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3993                                         config->rti.urange_c);
3994                 }
3995
3996                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3997                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
3998
3999                 if (config->rti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
4000                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
4001                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
4002                                                 config->rti.uec_a);
4003                 }
4004
4005                 if (config->rti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
4006                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
4007                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
4008                                                 config->rti.uec_b);
4009                 }
4010
4011                 if (config->rti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
4012                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
4013                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
4014                                                 config->rti.uec_c);
4015                 }
4016
4017                 if (config->rti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
4018                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
4019                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
4020                                                 config->rti.uec_d);
4021                 }
4022
4023                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
4024                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4025
4026                 if (config->rti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
4027                         if (config->rti.timer_ri_en)
4028                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4029                         else
4030                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4031                 }
4032
4033                 if (config->rti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4034                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4035                                         0x3ffffff);
4036                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4037                                         config->rti.rtimer_val);
4038                 }
4039
4040                 if (config->rti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
4041                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
4042                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
4043                                         config->rti.util_sel);
4044                 }
4045
4046                 if (config->rti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4047                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4048                                         0x3ffffff);
4049                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4050                                         config->rti.ltimer_val);
4051                 }
4052
4053                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4054         }
4055
4056         val64 = 0;
4057         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4058         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4059         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4060         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4061         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4062         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4063
4064         return status;
4065 }
4066
4067 /*
4068  * __vxge_hw_vpath_initialize
4069  * This routine is the final phase of init which initializes the
4070  * registers of the vpath using the configuration passed.
4071  */
4072 enum vxge_hw_status
4073 __vxge_hw_vpath_initialize(struct __vxge_hw_device *hldev, u32 vp_id)
4074 {
4075         u64 val64;
4076         u32 val32;
4077         enum vxge_hw_status status = VXGE_HW_OK;
4078         struct __vxge_hw_virtualpath *vpath;
4079         struct vxge_hw_vpath_reg __iomem *vp_reg;
4080
4081         vpath = &hldev->virtual_paths[vp_id];
4082
4083         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4084                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4085                 goto exit;
4086         }
4087         vp_reg = vpath->vp_reg;
4088
4089         status =  __vxge_hw_vpath_swapper_set(vpath->vp_reg);
4090
4091         if (status != VXGE_HW_OK)
4092                 goto exit;
4093
4094         status =  __vxge_hw_vpath_mac_configure(hldev, vp_id);
4095
4096         if (status != VXGE_HW_OK)
4097                 goto exit;
4098
4099         status =  __vxge_hw_vpath_kdfc_configure(hldev, vp_id);
4100
4101         if (status != VXGE_HW_OK)
4102                 goto exit;
4103
4104         status = __vxge_hw_vpath_tim_configure(hldev, vp_id);
4105
4106         if (status != VXGE_HW_OK)
4107                 goto exit;
4108
4109         writeq(0, &vp_reg->gendma_int);
4110
4111         val64 = readq(&vp_reg->rtdma_rd_optimization_ctrl);
4112
4113         /* Get MRRS value from device control */
4114         status  = __vxge_hw_vpath_pci_read(vpath, 1, 0x78, &val32);
4115
4116         if (status == VXGE_HW_OK) {
4117                 val32 = (val32 & VXGE_HW_PCI_EXP_DEVCTL_READRQ) >> 12;
4118                 val64 &=
4119                     ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(7));
4120                 val64 |=
4121                     VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(val32);
4122
4123                 val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_WAIT_FOR_SPACE;
4124         }
4125
4126         val64 &= ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(7));
4127         val64 |=
4128             VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(
4129                     VXGE_HW_MAX_PAYLOAD_SIZE_512);
4130
4131         val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY_EN;
4132         writeq(val64, &vp_reg->rtdma_rd_optimization_ctrl);
4133
4134 exit:
4135         return status;
4136 }
4137
4138 /*
4139  * __vxge_hw_vp_initialize - Initialize Virtual Path structure
4140  * This routine is the initial phase of init which resets the vpath and
4141  * initializes the software support structures.
4142  */
4143 enum vxge_hw_status
4144 __vxge_hw_vp_initialize(struct __vxge_hw_device *hldev, u32 vp_id,
4145                         struct vxge_hw_vp_config *config)
4146 {
4147         struct __vxge_hw_virtualpath *vpath;
4148         enum vxge_hw_status status = VXGE_HW_OK;
4149
4150         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4151                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4152                 goto exit;
4153         }
4154
4155         vpath = &hldev->virtual_paths[vp_id];
4156
4157         vpath->vp_id = vp_id;
4158         vpath->vp_open = VXGE_HW_VP_OPEN;
4159         vpath->hldev = hldev;
4160         vpath->vp_config = config;
4161         vpath->vp_reg = hldev->vpath_reg[vp_id];
4162         vpath->vpmgmt_reg = hldev->vpmgmt_reg[vp_id];
4163
4164         __vxge_hw_vpath_reset(hldev, vp_id);
4165
4166         status = __vxge_hw_vpath_reset_check(vpath);
4167
4168         if (status != VXGE_HW_OK) {
4169                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4170                 goto exit;
4171         }
4172
4173         status = __vxge_hw_vpath_mgmt_read(hldev, vpath);
4174
4175         if (status != VXGE_HW_OK) {
4176                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4177                 goto exit;
4178         }
4179
4180         INIT_LIST_HEAD(&vpath->vpath_handles);
4181
4182         vpath->sw_stats = &hldev->stats.sw_dev_info_stats.vpath_info[vp_id];
4183
4184         VXGE_HW_DEVICE_TIM_INT_MASK_SET(hldev->tim_int_mask0,
4185                 hldev->tim_int_mask1, vp_id);
4186
4187         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4188
4189         if (status != VXGE_HW_OK)
4190                 __vxge_hw_vp_terminate(hldev, vp_id);
4191 exit:
4192         return status;
4193 }
4194
4195 /*
4196  * __vxge_hw_vp_terminate - Terminate Virtual Path structure
4197  * This routine closes all channels it opened and freeup memory
4198  */
4199 void
4200 __vxge_hw_vp_terminate(struct __vxge_hw_device *hldev, u32 vp_id)
4201 {
4202         struct __vxge_hw_virtualpath *vpath;
4203
4204         vpath = &hldev->virtual_paths[vp_id];
4205
4206         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN)
4207                 goto exit;
4208
4209         VXGE_HW_DEVICE_TIM_INT_MASK_RESET(vpath->hldev->tim_int_mask0,
4210                 vpath->hldev->tim_int_mask1, vpath->vp_id);
4211         hldev->stats.hw_dev_info_stats.vpath_info[vpath->vp_id] = NULL;
4212
4213         memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4214 exit:
4215         return;
4216 }
4217
4218 /*
4219  * vxge_hw_vpath_mtu_set - Set MTU.
4220  * Set new MTU value. Example, to use jumbo frames:
4221  * vxge_hw_vpath_mtu_set(my_device, 9600);
4222  */
4223 enum vxge_hw_status
4224 vxge_hw_vpath_mtu_set(struct __vxge_hw_vpath_handle *vp, u32 new_mtu)
4225 {
4226         u64 val64;
4227         enum vxge_hw_status status = VXGE_HW_OK;
4228         struct __vxge_hw_virtualpath *vpath;
4229
4230         if (vp == NULL) {
4231                 status = VXGE_HW_ERR_INVALID_HANDLE;
4232                 goto exit;
4233         }
4234         vpath = vp->vpath;
4235
4236         new_mtu += VXGE_HW_MAC_HEADER_MAX_SIZE;
4237
4238         if ((new_mtu < VXGE_HW_MIN_MTU) || (new_mtu > vpath->max_mtu))
4239                 status = VXGE_HW_ERR_INVALID_MTU_SIZE;
4240
4241         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
4242
4243         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
4244         val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(new_mtu);
4245
4246         writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
4247
4248         vpath->vp_config->mtu = new_mtu - VXGE_HW_MAC_HEADER_MAX_SIZE;
4249
4250 exit:
4251         return status;
4252 }
4253
4254 /*
4255  * vxge_hw_vpath_open - Open a virtual path on a given adapter
4256  * This function is used to open access to virtual path of an
4257  * adapter for offload, GRO operations. This function returns
4258  * synchronously.
4259  */
4260 enum vxge_hw_status
4261 vxge_hw_vpath_open(struct __vxge_hw_device *hldev,
4262                    struct vxge_hw_vpath_attr *attr,
4263                    struct __vxge_hw_vpath_handle **vpath_handle)
4264 {
4265         struct __vxge_hw_virtualpath *vpath;
4266         struct __vxge_hw_vpath_handle *vp;
4267         enum vxge_hw_status status;
4268
4269         vpath = &hldev->virtual_paths[attr->vp_id];
4270
4271         if (vpath->vp_open == VXGE_HW_VP_OPEN) {
4272                 status = VXGE_HW_ERR_INVALID_STATE;
4273                 goto vpath_open_exit1;
4274         }
4275
4276         status = __vxge_hw_vp_initialize(hldev, attr->vp_id,
4277                         &hldev->config.vp_config[attr->vp_id]);
4278
4279         if (status != VXGE_HW_OK)
4280                 goto vpath_open_exit1;
4281
4282         vp = (struct __vxge_hw_vpath_handle *)
4283                 vmalloc(sizeof(struct __vxge_hw_vpath_handle));
4284         if (vp == NULL) {
4285                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4286                 goto vpath_open_exit2;
4287         }
4288
4289         memset(vp, 0, sizeof(struct __vxge_hw_vpath_handle));
4290
4291         vp->vpath = vpath;
4292
4293         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
4294                 status = __vxge_hw_fifo_create(vp, &attr->fifo_attr);
4295                 if (status != VXGE_HW_OK)
4296                         goto vpath_open_exit6;
4297         }
4298
4299         if (vpath->vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
4300                 status = __vxge_hw_ring_create(vp, &attr->ring_attr);
4301                 if (status != VXGE_HW_OK)
4302                         goto vpath_open_exit7;
4303
4304                 __vxge_hw_vpath_prc_configure(hldev, attr->vp_id);
4305         }
4306
4307         vpath->fifoh->tx_intr_num =
4308                 (attr->vp_id * VXGE_HW_MAX_INTR_PER_VP)  +
4309                         VXGE_HW_VPATH_INTR_TX;
4310
4311         vpath->stats_block = __vxge_hw_blockpool_block_allocate(hldev,
4312                                 VXGE_HW_BLOCK_SIZE);
4313
4314         if (vpath->stats_block == NULL) {
4315                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4316                 goto vpath_open_exit8;
4317         }
4318
4319         vpath->hw_stats = (struct vxge_hw_vpath_stats_hw_info *)vpath->
4320                         stats_block->memblock;
4321         memset(vpath->hw_stats, 0,
4322                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4323
4324         hldev->stats.hw_dev_info_stats.vpath_info[attr->vp_id] =
4325                                                 vpath->hw_stats;
4326
4327         vpath->hw_stats_sav =
4328                 &hldev->stats.hw_dev_info_stats.vpath_info_sav[attr->vp_id];
4329         memset(vpath->hw_stats_sav, 0,
4330                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4331
4332         writeq(vpath->stats_block->dma_addr, &vpath->vp_reg->stats_cfg);
4333
4334         status = vxge_hw_vpath_stats_enable(vp);
4335         if (status != VXGE_HW_OK)
4336                 goto vpath_open_exit8;
4337
4338         list_add(&vp->item, &vpath->vpath_handles);
4339
4340         hldev->vpaths_deployed |= vxge_mBIT(vpath->vp_id);
4341
4342         *vpath_handle = vp;
4343
4344         attr->fifo_attr.userdata = vpath->fifoh;
4345         attr->ring_attr.userdata = vpath->ringh;
4346
4347         return VXGE_HW_OK;
4348
4349 vpath_open_exit8:
4350         if (vpath->ringh != NULL)
4351                 __vxge_hw_ring_delete(vp);
4352 vpath_open_exit7:
4353         if (vpath->fifoh != NULL)
4354                 __vxge_hw_fifo_delete(vp);
4355 vpath_open_exit6:
4356         vfree(vp);
4357 vpath_open_exit2:
4358         __vxge_hw_vp_terminate(hldev, attr->vp_id);
4359 vpath_open_exit1:
4360
4361         return status;
4362 }
4363
4364 /**
4365  * vxge_hw_vpath_rx_doorbell_post - Close the handle got from previous vpath
4366  * (vpath) open
4367  * @vp: Handle got from previous vpath open
4368  *
4369  * This function is used to close access to virtual path opened
4370  * earlier.
4371  */
4372 void
4373 vxge_hw_vpath_rx_doorbell_init(struct __vxge_hw_vpath_handle *vp)
4374 {
4375         struct __vxge_hw_virtualpath *vpath = NULL;
4376         u64 new_count, val64, val164;
4377         struct __vxge_hw_ring *ring;
4378
4379         vpath = vp->vpath;
4380         ring = vpath->ringh;
4381
4382         new_count = readq(&vpath->vp_reg->rxdmem_size);
4383         new_count &= 0x1fff;
4384         val164 = (VXGE_HW_RXDMEM_SIZE_PRC_RXDMEM_SIZE(new_count));
4385
4386         writeq(VXGE_HW_PRC_RXD_DOORBELL_NEW_QW_CNT(val164),
4387                 &vpath->vp_reg->prc_rxd_doorbell);
4388         readl(&vpath->vp_reg->prc_rxd_doorbell);
4389
4390         val164 /= 2;
4391         val64 = readq(&vpath->vp_reg->prc_cfg6);
4392         val64 = VXGE_HW_PRC_CFG6_RXD_SPAT(val64);
4393         val64 &= 0x1ff;
4394
4395         /*
4396          * Each RxD is of 4 qwords
4397          */
4398         new_count -= (val64 + 1);
4399         val64 = min(val164, new_count) / 4;
4400
4401         ring->rxds_limit = min(ring->rxds_limit, val64);
4402         if (ring->rxds_limit < 4)
4403                 ring->rxds_limit = 4;
4404 }
4405
4406 /*
4407  * vxge_hw_vpath_close - Close the handle got from previous vpath (vpath) open
4408  * This function is used to close access to virtual path opened
4409  * earlier.
4410  */
4411 enum vxge_hw_status vxge_hw_vpath_close(struct __vxge_hw_vpath_handle *vp)
4412 {
4413         struct __vxge_hw_virtualpath *vpath = NULL;
4414         struct __vxge_hw_device *devh = NULL;
4415         u32 vp_id = vp->vpath->vp_id;
4416         u32 is_empty = TRUE;
4417         enum vxge_hw_status status = VXGE_HW_OK;
4418
4419         vpath = vp->vpath;
4420         devh = vpath->hldev;
4421
4422         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4423                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4424                 goto vpath_close_exit;
4425         }
4426
4427         list_del(&vp->item);
4428
4429         if (!list_empty(&vpath->vpath_handles)) {
4430                 list_add(&vp->item, &vpath->vpath_handles);
4431                 is_empty = FALSE;
4432         }
4433
4434         if (!is_empty) {
4435                 status = VXGE_HW_FAIL;
4436                 goto vpath_close_exit;
4437         }
4438
4439         devh->vpaths_deployed &= ~vxge_mBIT(vp_id);
4440
4441         if (vpath->ringh != NULL)
4442                 __vxge_hw_ring_delete(vp);
4443
4444         if (vpath->fifoh != NULL)
4445                 __vxge_hw_fifo_delete(vp);
4446
4447         if (vpath->stats_block != NULL)
4448                 __vxge_hw_blockpool_block_free(devh, vpath->stats_block);
4449
4450         vfree(vp);
4451
4452         __vxge_hw_vp_terminate(devh, vp_id);
4453
4454         vpath->vp_open = VXGE_HW_VP_NOT_OPEN;
4455
4456 vpath_close_exit:
4457         return status;
4458 }
4459
4460 /*
4461  * vxge_hw_vpath_reset - Resets vpath
4462  * This function is used to request a reset of vpath
4463  */
4464 enum vxge_hw_status vxge_hw_vpath_reset(struct __vxge_hw_vpath_handle *vp)
4465 {
4466         enum vxge_hw_status status;
4467         u32 vp_id;
4468         struct __vxge_hw_virtualpath *vpath = vp->vpath;
4469
4470         vp_id = vpath->vp_id;
4471
4472         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4473                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4474                 goto exit;
4475         }
4476
4477         status = __vxge_hw_vpath_reset(vpath->hldev, vp_id);
4478         if (status == VXGE_HW_OK)
4479                 vpath->sw_stats->soft_reset_cnt++;
4480 exit:
4481         return status;
4482 }
4483
4484 /*
4485  * vxge_hw_vpath_recover_from_reset - Poll for reset complete and re-initialize.
4486  * This function poll's for the vpath reset completion and re initializes
4487  * the vpath.
4488  */
4489 enum vxge_hw_status
4490 vxge_hw_vpath_recover_from_reset(struct __vxge_hw_vpath_handle *vp)
4491 {
4492         struct __vxge_hw_virtualpath *vpath = NULL;
4493         enum vxge_hw_status status;
4494         struct __vxge_hw_device *hldev;
4495         u32 vp_id;
4496
4497         vp_id = vp->vpath->vp_id;
4498         vpath = vp->vpath;
4499         hldev = vpath->hldev;
4500
4501         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4502                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4503                 goto exit;
4504         }
4505
4506         status = __vxge_hw_vpath_reset_check(vpath);
4507         if (status != VXGE_HW_OK)
4508                 goto exit;
4509
4510         status = __vxge_hw_vpath_sw_reset(hldev, vp_id);
4511         if (status != VXGE_HW_OK)
4512                 goto exit;
4513
4514         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4515         if (status != VXGE_HW_OK)
4516                 goto exit;
4517
4518         if (vpath->ringh != NULL)
4519                 __vxge_hw_vpath_prc_configure(hldev, vp_id);
4520
4521         memset(vpath->hw_stats, 0,
4522                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4523
4524         memset(vpath->hw_stats_sav, 0,
4525                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4526
4527         writeq(vpath->stats_block->dma_addr,
4528                 &vpath->vp_reg->stats_cfg);
4529
4530         status = vxge_hw_vpath_stats_enable(vp);
4531
4532 exit:
4533         return status;
4534 }
4535
4536 /*
4537  * vxge_hw_vpath_enable - Enable vpath.
4538  * This routine clears the vpath reset thereby enabling a vpath
4539  * to start forwarding frames and generating interrupts.
4540  */
4541 void
4542 vxge_hw_vpath_enable(struct __vxge_hw_vpath_handle *vp)
4543 {
4544         struct __vxge_hw_device *hldev;
4545         u64 val64;
4546
4547         hldev = vp->vpath->hldev;
4548
4549         val64 = VXGE_HW_CMN_RSTHDLR_CFG1_CLR_VPATH_RESET(
4550                 1 << (16 - vp->vpath->vp_id));
4551
4552         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
4553                 &hldev->common_reg->cmn_rsthdlr_cfg1);
4554 }
4555
4556 /*
4557  * vxge_hw_vpath_stats_enable - Enable vpath h/wstatistics.
4558  * Enable the DMA vpath statistics. The function is to be called to re-enable
4559  * the adapter to update stats into the host memory
4560  */
4561 enum vxge_hw_status
4562 vxge_hw_vpath_stats_enable(struct __vxge_hw_vpath_handle *vp)
4563 {
4564         enum vxge_hw_status status = VXGE_HW_OK;
4565         struct __vxge_hw_virtualpath *vpath;
4566
4567         vpath = vp->vpath;
4568
4569         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4570                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4571                 goto exit;
4572         }
4573
4574         memcpy(vpath->hw_stats_sav, vpath->hw_stats,
4575                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4576
4577         status = __vxge_hw_vpath_stats_get(vpath, vpath->hw_stats);
4578 exit:
4579         return status;
4580 }
4581
4582 /*
4583  * __vxge_hw_vpath_stats_access - Get the statistics from the given location
4584  *                           and offset and perform an operation
4585  */
4586 enum vxge_hw_status
4587 __vxge_hw_vpath_stats_access(struct __vxge_hw_virtualpath *vpath,
4588                              u32 operation, u32 offset, u64 *stat)
4589 {
4590         u64 val64;
4591         enum vxge_hw_status status = VXGE_HW_OK;
4592         struct vxge_hw_vpath_reg __iomem *vp_reg;
4593
4594         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4595                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4596                 goto vpath_stats_access_exit;
4597         }
4598
4599         vp_reg = vpath->vp_reg;
4600
4601         val64 =  VXGE_HW_XMAC_STATS_ACCESS_CMD_OP(operation) |
4602                  VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE |
4603                  VXGE_HW_XMAC_STATS_ACCESS_CMD_OFFSET_SEL(offset);
4604
4605         status = __vxge_hw_pio_mem_write64(val64,
4606                                 &vp_reg->xmac_stats_access_cmd,
4607                                 VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE,
4608                                 vpath->hldev->config.device_poll_millis);
4609
4610         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
4611                 *stat = readq(&vp_reg->xmac_stats_access_data);
4612         else
4613                 *stat = 0;
4614
4615 vpath_stats_access_exit:
4616         return status;
4617 }
4618
4619 /*
4620  * __vxge_hw_vpath_xmac_tx_stats_get - Get the TX Statistics of a vpath
4621  */
4622 enum vxge_hw_status
4623 __vxge_hw_vpath_xmac_tx_stats_get(
4624         struct __vxge_hw_virtualpath *vpath,
4625         struct vxge_hw_xmac_vpath_tx_stats *vpath_tx_stats)
4626 {
4627         u64 *val64;
4628         int i;
4629         u32 offset = VXGE_HW_STATS_VPATH_TX_OFFSET;
4630         enum vxge_hw_status status = VXGE_HW_OK;
4631
4632         val64 = (u64 *) vpath_tx_stats;
4633
4634         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4635                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4636                 goto exit;
4637         }
4638
4639         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_tx_stats) / 8; i++) {
4640                 status = __vxge_hw_vpath_stats_access(vpath,
4641                                         VXGE_HW_STATS_OP_READ,
4642                                         offset, val64);
4643                 if (status != VXGE_HW_OK)
4644                         goto exit;
4645                 offset++;
4646                 val64++;
4647         }
4648 exit:
4649         return status;
4650 }
4651
4652 /*
4653  * __vxge_hw_vpath_xmac_rx_stats_get - Get the RX Statistics of a vpath
4654  */
4655 enum vxge_hw_status
4656 __vxge_hw_vpath_xmac_rx_stats_get(struct __vxge_hw_virtualpath *vpath,
4657                         struct vxge_hw_xmac_vpath_rx_stats *vpath_rx_stats)
4658 {
4659         u64 *val64;
4660         enum vxge_hw_status status = VXGE_HW_OK;
4661         int i;
4662         u32 offset = VXGE_HW_STATS_VPATH_RX_OFFSET;
4663         val64 = (u64 *) vpath_rx_stats;
4664
4665         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4666                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4667                 goto exit;
4668         }
4669         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_rx_stats) / 8; i++) {
4670                 status = __vxge_hw_vpath_stats_access(vpath,
4671                                         VXGE_HW_STATS_OP_READ,
4672                                         offset >> 3, val64);
4673                 if (status != VXGE_HW_OK)
4674                         goto exit;
4675
4676                 offset += 8;
4677                 val64++;
4678         }
4679 exit:
4680         return status;
4681 }
4682
4683 /*
4684  * __vxge_hw_vpath_stats_get - Get the vpath hw statistics.
4685  */
4686 enum vxge_hw_status __vxge_hw_vpath_stats_get(
4687                         struct __vxge_hw_virtualpath *vpath,
4688                         struct vxge_hw_vpath_stats_hw_info *hw_stats)
4689 {
4690         u64 val64;
4691         enum vxge_hw_status status = VXGE_HW_OK;
4692         struct vxge_hw_vpath_reg __iomem *vp_reg;
4693
4694         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4695                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4696                 goto exit;
4697         }
4698         vp_reg = vpath->vp_reg;
4699
4700         val64 = readq(&vp_reg->vpath_debug_stats0);
4701         hw_stats->ini_num_mwr_sent =
4702                 (u32)VXGE_HW_VPATH_DEBUG_STATS0_GET_INI_NUM_MWR_SENT(val64);
4703
4704         val64 = readq(&vp_reg->vpath_debug_stats1);
4705         hw_stats->ini_num_mrd_sent =
4706                 (u32)VXGE_HW_VPATH_DEBUG_STATS1_GET_INI_NUM_MRD_SENT(val64);
4707
4708         val64 = readq(&vp_reg->vpath_debug_stats2);
4709         hw_stats->ini_num_cpl_rcvd =
4710                 (u32)VXGE_HW_VPATH_DEBUG_STATS2_GET_INI_NUM_CPL_RCVD(val64);
4711
4712         val64 = readq(&vp_reg->vpath_debug_stats3);
4713         hw_stats->ini_num_mwr_byte_sent =
4714                 VXGE_HW_VPATH_DEBUG_STATS3_GET_INI_NUM_MWR_BYTE_SENT(val64);
4715
4716         val64 = readq(&vp_reg->vpath_debug_stats4);
4717         hw_stats->ini_num_cpl_byte_rcvd =
4718                 VXGE_HW_VPATH_DEBUG_STATS4_GET_INI_NUM_CPL_BYTE_RCVD(val64);
4719
4720         val64 = readq(&vp_reg->vpath_debug_stats5);
4721         hw_stats->wrcrdtarb_xoff =
4722                 (u32)VXGE_HW_VPATH_DEBUG_STATS5_GET_WRCRDTARB_XOFF(val64);
4723
4724         val64 = readq(&vp_reg->vpath_debug_stats6);
4725         hw_stats->rdcrdtarb_xoff =
4726                 (u32)VXGE_HW_VPATH_DEBUG_STATS6_GET_RDCRDTARB_XOFF(val64);
4727
4728         val64 = readq(&vp_reg->vpath_genstats_count01);
4729         hw_stats->vpath_genstats_count0 =
4730         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT0(
4731                 val64);
4732
4733         val64 = readq(&vp_reg->vpath_genstats_count01);
4734         hw_stats->vpath_genstats_count1 =
4735         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT1(
4736                 val64);
4737
4738         val64 = readq(&vp_reg->vpath_genstats_count23);
4739         hw_stats->vpath_genstats_count2 =
4740         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT2(
4741                 val64);
4742
4743         val64 = readq(&vp_reg->vpath_genstats_count01);
4744         hw_stats->vpath_genstats_count3 =
4745         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT3(
4746                 val64);
4747
4748         val64 = readq(&vp_reg->vpath_genstats_count4);
4749         hw_stats->vpath_genstats_count4 =
4750         (u32)VXGE_HW_VPATH_GENSTATS_COUNT4_GET_PPIF_VPATH_GENSTATS_COUNT4(
4751                 val64);
4752
4753         val64 = readq(&vp_reg->vpath_genstats_count5);
4754         hw_stats->vpath_genstats_count5 =
4755         (u32)VXGE_HW_VPATH_GENSTATS_COUNT5_GET_PPIF_VPATH_GENSTATS_COUNT5(
4756                 val64);
4757
4758         status = __vxge_hw_vpath_xmac_tx_stats_get(vpath, &hw_stats->tx_stats);
4759         if (status != VXGE_HW_OK)
4760                 goto exit;
4761
4762         status = __vxge_hw_vpath_xmac_rx_stats_get(vpath, &hw_stats->rx_stats);
4763         if (status != VXGE_HW_OK)
4764                 goto exit;
4765
4766         VXGE_HW_VPATH_STATS_PIO_READ(
4767                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM0_OFFSET);
4768
4769         hw_stats->prog_event_vnum0 =
4770                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM0(val64);
4771
4772         hw_stats->prog_event_vnum1 =
4773                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM1(val64);
4774
4775         VXGE_HW_VPATH_STATS_PIO_READ(
4776                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM2_OFFSET);
4777
4778         hw_stats->prog_event_vnum2 =
4779                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM2(val64);
4780
4781         hw_stats->prog_event_vnum3 =
4782                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM3(val64);
4783
4784         val64 = readq(&vp_reg->rx_multi_cast_stats);
4785         hw_stats->rx_multi_cast_frame_discard =
4786                 (u16)VXGE_HW_RX_MULTI_CAST_STATS_GET_FRAME_DISCARD(val64);
4787
4788         val64 = readq(&vp_reg->rx_frm_transferred);
4789         hw_stats->rx_frm_transferred =
4790                 (u32)VXGE_HW_RX_FRM_TRANSFERRED_GET_RX_FRM_TRANSFERRED(val64);
4791
4792         val64 = readq(&vp_reg->rxd_returned);
4793         hw_stats->rxd_returned =
4794                 (u16)VXGE_HW_RXD_RETURNED_GET_RXD_RETURNED(val64);
4795
4796         val64 = readq(&vp_reg->dbg_stats_rx_mpa);
4797         hw_stats->rx_mpa_len_fail_frms =
4798                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_LEN_FAIL_FRMS(val64);
4799         hw_stats->rx_mpa_mrk_fail_frms =
4800                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_MRK_FAIL_FRMS(val64);
4801         hw_stats->rx_mpa_crc_fail_frms =
4802                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_CRC_FAIL_FRMS(val64);
4803
4804         val64 = readq(&vp_reg->dbg_stats_rx_fau);
4805         hw_stats->rx_permitted_frms =
4806                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_PERMITTED_FRMS(val64);
4807         hw_stats->rx_vp_reset_discarded_frms =
4808         (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_VP_RESET_DISCARDED_FRMS(val64);
4809         hw_stats->rx_wol_frms =
4810                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_WOL_FRMS(val64);
4811
4812         val64 = readq(&vp_reg->tx_vp_reset_discarded_frms);
4813         hw_stats->tx_vp_reset_discarded_frms =
4814         (u16)VXGE_HW_TX_VP_RESET_DISCARDED_FRMS_GET_TX_VP_RESET_DISCARDED_FRMS(
4815                 val64);
4816 exit:
4817         return status;
4818 }
4819
4820 /*
4821  * __vxge_hw_blockpool_create - Create block pool
4822  */
4823
4824 enum vxge_hw_status
4825 __vxge_hw_blockpool_create(struct __vxge_hw_device *hldev,
4826                            struct __vxge_hw_blockpool *blockpool,
4827                            u32 pool_size,
4828                            u32 pool_max)
4829 {
4830         u32 i;
4831         struct __vxge_hw_blockpool_entry *entry = NULL;
4832         void *memblock;
4833         dma_addr_t dma_addr;
4834         struct pci_dev *dma_handle;
4835         struct pci_dev *acc_handle;
4836         enum vxge_hw_status status = VXGE_HW_OK;
4837
4838         if (blockpool == NULL) {
4839                 status = VXGE_HW_FAIL;
4840                 goto blockpool_create_exit;
4841         }
4842
4843         blockpool->hldev = hldev;
4844         blockpool->block_size = VXGE_HW_BLOCK_SIZE;
4845         blockpool->pool_size = 0;
4846         blockpool->pool_max = pool_max;
4847         blockpool->req_out = 0;
4848
4849         INIT_LIST_HEAD(&blockpool->free_block_list);
4850         INIT_LIST_HEAD(&blockpool->free_entry_list);
4851
4852         for (i = 0; i < pool_size + pool_max; i++) {
4853                 entry = kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4854                                 GFP_KERNEL);
4855                 if (entry == NULL) {
4856                         __vxge_hw_blockpool_destroy(blockpool);
4857                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4858                         goto blockpool_create_exit;
4859                 }
4860                 list_add(&entry->item, &blockpool->free_entry_list);
4861         }
4862
4863         for (i = 0; i < pool_size; i++) {
4864
4865                 memblock = vxge_os_dma_malloc(
4866                                 hldev->pdev,
4867                                 VXGE_HW_BLOCK_SIZE,
4868                                 &dma_handle,
4869                                 &acc_handle);
4870
4871                 if (memblock == NULL) {
4872                         __vxge_hw_blockpool_destroy(blockpool);
4873                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4874                         goto blockpool_create_exit;
4875                 }
4876
4877                 dma_addr = pci_map_single(hldev->pdev, memblock,
4878                                 VXGE_HW_BLOCK_SIZE, PCI_DMA_BIDIRECTIONAL);
4879
4880                 if (unlikely(pci_dma_mapping_error(hldev->pdev,
4881                                 dma_addr))) {
4882
4883                         vxge_os_dma_free(hldev->pdev, memblock, &acc_handle);
4884                         __vxge_hw_blockpool_destroy(blockpool);
4885                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4886                         goto blockpool_create_exit;
4887                 }
4888
4889                 if (!list_empty(&blockpool->free_entry_list))
4890                         entry = (struct __vxge_hw_blockpool_entry *)
4891                                 list_first_entry(&blockpool->free_entry_list,
4892                                         struct __vxge_hw_blockpool_entry,
4893                                         item);
4894
4895                 if (entry == NULL)
4896                         entry =
4897                             kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4898                                         GFP_KERNEL);
4899                 if (entry != NULL) {
4900                         list_del(&entry->item);
4901                         entry->length = VXGE_HW_BLOCK_SIZE;
4902                         entry->memblock = memblock;
4903                         entry->dma_addr = dma_addr;
4904                         entry->acc_handle = acc_handle;
4905                         entry->dma_handle = dma_handle;
4906                         list_add(&entry->item,
4907                                           &blockpool->free_block_list);
4908                         blockpool->pool_size++;
4909                 } else {
4910                         __vxge_hw_blockpool_destroy(blockpool);
4911                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4912                         goto blockpool_create_exit;
4913                 }
4914         }
4915
4916 blockpool_create_exit:
4917         return status;
4918 }
4919
4920 /*
4921  * __vxge_hw_blockpool_destroy - Deallocates the block pool
4922  */
4923
4924 void __vxge_hw_blockpool_destroy(struct __vxge_hw_blockpool *blockpool)
4925 {
4926
4927         struct __vxge_hw_device *hldev;
4928         struct list_head *p, *n;
4929         u16 ret;
4930
4931         if (blockpool == NULL) {
4932                 ret = 1;
4933                 goto exit;
4934         }
4935
4936         hldev = blockpool->hldev;
4937
4938         list_for_each_safe(p, n, &blockpool->free_block_list) {
4939
4940                 pci_unmap_single(hldev->pdev,
4941                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
4942                         ((struct __vxge_hw_blockpool_entry *)p)->length,
4943                         PCI_DMA_BIDIRECTIONAL);
4944
4945                 vxge_os_dma_free(hldev->pdev,
4946                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
4947                         &((struct __vxge_hw_blockpool_entry *) p)->acc_handle);
4948
4949                 list_del(
4950                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4951                 kfree(p);
4952                 blockpool->pool_size--;
4953         }
4954
4955         list_for_each_safe(p, n, &blockpool->free_entry_list) {
4956                 list_del(
4957                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4958                 kfree((void *)p);
4959         }
4960         ret = 0;
4961 exit:
4962         return;
4963 }
4964
4965 /*
4966  * __vxge_hw_blockpool_blocks_add - Request additional blocks
4967  */
4968 static
4969 void __vxge_hw_blockpool_blocks_add(struct __vxge_hw_blockpool *blockpool)
4970 {
4971         u32 nreq = 0, i;
4972
4973         if ((blockpool->pool_size  +  blockpool->req_out) <
4974                 VXGE_HW_MIN_DMA_BLOCK_POOL_SIZE) {
4975                 nreq = VXGE_HW_INCR_DMA_BLOCK_POOL_SIZE;
4976                 blockpool->req_out += nreq;
4977         }
4978
4979         for (i = 0; i < nreq; i++)
4980                 vxge_os_dma_malloc_async(
4981                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
4982                         blockpool->hldev, VXGE_HW_BLOCK_SIZE);
4983 }
4984
4985 /*
4986  * __vxge_hw_blockpool_blocks_remove - Free additional blocks
4987  */
4988 static
4989 void __vxge_hw_blockpool_blocks_remove(struct __vxge_hw_blockpool *blockpool)
4990 {
4991         struct list_head *p, *n;
4992
4993         list_for_each_safe(p, n, &blockpool->free_block_list) {
4994
4995                 if (blockpool->pool_size < blockpool->pool_max)
4996                         break;
4997
4998                 pci_unmap_single(
4999                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
5000                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
5001                         ((struct __vxge_hw_blockpool_entry *)p)->length,
5002                         PCI_DMA_BIDIRECTIONAL);
5003
5004                 vxge_os_dma_free(
5005                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
5006                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
5007                         &((struct __vxge_hw_blockpool_entry *)p)->acc_handle);
5008
5009                 list_del(&((struct __vxge_hw_blockpool_entry *)p)->item);
5010
5011                 list_add(p, &blockpool->free_entry_list);
5012
5013                 blockpool->pool_size--;
5014
5015         }
5016 }
5017
5018 /*
5019  * vxge_hw_blockpool_block_add - callback for vxge_os_dma_malloc_async
5020  * Adds a block to block pool
5021  */
5022 void vxge_hw_blockpool_block_add(
5023                         struct __vxge_hw_device *devh,
5024                         void *block_addr,
5025                         u32 length,
5026                         struct pci_dev *dma_h,
5027                         struct pci_dev *acc_handle)
5028 {
5029         struct __vxge_hw_blockpool  *blockpool;
5030         struct __vxge_hw_blockpool_entry  *entry = NULL;
5031         dma_addr_t dma_addr;
5032         enum vxge_hw_status status = VXGE_HW_OK;
5033         u32 req_out;
5034
5035         blockpool = &devh->block_pool;
5036
5037         if (block_addr == NULL) {
5038                 blockpool->req_out--;
5039                 status = VXGE_HW_FAIL;
5040                 goto exit;
5041         }
5042
5043         dma_addr = pci_map_single(devh->pdev, block_addr, length,
5044                                 PCI_DMA_BIDIRECTIONAL);
5045
5046         if (unlikely(pci_dma_mapping_error(devh->pdev, dma_addr))) {
5047
5048                 vxge_os_dma_free(devh->pdev, block_addr, &acc_handle);
5049                 blockpool->req_out--;
5050                 status = VXGE_HW_FAIL;
5051                 goto exit;
5052         }
5053
5054
5055         if (!list_empty(&blockpool->free_entry_list))
5056                 entry = (struct __vxge_hw_blockpool_entry *)
5057                         list_first_entry(&blockpool->free_entry_list,
5058                                 struct __vxge_hw_blockpool_entry,
5059                                 item);
5060
5061         if (entry == NULL)
5062                 entry = (struct __vxge_hw_blockpool_entry *)
5063                         vmalloc(sizeof(struct __vxge_hw_blockpool_entry));
5064         else
5065                 list_del(&entry->item);
5066
5067         if (entry != NULL) {
5068                 entry->length = length;
5069                 entry->memblock = block_addr;
5070                 entry->dma_addr = dma_addr;
5071                 entry->acc_handle = acc_handle;
5072                 entry->dma_handle = dma_h;
5073                 list_add(&entry->item, &blockpool->free_block_list);
5074                 blockpool->pool_size++;
5075                 status = VXGE_HW_OK;
5076         } else
5077                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
5078
5079         blockpool->req_out--;
5080
5081         req_out = blockpool->req_out;
5082 exit:
5083         return;
5084 }
5085
5086 /*
5087  * __vxge_hw_blockpool_malloc - Allocate a memory block from pool
5088  * Allocates a block of memory of given size, either from block pool
5089  * or by calling vxge_os_dma_malloc()
5090  */
5091 void *
5092 __vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size,
5093                                 struct vxge_hw_mempool_dma *dma_object)
5094 {
5095         struct __vxge_hw_blockpool_entry *entry = NULL;
5096         struct __vxge_hw_blockpool  *blockpool;
5097         void *memblock = NULL;
5098         enum vxge_hw_status status = VXGE_HW_OK;
5099
5100         blockpool = &devh->block_pool;
5101
5102         if (size != blockpool->block_size) {
5103
5104                 memblock = vxge_os_dma_malloc(devh->pdev, size,
5105                                                 &dma_object->handle,
5106                                                 &dma_object->acc_handle);
5107
5108                 if (memblock == NULL) {
5109                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5110                         goto exit;
5111                 }
5112
5113                 dma_object->addr = pci_map_single(devh->pdev, memblock, size,
5114                                         PCI_DMA_BIDIRECTIONAL);
5115
5116                 if (unlikely(pci_dma_mapping_error(devh->pdev,
5117                                 dma_object->addr))) {
5118                         vxge_os_dma_free(devh->pdev, memblock,
5119                                 &dma_object->acc_handle);
5120                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5121                         goto exit;
5122                 }
5123
5124         } else {
5125
5126                 if (!list_empty(&blockpool->free_block_list))
5127                         entry = (struct __vxge_hw_blockpool_entry *)
5128                                 list_first_entry(&blockpool->free_block_list,
5129                                         struct __vxge_hw_blockpool_entry,
5130                                         item);
5131
5132                 if (entry != NULL) {
5133                         list_del(&entry->item);
5134                         dma_object->addr = entry->dma_addr;
5135                         dma_object->handle = entry->dma_handle;
5136                         dma_object->acc_handle = entry->acc_handle;
5137                         memblock = entry->memblock;
5138
5139                         list_add(&entry->item,
5140                                 &blockpool->free_entry_list);
5141                         blockpool->pool_size--;
5142                 }
5143
5144                 if (memblock != NULL)
5145                         __vxge_hw_blockpool_blocks_add(blockpool);
5146         }
5147 exit:
5148         return memblock;
5149 }
5150
5151 /*
5152  * __vxge_hw_blockpool_free - Frees the memory allcoated with
5153                                 __vxge_hw_blockpool_malloc
5154  */
5155 void
5156 __vxge_hw_blockpool_free(struct __vxge_hw_device *devh,
5157                         void *memblock, u32 size,
5158                         struct vxge_hw_mempool_dma *dma_object)
5159 {
5160         struct __vxge_hw_blockpool_entry *entry = NULL;
5161         struct __vxge_hw_blockpool  *blockpool;
5162         enum vxge_hw_status status = VXGE_HW_OK;
5163
5164         blockpool = &devh->block_pool;
5165
5166         if (size != blockpool->block_size) {
5167                 pci_unmap_single(devh->pdev, dma_object->addr, size,
5168                         PCI_DMA_BIDIRECTIONAL);
5169                 vxge_os_dma_free(devh->pdev, memblock, &dma_object->acc_handle);
5170         } else {
5171
5172                 if (!list_empty(&blockpool->free_entry_list))
5173                         entry = (struct __vxge_hw_blockpool_entry *)
5174                                 list_first_entry(&blockpool->free_entry_list,
5175                                         struct __vxge_hw_blockpool_entry,
5176                                         item);
5177
5178                 if (entry == NULL)
5179                         entry = (struct __vxge_hw_blockpool_entry *)
5180                                 vmalloc(sizeof(
5181                                         struct __vxge_hw_blockpool_entry));
5182                 else
5183                         list_del(&entry->item);
5184
5185                 if (entry != NULL) {
5186                         entry->length = size;
5187                         entry->memblock = memblock;
5188                         entry->dma_addr = dma_object->addr;
5189                         entry->acc_handle = dma_object->acc_handle;
5190                         entry->dma_handle = dma_object->handle;
5191                         list_add(&entry->item,
5192                                         &blockpool->free_block_list);
5193                         blockpool->pool_size++;
5194                         status = VXGE_HW_OK;
5195                 } else
5196                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5197
5198                 if (status == VXGE_HW_OK)
5199                         __vxge_hw_blockpool_blocks_remove(blockpool);
5200         }
5201
5202         return;
5203 }
5204
5205 /*
5206  * __vxge_hw_blockpool_block_allocate - Allocates a block from block pool
5207  * This function allocates a block from block pool or from the system
5208  */
5209 struct __vxge_hw_blockpool_entry *
5210 __vxge_hw_blockpool_block_allocate(struct __vxge_hw_device *devh, u32 size)
5211 {
5212         struct __vxge_hw_blockpool_entry *entry = NULL;
5213         struct __vxge_hw_blockpool  *blockpool;
5214
5215         blockpool = &devh->block_pool;
5216
5217         if (size == blockpool->block_size) {
5218
5219                 if (!list_empty(&blockpool->free_block_list))
5220                         entry = (struct __vxge_hw_blockpool_entry *)
5221                                 list_first_entry(&blockpool->free_block_list,
5222                                         struct __vxge_hw_blockpool_entry,
5223                                         item);
5224
5225                 if (entry != NULL) {
5226                         list_del(&entry->item);
5227                         blockpool->pool_size--;
5228                 }
5229         }
5230
5231         if (entry != NULL)
5232                 __vxge_hw_blockpool_blocks_add(blockpool);
5233
5234         return entry;
5235 }
5236
5237 /*
5238  * __vxge_hw_blockpool_block_free - Frees a block from block pool
5239  * @devh: Hal device
5240  * @entry: Entry of block to be freed
5241  *
5242  * This function frees a block from block pool
5243  */
5244 void
5245 __vxge_hw_blockpool_block_free(struct __vxge_hw_device *devh,
5246                         struct __vxge_hw_blockpool_entry *entry)
5247 {
5248         struct __vxge_hw_blockpool  *blockpool;
5249
5250         blockpool = &devh->block_pool;
5251
5252         if (entry->length == blockpool->block_size) {
5253                 list_add(&entry->item, &blockpool->free_block_list);
5254                 blockpool->pool_size++;
5255         }
5256
5257         __vxge_hw_blockpool_blocks_remove(blockpool);
5258
5259         return;
5260 }