drivers/net/niu.c

   1 /* niu.c: Neptune ethernet driver.
   2  *
   3  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
   4  */
   5
   6 #include <linux/module.h>
   7 #include <linux/init.h>
   8 #include <linux/pci.h>
   9 #include <linux/dma-mapping.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/ethtool.h>
  12 #include <linux/etherdevice.h>
  13 #include <linux/platform_device.h>
  14 #include <linux/delay.h>
  15 #include <linux/bitops.h>
  16 #include <linux/mii.h>
  17 #include <linux/if_ether.h>
  18 #include <linux/if_vlan.h>
  19 #include <linux/ip.h>
  20 #include <linux/in.h>
  21 #include <linux/ipv6.h>
  22 #include <linux/log2.h>
  23 #include <linux/jiffies.h>
  24 #include <linux/crc32.h>
  25
  26 #include <linux/io.h>
  27
  28 #ifdef CONFIG_SPARC64
  29 #include <linux/of_device.h>
  30 #endif
  31
  32 #include "niu.h"
  33
  34 #define DRV_MODULE_NAME         "niu"
  35 #define PFX DRV_MODULE_NAME     ": "
  36 #define DRV_MODULE_VERSION      "0.5"
  37 #define DRV_MODULE_RELDATE      "October 5, 2007"
  38
  39 static char version[] __devinitdata =
  40         DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
  41
  42 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
  43 MODULE_DESCRIPTION("NIU ethernet driver");
  44 MODULE_LICENSE("GPL");
  45 MODULE_VERSION(DRV_MODULE_VERSION);
  46
  47 #ifndef DMA_44BIT_MASK
  48 #define DMA_44BIT_MASK  0x00000fffffffffffULL
  49 #endif
  50
  51 #ifndef readq
  52 static u64 readq(void __iomem *reg)
  53 {
  54         return (((u64)readl(reg + 0x4UL) << 32) |
  55                 (u64)readl(reg));
  56 }
  57
  58 static void writeq(u64 val, void __iomem *reg)
  59 {
  60         writel(val & 0xffffffff, reg);
  61         writel(val >> 32, reg + 0x4UL);
  62 }
  63 #endif
  64
  65 static struct pci_device_id niu_pci_tbl[] = {
  66         {PCI_DEVICE(PCI_VENDOR_ID_SUN, 0xabcd)},
  67         {}
  68 };
  69
  70 MODULE_DEVICE_TABLE(pci, niu_pci_tbl);
  71
  72 #define NIU_TX_TIMEOUT                  (5 * HZ)
  73
  74 #define nr64(reg)               readq(np->regs + (reg))
  75 #define nw64(reg, val)          writeq((val), np->regs + (reg))
  76
  77 #define nr64_mac(reg)           readq(np->mac_regs + (reg))
  78 #define nw64_mac(reg, val)      writeq((val), np->mac_regs + (reg))
  79
  80 #define nr64_ipp(reg)           readq(np->regs + np->ipp_off + (reg))
  81 #define nw64_ipp(reg, val)      writeq((val), np->regs + np->ipp_off + (reg))
  82
  83 #define nr64_pcs(reg)           readq(np->regs + np->pcs_off + (reg))
  84 #define nw64_pcs(reg, val)      writeq((val), np->regs + np->pcs_off + (reg))
  85
  86 #define nr64_xpcs(reg)          readq(np->regs + np->xpcs_off + (reg))
  87 #define nw64_xpcs(reg, val)     writeq((val), np->regs + np->xpcs_off + (reg))
  88
  89 #define NIU_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  90
  91 static int niu_debug;
  92 static int debug = -1;
  93 module_param(debug, int, 0);
  94 MODULE_PARM_DESC(debug, "NIU debug level");
  95
  96 #define niudbg(TYPE, f, a...) \
  97 do {    if ((np)->msg_enable & NETIF_MSG_##TYPE) \
  98                 printk(KERN_DEBUG PFX f, ## a); \
  99 } while (0)
 100
 101 #define niuinfo(TYPE, f, a...) \
 102 do {    if ((np)->msg_enable & NETIF_MSG_##TYPE) \
 103                 printk(KERN_INFO PFX f, ## a); \
 104 } while (0)
 105
 106 #define niuwarn(TYPE, f, a...) \
 107 do {    if ((np)->msg_enable & NETIF_MSG_##TYPE) \
 108                 printk(KERN_WARNING PFX f, ## a); \
 109 } while (0)
 110
 111 #define niu_lock_parent(np, flags) \
 112         spin_lock_irqsave(&np->parent->lock, flags)
 113 #define niu_unlock_parent(np, flags) \
 114         spin_unlock_irqrestore(&np->parent->lock, flags)
 115
 116 static int __niu_wait_bits_clear_mac(struct niu *np, unsigned long reg,
 117                                      u64 bits, int limit, int delay)
 118 {
 119         while (--limit >= 0) {
 120                 u64 val = nr64_mac(reg);
 121
 122                 if (!(val & bits))
 123                         break;
 124                 udelay(delay);
 125         }
 126         if (limit < 0)
 127                 return -ENODEV;
 128         return 0;
 129 }
 130
 131 static int __niu_set_and_wait_clear_mac(struct niu *np, unsigned long reg,
 132                                         u64 bits, int limit, int delay,
 133                                         const char *reg_name)
 134 {
 135         int err;
 136
 137         nw64_mac(reg, bits);
 138         err = __niu_wait_bits_clear_mac(np, reg, bits, limit, delay);
 139         if (err)
 140                 dev_err(np->device, PFX "%s: bits (%llx) of register %s "
 141                         "would not clear, val[%llx]\n",
 142                         np->dev->name, (unsigned long long) bits, reg_name,
 143                         (unsigned long long) nr64_mac(reg));
 144         return err;
 145 }
 146
 147 #define niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
 148 ({      BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
 149         __niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
 150 })
 151
 152 static int __niu_wait_bits_clear_ipp(struct niu *np, unsigned long reg,
 153                                      u64 bits, int limit, int delay)
 154 {
 155         while (--limit >= 0) {
 156                 u64 val = nr64_ipp(reg);
 157
 158                 if (!(val & bits))
 159                         break;
 160                 udelay(delay);
 161         }
 162         if (limit < 0)
 163                 return -ENODEV;
 164         return 0;
 165 }
 166
 167 static int __niu_set_and_wait_clear_ipp(struct niu *np, unsigned long reg,
 168                                         u64 bits, int limit, int delay,
 169                                         const char *reg_name)
 170 {
 171         int err;
 172         u64 val;
 173
 174         val = nr64_ipp(reg);
 175         val |= bits;
 176         nw64_ipp(reg, val);
 177
 178         err = __niu_wait_bits_clear_ipp(np, reg, bits, limit, delay);
 179         if (err)
 180                 dev_err(np->device, PFX "%s: bits (%llx) of register %s "
 181                         "would not clear, val[%llx]\n",
 182                         np->dev->name, (unsigned long long) bits, reg_name,
 183                         (unsigned long long) nr64_ipp(reg));
 184         return err;
 185 }
 186
 187 #define niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
 188 ({      BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
 189         __niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
 190 })
 191
 192 static int __niu_wait_bits_clear(struct niu *np, unsigned long reg,
 193                                  u64 bits, int limit, int delay)
 194 {
 195         while (--limit >= 0) {
 196                 u64 val = nr64(reg);
 197
 198                 if (!(val & bits))
 199                         break;
 200                 udelay(delay);
 201         }
 202         if (limit < 0)
 203                 return -ENODEV;
 204         return 0;
 205 }
 206
 207 #define niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY) \
 208 ({      BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
 209         __niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY); \
 210 })
 211
 212 static int __niu_set_and_wait_clear(struct niu *np, unsigned long reg,
 213                                     u64 bits, int limit, int delay,
 214                                     const char *reg_name)
 215 {
 216         int err;
 217
 218         nw64(reg, bits);
 219         err = __niu_wait_bits_clear(np, reg, bits, limit, delay);
 220         if (err)
 221                 dev_err(np->device, PFX "%s: bits (%llx) of register %s "
 222                         "would not clear, val[%llx]\n",
 223                         np->dev->name, (unsigned long long) bits, reg_name,
 224                         (unsigned long long) nr64(reg));
 225         return err;
 226 }
 227
 228 #define niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
 229 ({      BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
 230         __niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
 231 })
 232
 233 static void niu_ldg_rearm(struct niu *np, struct niu_ldg *lp, int on)
 234 {
 235         u64 val = (u64) lp->timer;
 236
 237         if (on)
 238                 val |= LDG_IMGMT_ARM;
 239
 240         nw64(LDG_IMGMT(lp->ldg_num), val);
 241 }
 242
 243 static int niu_ldn_irq_enable(struct niu *np, int ldn, int on)
 244 {
 245         unsigned long mask_reg, bits;
 246         u64 val;
 247
 248         if (ldn < 0 || ldn > LDN_MAX)
 249                 return -EINVAL;
 250
 251         if (ldn < 64) {
 252                 mask_reg = LD_IM0(ldn);
 253                 bits = LD_IM0_MASK;
 254         } else {
 255                 mask_reg = LD_IM1(ldn - 64);
 256                 bits = LD_IM1_MASK;
 257         }
 258
 259         val = nr64(mask_reg);
 260         if (on)
 261                 val &= ~bits;
 262         else
 263                 val |= bits;
 264         nw64(mask_reg, val);
 265
 266         return 0;
 267 }
 268
 269 static int niu_enable_ldn_in_ldg(struct niu *np, struct niu_ldg *lp, int on)
 270 {
 271         struct niu_parent *parent = np->parent;
 272         int i;
 273
 274         for (i = 0; i <= LDN_MAX; i++) {
 275                 int err;
 276
 277                 if (parent->ldg_map[i] != lp->ldg_num)
 278                         continue;
 279
 280                 err = niu_ldn_irq_enable(np, i, on);
 281                 if (err)
 282                         return err;
 283         }
 284         return 0;
 285 }
 286
 287 static int niu_enable_interrupts(struct niu *np, int on)
 288 {
 289         int i;
 290
 291         for (i = 0; i < np->num_ldg; i++) {
 292                 struct niu_ldg *lp = &np->ldg[i];
 293                 int err;
 294
 295                 err = niu_enable_ldn_in_ldg(np, lp, on);
 296                 if (err)
 297                         return err;
 298         }
 299         for (i = 0; i < np->num_ldg; i++)
 300                 niu_ldg_rearm(np, &np->ldg[i], on);
 301
 302         return 0;
 303 }
 304
 305 static u32 phy_encode(u32 type, int port)
 306 {
 307         return (type << (port * 2));
 308 }
 309
 310 static u32 phy_decode(u32 val, int port)
 311 {
 312         return (val >> (port * 2)) & PORT_TYPE_MASK;
 313 }
 314
 315 static int mdio_wait(struct niu *np)
 316 {
 317         int limit = 1000;
 318         u64 val;
 319
 320         while (--limit > 0) {
 321                 val = nr64(MIF_FRAME_OUTPUT);
 322                 if ((val >> MIF_FRAME_OUTPUT_TA_SHIFT) & 0x1)
 323                         return val & MIF_FRAME_OUTPUT_DATA;
 324
 325                 udelay(10);
 326         }
 327
 328         return -ENODEV;
 329 }
 330
 331 static int mdio_read(struct niu *np, int port, int dev, int reg)
 332 {
 333         int err;
 334
 335         nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
 336         err = mdio_wait(np);
 337         if (err < 0)
 338                 return err;
 339
 340         nw64(MIF_FRAME_OUTPUT, MDIO_READ_OP(port, dev));
 341         return mdio_wait(np);
 342 }
 343
 344 static int mdio_write(struct niu *np, int port, int dev, int reg, int data)
 345 {
 346         int err;
 347
 348         nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
 349         err = mdio_wait(np);
 350         if (err < 0)
 351                 return err;
 352
 353         nw64(MIF_FRAME_OUTPUT, MDIO_WRITE_OP(port, dev, data));
 354         err = mdio_wait(np);
 355         if (err < 0)
 356                 return err;
 357
 358         return 0;
 359 }
 360
 361 static int mii_read(struct niu *np, int port, int reg)
 362 {
 363         nw64(MIF_FRAME_OUTPUT, MII_READ_OP(port, reg));
 364         return mdio_wait(np);
 365 }
 366
 367 static int mii_write(struct niu *np, int port, int reg, int data)
 368 {
 369         int err;
 370
 371         nw64(MIF_FRAME_OUTPUT, MII_WRITE_OP(port, reg, data));
 372         err = mdio_wait(np);
 373         if (err < 0)
 374                 return err;
 375
 376         return 0;
 377 }
 378
 379 static int esr2_set_tx_cfg(struct niu *np, unsigned long channel, u32 val)
 380 {
 381         int err;
 382
 383         err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
 384                          ESR2_TI_PLL_TX_CFG_L(channel),
 385                          val & 0xffff);
 386         if (!err)
 387                 err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
 388                                  ESR2_TI_PLL_TX_CFG_H(channel),
 389                                  val >> 16);
 390         return err;
 391 }
 392
 393 static int esr2_set_rx_cfg(struct niu *np, unsigned long channel, u32 val)
 394 {
 395         int err;
 396
 397         err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
 398                          ESR2_TI_PLL_RX_CFG_L(channel),
 399                          val & 0xffff);
 400         if (!err)
 401                 err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
 402                                  ESR2_TI_PLL_RX_CFG_H(channel),
 403                                  val >> 16);
 404         return err;
 405 }
 406
 407 /* Mode is always 10G fiber.  */
 408 static int serdes_init_niu(struct niu *np)
 409 {
 410         struct niu_link_config *lp = &np->link_config;
 411         u32 tx_cfg, rx_cfg;
 412         unsigned long i;
 413
 414         tx_cfg = (PLL_TX_CFG_ENTX | PLL_TX_CFG_SWING_1375MV);
 415         rx_cfg = (PLL_RX_CFG_ENRX | PLL_RX_CFG_TERM_0P8VDDT |
 416                   PLL_RX_CFG_ALIGN_ENA | PLL_RX_CFG_LOS_LTHRESH |
 417                   PLL_RX_CFG_EQ_LP_ADAPTIVE);
 418
 419         if (lp->loopback_mode == LOOPBACK_PHY) {
 420                 u16 test_cfg = PLL_TEST_CFG_LOOPBACK_CML_DIS;
 421
 422                 mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
 423                            ESR2_TI_PLL_TEST_CFG_L, test_cfg);
 424
 425                 tx_cfg |= PLL_TX_CFG_ENTEST;
 426                 rx_cfg |= PLL_RX_CFG_ENTEST;
 427         }
 428
 429         /* Initialize all 4 lanes of the SERDES.  */
 430         for (i = 0; i < 4; i++) {
 431                 int err = esr2_set_tx_cfg(np, i, tx_cfg);
 432                 if (err)
 433                         return err;
 434         }
 435
 436         for (i = 0; i < 4; i++) {
 437                 int err = esr2_set_rx_cfg(np, i, rx_cfg);
 438                 if (err)
 439                         return err;
 440         }
 441
 442         return 0;
 443 }
 444
 445 static int esr_read_rxtx_ctrl(struct niu *np, unsigned long chan, u32 *val)
 446 {
 447         int err;
 448
 449         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR, ESR_RXTX_CTRL_L(chan));
 450         if (err >= 0) {
 451                 *val = (err & 0xffff);
 452                 err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
 453                                 ESR_RXTX_CTRL_H(chan));
 454                 if (err >= 0)
 455                         *val |= ((err & 0xffff) << 16);
 456                 err = 0;
 457         }
 458         return err;
 459 }
 460
 461 static int esr_read_glue0(struct niu *np, unsigned long chan, u32 *val)
 462 {
 463         int err;
 464
 465         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
 466                         ESR_GLUE_CTRL0_L(chan));
 467         if (err >= 0) {
 468                 *val = (err & 0xffff);
 469                 err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
 470                                 ESR_GLUE_CTRL0_H(chan));
 471                 if (err >= 0) {
 472                         *val |= ((err & 0xffff) << 16);
 473                         err = 0;
 474                 }
 475         }
 476         return err;
 477 }
 478
 479 static int esr_read_reset(struct niu *np, u32 *val)
 480 {
 481         int err;
 482
 483         err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
 484                         ESR_RXTX_RESET_CTRL_L);
 485         if (err >= 0) {
 486                 *val = (err & 0xffff);
 487                 err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
 488                                 ESR_RXTX_RESET_CTRL_H);
 489                 if (err >= 0) {
 490                         *val |= ((err & 0xffff) << 16);
 491                         err = 0;
 492                 }
 493         }
 494         return err;
 495 }
 496
 497 static int esr_write_rxtx_ctrl(struct niu *np, unsigned long chan, u32 val)
 498 {
 499         int err;
 500
 501         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 502                          ESR_RXTX_CTRL_L(chan), val & 0xffff);
 503         if (!err)
 504                 err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 505                                  ESR_RXTX_CTRL_H(chan), (val >> 16));
 506         return err;
 507 }
 508
 509 static int esr_write_glue0(struct niu *np, unsigned long chan, u32 val)
 510 {
 511         int err;
 512
 513         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 514                         ESR_GLUE_CTRL0_L(chan), val & 0xffff);
 515         if (!err)
 516                 err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 517                                  ESR_GLUE_CTRL0_H(chan), (val >> 16));
 518         return err;
 519 }
 520
 521 static int esr_reset(struct niu *np)
 522 {
 523         u32 reset;
 524         int err;
 525
 526         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 527                          ESR_RXTX_RESET_CTRL_L, 0x0000);
 528         if (err)
 529                 return err;
 530         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 531                          ESR_RXTX_RESET_CTRL_H, 0xffff);
 532         if (err)
 533                 return err;
 534         udelay(200);
 535
 536         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 537                          ESR_RXTX_RESET_CTRL_L, 0xffff);
 538         if (err)
 539                 return err;
 540         udelay(200);
 541
 542         err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
 543                          ESR_RXTX_RESET_CTRL_H, 0x0000);
 544         if (err)
 545                 return err;
 546         udelay(200);
 547
 548         err = esr_read_reset(np, &reset);
 549         if (err)
 550                 return err;
 551         if (reset != 0) {
 552                 dev_err(np->device, PFX "Port %u ESR_RESET "
 553                         "did not clear [%08x]\n",
 554                         np->port, reset);
 555                 return -ENODEV;
 556         }
 557
 558         return 0;
 559 }
 560
 561 static int serdes_init_10g(struct niu *np)
 562 {
 563         struct niu_link_config *lp = &np->link_config;
 564         unsigned long ctrl_reg, test_cfg_reg, i;
 565         u64 ctrl_val, test_cfg_val, sig, mask, val;
 566         int err;
 567
 568         switch (np->port) {
 569         case 0:
 570                 ctrl_reg = ENET_SERDES_0_CTRL_CFG;
 571                 test_cfg_reg = ENET_SERDES_0_TEST_CFG;
 572                 break;
 573         case 1:
 574                 ctrl_reg = ENET_SERDES_1_CTRL_CFG;
 575                 test_cfg_reg = ENET_SERDES_1_TEST_CFG;
 576                 break;
 577
 578         default:
 579                 return -EINVAL;
 580         }
 581         ctrl_val = (ENET_SERDES_CTRL_SDET_0 |
 582                     ENET_SERDES_CTRL_SDET_1 |
 583                     ENET_SERDES_CTRL_SDET_2 |
 584                     ENET_SERDES_CTRL_SDET_3 |
 585                     (0x5 << ENET_SERDES_CTRL_EMPH_0_SHIFT) |
 586                     (0x5 << ENET_SERDES_CTRL_EMPH_1_SHIFT) |
 587                     (0x5 << ENET_SERDES_CTRL_EMPH_2_SHIFT) |
 588                     (0x5 << ENET_SERDES_CTRL_EMPH_3_SHIFT) |
 589                     (0x1 << ENET_SERDES_CTRL_LADJ_0_SHIFT) |
 590                     (0x1 << ENET_SERDES_CTRL_LADJ_1_SHIFT) |
 591                     (0x1 << ENET_SERDES_CTRL_LADJ_2_SHIFT) |
 592                     (0x1 << ENET_SERDES_CTRL_LADJ_3_SHIFT));
 593         test_cfg_val = 0;
 594
 595         if (lp->loopback_mode == LOOPBACK_PHY) {
 596                 test_cfg_val |= ((ENET_TEST_MD_PAD_LOOPBACK <<
 597                                   ENET_SERDES_TEST_MD_0_SHIFT) |
 598                                  (ENET_TEST_MD_PAD_LOOPBACK <<
 599                                   ENET_SERDES_TEST_MD_1_SHIFT) |
 600                                  (ENET_TEST_MD_PAD_LOOPBACK <<
 601                                   ENET_SERDES_TEST_MD_2_SHIFT) |
 602                                  (ENET_TEST_MD_PAD_LOOPBACK <<
 603                                   ENET_SERDES_TEST_MD_3_SHIFT));
 604         }
 605
 606         nw64(ctrl_reg, ctrl_val);
 607         nw64(test_cfg_reg, test_cfg_val);
 608
 609         /* Initialize all 4 lanes of the SERDES.  */
 610         for (i = 0; i < 4; i++) {
 611                 u32 rxtx_ctrl, glue0;
 612
 613                 err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);
 614                 if (err)
 615                         return err;
 616                 err = esr_read_glue0(np, i, &glue0);
 617                 if (err)
 618                         return err;
 619
 620                 rxtx_ctrl &= ~(ESR_RXTX_CTRL_VMUXLO);
 621                 rxtx_ctrl |= (ESR_RXTX_CTRL_ENSTRETCH |
 622                               (2 << ESR_RXTX_CTRL_VMUXLO_SHIFT));
 623
 624                 glue0 &= ~(ESR_GLUE_CTRL0_SRATE |
 625                            ESR_GLUE_CTRL0_THCNT |
 626                            ESR_GLUE_CTRL0_BLTIME);
 627                 glue0 |= (ESR_GLUE_CTRL0_RXLOSENAB |
 628                           (0xf << ESR_GLUE_CTRL0_SRATE_SHIFT) |
 629                           (0xff << ESR_GLUE_CTRL0_THCNT_SHIFT) |
 630                           (BLTIME_300_CYCLES <<
 631                            ESR_GLUE_CTRL0_BLTIME_SHIFT));
 632
 633                 err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);
 634                 if (err)
 635                         return err;
 636                 err = esr_write_glue0(np, i, glue0);
 637                 if (err)
 638                         return err;
 639         }
 640
 641         err = esr_reset(np);
 642         if (err)
 643                 return err;
 644
 645         sig = nr64(ESR_INT_SIGNALS);
 646         switch (np->port) {
 647         case 0:
 648                 mask = ESR_INT_SIGNALS_P0_BITS;
 649                 val = (ESR_INT_SRDY0_P0 |
 650                        ESR_INT_DET0_P0 |
 651                        ESR_INT_XSRDY_P0 |
 652                        ESR_INT_XDP_P0_CH3 |
 653                        ESR_INT_XDP_P0_CH2 |
 654                        ESR_INT_XDP_P0_CH1 |
 655                        ESR_INT_XDP_P0_CH0);
 656                 break;
 657
 658         case 1:
 659                 mask = ESR_INT_SIGNALS_P1_BITS;
 660                 val = (ESR_INT_SRDY0_P1 |
 661                        ESR_INT_DET0_P1 |
 662                        ESR_INT_XSRDY_P1 |
 663                        ESR_INT_XDP_P1_CH3 |
 664                        ESR_INT_XDP_P1_CH2 |
 665                        ESR_INT_XDP_P1_CH1 |
 666                        ESR_INT_XDP_P1_CH0);
 667                 break;
 668
 669         default:
 670                 return -EINVAL;
 671         }
 672
 673         if ((sig & mask) != val) {
 674                 dev_err(np->device, PFX "Port %u signal bits [%08x] are not "
 675                         "[%08x]\n", np->port, (int) (sig & mask), (int) val);
 676                 return -ENODEV;
 677         }
 678
 679         return 0;
 680 }
 681
 682 static int serdes_init_1g(struct niu *np)
 683 {
 684         u64 val;
 685
 686         val = nr64(ENET_SERDES_1_PLL_CFG);
 687         val &= ~ENET_SERDES_PLL_FBDIV2;
 688         switch (np->port) {
 689         case 0:
 690                 val |= ENET_SERDES_PLL_HRATE0;
 691                 break;
 692         case 1:
 693                 val |= ENET_SERDES_PLL_HRATE1;
 694                 break;
 695         case 2:
 696                 val |= ENET_SERDES_PLL_HRATE2;
 697                 break;
 698         case 3:
 699                 val |= ENET_SERDES_PLL_HRATE3;
 700                 break;
 701         default:
 702                 return -EINVAL;
 703         }
 704         nw64(ENET_SERDES_1_PLL_CFG, val);
 705
 706         return 0;
 707 }
 708
 709 static int bcm8704_reset(struct niu *np)
 710 {
 711         int err, limit;
 712
 713         err = mdio_read(np, np->phy_addr,
 714                         BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
 715         if (err < 0)
 716                 return err;
 717         err |= BMCR_RESET;
 718         err = mdio_write(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
 719                          MII_BMCR, err);
 720         if (err)
 721                 return err;
 722
 723         limit = 1000;
 724         while (--limit >= 0) {
 725                 err = mdio_read(np, np->phy_addr,
 726                                 BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
 727                 if (err < 0)
 728                         return err;
 729                 if (!(err & BMCR_RESET))
 730                         break;
 731         }
 732         if (limit < 0) {
 733                 dev_err(np->device, PFX "Port %u PHY will not reset "
 734                         "(bmcr=%04x)\n", np->port, (err & 0xffff));
 735                 return -ENODEV;
 736         }
 737         return 0;
 738 }
 739
 740 /* When written, certain PHY registers need to be read back twice
 741  * in order for the bits to settle properly.
 742  */
 743 static int bcm8704_user_dev3_readback(struct niu *np, int reg)
 744 {
 745         int err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
 746         if (err < 0)
 747                 return err;
 748         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
 749         if (err < 0)
 750                 return err;
 751         return 0;
 752 }
 753
 754 static int bcm8704_init_user_dev3(struct niu *np)
 755 {
 756         int err;
 757
 758         err = mdio_write(np, np->phy_addr,
 759                          BCM8704_USER_DEV3_ADDR, BCM8704_USER_CONTROL,
 760                          (USER_CONTROL_OPTXRST_LVL |
 761                           USER_CONTROL_OPBIASFLT_LVL |
 762                           USER_CONTROL_OBTMPFLT_LVL |
 763                           USER_CONTROL_OPPRFLT_LVL |
 764                           USER_CONTROL_OPTXFLT_LVL |
 765                           USER_CONTROL_OPRXLOS_LVL |
 766                           USER_CONTROL_OPRXFLT_LVL |
 767                           USER_CONTROL_OPTXON_LVL |
 768                           (0x3f << USER_CONTROL_RES1_SHIFT)));
 769         if (err)
 770                 return err;
 771
 772         err = mdio_write(np, np->phy_addr,
 773                          BCM8704_USER_DEV3_ADDR, BCM8704_USER_PMD_TX_CONTROL,
 774                          (USER_PMD_TX_CTL_XFP_CLKEN |
 775                           (1 << USER_PMD_TX_CTL_TX_DAC_TXD_SH) |
 776                           (2 << USER_PMD_TX_CTL_TX_DAC_TXCK_SH) |
 777                           USER_PMD_TX_CTL_TSCK_LPWREN));
 778         if (err)
 779                 return err;
 780
 781         err = bcm8704_user_dev3_readback(np, BCM8704_USER_CONTROL);
 782         if (err)
 783                 return err;
 784         err = bcm8704_user_dev3_readback(np, BCM8704_USER_PMD_TX_CONTROL);
 785         if (err)
 786                 return err;
 787
 788         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 789                         BCM8704_USER_OPT_DIGITAL_CTRL);
 790         if (err < 0)
 791                 return err;
 792         err &= ~USER_ODIG_CTRL_GPIOS;
 793         err |= (0x3 << USER_ODIG_CTRL_GPIOS_SHIFT);
 794         err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 795                          BCM8704_USER_OPT_DIGITAL_CTRL, err);
 796         if (err)
 797                 return err;
 798
 799         mdelay(1000);
 800
 801         return 0;
 802 }
 803
 804 static int xcvr_init_10g(struct niu *np)
 805 {
 806         struct niu_link_config *lp = &np->link_config;
 807         u16 analog_stat0, tx_alarm_status;
 808         int err;
 809         u64 val;
 810
 811         val = nr64_mac(XMAC_CONFIG);
 812         val &= ~XMAC_CONFIG_LED_POLARITY;
 813         val |= XMAC_CONFIG_FORCE_LED_ON;
 814         nw64_mac(XMAC_CONFIG, val);
 815
 816         /* XXX shared resource, lock parent XXX */
 817         val = nr64(MIF_CONFIG);
 818         val |= MIF_CONFIG_INDIRECT_MODE;
 819         nw64(MIF_CONFIG, val);
 820
 821         err = bcm8704_reset(np);
 822         if (err)
 823                 return err;
 824
 825         err = bcm8704_init_user_dev3(np);
 826         if (err)
 827                 return err;
 828
 829         err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
 830                         MII_BMCR);
 831         if (err < 0)
 832                 return err;
 833         err &= ~BMCR_LOOPBACK;
 834
 835         if (lp->loopback_mode == LOOPBACK_MAC)
 836                 err |= BMCR_LOOPBACK;
 837
 838         err = mdio_write(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
 839                          MII_BMCR, err);
 840         if (err)
 841                 return err;
 842
 843 #if 1
 844         err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
 845                         MII_STAT1000);
 846         if (err < 0)
 847                 return err;
 848         pr_info(PFX "Port %u PMA_PMD(MII_STAT1000) [%04x]\n",
 849                 np->port, err);
 850
 851         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, 0x20);
 852         if (err < 0)
 853                 return err;
 854         pr_info(PFX "Port %u USER_DEV3(0x20) [%04x]\n",
 855                 np->port, err);
 856
 857         err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
 858                         MII_NWAYTEST);
 859         if (err < 0)
 860                 return err;
 861         pr_info(PFX "Port %u PHYXS(MII_NWAYTEST) [%04x]\n",
 862                 np->port, err);
 863 #endif
 864
 865         /* XXX dig this out it might not be so useful XXX */
 866         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 867                         BCM8704_USER_ANALOG_STATUS0);
 868         if (err < 0)
 869                 return err;
 870         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 871                         BCM8704_USER_ANALOG_STATUS0);
 872         if (err < 0)
 873                 return err;
 874         analog_stat0 = err;
 875
 876         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 877                         BCM8704_USER_TX_ALARM_STATUS);
 878         if (err < 0)
 879                 return err;
 880         err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
 881                         BCM8704_USER_TX_ALARM_STATUS);
 882         if (err < 0)
 883                 return err;
 884         tx_alarm_status = err;
 885
 886         if (analog_stat0 != 0x03fc) {
 887                 if ((analog_stat0 == 0x43bc) && (tx_alarm_status != 0)) {
 888                         pr_info(PFX "Port %u cable not connected "
 889                                 "or bad cable.\n", np->port);
 890                 } else if (analog_stat0 == 0x639c) {
 891                         pr_info(PFX "Port %u optical module is bad "
 892                                 "or missing.\n", np->port);
 893                 }
 894         }
 895
 896         return 0;
 897 }
 898
 899 static int mii_reset(struct niu *np)
 900 {
 901         int limit, err;
 902
 903         err = mii_write(np, np->phy_addr, MII_BMCR, BMCR_RESET);
 904         if (err)
 905                 return err;
 906
 907         limit = 1000;
 908         while (--limit >= 0) {
 909                 udelay(500);
 910                 err = mii_read(np, np->phy_addr, MII_BMCR);
 911                 if (err < 0)
 912                         return err;
 913                 if (!(err & BMCR_RESET))
 914                         break;
 915         }
 916         if (limit < 0) {
 917                 dev_err(np->device, PFX "Port %u MII would not reset, "
 918                         "bmcr[%04x]\n", np->port, err);
 919                 return -ENODEV;
 920         }
 921
 922         return 0;
 923 }
 924
 925 static int mii_init_common(struct niu *np)
 926 {
 927         struct niu_link_config *lp = &np->link_config;
 928         u16 bmcr, bmsr, adv, estat;
 929         int err;
 930
 931         err = mii_reset(np);
 932         if (err)
 933                 return err;
 934
 935         err = mii_read(np, np->phy_addr, MII_BMSR);
 936         if (err < 0)
 937                 return err;
 938         bmsr = err;
 939
 940         estat = 0;
 941         if (bmsr & BMSR_ESTATEN) {
 942                 err = mii_read(np, np->phy_addr, MII_ESTATUS);
 943                 if (err < 0)
 944                         return err;
 945                 estat = err;
 946         }
 947
 948         bmcr = 0;
 949         err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
 950         if (err)
 951                 return err;
 952
 953         if (lp->loopback_mode == LOOPBACK_MAC) {
 954                 bmcr |= BMCR_LOOPBACK;
 955                 if (lp->active_speed == SPEED_1000)
 956                         bmcr |= BMCR_SPEED1000;
 957                 if (lp->active_duplex == DUPLEX_FULL)
 958                         bmcr |= BMCR_FULLDPLX;
 959         }
 960
 961         if (lp->loopback_mode == LOOPBACK_PHY) {
 962                 u16 aux;
 963
 964                 aux = (BCM5464R_AUX_CTL_EXT_LB |
 965                        BCM5464R_AUX_CTL_WRITE_1);
 966                 err = mii_write(np, np->phy_addr, BCM5464R_AUX_CTL, aux);
 967                 if (err)
 968                         return err;
 969         }
 970
 971         /* XXX configurable XXX */
 972         /* XXX for now don't advertise half-duplex or asym pause... XXX */
 973         adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
 974         if (bmsr & BMSR_10FULL)
 975                 adv |= ADVERTISE_10FULL;
 976         if (bmsr & BMSR_100FULL)
 977                 adv |= ADVERTISE_100FULL;
 978         err = mii_write(np, np->phy_addr, MII_ADVERTISE, adv);
 979         if (err)
 980                 return err;
 981
 982         if (bmsr & BMSR_ESTATEN) {
 983                 u16 ctrl1000 = 0;
 984
 985                 if (estat & ESTATUS_1000_TFULL)
 986                         ctrl1000 |= ADVERTISE_1000FULL;
 987                 err = mii_write(np, np->phy_addr, MII_CTRL1000, ctrl1000);
 988                 if (err)
 989                         return err;
 990         }
 991         bmcr |= (BMCR_ANENABLE | BMCR_ANRESTART);
 992
 993         err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
 994         if (err)
 995                 return err;
 996
 997         err = mii_read(np, np->phy_addr, MII_BMCR);
 998         if (err < 0)
 999                 return err;
1000         err = mii_read(np, np->phy_addr, MII_BMSR);
1001         if (err < 0)
1002                 return err;
1003 #if 0
1004         pr_info(PFX "Port %u after MII init bmcr[%04x] bmsr[%04x]\n",
1005                 np->port, bmcr, bmsr);
1006 #endif
1007
1008         return 0;
1009 }
1010
1011 static int xcvr_init_1g(struct niu *np)
1012 {
1013         u64 val;
1014
1015         /* XXX shared resource, lock parent XXX */
1016         val = nr64(MIF_CONFIG);
1017         val &= ~MIF_CONFIG_INDIRECT_MODE;
1018         nw64(MIF_CONFIG, val);
1019
1020         return mii_init_common(np);
1021 }
1022
1023 static int niu_xcvr_init(struct niu *np)
1024 {
1025         const struct niu_phy_ops *ops = np->phy_ops;
1026         int err;
1027
1028         err = 0;
1029         if (ops->xcvr_init)
1030                 err = ops->xcvr_init(np);
1031
1032         return err;
1033 }
1034
1035 static int niu_serdes_init(struct niu *np)
1036 {
1037         const struct niu_phy_ops *ops = np->phy_ops;
1038         int err;
1039
1040         err = 0;
1041         if (ops->serdes_init)
1042                 err = ops->serdes_init(np);
1043
1044         return err;
1045 }
1046
1047 static void niu_init_xif(struct niu *);
1048 static void niu_handle_led(struct niu *, int status);
1049
1050 static int niu_link_status_common(struct niu *np, int link_up)
1051 {
1052         struct niu_link_config *lp = &np->link_config;
1053         struct net_device *dev = np->dev;
1054         unsigned long flags;
1055
1056         if (!netif_carrier_ok(dev) && link_up) {
1057                 niuinfo(LINK, "%s: Link is up at %s, %s duplex\n",
1058                        dev->name,
1059                        (lp->active_speed == SPEED_10000 ?
1060                         "10Gb/sec" :
1061                         (lp->active_speed == SPEED_1000 ?
1062                          "1Gb/sec" :
1063                          (lp->active_speed == SPEED_100 ?
1064                           "100Mbit/sec" : "10Mbit/sec"))),
1065                        (lp->active_duplex == DUPLEX_FULL ?
1066                         "full" : "half"));
1067
1068                 spin_lock_irqsave(&np->lock, flags);
1069                 niu_init_xif(np);
1070                 niu_handle_led(np, 1);
1071                 spin_unlock_irqrestore(&np->lock, flags);
1072
1073                 netif_carrier_on(dev);
1074         } else if (netif_carrier_ok(dev) && !link_up) {
1075                 niuwarn(LINK, "%s: Link is down\n", dev->name);
1076                 spin_lock_irqsave(&np->lock, flags);
1077                 niu_handle_led(np, 0);
1078                 spin_unlock_irqrestore(&np->lock, flags);
1079                 netif_carrier_off(dev);
1080         }
1081
1082         return 0;
1083 }
1084
1085 static int link_status_10g(struct niu *np, int *link_up_p)
1086 {
1087         unsigned long flags;
1088         int err, link_up;
1089
1090         link_up = 0;
1091
1092         spin_lock_irqsave(&np->lock, flags);
1093
1094         err = -EINVAL;
1095         if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
1096                 goto out;
1097
1098         err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
1099                         BCM8704_PMD_RCV_SIGDET);
1100         if (err < 0)
1101                 goto out;
1102         if (!(err & PMD_RCV_SIGDET_GLOBAL)) {
1103                 err = 0;
1104                 goto out;
1105         }
1106
1107         err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
1108                         BCM8704_PCS_10G_R_STATUS);
1109         if (err < 0)
1110                 goto out;
1111         if (!(err & PCS_10G_R_STATUS_BLK_LOCK)) {
1112                 err = 0;
1113                 goto out;
1114         }
1115
1116         err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
1117                         BCM8704_PHYXS_XGXS_LANE_STAT);
1118         if (err < 0)
1119                 goto out;
1120
1121         if (err != (PHYXS_XGXS_LANE_STAT_ALINGED |
1122                     PHYXS_XGXS_LANE_STAT_MAGIC |
1123                     PHYXS_XGXS_LANE_STAT_LANE3 |
1124                     PHYXS_XGXS_LANE_STAT_LANE2 |
1125                     PHYXS_XGXS_LANE_STAT_LANE1 |
1126                     PHYXS_XGXS_LANE_STAT_LANE0)) {
1127                 err = 0;
1128                 goto out;
1129         }
1130
1131         link_up = 1;
1132         np->link_config.active_speed = SPEED_10000;
1133         np->link_config.active_duplex = DUPLEX_FULL;
1134         err = 0;
1135
1136 out:
1137         spin_unlock_irqrestore(&np->lock, flags);
1138
1139         *link_up_p = link_up;
1140         return err;
1141 }
1142
1143 static int link_status_1g(struct niu *np, int *link_up_p)
1144 {
1145         u16 current_speed, bmsr;
1146         unsigned long flags;
1147         u8 current_duplex;
1148         int err, link_up;
1149
1150         link_up = 0;
1151         current_speed = SPEED_INVALID;
1152         current_duplex = DUPLEX_INVALID;
1153
1154         spin_lock_irqsave(&np->lock, flags);
1155
1156         err = -EINVAL;
1157         if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
1158                 goto out;
1159
1160         err = mii_read(np, np->phy_addr, MII_BMSR);
1161         if (err < 0)
1162                 goto out;
1163
1164         bmsr = err;
1165         if (bmsr & BMSR_LSTATUS) {
1166                 u16 adv, lpa, common, estat;
1167
1168                 err = mii_read(np, np->phy_addr, MII_ADVERTISE);
1169                 if (err < 0)
1170                         goto out;
1171                 adv = err;
1172
1173                 err = mii_read(np, np->phy_addr, MII_LPA);
1174                 if (err < 0)
1175                         goto out;
1176                 lpa = err;
1177
1178                 common = adv & lpa;
1179
1180                 err = mii_read(np, np->phy_addr, MII_ESTATUS);
1181                 if (err < 0)
1182                         goto out;
1183                 estat = err;
1184
1185                 link_up = 1;
1186                 if (estat & (ESTATUS_1000_TFULL | ESTATUS_1000_THALF)) {
1187                         current_speed = SPEED_1000;
1188                         if (estat & ESTATUS_1000_TFULL)
1189                                 current_duplex = DUPLEX_FULL;
1190                         else
1191                                 current_duplex = DUPLEX_HALF;
1192                 } else {
1193                         if (common & ADVERTISE_100BASE4) {
1194                                 current_speed = SPEED_100;
1195                                 current_duplex = DUPLEX_HALF;
1196                         } else if (common & ADVERTISE_100FULL) {
1197                                 current_speed = SPEED_100;
1198                                 current_duplex = DUPLEX_FULL;
1199                         } else if (common & ADVERTISE_100HALF) {
1200                                 current_speed = SPEED_100;
1201                                 current_duplex = DUPLEX_HALF;
1202                         } else if (common & ADVERTISE_10FULL) {
1203                                 current_speed = SPEED_10;
1204                                 current_duplex = DUPLEX_FULL;
1205                         } else if (common & ADVERTISE_10HALF) {
1206                                 current_speed = SPEED_10;
1207                                 current_duplex = DUPLEX_HALF;
1208                         } else
1209                                 link_up = 0;
1210                 }
1211         }
1212         err = 0;
1213
1214 out:
1215         spin_unlock_irqrestore(&np->lock, flags);
1216
1217         *link_up_p = link_up;
1218         return err;
1219 }
1220
1221 static int niu_link_status(struct niu *np, int *link_up_p)
1222 {
1223         const struct niu_phy_ops *ops = np->phy_ops;
1224         int err;
1225
1226         err = 0;
1227         if (ops->link_status)
1228                 err = ops->link_status(np, link_up_p);
1229
1230         return err;
1231 }
1232
1233 static void niu_timer(unsigned long __opaque)
1234 {
1235         struct niu *np = (struct niu *) __opaque;
1236         unsigned long off;
1237         int err, link_up;
1238
1239         err = niu_link_status(np, &link_up);
1240         if (!err)
1241                 niu_link_status_common(np, link_up);
1242
1243         if (netif_carrier_ok(np->dev))
1244                 off = 5 * HZ;
1245         else
1246                 off = 1 * HZ;
1247         np->timer.expires = jiffies + off;
1248
1249         add_timer(&np->timer);
1250 }
1251
1252 static const struct niu_phy_ops phy_ops_10g_fiber_niu = {
1253         .serdes_init            = serdes_init_niu,
1254         .xcvr_init              = xcvr_init_10g,
1255         .link_status            = link_status_10g,
1256 };
1257
1258 static const struct niu_phy_ops phy_ops_10g_fiber = {
1259         .serdes_init            = serdes_init_10g,
1260         .xcvr_init              = xcvr_init_10g,
1261         .link_status            = link_status_10g,
1262 };
1263
1264 static const struct niu_phy_ops phy_ops_10g_copper = {
1265         .serdes_init            = serdes_init_10g,
1266         .link_status            = link_status_10g, /* XXX */
1267 };
1268
1269 static const struct niu_phy_ops phy_ops_1g_fiber = {
1270         .serdes_init            = serdes_init_1g,
1271         .xcvr_init              = xcvr_init_1g,
1272         .link_status            = link_status_1g,
1273 };
1274
1275 static const struct niu_phy_ops phy_ops_1g_copper = {
1276         .xcvr_init              = xcvr_init_1g,
1277         .link_status            = link_status_1g,
1278 };
1279
1280 struct niu_phy_template {
1281         const struct niu_phy_ops        *ops;
1282         u32                             phy_addr_base;
1283 };
1284
1285 static const struct niu_phy_template phy_template_niu = {
1286         .ops            = &phy_ops_10g_fiber_niu,
1287         .phy_addr_base  = 16,
1288 };
1289
1290 static const struct niu_phy_template phy_template_10g_fiber = {
1291         .ops            = &phy_ops_10g_fiber,
1292         .phy_addr_base  = 8,
1293 };
1294
1295 static const struct niu_phy_template phy_template_10g_copper = {
1296         .ops            = &phy_ops_10g_copper,
1297         .phy_addr_base  = 10,
1298 };
1299
1300 static const struct niu_phy_template phy_template_1g_fiber = {
1301         .ops            = &phy_ops_1g_fiber,
1302         .phy_addr_base  = 0,
1303 };
1304
1305 static const struct niu_phy_template phy_template_1g_copper = {
1306         .ops            = &phy_ops_1g_copper,
1307         .phy_addr_base  = 0,
1308 };
1309
1310 static int niu_determine_phy_disposition(struct niu *np)
1311 {
1312         struct niu_parent *parent = np->parent;
1313         u8 plat_type = parent->plat_type;
1314         const struct niu_phy_template *tp;
1315         u32 phy_addr_off = 0;
1316
1317         if (plat_type == PLAT_TYPE_NIU) {
1318                 tp = &phy_template_niu;
1319                 phy_addr_off += np->port;
1320         } else {
1321                 switch (np->flags & (NIU_FLAGS_10G | NIU_FLAGS_FIBER)) {
1322                 case 0:
1323                         /* 1G copper */
1324                         tp = &phy_template_1g_copper;
1325                         if (plat_type == PLAT_TYPE_VF_P0)
1326                                 phy_addr_off = 10;
1327                         else if (plat_type == PLAT_TYPE_VF_P1)
1328                                 phy_addr_off = 26;
1329
1330                         phy_addr_off += (np->port ^ 0x3);
1331                         break;
1332
1333                 case NIU_FLAGS_10G:
1334                         /* 10G copper */
1335                         tp = &phy_template_1g_copper;
1336                         break;
1337
1338                 case NIU_FLAGS_FIBER:
1339                         /* 1G fiber */
1340                         tp = &phy_template_1g_fiber;
1341                         break;
1342
1343                 case NIU_FLAGS_10G | NIU_FLAGS_FIBER:
1344                         /* 10G fiber */
1345                         tp = &phy_template_10g_fiber;
1346                         if (plat_type == PLAT_TYPE_VF_P0 ||
1347                             plat_type == PLAT_TYPE_VF_P1)
1348                                 phy_addr_off = 8;
1349                         phy_addr_off += np->port;
1350                         break;
1351
1352                 default:
1353                         return -EINVAL;
1354                 }
1355         }
1356
1357         np->phy_ops = tp->ops;
1358         np->phy_addr = tp->phy_addr_base + phy_addr_off;
1359
1360         return 0;
1361 }
1362
1363 static int niu_init_link(struct niu *np)
1364 {
1365         struct niu_parent *parent = np->parent;
1366         int err, ignore;
1367
1368         if (parent->plat_type == PLAT_TYPE_NIU) {
1369                 err = niu_xcvr_init(np);
1370                 if (err)
1371                         return err;
1372                 msleep(200);
1373         }
1374         err = niu_serdes_init(np);
1375         if (err)
1376                 return err;
1377         msleep(200);
1378         err = niu_xcvr_init(np);
1379         if (!err)
1380                 niu_link_status(np, &ignore);
1381         return 0;
1382 }
1383
1384 static void niu_set_primary_mac(struct niu *np, unsigned char *addr)
1385 {
1386         u16 reg0 = addr[4] << 8 | addr[5];
1387         u16 reg1 = addr[2] << 8 | addr[3];
1388         u16 reg2 = addr[0] << 8 | addr[1];
1389
1390         if (np->flags & NIU_FLAGS_XMAC) {
1391                 nw64_mac(XMAC_ADDR0, reg0);
1392                 nw64_mac(XMAC_ADDR1, reg1);
1393                 nw64_mac(XMAC_ADDR2, reg2);
1394         } else {
1395                 nw64_mac(BMAC_ADDR0, reg0);
1396                 nw64_mac(BMAC_ADDR1, reg1);
1397                 nw64_mac(BMAC_ADDR2, reg2);
1398         }
1399 }
1400
1401 static int niu_num_alt_addr(struct niu *np)
1402 {
1403         if (np->flags & NIU_FLAGS_XMAC)
1404                 return XMAC_NUM_ALT_ADDR;
1405         else
1406                 return BMAC_NUM_ALT_ADDR;
1407 }
1408
1409 static int niu_set_alt_mac(struct niu *np, int index, unsigned char *addr)
1410 {
1411         u16 reg0 = addr[4] << 8 | addr[5];
1412         u16 reg1 = addr[2] << 8 | addr[3];
1413         u16 reg2 = addr[0] << 8 | addr[1];
1414
1415         if (index >= niu_num_alt_addr(np))
1416                 return -EINVAL;
1417
1418         if (np->flags & NIU_FLAGS_XMAC) {
1419                 nw64_mac(XMAC_ALT_ADDR0(index), reg0);
1420                 nw64_mac(XMAC_ALT_ADDR1(index), reg1);
1421                 nw64_mac(XMAC_ALT_ADDR2(index), reg2);
1422         } else {
1423                 nw64_mac(BMAC_ALT_ADDR0(index), reg0);
1424                 nw64_mac(BMAC_ALT_ADDR1(index), reg1);
1425                 nw64_mac(BMAC_ALT_ADDR2(index), reg2);
1426         }
1427
1428         return 0;
1429 }
1430
1431 static int niu_enable_alt_mac(struct niu *np, int index, int on)
1432 {
1433         unsigned long reg;
1434         u64 val, mask;
1435
1436         if (index >= niu_num_alt_addr(np))
1437                 return -EINVAL;
1438
1439         if (np->flags & NIU_FLAGS_XMAC)
1440                 reg = XMAC_ADDR_CMPEN;
1441         else
1442                 reg = BMAC_ADDR_CMPEN;
1443
1444         mask = 1 << index;
1445
1446         val = nr64_mac(reg);
1447         if (on)
1448                 val |= mask;
1449         else
1450                 val &= ~mask;
1451         nw64_mac(reg, val);
1452
1453         return 0;
1454 }
1455
1456 static void __set_rdc_table_num_hw(struct niu *np, unsigned long reg,
1457                                    int num, int mac_pref)
1458 {
1459         u64 val = nr64_mac(reg);
1460         val &= ~(HOST_INFO_MACRDCTBLN | HOST_INFO_MPR);
1461         val |= num;
1462         if (mac_pref)
1463                 val |= HOST_INFO_MPR;
1464         nw64_mac(reg, val);
1465 }
1466
1467 static int __set_rdc_table_num(struct niu *np,
1468                                int xmac_index, int bmac_index,
1469                                int rdc_table_num, int mac_pref)
1470 {
1471         unsigned long reg;
1472
1473         if (rdc_table_num & ~HOST_INFO_MACRDCTBLN)
1474                 return -EINVAL;
1475         if (np->flags & NIU_FLAGS_XMAC)
1476                 reg = XMAC_HOST_INFO(xmac_index);
1477         else
1478                 reg = BMAC_HOST_INFO(bmac_index);
1479         __set_rdc_table_num_hw(np, reg, rdc_table_num, mac_pref);
1480         return 0;
1481 }
1482
1483 static int niu_set_primary_mac_rdc_table(struct niu *np, int table_num,
1484                                          int mac_pref)
1485 {
1486         return __set_rdc_table_num(np, 17, 0, table_num, mac_pref);
1487 }
1488
1489 static int niu_set_multicast_mac_rdc_table(struct niu *np, int table_num,
1490                                            int mac_pref)
1491 {
1492         return __set_rdc_table_num(np, 16, 8, table_num, mac_pref);
1493 }
1494
1495 static int niu_set_alt_mac_rdc_table(struct niu *np, int idx,
1496                                      int table_num, int mac_pref)
1497 {
1498         if (idx >= niu_num_alt_addr(np))
1499                 return -EINVAL;
1500         return __set_rdc_table_num(np, idx, idx + 1, table_num, mac_pref);
1501 }
1502
1503 static u64 vlan_entry_set_parity(u64 reg_val)
1504 {
1505         u64 port01_mask;
1506         u64 port23_mask;
1507
1508         port01_mask = 0x00ff;
1509         port23_mask = 0xff00;
1510
1511         if (hweight64(reg_val & port01_mask) & 1)
1512                 reg_val |= ENET_VLAN_TBL_PARITY0;
1513         else
1514                 reg_val &= ~ENET_VLAN_TBL_PARITY0;
1515
1516         if (hweight64(reg_val & port23_mask) & 1)
1517                 reg_val |= ENET_VLAN_TBL_PARITY1;
1518         else
1519                 reg_val &= ~ENET_VLAN_TBL_PARITY1;
1520
1521         return reg_val;
1522 }
1523
1524 static void vlan_tbl_write(struct niu *np, unsigned long index,
1525                            int port, int vpr, int rdc_table)
1526 {
1527         u64 reg_val = nr64(ENET_VLAN_TBL(index));
1528
1529         reg_val &= ~((ENET_VLAN_TBL_VPR |
1530                       ENET_VLAN_TBL_VLANRDCTBLN) <<
1531                      ENET_VLAN_TBL_SHIFT(port));
1532         if (vpr)
1533                 reg_val |= (ENET_VLAN_TBL_VPR <<
1534                             ENET_VLAN_TBL_SHIFT(port));
1535         reg_val |= (rdc_table << ENET_VLAN_TBL_SHIFT(port));
1536
1537         reg_val = vlan_entry_set_parity(reg_val);
1538
1539         nw64(ENET_VLAN_TBL(index), reg_val);
1540 }
1541
1542 static void vlan_tbl_clear(struct niu *np)
1543 {
1544         int i;
1545
1546         for (i = 0; i < ENET_VLAN_TBL_NUM_ENTRIES; i++)
1547                 nw64(ENET_VLAN_TBL(i), 0);
1548 }
1549
1550 static int tcam_wait_bit(struct niu *np, u64 bit)
1551 {
1552         int limit = 1000;
1553
1554         while (--limit > 0) {
1555                 if (nr64(TCAM_CTL) & bit)
1556                         break;
1557                 udelay(1);
1558         }
1559         if (limit < 0)
1560                 return -ENODEV;
1561
1562         return 0;
1563 }
1564
1565 static int tcam_flush(struct niu *np, int index)
1566 {
1567         nw64(TCAM_KEY_0, 0x00);
1568         nw64(TCAM_KEY_MASK_0, 0xff);
1569         nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_WRITE | index));
1570
1571         return tcam_wait_bit(np, TCAM_CTL_STAT);
1572 }
1573
1574 #if 0
1575 static int tcam_read(struct niu *np, int index,
1576                      u64 *key, u64 *mask)
1577 {
1578         int err;
1579
1580         nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_READ | index));
1581         err = tcam_wait_bit(np, TCAM_CTL_STAT);
1582         if (!err) {
1583                 key[0] = nr64(TCAM_KEY_0);
1584                 key[1] = nr64(TCAM_KEY_1);
1585                 key[2] = nr64(TCAM_KEY_2);
1586                 key[3] = nr64(TCAM_KEY_3);
1587                 mask[0] = nr64(TCAM_KEY_MASK_0);
1588                 mask[1] = nr64(TCAM_KEY_MASK_1);
1589                 mask[2] = nr64(TCAM_KEY_MASK_2);
1590                 mask[3] = nr64(TCAM_KEY_MASK_3);
1591         }
1592         return err;
1593 }
1594 #endif
1595
1596 static int tcam_write(struct niu *np, int index,
1597                       u64 *key, u64 *mask)
1598 {
1599         nw64(TCAM_KEY_0, key[0]);
1600         nw64(TCAM_KEY_1, key[1]);
1601         nw64(TCAM_KEY_2, key[2]);
1602         nw64(TCAM_KEY_3, key[3]);
1603         nw64(TCAM_KEY_MASK_0, mask[0]);
1604         nw64(TCAM_KEY_MASK_1, mask[1]);
1605         nw64(TCAM_KEY_MASK_2, mask[2]);
1606         nw64(TCAM_KEY_MASK_3, mask[3]);
1607         nw64(TCAM_CTL, (TCAM_CTL_RWC_TCAM_WRITE | index));
1608
1609         return tcam_wait_bit(np, TCAM_CTL_STAT);
1610 }
1611
1612 #if 0
1613 static int tcam_assoc_read(struct niu *np, int index, u64 *data)
1614 {
1615         int err;
1616
1617         nw64(TCAM_CTL, (TCAM_CTL_RWC_RAM_READ | index));
1618         err = tcam_wait_bit(np, TCAM_CTL_STAT);
1619         if (!err)
1620                 *data = nr64(TCAM_KEY_1);
1621
1622         return err;
1623 }
1624 #endif
1625
1626 static int tcam_assoc_write(struct niu *np, int index, u64 assoc_data)
1627 {
1628         nw64(TCAM_KEY_1, assoc_data);
1629         nw64(TCAM_CTL, (TCAM_CTL_RWC_RAM_WRITE | index));
1630
1631         return tcam_wait_bit(np, TCAM_CTL_STAT);
1632 }
1633
1634 static void tcam_enable(struct niu *np, int on)
1635 {
1636         u64 val = nr64(FFLP_CFG_1);
1637
1638         if (on)
1639                 val &= ~FFLP_CFG_1_TCAM_DIS;
1640         else
1641                 val |= FFLP_CFG_1_TCAM_DIS;
1642         nw64(FFLP_CFG_1, val);
1643 }
1644
1645 static void tcam_set_lat_and_ratio(struct niu *np, u64 latency, u64 ratio)
1646 {
1647         u64 val = nr64(FFLP_CFG_1);
1648
1649         val &= ~(FFLP_CFG_1_FFLPINITDONE |
1650                  FFLP_CFG_1_CAMLAT |
1651                  FFLP_CFG_1_CAMRATIO);
1652         val |= (latency << FFLP_CFG_1_CAMLAT_SHIFT);
1653         val |= (ratio << FFLP_CFG_1_CAMRATIO_SHIFT);
1654         nw64(FFLP_CFG_1, val);
1655
1656         val = nr64(FFLP_CFG_1);
1657         val |= FFLP_CFG_1_FFLPINITDONE;
1658         nw64(FFLP_CFG_1, val);
1659 }
1660
1661 static int tcam_user_eth_class_enable(struct niu *np, unsigned long class,
1662                                       int on)
1663 {
1664         unsigned long reg;
1665         u64 val;
1666
1667         if (class < CLASS_CODE_ETHERTYPE1 ||
1668             class > CLASS_CODE_ETHERTYPE2)
1669                 return -EINVAL;
1670
1671         reg = L2_CLS(class - CLASS_CODE_ETHERTYPE1);
1672         val = nr64(reg);
1673         if (on)
1674                 val |= L2_CLS_VLD;
1675         else
1676                 val &= ~L2_CLS_VLD;
1677         nw64(reg, val);
1678
1679         return 0;
1680 }
1681
1682 #if 0
1683 static int tcam_user_eth_class_set(struct niu *np, unsigned long class,
1684                                    u64 ether_type)
1685 {
1686         unsigned long reg;
1687         u64 val;
1688
1689         if (class < CLASS_CODE_ETHERTYPE1 ||
1690             class > CLASS_CODE_ETHERTYPE2 ||
1691             (ether_type & ~(u64)0xffff) != 0)
1692                 return -EINVAL;
1693
1694         reg = L2_CLS(class - CLASS_CODE_ETHERTYPE1);
1695         val = nr64(reg);
1696         val &= ~L2_CLS_ETYPE;
1697         val |= (ether_type << L2_CLS_ETYPE_SHIFT);
1698         nw64(reg, val);
1699
1700         return 0;
1701 }
1702 #endif
1703
1704 static int tcam_user_ip_class_enable(struct niu *np, unsigned long class,
1705                                      int on)
1706 {
1707         unsigned long reg;
1708         u64 val;
1709
1710         if (class < CLASS_CODE_USER_PROG1 ||
1711             class > CLASS_CODE_USER_PROG4)
1712                 return -EINVAL;
1713
1714         reg = L3_CLS(class - CLASS_CODE_USER_PROG1);
1715         val = nr64(reg);
1716         if (on)
1717                 val |= L3_CLS_VALID;
1718         else
1719                 val &= ~L3_CLS_VALID;
1720         nw64(reg, val);
1721
1722         return 0;
1723 }
1724
1725 #if 0
1726 static int tcam_user_ip_class_set(struct niu *np, unsigned long class,
1727                                   int ipv6, u64 protocol_id,
1728                                   u64 tos_mask, u64 tos_val)
1729 {
1730         unsigned long reg;
1731         u64 val;
1732
1733         if (class < CLASS_CODE_USER_PROG1 ||
1734             class > CLASS_CODE_USER_PROG4 ||
1735             (protocol_id & ~(u64)0xff) != 0 ||
1736             (tos_mask & ~(u64)0xff) != 0 ||
1737             (tos_val & ~(u64)0xff) != 0)
1738                 return -EINVAL;
1739
1740         reg = L3_CLS(class - CLASS_CODE_USER_PROG1);
1741         val = nr64(reg);
1742         val &= ~(L3_CLS_IPVER | L3_CLS_PID |
1743                  L3_CLS_TOSMASK | L3_CLS_TOS);
1744         if (ipv6)
1745                 val |= L3_CLS_IPVER;
1746         val |= (protocol_id << L3_CLS_PID_SHIFT);
1747         val |= (tos_mask << L3_CLS_TOSMASK_SHIFT);
1748         val |= (tos_val << L3_CLS_TOS_SHIFT);
1749         nw64(reg, val);
1750
1751         return 0;
1752 }
1753 #endif
1754
1755 static int tcam_early_init(struct niu *np)
1756 {
1757         unsigned long i;
1758         int err;
1759
1760         tcam_enable(np, 0);
1761         tcam_set_lat_and_ratio(np,
1762                                DEFAULT_TCAM_LATENCY,
1763                                DEFAULT_TCAM_ACCESS_RATIO);
1764         for (i = CLASS_CODE_ETHERTYPE1; i <= CLASS_CODE_ETHERTYPE2; i++) {
1765                 err = tcam_user_eth_class_enable(np, i, 0);
1766                 if (err)
1767                         return err;
1768         }
1769         for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_USER_PROG4; i++) {
1770                 err = tcam_user_ip_class_enable(np, i, 0);
1771                 if (err)
1772                         return err;
1773         }
1774
1775         return 0;
1776 }
1777
1778 static int tcam_flush_all(struct niu *np)
1779 {
1780         unsigned long i;
1781
1782         for (i = 0; i < np->parent->tcam_num_entries; i++) {
1783                 int err = tcam_flush(np, i);
1784                 if (err)
1785                         return err;
1786         }
1787         return 0;
1788 }
1789
1790 static u64 hash_addr_regval(unsigned long index, unsigned long num_entries)
1791 {
1792         return ((u64)index | (num_entries == 1 ?
1793                               HASH_TBL_ADDR_AUTOINC : 0));
1794 }
1795
1796 #if 0
1797 static int hash_read(struct niu *np, unsigned long partition,
1798                      unsigned long index, unsigned long num_entries,
1799                      u64 *data)
1800 {
1801         u64 val = hash_addr_regval(index, num_entries);
1802         unsigned long i;
1803
1804         if (partition >= FCRAM_NUM_PARTITIONS ||
1805             index + num_entries > FCRAM_SIZE)
1806                 return -EINVAL;
1807
1808         nw64(HASH_TBL_ADDR(partition), val);
1809         for (i = 0; i < num_entries; i++)
1810                 data[i] = nr64(HASH_TBL_DATA(partition));
1811
1812         return 0;
1813 }
1814 #endif
1815
1816 static int hash_write(struct niu *np, unsigned long partition,
1817                       unsigned long index, unsigned long num_entries,
1818                       u64 *data)
1819 {
1820         u64 val = hash_addr_regval(index, num_entries);
1821         unsigned long i;
1822
1823         if (partition >= FCRAM_NUM_PARTITIONS ||
1824             index + (num_entries * 8) > FCRAM_SIZE)
1825                 return -EINVAL;
1826
1827         nw64(HASH_TBL_ADDR(partition), val);
1828         for (i = 0; i < num_entries; i++)
1829                 nw64(HASH_TBL_DATA(partition), data[i]);
1830
1831         return 0;
1832 }
1833
1834 static void fflp_reset(struct niu *np)
1835 {
1836         u64 val;
1837
1838         nw64(FFLP_CFG_1, FFLP_CFG_1_PIO_FIO_RST);
1839         udelay(10);
1840         nw64(FFLP_CFG_1, 0);
1841
1842         val = FFLP_CFG_1_FCRAMOUTDR_NORMAL | FFLP_CFG_1_FFLPINITDONE;
1843         nw64(FFLP_CFG_1, val);
1844 }
1845
1846 static void fflp_set_timings(struct niu *np)
1847 {
1848         u64 val = nr64(FFLP_CFG_1);
1849
1850         val &= ~FFLP_CFG_1_FFLPINITDONE;
1851         val |= (DEFAULT_FCRAMRATIO << FFLP_CFG_1_FCRAMRATIO_SHIFT);
1852         nw64(FFLP_CFG_1, val);
1853
1854         val = nr64(FFLP_CFG_1);
1855         val |= FFLP_CFG_1_FFLPINITDONE;
1856         nw64(FFLP_CFG_1, val);
1857
1858         val = nr64(FCRAM_REF_TMR);
1859         val &= ~(FCRAM_REF_TMR_MAX | FCRAM_REF_TMR_MIN);
1860         val |= (DEFAULT_FCRAM_REFRESH_MAX << FCRAM_REF_TMR_MAX_SHIFT);
1861         val |= (DEFAULT_FCRAM_REFRESH_MIN << FCRAM_REF_TMR_MIN_SHIFT);
1862         nw64(FCRAM_REF_TMR, val);
1863 }
1864
1865 static int fflp_set_partition(struct niu *np, u64 partition,
1866                               u64 mask, u64 base, int enable)
1867 {
1868         unsigned long reg;
1869         u64 val;
1870
1871         if (partition >= FCRAM_NUM_PARTITIONS ||
1872             (mask & ~(u64)0x1f) != 0 ||
1873             (base & ~(u64)0x1f) != 0)
1874                 return -EINVAL;
1875
1876         reg = FLW_PRT_SEL(partition);
1877
1878         val = nr64(reg);
1879         val &= ~(FLW_PRT_SEL_EXT | FLW_PRT_SEL_MASK | FLW_PRT_SEL_BASE);
1880         val |= (mask << FLW_PRT_SEL_MASK_SHIFT);
1881         val |= (base << FLW_PRT_SEL_BASE_SHIFT);
1882         if (enable)
1883                 val |= FLW_PRT_SEL_EXT;
1884         nw64(reg, val);
1885
1886         return 0;
1887 }
1888
1889 static int fflp_disable_all_partitions(struct niu *np)
1890 {
1891         unsigned long i;
1892
1893         for (i = 0; i < FCRAM_NUM_PARTITIONS; i++) {
1894                 int err = fflp_set_partition(np, 0, 0, 0, 0);
1895                 if (err)
1896                         return err;
1897         }
1898         return 0;
1899 }
1900
1901 static void fflp_llcsnap_enable(struct niu *np, int on)
1902 {
1903         u64 val = nr64(FFLP_CFG_1);
1904
1905         if (on)
1906                 val |= FFLP_CFG_1_LLCSNAP;
1907         else
1908                 val &= ~FFLP_CFG_1_LLCSNAP;
1909         nw64(FFLP_CFG_1, val);
1910 }
1911
1912 static void fflp_errors_enable(struct niu *np, int on)
1913 {
1914         u64 val = nr64(FFLP_CFG_1);
1915
1916         if (on)
1917                 val &= ~FFLP_CFG_1_ERRORDIS;
1918         else
1919                 val |= FFLP_CFG_1_ERRORDIS;
1920         nw64(FFLP_CFG_1, val);
1921 }
1922
1923 static int fflp_hash_clear(struct niu *np)
1924 {
1925         struct fcram_hash_ipv4 ent;
1926         unsigned long i;
1927
1928         /* IPV4 hash entry with valid bit clear, rest is don't care.  */
1929         memset(&ent, 0, sizeof(ent));
1930         ent.header = HASH_HEADER_EXT;
1931
1932         for (i = 0; i < FCRAM_SIZE; i += sizeof(ent)) {
1933                 int err = hash_write(np, 0, i, 1, (u64 *) &ent);
1934                 if (err)
1935                         return err;
1936         }
1937         return 0;
1938 }
1939
1940 static int fflp_early_init(struct niu *np)
1941 {
1942         struct niu_parent *parent;
1943         unsigned long flags;
1944         int err;
1945
1946         niu_lock_parent(np, flags);
1947
1948         parent = np->parent;
1949         err = 0;
1950         if (!(parent->flags & PARENT_FLGS_CLS_HWINIT)) {
1951                 niudbg(PROBE, "fflp_early_init: Initting hw on port %u\n",
1952                        np->port);
1953                 if (np->parent->plat_type != PLAT_TYPE_NIU) {
1954                         fflp_reset(np);
1955                         fflp_set_timings(np);
1956                         err = fflp_disable_all_partitions(np);
1957                         if (err) {
1958                                 niudbg(PROBE, "fflp_disable_all_partitions "
1959                                        "failed, err=%d\n", err);
1960                                 goto out;
1961                         }
1962                 }
1963
1964                 err = tcam_early_init(np);
1965                 if (err) {
1966                         niudbg(PROBE, "tcam_early_init failed, err=%d\n",
1967                                err);
1968                         goto out;
1969                 }
1970                 fflp_llcsnap_enable(np, 1);
1971                 fflp_errors_enable(np, 0);
1972                 nw64(H1POLY, 0);
1973                 nw64(H2POLY, 0);
1974
1975                 err = tcam_flush_all(np);
1976                 if (err) {
1977                         niudbg(PROBE, "tcam_flush_all failed, err=%d\n",
1978                                err);
1979                         goto out;
1980                 }
1981                 if (np->parent->plat_type != PLAT_TYPE_NIU) {
1982                         err = fflp_hash_clear(np);
1983                         if (err) {
1984                                 niudbg(PROBE, "fflp_hash_clear failed, "
1985                                        "err=%d\n", err);
1986                                 goto out;
1987                         }
1988                 }
1989
1990                 vlan_tbl_clear(np);
1991
1992                 niudbg(PROBE, "fflp_early_init: Success\n");
1993                 parent->flags |= PARENT_FLGS_CLS_HWINIT;
1994         }
1995 out:
1996         niu_unlock_parent(np, flags);
1997         return err;
1998 }
1999
2000 static int niu_set_flow_key(struct niu *np, unsigned long class_code, u64 key)
2001 {
2002         if (class_code < CLASS_CODE_USER_PROG1 ||
2003             class_code > CLASS_CODE_SCTP_IPV6)
2004                 return -EINVAL;
2005
2006         nw64(FLOW_KEY(class_code - CLASS_CODE_USER_PROG1), key);
2007         return 0;
2008 }
2009
2010 static int niu_set_tcam_key(struct niu *np, unsigned long class_code, u64 key)
2011 {
2012         if (class_code < CLASS_CODE_USER_PROG1 ||
2013             class_code > CLASS_CODE_SCTP_IPV6)
2014                 return -EINVAL;
2015
2016         nw64(TCAM_KEY(class_code - CLASS_CODE_USER_PROG1), key);
2017         return 0;
2018 }
2019
2020 static void niu_rx_skb_append(struct sk_buff *skb, struct page *page,
2021                               u32 offset, u32 size)
2022 {
2023         int i = skb_shinfo(skb)->nr_frags;
2024         skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2025
2026         frag->page = page;
2027         frag->page_offset = offset;
2028         frag->size = size;
2029
2030         skb->len += size;
2031         skb->data_len += size;
2032         skb->truesize += size;
2033
2034         skb_shinfo(skb)->nr_frags = i + 1;
2035 }
2036
2037 static unsigned int niu_hash_rxaddr(struct rx_ring_info *rp, u64 a)
2038 {
2039         a >>= PAGE_SHIFT;
2040         a ^= (a >> ilog2(MAX_RBR_RING_SIZE));
2041
2042         return (a & (MAX_RBR_RING_SIZE - 1));
2043 }
2044
2045 static struct page *niu_find_rxpage(struct rx_ring_info *rp, u64 addr,
2046                                     struct page ***link)
2047 {
2048         unsigned int h = niu_hash_rxaddr(rp, addr);
2049         struct page *p, **pp;
2050
2051         addr &= PAGE_MASK;
2052         pp = &rp->rxhash[h];
2053         for (; (p = *pp) != NULL; pp = (struct page **) &p->mapping) {
2054                 if (p->index == addr) {
2055                         *link = pp;
2056                         break;
2057                 }
2058         }
2059
2060         return p;
2061 }
2062
2063 static void niu_hash_page(struct rx_ring_info *rp, struct page *page, u64 base)
2064 {
2065         unsigned int h = niu_hash_rxaddr(rp, base);
2066
2067         page->index = base;
2068         page->mapping = (struct address_space *) rp->rxhash[h];
2069         rp->rxhash[h] = page;
2070 }
2071
2072 static int niu_rbr_add_page(struct niu *np, struct rx_ring_info *rp,
2073                             gfp_t mask, int start_index)
2074 {
2075         struct page *page;
2076         u64 addr;
2077         int i;
2078
2079         page = alloc_page(mask);
2080         if (!page)
2081                 return -ENOMEM;
2082
2083         addr = np->ops->map_page(np->device, page, 0,
2084                                  PAGE_SIZE, DMA_FROM_DEVICE);
2085
2086         niu_hash_page(rp, page, addr);
2087         if (rp->rbr_blocks_per_page > 1)
2088                 atomic_add(rp->rbr_blocks_per_page - 1,
2089                            &compound_head(page)->_count);
2090
2091         for (i = 0; i < rp->rbr_blocks_per_page; i++) {
2092                 __le32 *rbr = &rp->rbr[start_index + i];
2093
2094                 *rbr = cpu_to_le32(addr >> RBR_DESCR_ADDR_SHIFT);
2095                 addr += rp->rbr_block_size;
2096         }
2097
2098         return 0;
2099 }
2100
2101 static void niu_rbr_refill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)
2102 {
2103         int index = rp->rbr_index;
2104
2105         rp->rbr_pending++;
2106         if ((rp->rbr_pending % rp->rbr_blocks_per_page) == 0) {
2107                 int err = niu_rbr_add_page(np, rp, mask, index);
2108
2109                 if (unlikely(err)) {
2110                         rp->rbr_pending--;
2111                         return;
2112                 }
2113
2114                 rp->rbr_index += rp->rbr_blocks_per_page;
2115                 BUG_ON(rp->rbr_index > rp->rbr_table_size);
2116                 if (rp->rbr_index == rp->rbr_table_size)
2117                         rp->rbr_index = 0;
2118
2119                 if (rp->rbr_pending >= rp->rbr_kick_thresh) {
2120                         nw64(RBR_KICK(rp->rx_channel), rp->rbr_pending);
2121                         rp->rbr_pending = 0;
2122                 }
2123         }
2124 }
2125
2126 static int niu_rx_pkt_ignore(struct niu *np, struct rx_ring_info *rp)
2127 {
2128         unsigned int index = rp->rcr_index;
2129         int num_rcr = 0;
2130
2131         rp->rx_dropped++;
2132         while (1) {
2133                 struct page *page, **link;
2134                 u64 addr, val;
2135                 u32 rcr_size;
2136
2137                 num_rcr++;
2138
2139                 val = le64_to_cpup(&rp->rcr[index]);
2140                 addr = (val & RCR_ENTRY_PKT_BUF_ADDR) <<
2141                         RCR_ENTRY_PKT_BUF_ADDR_SHIFT;
2142                 page = niu_find_rxpage(rp, addr, &link);
2143
2144                 rcr_size = rp->rbr_sizes[(val & RCR_ENTRY_PKTBUFSZ) >>
2145                                          RCR_ENTRY_PKTBUFSZ_SHIFT];
2146                 if ((page->index + PAGE_SIZE) - rcr_size == addr) {
2147                         *link = (struct page *) page->mapping;
2148                         np->ops->unmap_page(np->device, page->index,
2149                                             PAGE_SIZE, DMA_FROM_DEVICE);
2150                         page->index = 0;
2151                         page->mapping = NULL;
2152                         __free_page(page);
2153                         rp->rbr_refill_pending++;
2154                 }
2155
2156                 index = NEXT_RCR(rp, index);
2157                 if (!(val & RCR_ENTRY_MULTI))
2158                         break;
2159
2160         }
2161         rp->rcr_index = index;
2162
2163         return num_rcr;
2164 }
2165
2166 static int niu_process_rx_pkt(struct niu *np, struct rx_ring_info *rp)
2167 {
2168         unsigned int index = rp->rcr_index;
2169         struct sk_buff *skb;
2170         int len, num_rcr;
2171
2172         skb = netdev_alloc_skb(np->dev, RX_SKB_ALLOC_SIZE);
2173         if (unlikely(!skb))
2174                 return niu_rx_pkt_ignore(np, rp);
2175
2176         num_rcr = 0;
2177         while (1) {
2178                 struct page *page, **link;
2179                 u32 rcr_size, append_size;
2180                 u64 addr, val, off;
2181
2182                 num_rcr++;
2183
2184                 val = le64_to_cpup(&rp->rcr[index]);
2185
2186                 len = (val & RCR_ENTRY_L2_LEN) >>
2187                         RCR_ENTRY_L2_LEN_SHIFT;
2188                 len -= ETH_FCS_LEN;
2189
2190                 addr = (val & RCR_ENTRY_PKT_BUF_ADDR) <<
2191                         RCR_ENTRY_PKT_BUF_ADDR_SHIFT;
2192                 page = niu_find_rxpage(rp, addr, &link);
2193
2194                 rcr_size = rp->rbr_sizes[(val & RCR_ENTRY_PKTBUFSZ) >>
2195                                          RCR_ENTRY_PKTBUFSZ_SHIFT];
2196
2197                 off = addr & ~PAGE_MASK;
2198                 append_size = rcr_size;
2199                 if (num_rcr == 1) {
2200                         int ptype;
2201
2202                         off += 2;
2203                         append_size -= 2;
2204
2205                         ptype = (val >> RCR_ENTRY_PKT_TYPE_SHIFT);
2206                         if ((ptype == RCR_PKT_TYPE_TCP ||
2207                              ptype == RCR_PKT_TYPE_UDP) &&
2208                             !(val & (RCR_ENTRY_NOPORT |
2209                                      RCR_ENTRY_ERROR)))
2210                                 skb->ip_summed = CHECKSUM_UNNECESSARY;
2211                         else
2212                                 skb->ip_summed = CHECKSUM_NONE;
2213                 }
2214                 if (!(val & RCR_ENTRY_MULTI))
2215                         append_size = len - skb->len;
2216
2217                 niu_rx_skb_append(skb, page, off, append_size);
2218                 if ((page->index + rp->rbr_block_size) - rcr_size == addr) {
2219                         *link = (struct page *) page->mapping;
2220                         np->ops->unmap_page(np->device, page->index,
2221                                             PAGE_SIZE, DMA_FROM_DEVICE);
2222                         page->index = 0;
2223                         page->mapping = NULL;
2224                         rp->rbr_refill_pending++;
2225                 } else
2226                         get_page(page);
2227
2228                 index = NEXT_RCR(rp, index);
2229                 if (!(val & RCR_ENTRY_MULTI))
2230                         break;
2231
2232         }
2233         rp->rcr_index = index;
2234
2235         skb_reserve(skb, NET_IP_ALIGN);
2236         __pskb_pull_tail(skb, min(len, NIU_RXPULL_MAX));
2237
2238         rp->rx_packets++;
2239         rp->rx_bytes += skb->len;
2240
2241         skb->protocol = eth_type_trans(skb, np->dev);
2242         netif_receive_skb(skb);
2243
2244         return num_rcr;
2245 }
2246
2247 static int niu_rbr_fill(struct niu *np, struct rx_ring_info *rp, gfp_t mask)
2248 {
2249         int blocks_per_page = rp->rbr_blocks_per_page;
2250         int err, index = rp->rbr_index;
2251
2252         err = 0;
2253         while (index < (rp->rbr_table_size - blocks_per_page)) {
2254                 err = niu_rbr_add_page(np, rp, mask, index);
2255                 if (err)
2256                         break;
2257
2258                 index += blocks_per_page;
2259         }
2260
2261         rp->rbr_index = index;
2262         return err;
2263 }
2264
2265 static void niu_rbr_free(struct niu *np, struct rx_ring_info *rp)
2266 {
2267         int i;
2268
2269         for (i = 0; i < MAX_RBR_RING_SIZE; i++) {
2270                 struct page *page;
2271
2272                 page = rp->rxhash[i];
2273                 while (page) {
2274                         struct page *next = (struct page *) page->mapping;
2275                         u64 base = page->index;
2276
2277                         np->ops->unmap_page(np->device, base, PAGE_SIZE,
2278                                             DMA_FROM_DEVICE);
2279                         page->index = 0;
2280                         page->mapping = NULL;
2281
2282                         __free_page(page);
2283
2284                         page = next;
2285                 }
2286         }
2287
2288         for (i = 0; i < rp->rbr_table_size; i++)
2289                 rp->rbr[i] = cpu_to_le32(0);
2290         rp->rbr_index = 0;
2291 }
2292
2293 static int release_tx_packet(struct niu *np, struct tx_ring_info *rp, int idx)
2294 {
2295         struct tx_buff_info *tb = &rp->tx_buffs[idx];
2296         struct sk_buff *skb = tb->skb;
2297         struct tx_pkt_hdr *tp;
2298         u64 tx_flags;
2299         int i, len;
2300
2301         tp = (struct tx_pkt_hdr *) skb->data;
2302         tx_flags = le64_to_cpup(&tp->flags);
2303
2304         rp->tx_packets++;
2305         rp->tx_bytes += (((tx_flags & TXHDR_LEN) >> TXHDR_LEN_SHIFT) -
2306                          ((tx_flags & TXHDR_PAD) / 2));
2307
2308         len = skb_headlen(skb);
2309         np->ops->unmap_single(np->device, tb->mapping,
2310                               len, DMA_TO_DEVICE);
2311
2312         if (le64_to_cpu(rp->descr[idx]) & TX_DESC_MARK)
2313                 rp->mark_pending--;
2314
2315         tb->skb = NULL;
2316         do {
2317                 idx = NEXT_TX(rp, idx);
2318                 len -= MAX_TX_DESC_LEN;
2319         } while (len > 0);
2320
2321         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
2322                 tb = &rp->tx_buffs[idx];
2323                 BUG_ON(tb->skb != NULL);
2324                 np->ops->unmap_page(np->device, tb->mapping,
2325                                     skb_shinfo(skb)->frags[i].size,
2326                                     DMA_TO_DEVICE);
2327                 idx = NEXT_TX(rp, idx);
2328         }
2329
2330         dev_kfree_skb(skb);
2331
2332         return idx;
2333 }
2334
2335 #define NIU_TX_WAKEUP_THRESH(rp)                ((rp)->pending / 4)
2336
2337 static void niu_tx_work(struct niu *np, struct tx_ring_info *rp)
2338 {
2339         u16 pkt_cnt, tmp;
2340         int cons;
2341         u64 cs;
2342
2343         cs = rp->tx_cs;
2344         if (unlikely(!(cs & (TX_CS_MK | TX_CS_MMK))))
2345                 goto out;
2346
2347         tmp = pkt_cnt = (cs & TX_CS_PKT_CNT) >> TX_CS_PKT_CNT_SHIFT;
2348         pkt_cnt = (pkt_cnt - rp->last_pkt_cnt) &
2349                 (TX_CS_PKT_CNT >> TX_CS_PKT_CNT_SHIFT);
2350
2351         rp->last_pkt_cnt = tmp;
2352
2353         cons = rp->cons;
2354
2355         niudbg(TX_DONE, "%s: niu_tx_work() pkt_cnt[%u] cons[%d]\n",
2356                np->dev->name, pkt_cnt, cons);
2357
2358         while (pkt_cnt--)
2359                 cons = release_tx_packet(np, rp, cons);
2360
2361         rp->cons = cons;
2362         smp_mb();
2363
2364 out:
2365         if (unlikely(netif_queue_stopped(np->dev) &&
2366                      (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp)))) {
2367                 netif_tx_lock(np->dev);
2368                 if (netif_queue_stopped(np->dev) &&
2369                     (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp)))
2370                         netif_wake_queue(np->dev);
2371                 netif_tx_unlock(np->dev);
2372         }
2373 }
2374
2375 static int niu_rx_work(struct niu *np, struct rx_ring_info *rp, int budget)
2376 {
2377         int qlen, rcr_done = 0, work_done = 0;
2378         struct rxdma_mailbox *mbox = rp->mbox;
2379         u64 stat;
2380
2381 #if 1
2382         stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
2383         qlen = nr64(RCRSTAT_A(rp->rx_channel)) & RCRSTAT_A_QLEN;
2384 #else
2385         stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
2386         qlen = (le64_to_cpup(&mbox->rcrstat_a) & RCRSTAT_A_QLEN);
2387 #endif
2388         mbox->rx_dma_ctl_stat = 0;
2389         mbox->rcrstat_a = 0;
2390
2391         niudbg(RX_STATUS, "%s: niu_rx_work(chan[%d]), stat[%llx] qlen=%d\n",
2392                np->dev->name, rp->rx_channel, (unsigned long long) stat, qlen);
2393
2394         rcr_done = work_done = 0;
2395         qlen = min(qlen, budget);
2396         while (work_done < qlen) {
2397                 rcr_done += niu_process_rx_pkt(np, rp);
2398                 work_done++;
2399         }
2400
2401         if (rp->rbr_refill_pending >= rp->rbr_kick_thresh) {
2402                 unsigned int i;
2403
2404                 for (i = 0; i < rp->rbr_refill_pending; i++)
2405                         niu_rbr_refill(np, rp, GFP_ATOMIC);
2406                 rp->rbr_refill_pending = 0;
2407         }
2408
2409         stat = (RX_DMA_CTL_STAT_MEX |
2410                 ((u64)work_done << RX_DMA_CTL_STAT_PKTREAD_SHIFT) |
2411                 ((u64)rcr_done << RX_DMA_CTL_STAT_PTRREAD_SHIFT));
2412
2413         nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat);
2414
2415         return work_done;
2416 }
2417
2418 static int niu_poll_core(struct niu *np, struct niu_ldg *lp, int budget)
2419 {
2420         u64 v0 = lp->v0;
2421         u32 tx_vec = (v0 >> 32);
2422         u32 rx_vec = (v0 & 0xffffffff);
2423         int i, work_done = 0;
2424
2425         niudbg(INTR, "%s: niu_poll_core() v0[%016llx]\n",
2426                np->dev->name, (unsigned long long) v0);
2427
2428         for (i = 0; i < np->num_tx_rings; i++) {
2429                 struct tx_ring_info *rp = &np->tx_rings[i];
2430                 if (tx_vec & (1 << rp->tx_channel))
2431                         niu_tx_work(np, rp);
2432                 nw64(LD_IM0(LDN_TXDMA(rp->tx_channel)), 0);
2433         }
2434
2435         for (i = 0; i < np->num_rx_rings; i++) {
2436                 struct rx_ring_info *rp = &np->rx_rings[i];
2437
2438                 if (rx_vec & (1 << rp->rx_channel)) {
2439                         int this_work_done;
2440
2441                         this_work_done = niu_rx_work(np, rp,
2442                                                      budget);
2443
2444                         budget -= this_work_done;
2445                         work_done += this_work_done;
2446                 }
2447                 nw64(LD_IM0(LDN_RXDMA(rp->rx_channel)), 0);
2448         }
2449
2450         return work_done;
2451 }
2452
2453 static int niu_poll(struct napi_struct *napi, int budget)
2454 {
2455         struct niu_ldg *lp = container_of(napi, struct niu_ldg, napi);
2456         struct niu *np = lp->np;
2457         int work_done;
2458
2459         work_done = niu_poll_core(np, lp, budget);
2460
2461         if (work_done < budget) {
2462                 netif_rx_complete(np->dev, napi);
2463                 niu_ldg_rearm(np, lp, 1);
2464         }
2465         return work_done;
2466 }
2467
2468 static void niu_log_rxchan_errors(struct niu *np, struct rx_ring_info *rp,
2469                                   u64 stat)
2470 {
2471         dev_err(np->device, PFX "%s: RX channel %u errors ( ",
2472                 np->dev->name, rp->rx_channel);
2473
2474         if (stat & RX_DMA_CTL_STAT_RBR_TMOUT)
2475                 printk("RBR_TMOUT ");
2476         if (stat & RX_DMA_CTL_STAT_RSP_CNT_ERR)
2477                 printk("RSP_CNT ");
2478         if (stat & RX_DMA_CTL_STAT_BYTE_EN_BUS)
2479                 printk("BYTE_EN_BUS ");
2480         if (stat & RX_DMA_CTL_STAT_RSP_DAT_ERR)
2481                 printk("RSP_DAT ");
2482         if (stat & RX_DMA_CTL_STAT_RCR_ACK_ERR)
2483                 printk("RCR_ACK ");
2484         if (stat & RX_DMA_CTL_STAT_RCR_SHA_PAR)
2485                 printk("RCR_SHA_PAR ");
2486         if (stat & RX_DMA_CTL_STAT_RBR_PRE_PAR)
2487                 printk("RBR_PRE_PAR ");
2488         if (stat & RX_DMA_CTL_STAT_CONFIG_ERR)
2489                 printk("CONFIG ");
2490         if (stat & RX_DMA_CTL_STAT_RCRINCON)
2491                 printk("RCRINCON ");
2492         if (stat & RX_DMA_CTL_STAT_RCRFULL)
2493                 printk("RCRFULL ");
2494         if (stat & RX_DMA_CTL_STAT_RBRFULL)
2495                 printk("RBRFULL ");
2496         if (stat & RX_DMA_CTL_STAT_RBRLOGPAGE)
2497                 printk("RBRLOGPAGE ");
2498         if (stat & RX_DMA_CTL_STAT_CFIGLOGPAGE)
2499                 printk("CFIGLOGPAGE ");
2500         if (stat & RX_DMA_CTL_STAT_DC_FIFO_ERR)
2501                 printk("DC_FIDO ");
2502
2503         printk(")\n");
2504 }
2505
2506 static int niu_rx_error(struct niu *np, struct rx_ring_info *rp)
2507 {
2508         u64 stat = nr64(RX_DMA_CTL_STAT(rp->rx_channel));
2509         int err = 0;
2510
2511         dev_err(np->device, PFX "%s: RX channel %u error, stat[%llx]\n",
2512                 np->dev->name, rp->rx_channel, (unsigned long long) stat);
2513
2514         niu_log_rxchan_errors(np, rp, stat);
2515
2516         if (stat & (RX_DMA_CTL_STAT_CHAN_FATAL |
2517                     RX_DMA_CTL_STAT_PORT_FATAL))
2518                 err = -EINVAL;
2519
2520         nw64(RX_DMA_CTL_STAT(rp->rx_channel),
2521              stat & RX_DMA_CTL_WRITE_CLEAR_ERRS);
2522
2523         return err;
2524 }
2525
2526 static void niu_log_txchan_errors(struct niu *np, struct tx_ring_info *rp,
2527                                   u64 cs)
2528 {
2529         dev_err(np->device, PFX "%s: TX channel %u errors ( ",
2530                 np->dev->name, rp->tx_channel);
2531
2532         if (cs & TX_CS_MBOX_ERR)
2533                 printk("MBOX ");
2534         if (cs & TX_CS_PKT_SIZE_ERR)
2535                 printk("PKT_SIZE ");
2536         if (cs & TX_CS_TX_RING_OFLOW)
2537                 printk("TX_RING_OFLOW ");
2538         if (cs & TX_CS_PREF_BUF_PAR_ERR)
2539                 printk("PREF_BUF_PAR ");
2540         if (cs & TX_CS_NACK_PREF)
2541                 printk("NACK_PREF ");
2542         if (cs & TX_CS_NACK_PKT_RD)
2543                 printk("NACK_PKT_RD ");
2544         if (cs & TX_CS_CONF_PART_ERR)
2545                 printk("CONF_PART ");
2546         if (cs & TX_CS_PKT_PRT_ERR)
2547                 printk("PKT_PTR ");
2548
2549         printk(")\n");
2550 }
2551
2552 static int niu_tx_error(struct niu *np, struct tx_ring_info *rp)
2553 {
2554         u64 cs, logh, logl;
2555
2556         cs = nr64(TX_CS(rp->tx_channel));
2557         logh = nr64(TX_RNG_ERR_LOGH(rp->tx_channel));
2558         logl = nr64(TX_RNG_ERR_LOGL(rp->tx_channel));
2559
2560         dev_err(np->device, PFX "%s: TX channel %u error, "
2561                 "cs[%llx] logh[%llx] logl[%llx]\n",
2562                 np->dev->name, rp->tx_channel,
2563                 (unsigned long long) cs,
2564                 (unsigned long long) logh,
2565                 (unsigned long long) logl);
2566
2567         niu_log_txchan_errors(np, rp, cs);
2568
2569         return -ENODEV;
2570 }
2571
2572 static int niu_mif_interrupt(struct niu *np)
2573 {
2574         u64 mif_status = nr64(MIF_STATUS);
2575         int phy_mdint = 0;
2576
2577         if (np->flags & NIU_FLAGS_XMAC) {
2578                 u64 xrxmac_stat = nr64_mac(XRXMAC_STATUS);
2579
2580                 if (xrxmac_stat & XRXMAC_STATUS_PHY_MDINT)
2581                         phy_mdint = 1;
2582         }
2583
2584         dev_err(np->device, PFX "%s: MIF interrupt, "
2585                 "stat[%llx] phy_mdint(%d)\n",
2586                 np->dev->name, (unsigned long long) mif_status, phy_mdint);
2587
2588         return -ENODEV;
2589 }
2590
2591 static void niu_xmac_interrupt(struct niu *np)
2592 {
2593         struct niu_xmac_stats *mp = &np->mac_stats.xmac;
2594         u64 val;
2595
2596         val = nr64_mac(XTXMAC_STATUS);
2597         if (val & XTXMAC_STATUS_FRAME_CNT_EXP)
2598                 mp->tx_frames += TXMAC_FRM_CNT_COUNT;
2599         if (val & XTXMAC_STATUS_BYTE_CNT_EXP)
2600                 mp->tx_bytes += TXMAC_BYTE_CNT_COUNT;
2601         if (val & XTXMAC_STATUS_TXFIFO_XFR_ERR)
2602                 mp->tx_fifo_errors++;
2603         if (val & XTXMAC_STATUS_TXMAC_OFLOW)
2604                 mp->tx_overflow_errors++;
2605         if (val & XTXMAC_STATUS_MAX_PSIZE_ERR)
2606                 mp->tx_max_pkt_size_errors++;
2607         if (val & XTXMAC_STATUS_TXMAC_UFLOW)
2608                 mp->tx_underflow_errors++;
2609
2610         val = nr64_mac(XRXMAC_STATUS);
2611         if (val & XRXMAC_STATUS_LCL_FLT_STATUS)
2612                 mp->rx_local_faults++;
2613         if (val & XRXMAC_STATUS_RFLT_DET)
2614                 mp->rx_remote_faults++;
2615         if (val & XRXMAC_STATUS_LFLT_CNT_EXP)
2616                 mp->rx_link_faults += LINK_FAULT_CNT_COUNT;
2617         if (val & XRXMAC_STATUS_ALIGNERR_CNT_EXP)
2618                 mp->rx_align_errors += RXMAC_ALIGN_ERR_CNT_COUNT;
2619         if (val & XRXMAC_STATUS_RXFRAG_CNT_EXP)
2620                 mp->rx_frags += RXMAC_FRAG_CNT_COUNT;
2621         if (val & XRXMAC_STATUS_RXMULTF_CNT_EXP)
2622                 mp->rx_mcasts += RXMAC_MC_FRM_CNT_COUNT;
2623         if (val & XRXMAC_STATUS_RXBCAST_CNT_EXP)
2624                 mp->rx_bcasts += RXMAC_BC_FRM_CNT_COUNT;
2625         if (val & XRXMAC_STATUS_RXBCAST_CNT_EXP)
2626                 mp->rx_bcasts += RXMAC_BC_FRM_CNT_COUNT;
2627         if (val & XRXMAC_STATUS_RXHIST1_CNT_EXP)
2628                 mp->rx_hist_cnt1 += RXMAC_HIST_CNT1_COUNT;
2629         if (val & XRXMAC_STATUS_RXHIST2_CNT_EXP)
2630                 mp->rx_hist_cnt2 += RXMAC_HIST_CNT2_COUNT;
2631         if (val & XRXMAC_STATUS_RXHIST3_CNT_EXP)
2632                 mp->rx_hist_cnt3 += RXMAC_HIST_CNT3_COUNT;
2633         if (val & XRXMAC_STATUS_RXHIST4_CNT_EXP)
2634                 mp->rx_hist_cnt4 += RXMAC_HIST_CNT4_COUNT;
2635         if (val & XRXMAC_STATUS_RXHIST5_CNT_EXP)
2636                 mp->rx_hist_cnt5 += RXMAC_HIST_CNT5_COUNT;
2637         if (val & XRXMAC_STATUS_RXHIST6_CNT_EXP)
2638                 mp->rx_hist_cnt6 += RXMAC_HIST_CNT6_COUNT;
2639         if (val & XRXMAC_STATUS_RXHIST7_CNT_EXP)
2640                 mp->rx_hist_cnt7 += RXMAC_HIST_CNT7_COUNT;
2641         if (val & XRXMAC_STAT_MSK_RXOCTET_CNT_EXP)
2642                 mp->rx_octets += RXMAC_BT_CNT_COUNT;
2643         if (val & XRXMAC_STATUS_CVIOLERR_CNT_EXP)
2644                 mp->rx_code_violations += RXMAC_CD_VIO_CNT_COUNT;
2645         if (val & XRXMAC_STATUS_LENERR_CNT_EXP)
2646                 mp->rx_len_errors += RXMAC_MPSZER_CNT_COUNT;
2647         if (val & XRXMAC_STATUS_CRCERR_CNT_EXP)
2648                 mp->rx_crc_errors += RXMAC_CRC_ER_CNT_COUNT;
2649         if (val & XRXMAC_STATUS_RXUFLOW)
2650                 mp->rx_underflows++;
2651         if (val & XRXMAC_STATUS_RXOFLOW)
2652                 mp->rx_overflows++;
2653
2654         val = nr64_mac(XMAC_FC_STAT);
2655         if (val & XMAC_FC_STAT_TX_MAC_NPAUSE)
2656                 mp->pause_off_state++;
2657         if (val & XMAC_FC_STAT_TX_MAC_PAUSE)
2658                 mp->pause_on_state++;
2659         if (val & XMAC_FC_STAT_RX_MAC_RPAUSE)
2660                 mp->pause_received++;
2661 }
2662
2663 static void niu_bmac_interrupt(struct niu *np)
2664 {
2665         struct niu_bmac_stats *mp = &np->mac_stats.bmac;
2666         u64 val;
2667
2668         val = nr64_mac(BTXMAC_STATUS);
2669         if (val & BTXMAC_STATUS_UNDERRUN)
2670                 mp->tx_underflow_errors++;
2671         if (val & BTXMAC_STATUS_MAX_PKT_ERR)
2672                 mp->tx_max_pkt_size_errors++;
2673         if (val & BTXMAC_STATUS_BYTE_CNT_EXP)
2674                 mp->tx_bytes += BTXMAC_BYTE_CNT_COUNT;
2675         if (val & BTXMAC_STATUS_FRAME_CNT_EXP)
2676                 mp->tx_frames += BTXMAC_FRM_CNT_COUNT;
2677
2678         val = nr64_mac(BRXMAC_STATUS);
2679         if (val & BRXMAC_STATUS_OVERFLOW)
2680                 mp->rx_overflows++;
2681         if (val & BRXMAC_STATUS_FRAME_CNT_EXP)
2682                 mp->rx_frames += BRXMAC_FRAME_CNT_COUNT;
2683         if (val & BRXMAC_STATUS_ALIGN_ERR_EXP)
2684                 mp->rx_align_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
2685         if (val & BRXMAC_STATUS_CRC_ERR_EXP)
2686                 mp->rx_crc_errors += BRXMAC_ALIGN_ERR_CNT_COUNT;
2687         if (val & BRXMAC_STATUS_LEN_ERR_EXP)
2688                 mp->rx_len_errors += BRXMAC_CODE_VIOL_ERR_CNT_COUNT;
2689
2690         val = nr64_mac(BMAC_CTRL_STATUS);
2691         if (val & BMAC_CTRL_STATUS_NOPAUSE)
2692                 mp->pause_off_state++;
2693         if (val & BMAC_CTRL_STATUS_PAUSE)
2694                 mp->pause_on_state++;
2695         if (val & BMAC_CTRL_STATUS_PAUSE_RECV)
2696                 mp->pause_received++;
2697 }
2698
2699 static int niu_mac_interrupt(struct niu *np)
2700 {
2701         if (np->flags & NIU_FLAGS_XMAC)
2702                 niu_xmac_interrupt(np);
2703         else
2704                 niu_bmac_interrupt(np);
2705
2706         return 0;
2707 }
2708
2709 static void niu_log_device_error(struct niu *np, u64 stat)
2710 {
2711         dev_err(np->device, PFX "%s: Core device errors ( ",
2712                 np->dev->name);
2713
2714         if (stat & SYS_ERR_MASK_META2)
2715                 printk("META2 ");
2716         if (stat & SYS_ERR_MASK_META1)
2717                 printk("META1 ");
2718         if (stat & SYS_ERR_MASK_PEU)
2719                 printk("PEU ");
2720         if (stat & SYS_ERR_MASK_TXC)
2721                 printk("TXC ");
2722         if (stat & SYS_ERR_MASK_RDMC)
2723                 printk("RDMC ");
2724         if (stat & SYS_ERR_MASK_TDMC)
2725                 printk("TDMC ");
2726         if (stat & SYS_ERR_MASK_ZCP)
2727                 printk("ZCP ");
2728         if (stat & SYS_ERR_MASK_FFLP)
2729                 printk("FFLP ");
2730         if (stat & SYS_ERR_MASK_IPP)
2731                 printk("IPP ");
2732         if (stat & SYS_ERR_MASK_MAC)
2733                 printk("MAC ");
2734         if (stat & SYS_ERR_MASK_SMX)
2735                 printk("SMX ");
2736
2737         printk(")\n");
2738 }
2739
2740 static int niu_device_error(struct niu *np)
2741 {
2742         u64 stat = nr64(SYS_ERR_STAT);
2743
2744         dev_err(np->device, PFX "%s: Core device error, stat[%llx]\n",
2745                 np->dev->name, (unsigned long long) stat);
2746
2747         niu_log_device_error(np, stat);
2748
2749         return -ENODEV;
2750 }
2751
2752 static int niu_slowpath_interrupt(struct niu *np, struct niu_ldg *lp)
2753 {
2754         u64 v0 = lp->v0;
2755         u64 v1 = lp->v1;
2756         u64 v2 = lp->v2;
2757         int i, err = 0;
2758
2759         if (v1 & 0x00000000ffffffffULL) {
2760                 u32 rx_vec = (v1 & 0xffffffff);
2761
2762                 for (i = 0; i < np->num_rx_rings; i++) {
2763                         struct rx_ring_info *rp = &np->rx_rings[i];
2764
2765                         if (rx_vec & (1 << rp->rx_channel)) {
2766                                 int r = niu_rx_error(np, rp);
2767                                 if (r)
2768                                         err = r;
2769                         }
2770                 }
2771         }
2772         if (v1 & 0x7fffffff00000000ULL) {
2773                 u32 tx_vec = (v1 >> 32) & 0x7fffffff;
2774
2775                 for (i = 0; i < np->num_tx_rings; i++) {
2776                         struct tx_ring_info *rp = &np->tx_rings[i];
2777
2778                         if (tx_vec & (1 << rp->tx_channel)) {
2779                                 int r = niu_tx_error(np, rp);
2780                                 if (r)
2781                                         err = r;
2782                         }
2783                 }
2784         }
2785         if ((v0 | v1) & 0x8000000000000000ULL) {
2786                 int r = niu_mif_interrupt(np);
2787                 if (r)
2788                         err = r;
2789         }
2790         if (v2) {
2791                 if (v2 & 0x01ef) {
2792                         int r = niu_mac_interrupt(np);
2793                         if (r)
2794                                 err = r;
2795                 }
2796                 if (v2 & 0x0210) {
2797                         int r = niu_device_error(np);
2798                         if (r)
2799                                 err = r;
2800                 }
2801         }
2802
2803         if (err)
2804                 niu_enable_interrupts(np, 0);
2805
2806         return -EINVAL;
2807 }
2808
2809 static void niu_rxchan_intr(struct niu *np, struct rx_ring_info *rp,
2810                             int ldn)
2811 {
2812         struct rxdma_mailbox *mbox = rp->mbox;
2813         u64 stat_write, stat = le64_to_cpup(&mbox->rx_dma_ctl_stat);
2814
2815         stat_write = (RX_DMA_CTL_STAT_RCRTHRES |
2816                       RX_DMA_CTL_STAT_RCRTO);
2817         nw64(RX_DMA_CTL_STAT(rp->rx_channel), stat_write);
2818
2819         niudbg(INTR, "%s: rxchan_intr stat[%llx]\n",
2820                np->dev->name, (unsigned long long) stat);
2821 }
2822
2823 static void niu_txchan_intr(struct niu *np, struct tx_ring_info *rp,
2824                             int ldn)
2825 {
2826         rp->tx_cs = nr64(TX_CS(rp->tx_channel));
2827
2828         niudbg(INTR, "%s: txchan_intr cs[%llx]\n",
2829                np->dev->name, (unsigned long long) rp->tx_cs);
2830 }
2831
2832 static void __niu_fastpath_interrupt(struct niu *np, int ldg, u64 v0)
2833 {
2834         struct niu_parent *parent = np->parent;
2835         u32 rx_vec, tx_vec;
2836         int i;
2837
2838         tx_vec = (v0 >> 32);
2839         rx_vec = (v0 & 0xffffffff);
2840
2841         for (i = 0; i < np->num_rx_rings; i++) {
2842                 struct rx_ring_info *rp = &np->rx_rings[i];
2843                 int ldn = LDN_RXDMA(rp->rx_channel);
2844
2845                 if (parent->ldg_map[ldn] != ldg)
2846                         continue;
2847
2848                 nw64(LD_IM0(ldn), LD_IM0_MASK);
2849                 if (rx_vec & (1 << rp->rx_channel))
2850                         niu_rxchan_intr(np, rp, ldn);
2851         }
2852
2853         for (i = 0; i < np->num_tx_rings; i++) {
2854                 struct tx_ring_info *rp = &np->tx_rings[i];
2855                 int ldn = LDN_TXDMA(rp->tx_channel);
2856
2857                 if (parent->ldg_map[ldn] != ldg)
2858                         continue;
2859
2860                 nw64(LD_IM0(ldn), LD_IM0_MASK);
2861                 if (tx_vec & (1 << rp->tx_channel))
2862                         niu_txchan_intr(np, rp, ldn);
2863         }
2864 }
2865
2866 static void niu_schedule_napi(struct niu *np, struct niu_ldg *lp,
2867                               u64 v0, u64 v1, u64 v2)
2868 {
2869         if (likely(netif_rx_schedule_prep(np->dev, &lp->napi))) {
2870                 lp->v0 = v0;
2871                 lp->v1 = v1;
2872                 lp->v2 = v2;
2873                 __niu_fastpath_interrupt(np, lp->ldg_num, v0);
2874                 __netif_rx_schedule(np->dev, &lp->napi);
2875         }
2876 }
2877
2878 static irqreturn_t niu_interrupt(int irq, void *dev_id)
2879 {
2880         struct niu_ldg *lp = dev_id;
2881         struct niu *np = lp->np;
2882         int ldg = lp->ldg_num;
2883         unsigned long flags;
2884         u64 v0, v1, v2;
2885
2886         if (netif_msg_intr(np))
2887                 printk(KERN_DEBUG PFX "niu_interrupt() ldg[%p](%d) ",
2888                        lp, ldg);
2889
2890         spin_lock_irqsave(&np->lock, flags);
2891
2892         v0 = nr64(LDSV0(ldg));
2893         v1 = nr64(LDSV1(ldg));
2894         v2 = nr64(LDSV2(ldg));
2895
2896         if (netif_msg_intr(np))
2897                 printk("v0[%llx] v1[%llx] v2[%llx]\n",
2898                        (unsigned long long) v0,
2899                        (unsigned long long) v1,
2900                        (unsigned long long) v2);
2901
2902         if (unlikely(!v0 && !v1 && !v2)) {
2903                 spin_unlock_irqrestore(&np->lock, flags);
2904                 return IRQ_NONE;
2905         }
2906
2907         if (unlikely((v0 & ((u64)1 << LDN_MIF)) || v1 || v2)) {
2908                 int err = niu_slowpath_interrupt(np, lp);
2909                 if (err)
2910                         goto out;
2911         }
2912         if (likely(v0 & ~((u64)1 << LDN_MIF)))
2913                 niu_schedule_napi(np, lp, v0, v1, v2);
2914         else
2915                 niu_ldg_rearm(np, lp, 1);
2916 out:
2917         spin_unlock_irqrestore(&np->lock, flags);
2918
2919         return IRQ_HANDLED;
2920 }
2921
2922 static void niu_free_rx_ring_info(struct niu *np, struct rx_ring_info *rp)
2923 {
2924         if (rp->mbox) {
2925                 np->ops->free_coherent(np->device,
2926                                        sizeof(struct rxdma_mailbox),
2927                                        rp->mbox, rp->mbox_dma);
2928                 rp->mbox = NULL;
2929         }
2930         if (rp->rcr) {
2931                 np->ops->free_coherent(np->device,
2932                                        MAX_RCR_RING_SIZE * sizeof(__le64),
2933                                        rp->rcr, rp->rcr_dma);
2934                 rp->rcr = NULL;
2935                 rp->rcr_table_size = 0;
2936                 rp->rcr_index = 0;
2937         }
2938         if (rp->rbr) {
2939                 niu_rbr_free(np, rp);
2940
2941                 np->ops->free_coherent(np->device,
2942                                        MAX_RBR_RING_SIZE * sizeof(__le32),
2943                                        rp->rbr, rp->rbr_dma);
2944                 rp->rbr = NULL;
2945                 rp->rbr_table_size = 0;
2946                 rp->rbr_index = 0;
2947         }
2948         kfree(rp->rxhash);
2949         rp->rxhash = NULL;
2950 }
2951
2952 static void niu_free_tx_ring_info(struct niu *np, struct tx_ring_info *rp)
2953 {
2954         if (rp->mbox) {
2955                 np->ops->free_coherent(np->device,
2956                                        sizeof(struct txdma_mailbox),
2957                                        rp->mbox, rp->mbox_dma);
2958                 rp->mbox = NULL;
2959         }
2960         if (rp->descr) {
2961                 int i;
2962
2963                 for (i = 0; i < MAX_TX_RING_SIZE; i++) {
2964                         if (rp->tx_buffs[i].skb)
2965                                 (void) release_tx_packet(np, rp, i);
2966                 }
2967
2968                 np->ops->free_coherent(np->device,
2969                                        MAX_TX_RING_SIZE * sizeof(__le64),
2970                                        rp->descr, rp->descr_dma);
2971                 rp->descr = NULL;
2972                 rp->pending = 0;
2973                 rp->prod = 0;
2974                 rp->cons = 0;
2975                 rp->wrap_bit = 0;
2976         }
2977 }
2978
2979 static void niu_free_channels(struct niu *np)
2980 {
2981         int i;
2982
2983         if (np->rx_rings) {
2984                 for (i = 0; i < np->num_rx_rings; i++) {
2985                         struct rx_ring_info *rp = &np->rx_rings[i];
2986
2987                         niu_free_rx_ring_info(np, rp);
2988                 }
2989                 kfree(np->rx_rings);
2990                 np->rx_rings = NULL;
2991                 np->num_rx_rings = 0;
2992         }
2993
2994         if (np->tx_rings) {
2995                 for (i = 0; i < np->num_tx_rings; i++) {
2996                         struct tx_ring_info *rp = &np->tx_rings[i];
2997
2998                         niu_free_tx_ring_info(np, rp);
2999                 }
3000                 kfree(np->tx_rings);
3001                 np->tx_rings = NULL;
3002                 np->num_tx_rings = 0;
3003         }
3004 }
3005
3006 static int niu_alloc_rx_ring_info(struct niu *np,
3007                                   struct rx_ring_info *rp)
3008 {
3009         BUILD_BUG_ON(sizeof(struct rxdma_mailbox) != 64);
3010
3011         rp->rxhash = kzalloc(MAX_RBR_RING_SIZE * sizeof(struct page *),
3012                              GFP_KERNEL);
3013         if (!rp->rxhash)
3014                 return -ENOMEM;
3015
3016         rp->mbox = np->ops->alloc_coherent(np->device,
3017                                            sizeof(struct rxdma_mailbox),
3018                                            &rp->mbox_dma, GFP_KERNEL);
3019         if (!rp->mbox)
3020                 return -ENOMEM;
3021         if ((unsigned long)rp->mbox & (64UL - 1)) {
3022                 dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
3023                         "RXDMA mailbox %p\n", np->dev->name, rp->mbox);
3024                 return -EINVAL;
3025         }
3026
3027         rp->rcr = np->ops->alloc_coherent(np->device,
3028                                           MAX_RCR_RING_SIZE * sizeof(__le64),
3029                                           &rp->rcr_dma, GFP_KERNEL);
3030         if (!rp->rcr)
3031                 return -ENOMEM;
3032         if ((unsigned long)rp->rcr & (64UL - 1)) {
3033                 dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
3034                         "RXDMA RCR table %p\n", np->dev->name, rp->rcr);
3035                 return -EINVAL;
3036         }
3037         rp->rcr_table_size = MAX_RCR_RING_SIZE;
3038         rp->rcr_index = 0;
3039
3040         rp->rbr = np->ops->alloc_coherent(np->device,
3041                                           MAX_RBR_RING_SIZE * sizeof(__le32),
3042                                           &rp->rbr_dma, GFP_KERNEL);
3043         if (!rp->rbr)
3044                 return -ENOMEM;
3045         if ((unsigned long)rp->rbr & (64UL - 1)) {
3046                 dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
3047                         "RXDMA RBR table %p\n", np->dev->name, rp->rbr);
3048                 return -EINVAL;
3049         }
3050         rp->rbr_table_size = MAX_RBR_RING_SIZE;
3051         rp->rbr_index = 0;
3052         rp->rbr_pending = 0;
3053
3054         return 0;
3055 }
3056
3057 static void niu_set_max_burst(struct niu *np, struct tx_ring_info *rp)
3058 {
3059         int mtu = np->dev->mtu;
3060
3061         /* These values are recommended by the HW designers for fair
3062          * utilization of DRR amongst the rings.
3063          */
3064         rp->max_burst = mtu + 32;
3065         if (rp->max_burst > 4096)
3066                 rp->max_burst = 4096;
3067 }
3068
3069 static int niu_alloc_tx_ring_info(struct niu *np,
3070                                   struct tx_ring_info *rp)
3071 {
3072         BUILD_BUG_ON(sizeof(struct txdma_mailbox) != 64);
3073
3074         rp->mbox = np->ops->alloc_coherent(np->device,
3075                                            sizeof(struct txdma_mailbox),
3076                                            &rp->mbox_dma, GFP_KERNEL);
3077         if (!rp->mbox)
3078                 return -ENOMEM;
3079         if ((unsigned long)rp->mbox & (64UL - 1)) {
3080                 dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
3081                         "TXDMA mailbox %p\n", np->dev->name, rp->mbox);
3082                 return -EINVAL;
3083         }
3084
3085         rp->descr = np->ops->alloc_coherent(np->device,
3086                                             MAX_TX_RING_SIZE * sizeof(__le64),
3087                                             &rp->descr_dma, GFP_KERNEL);
3088         if (!rp->descr)
3089                 return -ENOMEM;
3090         if ((unsigned long)rp->descr & (64UL - 1)) {
3091                 dev_err(np->device, PFX "%s: Coherent alloc gives misaligned "
3092                         "TXDMA descr table %p\n", np->dev->name, rp->descr);
3093                 return -EINVAL;
3094         }
3095
3096         rp->pending = MAX_TX_RING_SIZE;
3097         rp->prod = 0;
3098         rp->cons = 0;
3099         rp->wrap_bit = 0;
3100
3101         /* XXX make these configurable... XXX */
3102         rp->mark_freq = rp->pending / 4;
3103
3104         niu_set_max_burst(np, rp);
3105
3106         return 0;
3107 }
3108
3109 static void niu_size_rbr(struct niu *np, struct rx_ring_info *rp)
3110 {
3111         u16 bss;
3112
3113         bss = min(PAGE_SHIFT, 15);
3114
3115         rp->rbr_block_size = 1 << bss;
3116         rp->rbr_blocks_per_page = 1 << (PAGE_SHIFT-bss);
3117
3118         rp->rbr_sizes[0] = 256;
3119         rp->rbr_sizes[1] = 1024;
3120         if (np->dev->mtu > ETH_DATA_LEN) {
3121                 switch (PAGE_SIZE) {
3122                 case 4 * 1024:
3123                         rp->rbr_sizes[2] = 4096;
3124                         break;
3125
3126                 default:
3127                         rp->rbr_sizes[2] = 8192;
3128                         break;
3129                 }
3130         } else {
3131                 rp->rbr_sizes[2] = 2048;
3132         }
3133         rp->rbr_sizes[3] = rp->rbr_block_size;
3134 }
3135
3136 static int niu_alloc_channels(struct niu *np)
3137 {
3138         struct niu_parent *parent = np->parent;
3139         int first_rx_channel, first_tx_channel;
3140         int i, port, err;
3141
3142         port = np->port;
3143         first_rx_channel = first_tx_channel = 0;
3144         for (i = 0; i < port; i++) {
3145                 first_rx_channel += parent->rxchan_per_port[i];
3146                 first_tx_channel += parent->txchan_per_port[i];
3147         }
3148
3149         np->num_rx_rings = parent->rxchan_per_port[port];
3150         np->num_tx_rings = parent->txchan_per_port[port];
3151
3152         np->rx_rings = kzalloc(np->num_rx_rings * sizeof(struct rx_ring_info),
3153                                GFP_KERNEL);
3154         err = -ENOMEM;
3155         if (!np->rx_rings)
3156                 goto out_err;
3157
3158         for (i = 0; i < np->num_rx_rings; i++) {
3159                 struct rx_ring_info *rp = &np->rx_rings[i];
3160
3161                 rp->np = np;
3162                 rp->rx_channel = first_rx_channel + i;
3163
3164                 err = niu_alloc_rx_ring_info(np, rp);
3165                 if (err)
3166                         goto out_err;
3167
3168                 niu_size_rbr(np, rp);
3169
3170                 /* XXX better defaults, configurable, etc... XXX */
3171                 rp->nonsyn_window = 64;
3172                 rp->nonsyn_threshold = rp->rcr_table_size - 64;
3173                 rp->syn_window = 64;
3174                 rp->syn_threshold = rp->rcr_table_size - 64;
3175                 rp->rcr_pkt_threshold = 16;
3176                 rp->rcr_timeout = 8;
3177                 rp->rbr_kick_thresh = RBR_REFILL_MIN;
3178                 if (rp->rbr_kick_thresh < rp->rbr_blocks_per_page)
3179                         rp->rbr_kick_thresh = rp->rbr_blocks_per_page;
3180
3181                 err = niu_rbr_fill(np, rp, GFP_KERNEL);
3182                 if (err)
3183                         return err;
3184         }
3185
3186         np->tx_rings = kzalloc(np->num_tx_rings * sizeof(struct tx_ring_info),
3187                                GFP_KERNEL);
3188         err = -ENOMEM;
3189         if (!np->tx_rings)
3190                 goto out_err;
3191
3192         for (i = 0; i < np->num_tx_rings; i++) {
3193                 struct tx_ring_info *rp = &np->tx_rings[i];
3194
3195                 rp->np = np;
3196                 rp->tx_channel = first_tx_channel + i;
3197
3198                 err = niu_alloc_tx_ring_info(np, rp);
3199                 if (err)
3200                         goto out_err;
3201         }
3202
3203         return 0;
3204
3205 out_err:
3206         niu_free_channels(np);
3207         return err;
3208 }
3209
3210 static int niu_tx_cs_sng_poll(struct niu *np, int channel)
3211 {
3212         int limit = 1000;
3213
3214         while (--limit > 0) {
3215                 u64 val = nr64(TX_CS(channel));
3216                 if (val & TX_CS_SNG_STATE)
3217                         return 0;
3218         }
3219         return -ENODEV;
3220 }
3221
3222 static int niu_tx_channel_stop(struct niu *np, int channel)
3223 {
3224         u64 val = nr64(TX_CS(channel));
3225
3226         val |= TX_CS_STOP_N_GO;
3227         nw64(TX_CS(channel), val);
3228
3229         return niu_tx_cs_sng_poll(np, channel);
3230 }
3231
3232 static int niu_tx_cs_reset_poll(struct niu *np, int channel)
3233 {
3234         int limit = 1000;
3235
3236         while (--limit > 0) {
3237                 u64 val = nr64(TX_CS(channel));
3238                 if (!(val & TX_CS_RST))
3239                         return 0;
3240         }
3241         return -ENODEV;
3242 }
3243
3244 static int niu_tx_channel_reset(struct niu *np, int channel)
3245 {
3246         u64 val = nr64(TX_CS(channel));
3247         int err;
3248
3249         val |= TX_CS_RST;
3250         nw64(TX_CS(channel), val);
3251
3252         err = niu_tx_cs_reset_poll(np, channel);
3253         if (!err)
3254                 nw64(TX_RING_KICK(channel), 0);
3255
3256         return err;
3257 }
3258
3259 static int niu_tx_channel_lpage_init(struct niu *np, int channel)
3260 {
3261         u64 val;
3262
3263         nw64(TX_LOG_MASK1(channel), 0);
3264         nw64(TX_LOG_VAL1(channel), 0);
3265         nw64(TX_LOG_MASK2(channel), 0);
3266         nw64(TX_LOG_VAL2(channel), 0);
3267         nw64(TX_LOG_PAGE_RELO1(channel), 0);
3268         nw64(TX_LOG_PAGE_RELO2(channel), 0);
3269         nw64(TX_LOG_PAGE_HDL(channel), 0);
3270
3271         val  = (u64)np->port << TX_LOG_PAGE_VLD_FUNC_SHIFT;
3272         val |= (TX_LOG_PAGE_VLD_PAGE0 | TX_LOG_PAGE_VLD_PAGE1);
3273         nw64(TX_LOG_PAGE_VLD(channel), val);
3274
3275         /* XXX TXDMA 32bit mode? XXX */
3276
3277         return 0;
3278 }
3279
3280 static void niu_txc_enable_port(struct niu *np, int on)
3281 {
3282         unsigned long flags;
3283         u64 val, mask;
3284
3285         niu_lock_parent(np, flags);
3286         val = nr64(TXC_CONTROL);
3287         mask = (u64)1 << np->port;
3288         if (on) {
3289                 val |= TXC_CONTROL_ENABLE | mask;
3290         } else {
3291                 val &= ~mask;
3292                 if ((val & ~TXC_CONTROL_ENABLE) == 0)
3293                         val &= ~TXC_CONTROL_ENABLE;
3294         }
3295         nw64(TXC_CONTROL, val);
3296         niu_unlock_parent(np, flags);
3297 }
3298
3299 static void niu_txc_set_imask(struct niu *np, u64 imask)
3300 {
3301         unsigned long flags;
3302         u64 val;
3303
3304         niu_lock_parent(np, flags);
3305         val = nr64(TXC_INT_MASK);
3306         val &= ~TXC_INT_MASK_VAL(np->port);
3307         val |= (imask << TXC_INT_MASK_VAL_SHIFT(np->port));
3308         niu_unlock_parent(np, flags);
3309 }
3310
3311 static void niu_txc_port_dma_enable(struct niu *np, int on)
3312 {
3313         u64 val = 0;
3314
3315         if (on) {
3316                 int i;
3317
3318                 for (i = 0; i < np->num_tx_rings; i++)
3319                         val |= (1 << np->tx_rings[i].tx_channel);
3320         }
3321         nw64(TXC_PORT_DMA(np->port), val);
3322 }
3323
3324 static int niu_init_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
3325 {
3326         int err, channel = rp->tx_channel;
3327         u64 val, ring_len;
3328
3329         err = niu_tx_channel_stop(np, channel);
3330         if (err)
3331                 return err;
3332
3333         err = niu_tx_channel_reset(np, channel);
3334         if (err)
3335                 return err;
3336
3337         err = niu_tx_channel_lpage_init(np, channel);
3338         if (err)
3339                 return err;
3340
3341         nw64(TXC_DMA_MAX(channel), rp->max_burst);
3342         nw64(TX_ENT_MSK(channel), 0);
3343
3344         if (rp->descr_dma & ~(TX_RNG_CFIG_STADDR_BASE |
3345                               TX_RNG_CFIG_STADDR)) {
3346                 dev_err(np->device, PFX "%s: TX ring channel %d "
3347                         "DMA addr (%llx) is not aligned.\n",
3348                         np->dev->name, channel,
3349                         (unsigned long long) rp->descr_dma);
3350                 return -EINVAL;
3351         }
3352
3353         /* The length field in TX_RNG_CFIG is measured in 64-byte
3354          * blocks.  rp->pending is the number of TX descriptors in
3355          * our ring, 8 bytes each, thus we divide by 8 bytes more
3356          * to get the proper value the chip wants.
3357          */
3358         ring_len = (rp->pending / 8);
3359
3360         val = ((ring_len << TX_RNG_CFIG_LEN_SHIFT) |
3361                rp->descr_dma);
3362         nw64(TX_RNG_CFIG(channel), val);
3363
3364         if (((rp->mbox_dma >> 32) & ~TXDMA_MBH_MBADDR) ||
3365             ((u32)rp->mbox_dma & ~TXDMA_MBL_MBADDR)) {
3366                 dev_err(np->device, PFX "%s: TX ring channel %d "
3367                         "MBOX addr (%llx) is has illegal bits.\n",
3368                         np->dev->name, channel,
3369                         (unsigned long long) rp->mbox_dma);
3370                 return -EINVAL;
3371         }
3372         nw64(TXDMA_MBH(channel), rp->mbox_dma >> 32);
3373         nw64(TXDMA_MBL(channel), rp->mbox_dma & TXDMA_MBL_MBADDR);
3374
3375         nw64(TX_CS(channel), 0);
3376
3377         rp->last_pkt_cnt = 0;
3378
3379         return 0;
3380 }
3381
3382 static void niu_init_rdc_groups(struct niu *np)
3383 {
3384         struct niu_rdc_tables *tp = &np->parent->rdc_group_cfg[np->port];
3385         int i, first_table_num = tp->first_table_num;
3386
3387         for (i = 0; i < tp->num_tables; i++) {
3388                 struct rdc_table *tbl = &tp->tables[i];
3389                 int this_table = first_table_num + i;
3390                 int slot;
3391
3392                 for (slot = 0; slot < NIU_RDC_TABLE_SLOTS; slot++)
3393                         nw64(RDC_TBL(this_table, slot),
3394                              tbl->rxdma_channel[slot]);
3395         }
3396
3397         nw64(DEF_RDC(np->port), np->parent->rdc_default[np->port]);
3398 }
3399
3400 static void niu_init_drr_weight(struct niu *np)
3401 {
3402         int type = phy_decode(np->parent->port_phy, np->port);
3403         u64 val;
3404
3405         switch (type) {
3406         case PORT_TYPE_10G:
3407                 val = PT_DRR_WEIGHT_DEFAULT_10G;
3408                 break;
3409
3410         case PORT_TYPE_1G:
3411         default:
3412                 val = PT_DRR_WEIGHT_DEFAULT_1G;
3413                 break;
3414         }
3415         nw64(PT_DRR_WT(np->port), val);
3416 }
3417
3418 static int niu_init_hostinfo(struct niu *np)
3419 {
3420         struct niu_parent *parent = np->parent;
3421         struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
3422         int i, err, num_alt = niu_num_alt_addr(np);
3423         int first_rdc_table = tp->first_table_num;
3424
3425         err = niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
3426         if (err)
3427                 return err;
3428
3429         err = niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
3430         if (err)
3431                 return err;
3432
3433         for (i = 0; i < num_alt; i++) {
3434                 err = niu_set_alt_mac_rdc_table(np, i, first_rdc_table, 1);
3435                 if (err)
3436                         return err;
3437         }
3438
3439         return 0;
3440 }
3441
3442 static int niu_rx_channel_reset(struct niu *np, int channel)
3443 {
3444         return niu_set_and_wait_clear(np, RXDMA_CFIG1(channel),
3445                                       RXDMA_CFIG1_RST, 1000, 10,
3446                                       "RXDMA_CFIG1");
3447 }
3448
3449 static int niu_rx_channel_lpage_init(struct niu *np, int channel)
3450 {
3451         u64 val;
3452
3453         nw64(RX_LOG_MASK1(channel), 0);
3454         nw64(RX_LOG_VAL1(channel), 0);
3455         nw64(RX_LOG_MASK2(channel), 0);
3456         nw64(RX_LOG_VAL2(channel), 0);
3457         nw64(RX_LOG_PAGE_RELO1(channel), 0);
3458         nw64(RX_LOG_PAGE_RELO2(channel), 0);
3459         nw64(RX_LOG_PAGE_HDL(channel), 0);
3460
3461         val  = (u64)np->port << RX_LOG_PAGE_VLD_FUNC_SHIFT;
3462         val |= (RX_LOG_PAGE_VLD_PAGE0 | RX_LOG_PAGE_VLD_PAGE1);
3463         nw64(RX_LOG_PAGE_VLD(channel), val);
3464
3465         return 0;
3466 }
3467
3468 static void niu_rx_channel_wred_init(struct niu *np, struct rx_ring_info *rp)
3469 {
3470         u64 val;
3471
3472         val = (((u64)rp->nonsyn_window << RDC_RED_PARA_WIN_SHIFT) |
3473                ((u64)rp->nonsyn_threshold << RDC_RED_PARA_THRE_SHIFT) |
3474                ((u64)rp->syn_window << RDC_RED_PARA_WIN_SYN_SHIFT) |
3475                ((u64)rp->syn_threshold << RDC_RED_PARA_THRE_SYN_SHIFT));
3476         nw64(RDC_RED_PARA(rp->rx_channel), val);
3477 }
3478
3479 static int niu_compute_rbr_cfig_b(struct rx_ring_info *rp, u64 *ret)
3480 {
3481         u64 val = 0;
3482
3483         switch (rp->rbr_block_size) {
3484         case 4 * 1024:
3485                 val |= (RBR_BLKSIZE_4K << RBR_CFIG_B_BLKSIZE_SHIFT);
3486                 break;
3487         case 8 * 1024:
3488                 val |= (RBR_BLKSIZE_8K << RBR_CFIG_B_BLKSIZE_SHIFT);
3489                 break;
3490         case 16 * 1024:
3491                 val |= (RBR_BLKSIZE_16K << RBR_CFIG_B_BLKSIZE_SHIFT);
3492                 break;
3493         case 32 * 1024:
3494                 val |= (RBR_BLKSIZE_32K << RBR_CFIG_B_BLKSIZE_SHIFT);
3495                 break;
3496         default:
3497                 return -EINVAL;
3498         }
3499         val |= RBR_CFIG_B_VLD2;
3500         switch (rp->rbr_sizes[2]) {
3501         case 2 * 1024:
3502                 val |= (RBR_BUFSZ2_2K << RBR_CFIG_B_BUFSZ2_SHIFT);
3503                 break;
3504         case 4 * 1024:
3505                 val |= (RBR_BUFSZ2_4K << RBR_CFIG_B_BUFSZ2_SHIFT);
3506                 break;
3507         case 8 * 1024:
3508                 val |= (RBR_BUFSZ2_8K << RBR_CFIG_B_BUFSZ2_SHIFT);
3509                 break;
3510         case 16 * 1024:
3511                 val |= (RBR_BUFSZ2_16K << RBR_CFIG_B_BUFSZ2_SHIFT);
3512                 break;
3513
3514         default:
3515                 return -EINVAL;
3516         }
3517         val |= RBR_CFIG_B_VLD1;
3518         switch (rp->rbr_sizes[1]) {
3519         case 1 * 1024:
3520                 val |= (RBR_BUFSZ1_1K << RBR_CFIG_B_BUFSZ1_SHIFT);
3521                 break;
3522         case 2 * 1024:
3523                 val |= (RBR_BUFSZ1_2K << RBR_CFIG_B_BUFSZ1_SHIFT);
3524                 break;
3525         case 4 * 1024:
3526                 val |= (RBR_BUFSZ1_4K << RBR_CFIG_B_BUFSZ1_SHIFT);
3527                 break;
3528         case 8 * 1024:
3529                 val |= (RBR_BUFSZ1_8K << RBR_CFIG_B_BUFSZ1_SHIFT);
3530                 break;
3531
3532         default:
3533                 return -EINVAL;
3534         }
3535         val |= RBR_CFIG_B_VLD0;
3536         switch (rp->rbr_sizes[0]) {
3537         case 256:
3538                 val |= (RBR_BUFSZ0_256 << RBR_CFIG_B_BUFSZ0_SHIFT);
3539                 break;
3540         case 512:
3541                 val |= (RBR_BUFSZ0_512 << RBR_CFIG_B_BUFSZ0_SHIFT);
3542                 break;
3543         case 1 * 1024:
3544                 val |= (RBR_BUFSZ0_1K << RBR_CFIG_B_BUFSZ0_SHIFT);
3545                 break;
3546         case 2 * 1024:
3547                 val |= (RBR_BUFSZ0_2K << RBR_CFIG_B_BUFSZ0_SHIFT);
3548                 break;
3549
3550         default:
3551                 return -EINVAL;
3552         }
3553
3554         *ret = val;
3555         return 0;
3556 }
3557
3558 static int niu_enable_rx_channel(struct niu *np, int channel, int on)
3559 {
3560         u64 val = nr64(RXDMA_CFIG1(channel));
3561         int limit;
3562
3563         if (on)
3564                 val |= RXDMA_CFIG1_EN;
3565         else
3566                 val &= ~RXDMA_CFIG1_EN;
3567         nw64(RXDMA_CFIG1(channel), val);
3568
3569         limit = 1000;
3570         while (--limit > 0) {
3571                 if (nr64(RXDMA_CFIG1(channel)) & RXDMA_CFIG1_QST)
3572                         break;
3573                 udelay(10);
3574         }
3575         if (limit <= 0)
3576                 return -ENODEV;
3577         return 0;
3578 }
3579
3580 static int niu_init_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
3581 {
3582         int err, channel = rp->rx_channel;
3583         u64 val;
3584
3585         err = niu_rx_channel_reset(np, channel);
3586         if (err)
3587                 return err;
3588
3589         err = niu_rx_channel_lpage_init(np, channel);
3590         if (err)
3591                 return err;
3592
3593         niu_rx_channel_wred_init(np, rp);
3594
3595         nw64(RX_DMA_ENT_MSK(channel), RX_DMA_ENT_MSK_RBR_EMPTY);
3596         nw64(RX_DMA_CTL_STAT(channel),
3597              (RX_DMA_CTL_STAT_MEX |
3598               RX_DMA_CTL_STAT_RCRTHRES |
3599               RX_DMA_CTL_STAT_RCRTO |
3600               RX_DMA_CTL_STAT_RBR_EMPTY));
3601         nw64(RXDMA_CFIG1(channel), rp->mbox_dma >> 32);
3602         nw64(RXDMA_CFIG2(channel), (rp->mbox_dma & 0x00000000ffffffc0));
3603         nw64(RBR_CFIG_A(channel),
3604              ((u64)rp->rbr_table_size << RBR_CFIG_A_LEN_SHIFT) |
3605              (rp->rbr_dma & (RBR_CFIG_A_STADDR_BASE | RBR_CFIG_A_STADDR)));
3606         err = niu_compute_rbr_cfig_b(rp, &val);
3607         if (err)
3608                 return err;
3609         nw64(RBR_CFIG_B(channel), val);
3610         nw64(RCRCFIG_A(channel),
3611              ((u64)rp->rcr_table_size << RCRCFIG_A_LEN_SHIFT) |
3612              (rp->rcr_dma & (RCRCFIG_A_STADDR_BASE | RCRCFIG_A_STADDR)));
3613         nw64(RCRCFIG_B(channel),
3614              ((u64)rp->rcr_pkt_threshold << RCRCFIG_B_PTHRES_SHIFT) |
3615              RCRCFIG_B_ENTOUT |
3616              ((u64)rp->rcr_timeout << RCRCFIG_B_TIMEOUT_SHIFT));
3617
3618         err = niu_enable_rx_channel(np, channel, 1);
3619         if (err)
3620                 return err;
3621
3622         nw64(RBR_KICK(channel), rp->rbr_index);
3623
3624         val = nr64(RX_DMA_CTL_STAT(channel));
3625         val |= RX_DMA_CTL_STAT_RBR_EMPTY;
3626         nw64(RX_DMA_CTL_STAT(channel), val);
3627
3628         return 0;
3629 }
3630
3631 static int niu_init_rx_channels(struct niu *np)
3632 {
3633         unsigned long flags;
3634         u64 seed = jiffies_64;
3635         int err, i;
3636
3637         niu_lock_parent(np, flags);
3638         nw64(RX_DMA_CK_DIV, np->parent->rxdma_clock_divider);
3639         nw64(RED_RAN_INIT, RED_RAN_INIT_OPMODE | (seed & RED_RAN_INIT_VAL));
3640         niu_unlock_parent(np, flags);
3641
3642         /* XXX RXDMA 32bit mode? XXX */
3643
3644         niu_init_rdc_groups(np);
3645         niu_init_drr_weight(np);
3646
3647         err = niu_init_hostinfo(np);
3648         if (err)
3649                 return err;
3650
3651         for (i = 0; i < np->num_rx_rings; i++) {
3652                 struct rx_ring_info *rp = &np->rx_rings[i];
3653
3654                 err = niu_init_one_rx_channel(np, rp);
3655                 if (err)
3656                         return err;
3657         }
3658
3659         return 0;
3660 }
3661
3662 static int niu_set_ip_frag_rule(struct niu *np)
3663 {
3664         struct niu_parent *parent = np->parent;
3665         struct niu_classifier *cp = &np->clas;
3666         struct niu_tcam_entry *tp;
3667         int index, err;
3668
3669         /* XXX fix this allocation scheme XXX */
3670         index = cp->tcam_index;
3671         tp = &parent->tcam[index];
3672
3673         /* Note that the noport bit is the same in both ipv4 and
3674          * ipv6 format TCAM entries.
3675          */
3676         memset(tp, 0, sizeof(*tp));
3677         tp->key[1] = TCAM_V4KEY1_NOPORT;
3678         tp->key_mask[1] = TCAM_V4KEY1_NOPORT;
3679         tp->assoc_data = (TCAM_ASSOCDATA_TRES_USE_OFFSET |
3680                           ((u64)0 << TCAM_ASSOCDATA_OFFSET_SHIFT));
3681         err = tcam_write(np, index, tp->key, tp->key_mask);
3682         if (err)
3683                 return err;
3684         err = tcam_assoc_write(np, index, tp->assoc_data);
3685         if (err)
3686                 return err;
3687
3688         return 0;
3689 }
3690
3691 static int niu_init_classifier_hw(struct niu *np)
3692 {
3693         struct niu_parent *parent = np->parent;
3694         struct niu_classifier *cp = &np->clas;
3695         int i, err;
3696
3697         nw64(H1POLY, cp->h1_init);
3698         nw64(H2POLY, cp->h2_init);
3699
3700         err = niu_init_hostinfo(np);
3701         if (err)
3702                 return err;
3703
3704         for (i = 0; i < ENET_VLAN_TBL_NUM_ENTRIES; i++) {
3705                 struct niu_vlan_rdc *vp = &cp->vlan_mappings[i];
3706
3707                 vlan_tbl_write(np, i, np->port,
3708                                vp->vlan_pref, vp->rdc_num);
3709         }
3710
3711         for (i = 0; i < cp->num_alt_mac_mappings; i++) {
3712                 struct niu_altmac_rdc *ap = &cp->alt_mac_mappings[i];
3713
3714                 err = niu_set_alt_mac_rdc_table(np, ap->alt_mac_num,
3715                                                 ap->rdc_num, ap->mac_pref);
3716                 if (err)
3717                         return err;
3718         }
3719
3720         for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_SCTP_IPV6; i++) {
3721                 int index = i - CLASS_CODE_USER_PROG1;
3722
3723                 err = niu_set_tcam_key(np, i, parent->tcam_key[index]);
3724                 if (err)
3725                         return err;
3726                 err = niu_set_flow_key(np, i, parent->flow_key[index]);
3727                 if (err)
3728                         return err;
3729         }
3730
3731         err = niu_set_ip_frag_rule(np);
3732         if (err)
3733                 return err;
3734
3735         tcam_enable(np, 1);
3736
3737         return 0;
3738 }
3739
3740 static int niu_zcp_write(struct niu *np, int index, u64 *data)
3741 {
3742         nw64(ZCP_RAM_DATA0, data[0]);
3743         nw64(ZCP_RAM_DATA1, data[1]);
3744         nw64(ZCP_RAM_DATA2, data[2]);
3745         nw64(ZCP_RAM_DATA3, data[3]);
3746         nw64(ZCP_RAM_DATA4, data[4]);
3747         nw64(ZCP_RAM_BE, ZCP_RAM_BE_VAL);
3748         nw64(ZCP_RAM_ACC,
3749              (ZCP_RAM_ACC_WRITE |
3750               (0 << ZCP_RAM_ACC_ZFCID_SHIFT) |
3751               (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));
3752
3753         return niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
3754                                    1000, 100);
3755 }
3756
3757 static int niu_zcp_read(struct niu *np, int index, u64 *data)
3758 {
3759         int err;
3760
3761         err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
3762                                   1000, 100);
3763         if (err) {
3764                 dev_err(np->device, PFX "%s: ZCP read busy won't clear, "
3765                         "ZCP_RAM_ACC[%llx]\n", np->dev->name,
3766                         (unsigned long long) nr64(ZCP_RAM_ACC));
3767                 return err;
3768         }
3769
3770         nw64(ZCP_RAM_ACC,
3771              (ZCP_RAM_ACC_READ |
3772               (0 << ZCP_RAM_ACC_ZFCID_SHIFT) |
3773               (ZCP_RAM_SEL_CFIFO(np->port) << ZCP_RAM_ACC_RAM_SEL_SHIFT)));
3774
3775         err = niu_wait_bits_clear(np, ZCP_RAM_ACC, ZCP_RAM_ACC_BUSY,
3776                                   1000, 100);
3777         if (err) {
3778                 dev_err(np->device, PFX "%s: ZCP read busy2 won't clear, "
3779                         "ZCP_RAM_ACC[%llx]\n", np->dev->name,
3780                         (unsigned long long) nr64(ZCP_RAM_ACC));
3781                 return err;
3782         }
3783
3784         data[0] = nr64(ZCP_RAM_DATA0);
3785         data[1] = nr64(ZCP_RAM_DATA1);
3786         data[2] = nr64(ZCP_RAM_DATA2);
3787         data[3] = nr64(ZCP_RAM_DATA3);
3788         data[4] = nr64(ZCP_RAM_DATA4);
3789
3790         return 0;
3791 }
3792
3793 static void niu_zcp_cfifo_reset(struct niu *np)
3794 {
3795         u64 val = nr64(RESET_CFIFO);
3796
3797         val |= RESET_CFIFO_RST(np->port);
3798         nw64(RESET_CFIFO, val);
3799         udelay(10);
3800
3801         val &= ~RESET_CFIFO_RST(np->port);
3802         nw64(RESET_CFIFO, val);
3803 }
3804
3805 static int niu_init_zcp(struct niu *np)
3806 {
3807         u64 data[5], rbuf[5];
3808         int i, max, err;
3809
3810         if (np->parent->plat_type != PLAT_TYPE_NIU) {
3811                 if (np->port == 0 || np->port == 1)
3812                         max = ATLAS_P0_P1_CFIFO_ENTRIES;
3813                 else
3814                         max = ATLAS_P2_P3_CFIFO_ENTRIES;
3815         } else
3816                 max = NIU_CFIFO_ENTRIES;
3817
3818         data[0] = 0;
3819         data[1] = 0;
3820         data[2] = 0;
3821         data[3] = 0;
3822         data[4] = 0;
3823
3824         for (i = 0; i < max; i++) {
3825                 err = niu_zcp_write(np, i, data);
3826                 if (err)
3827                         return err;
3828                 err = niu_zcp_read(np, i, rbuf);
3829                 if (err)
3830                         return err;
3831         }
3832
3833         niu_zcp_cfifo_reset(np);
3834         nw64(CFIFO_ECC(np->port), 0);
3835         nw64(ZCP_INT_STAT, ZCP_INT_STAT_ALL);
3836         (void) nr64(ZCP_INT_STAT);
3837         nw64(ZCP_INT_MASK, ZCP_INT_MASK_ALL);
3838
3839         return 0;
3840 }
3841
3842 static void niu_ipp_write(struct niu *np, int index, u64 *data)
3843 {
3844         u64 val = nr64_ipp(IPP_CFIG);
3845
3846         nw64_ipp(IPP_CFIG, val | IPP_CFIG_DFIFO_PIO_W);
3847         nw64_ipp(IPP_DFIFO_WR_PTR, index);
3848         nw64_ipp(IPP_DFIFO_WR0, data[0]);
3849         nw64_ipp(IPP_DFIFO_WR1, data[1]);
3850         nw64_ipp(IPP_DFIFO_WR2, data[2]);
3851         nw64_ipp(IPP_DFIFO_WR3, data[3]);
3852         nw64_ipp(IPP_DFIFO_WR4, data[4]);
3853         nw64_ipp(IPP_CFIG, val & ~IPP_CFIG_DFIFO_PIO_W);
3854 }
3855
3856 static void niu_ipp_read(struct niu *np, int index, u64 *data)
3857 {
3858         nw64_ipp(IPP_DFIFO_RD_PTR, index);
3859         data[0] = nr64_ipp(IPP_DFIFO_RD0);
3860         data[1] = nr64_ipp(IPP_DFIFO_RD1);
3861         data[2] = nr64_ipp(IPP_DFIFO_RD2);
3862         data[3] = nr64_ipp(IPP_DFIFO_RD3);
3863         data[4] = nr64_ipp(IPP_DFIFO_RD4);
3864 }
3865
3866 static int niu_ipp_reset(struct niu *np)
3867 {
3868         return niu_set_and_wait_clear_ipp(np, IPP_CFIG, IPP_CFIG_SOFT_RST,
3869                                           1000, 100, "IPP_CFIG");
3870 }
3871
3872 static int niu_init_ipp(struct niu *np)
3873 {
3874         u64 data[5], rbuf[5], val;
3875         int i, max, err;
3876
3877         if (np->parent->plat_type != PLAT_TYPE_NIU) {
3878                 if (np->port == 0 || np->port == 1)
3879                         max = ATLAS_P0_P1_DFIFO_ENTRIES;
3880                 else
3881                         max = ATLAS_P2_P3_DFIFO_ENTRIES;
3882         } else
3883                 max = NIU_DFIFO_ENTRIES;
3884
3885         data[0] = 0;
3886         data[1] = 0;
3887         data[2] = 0;
3888         data[3] = 0;
3889         data[4] = 0;
3890
3891         for (i = 0; i < max; i++) {
3892                 niu_ipp_write(np, i, data);
3893                 niu_ipp_read(np, i, rbuf);
3894         }
3895
3896         (void) nr64_ipp(IPP_INT_STAT);
3897         (void) nr64_ipp(IPP_INT_STAT);
3898
3899         err = niu_ipp_reset(np);
3900         if (err)
3901                 return err;
3902
3903         (void) nr64_ipp(IPP_PKT_DIS);
3904         (void) nr64_ipp(IPP_BAD_CS_CNT);
3905         (void) nr64_ipp(IPP_ECC);
3906
3907         (void) nr64_ipp(IPP_INT_STAT);
3908
3909         nw64_ipp(IPP_MSK, ~IPP_MSK_ALL);
3910
3911         val = nr64_ipp(IPP_CFIG);
3912         val &= ~IPP_CFIG_IP_MAX_PKT;
3913         val |= (IPP_CFIG_IPP_ENABLE |
3914                 IPP_CFIG_DFIFO_ECC_EN |
3915                 IPP_CFIG_DROP_BAD_CRC |
3916                 IPP_CFIG_CKSUM_EN |
3917                 (0x1ffff << IPP_CFIG_IP_MAX_PKT_SHIFT));
3918         nw64_ipp(IPP_CFIG, val);
3919
3920         return 0;
3921 }
3922
3923 static void niu_handle_led(struct niu *np, int status)
3924 {
3925         u64 val;
3926         val = nr64_mac(XMAC_CONFIG);
3927
3928         if ((np->flags & NIU_FLAGS_10G) != 0 &&
3929             (np->flags & NIU_FLAGS_FIBER) != 0) {
3930                 if (status) {
3931                         val |= XMAC_CONFIG_LED_POLARITY;
3932                         val &= ~XMAC_CONFIG_FORCE_LED_ON;
3933                 } else {
3934                         val |= XMAC_CONFIG_FORCE_LED_ON;
3935                         val &= ~XMAC_CONFIG_LED_POLARITY;
3936                 }
3937         }
3938
3939         nw64_mac(XMAC_CONFIG, val);
3940 }
3941
3942 static void niu_init_xif_xmac(struct niu *np)
3943 {
3944         struct niu_link_config *lp = &np->link_config;
3945         u64 val;
3946
3947         val = nr64_mac(XMAC_CONFIG);
3948         val &= ~XMAC_CONFIG_SEL_POR_CLK_SRC;
3949
3950         val |= XMAC_CONFIG_TX_OUTPUT_EN;
3951
3952         if (lp->loopback_mode == LOOPBACK_MAC) {
3953                 val &= ~XMAC_CONFIG_SEL_POR_CLK_SRC;
3954                 val |= XMAC_CONFIG_LOOPBACK;
3955         } else {
3956                 val &= ~XMAC_CONFIG_LOOPBACK;
3957         }
3958
3959         if (np->flags & NIU_FLAGS_10G) {
3960                 val &= ~XMAC_CONFIG_LFS_DISABLE;
3961         } else {
3962                 val |= XMAC_CONFIG_LFS_DISABLE;
3963                 if (!(np->flags & NIU_FLAGS_FIBER))
3964                         val |= XMAC_CONFIG_1G_PCS_BYPASS;
3965                 else
3966                         val &= ~XMAC_CONFIG_1G_PCS_BYPASS;
3967         }
3968
3969         val &= ~XMAC_CONFIG_10G_XPCS_BYPASS;
3970
3971         if (lp->active_speed == SPEED_100)
3972                 val |= XMAC_CONFIG_SEL_CLK_25MHZ;
3973         else
3974                 val &= ~XMAC_CONFIG_SEL_CLK_25MHZ;
3975
3976         nw64_mac(XMAC_CONFIG, val);
3977
3978         val = nr64_mac(XMAC_CONFIG);
3979         val &= ~XMAC_CONFIG_MODE_MASK;
3980         if (np->flags & NIU_FLAGS_10G) {
3981                 val |= XMAC_CONFIG_MODE_XGMII;
3982         } else {
3983                 if (lp->active_speed == SPEED_100)
3984                         val |= XMAC_CONFIG_MODE_MII;
3985                 else
3986                         val |= XMAC_CONFIG_MODE_GMII;
3987         }
3988
3989         nw64_mac(XMAC_CONFIG, val);
3990 }
3991
3992 static void niu_init_xif_bmac(struct niu *np)
3993 {
3994         struct niu_link_config *lp = &np->link_config;
3995         u64 val;
3996
3997         val = BMAC_XIF_CONFIG_TX_OUTPUT_EN;
3998
3999         if (lp->loopback_mode == LOOPBACK_MAC)
4000                 val |= BMAC_XIF_CONFIG_MII_LOOPBACK;
4001         else
4002                 val &= ~BMAC_XIF_CONFIG_MII_LOOPBACK;
4003
4004         if (lp->active_speed == SPEED_1000)
4005                 val |= BMAC_XIF_CONFIG_GMII_MODE;
4006         else
4007                 val &= ~BMAC_XIF_CONFIG_GMII_MODE;
4008
4009         val &= ~(BMAC_XIF_CONFIG_LINK_LED |
4010                  BMAC_XIF_CONFIG_LED_POLARITY);
4011
4012         if (!(np->flags & NIU_FLAGS_10G) &&
4013             !(np->flags & NIU_FLAGS_FIBER) &&
4014             lp->active_speed == SPEED_100)
4015                 val |= BMAC_XIF_CONFIG_25MHZ_CLOCK;
4016         else
4017                 val &= ~BMAC_XIF_CONFIG_25MHZ_CLOCK;
4018
4019         nw64_mac(BMAC_XIF_CONFIG, val);
4020 }
4021
4022 static void niu_init_xif(struct niu *np)
4023 {
4024         if (np->flags & NIU_FLAGS_XMAC)
4025                 niu_init_xif_xmac(np);
4026         else
4027                 niu_init_xif_bmac(np);
4028 }
4029
4030 static void niu_pcs_mii_reset(struct niu *np)
4031 {
4032         u64 val = nr64_pcs(PCS_MII_CTL);
4033         val |= PCS_MII_CTL_RST;
4034         nw64_pcs(PCS_MII_CTL, val);
4035 }
4036
4037 static void niu_xpcs_reset(struct niu *np)
4038 {
4039         u64 val = nr64_xpcs(XPCS_CONTROL1);
4040         val |= XPCS_CONTROL1_RESET;
4041         nw64_xpcs(XPCS_CONTROL1, val);
4042 }
4043
4044 static int niu_init_pcs(struct niu *np)
4045 {
4046         struct niu_link_config *lp = &np->link_config;
4047         u64 val;
4048
4049         switch (np->flags & (NIU_FLAGS_10G | NIU_FLAGS_FIBER)) {
4050         case NIU_FLAGS_FIBER:
4051                 /* 1G fiber */
4052                 nw64_pcs(PCS_CONF, PCS_CONF_MASK | PCS_CONF_ENABLE);
4053                 nw64_pcs(PCS_DPATH_MODE, 0);
4054                 niu_pcs_mii_reset(np);
4055                 break;
4056
4057         case NIU_FLAGS_10G:
4058         case NIU_FLAGS_10G | NIU_FLAGS_FIBER:
4059                 if (!(np->flags & NIU_FLAGS_XMAC))
4060                         return -EINVAL;
4061
4062                 /* 10G copper or fiber */
4063                 val = nr64_mac(XMAC_CONFIG);
4064                 val &= ~XMAC_CONFIG_10G_XPCS_BYPASS;
4065                 nw64_mac(XMAC_CONFIG, val);
4066
4067                 niu_xpcs_reset(np);
4068
4069                 val = nr64_xpcs(XPCS_CONTROL1);
4070                 if (lp->loopback_mode == LOOPBACK_PHY)
4071                         val |= XPCS_CONTROL1_LOOPBACK;
4072                 else
4073                         val &= ~XPCS_CONTROL1_LOOPBACK;
4074                 nw64_xpcs(XPCS_CONTROL1, val);
4075
4076                 nw64_xpcs(XPCS_DESKEW_ERR_CNT, 0);
4077                 (void) nr64_xpcs(XPCS_SYMERR_CNT01);
4078                 (void) nr64_xpcs(XPCS_SYMERR_CNT23);
4079                 break;
4080
4081         case 0:
4082                 /* 1G copper */
4083                 nw64_pcs(PCS_DPATH_MODE, PCS_DPATH_MODE_MII);
4084                 niu_pcs_mii_reset(np);
4085                 break;
4086
4087         default:
4088                 return -EINVAL;
4089         }
4090
4091         return 0;
4092 }
4093
4094 static int niu_reset_tx_xmac(struct niu *np)
4095 {
4096         return niu_set_and_wait_clear_mac(np, XTXMAC_SW_RST,
4097                                           (XTXMAC_SW_RST_REG_RS |
4098                                            XTXMAC_SW_RST_SOFT_RST),
4099                                           1000, 100, "XTXMAC_SW_RST");
4100 }
4101
4102 static int niu_reset_tx_bmac(struct niu *np)
4103 {
4104         int limit;
4105
4106         nw64_mac(BTXMAC_SW_RST, BTXMAC_SW_RST_RESET);
4107         limit = 1000;
4108         while (--limit >= 0) {
4109                 if (!(nr64_mac(BTXMAC_SW_RST) & BTXMAC_SW_RST_RESET))
4110                         break;
4111                 udelay(100);
4112         }
4113         if (limit < 0) {
4114                 dev_err(np->device, PFX "Port %u TX BMAC would not reset, "
4115                         "BTXMAC_SW_RST[%llx]\n",
4116                         np->port,
4117                         (unsigned long long) nr64_mac(BTXMAC_SW_RST));
4118                 return -ENODEV;
4119         }
4120
4121         return 0;
4122 }
4123
4124 static int niu_reset_tx_mac(struct niu *np)
4125 {
4126         if (np->flags & NIU_FLAGS_XMAC)
4127                 return niu_reset_tx_xmac(np);
4128         else
4129                 return niu_reset_tx_bmac(np);
4130 }
4131
4132 static void niu_init_tx_xmac(struct niu *np, u64 min, u64 max)
4133 {
4134         u64 val;
4135
4136         val = nr64_mac(XMAC_MIN);
4137         val &= ~(XMAC_MIN_TX_MIN_PKT_SIZE |
4138                  XMAC_MIN_RX_MIN_PKT_SIZE);
4139         val |= (min << XMAC_MIN_RX_MIN_PKT_SIZE_SHFT);
4140         val |= (min << XMAC_MIN_TX_MIN_PKT_SIZE_SHFT);
4141         nw64_mac(XMAC_MIN, val);
4142
4143         nw64_mac(XMAC_MAX, max);
4144
4145         nw64_mac(XTXMAC_STAT_MSK, ~(u64)0);
4146
4147         val = nr64_mac(XMAC_IPG);
4148         if (np->flags & NIU_FLAGS_10G) {
4149                 val &= ~XMAC_IPG_IPG_XGMII;
4150                 val |= (IPG_12_15_XGMII << XMAC_IPG_IPG_XGMII_SHIFT);
4151         } else {
4152                 val &= ~XMAC_IPG_IPG_MII_GMII;
4153                 val |= (IPG_12_MII_GMII << XMAC_IPG_IPG_MII_GMII_SHIFT);
4154         }
4155         nw64_mac(XMAC_IPG, val);
4156
4157         val = nr64_mac(XMAC_CONFIG);
4158         val &= ~(XMAC_CONFIG_ALWAYS_NO_CRC |
4159                  XMAC_CONFIG_STRETCH_MODE |
4160                  XMAC_CONFIG_VAR_MIN_IPG_EN |
4161                  XMAC_CONFIG_TX_ENABLE);
4162         nw64_mac(XMAC_CONFIG, val);
4163
4164         nw64_mac(TXMAC_FRM_CNT, 0);
4165         nw64_mac(TXMAC_BYTE_CNT, 0);
4166 }
4167
4168 static void niu_init_tx_bmac(struct niu *np, u64 min, u64 max)
4169 {
4170         u64 val;
4171
4172         nw64_mac(BMAC_MIN_FRAME, min);
4173         nw64_mac(BMAC_MAX_FRAME, max);
4174
4175         nw64_mac(BTXMAC_STATUS_MASK, ~(u64)0);
4176         nw64_mac(BMAC_CTRL_TYPE, 0x8808);
4177         nw64_mac(BMAC_PREAMBLE_SIZE, 7);
4178
4179         val = nr64_mac(BTXMAC_CONFIG);
4180         val &= ~(BTXMAC_CONFIG_FCS_DISABLE |
4181                  BTXMAC_CONFIG_ENABLE);
4182         nw64_mac(BTXMAC_CONFIG, val);
4183 }
4184
4185 static void niu_init_tx_mac(struct niu *np)
4186 {
4187         u64 min, max;
4188
4189         min = 64;
4190         if (np->dev->mtu > ETH_DATA_LEN)
4191                 max = 9216;
4192         else
4193                 max = 1522;
4194
4195         /* The XMAC_MIN register only accepts values for TX min which
4196          * have the low 3 bits cleared.
4197          */
4198         BUILD_BUG_ON(min & 0x7);
4199
4200         if (np->flags & NIU_FLAGS_XMAC)
4201                 niu_init_tx_xmac(np, min, max);
4202         else
4203                 niu_init_tx_bmac(np, min, max);
4204 }
4205
4206 static int niu_reset_rx_xmac(struct niu *np)
4207 {
4208         int limit;
4209
4210         nw64_mac(XRXMAC_SW_RST,
4211                  XRXMAC_SW_RST_REG_RS | XRXMAC_SW_RST_SOFT_RST);
4212         limit = 1000;
4213         while (--limit >= 0) {
4214                 if (!(nr64_mac(XRXMAC_SW_RST) & (XRXMAC_SW_RST_REG_RS |
4215                                                  XRXMAC_SW_RST_SOFT_RST)))
4216                     break;
4217                 udelay(100);
4218         }
4219         if (limit < 0) {
4220                 dev_err(np->device, PFX "Port %u RX XMAC would not reset, "
4221                         "XRXMAC_SW_RST[%llx]\n",
4222                         np->port,
4223                         (unsigned long long) nr64_mac(XRXMAC_SW_RST));
4224                 return -ENODEV;
4225         }
4226
4227         return 0;
4228 }
4229
4230 static int niu_reset_rx_bmac(struct niu *np)
4231 {
4232         int limit;
4233
4234         nw64_mac(BRXMAC_SW_RST, BRXMAC_SW_RST_RESET);
4235         limit = 1000;
4236         while (--limit >= 0) {
4237                 if (!(nr64_mac(BRXMAC_SW_RST) & BRXMAC_SW_RST_RESET))
4238                         break;
4239                 udelay(100);
4240         }
4241         if (limit < 0) {
4242                 dev_err(np->device, PFX "Port %u RX BMAC would not reset, "
4243                         "BRXMAC_SW_RST[%llx]\n",
4244                         np->port,
4245                         (unsigned long long) nr64_mac(BRXMAC_SW_RST));
4246                 return -ENODEV;
4247         }
4248
4249         return 0;
4250 }
4251
4252 static int niu_reset_rx_mac(struct niu *np)
4253 {
4254         if (np->flags & NIU_FLAGS_XMAC)
4255                 return niu_reset_rx_xmac(np);
4256         else
4257                 return niu_reset_rx_bmac(np);
4258 }
4259
4260 static void niu_init_rx_xmac(struct niu *np)
4261 {
4262         struct niu_parent *parent = np->parent;
4263         struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
4264         int first_rdc_table = tp->first_table_num;
4265         unsigned long i;
4266         u64 val;
4267
4268         nw64_mac(XMAC_ADD_FILT0, 0);
4269         nw64_mac(XMAC_ADD_FILT1, 0);
4270         nw64_mac(XMAC_ADD_FILT2, 0);
4271         nw64_mac(XMAC_ADD_FILT12_MASK, 0);
4272         nw64_mac(XMAC_ADD_FILT00_MASK, 0);
4273         for (i = 0; i < MAC_NUM_HASH; i++)
4274                 nw64_mac(XMAC_HASH_TBL(i), 0);
4275         nw64_mac(XRXMAC_STAT_MSK, ~(u64)0);
4276         niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
4277         niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
4278
4279         val = nr64_mac(XMAC_CONFIG);
4280         val &= ~(XMAC_CONFIG_RX_MAC_ENABLE |
4281                  XMAC_CONFIG_PROMISCUOUS |
4282                  XMAC_CONFIG_PROMISC_GROUP |
4283                  XMAC_CONFIG_ERR_CHK_DIS |
4284                  XMAC_CONFIG_RX_CRC_CHK_DIS |
4285                  XMAC_CONFIG_RESERVED_MULTICAST |
4286                  XMAC_CONFIG_RX_CODEV_CHK_DIS |
4287                  XMAC_CONFIG_ADDR_FILTER_EN |
4288                  XMAC_CONFIG_RCV_PAUSE_ENABLE |
4289                  XMAC_CONFIG_STRIP_CRC |
4290                  XMAC_CONFIG_PASS_FLOW_CTRL |
4291                  XMAC_CONFIG_MAC2IPP_PKT_CNT_EN);
4292         val |= (XMAC_CONFIG_HASH_FILTER_EN);
4293         nw64_mac(XMAC_CONFIG, val);
4294
4295         nw64_mac(RXMAC_BT_CNT, 0);
4296         nw64_mac(RXMAC_BC_FRM_CNT, 0);
4297         nw64_mac(RXMAC_MC_FRM_CNT, 0);
4298         nw64_mac(RXMAC_FRAG_CNT, 0);
4299         nw64_mac(RXMAC_HIST_CNT1, 0);
4300         nw64_mac(RXMAC_HIST_CNT2, 0);
4301         nw64_mac(RXMAC_HIST_CNT3, 0);
4302         nw64_mac(RXMAC_HIST_CNT4, 0);
4303         nw64_mac(RXMAC_HIST_CNT5, 0);
4304         nw64_mac(RXMAC_HIST_CNT6, 0);
4305         nw64_mac(RXMAC_HIST_CNT7, 0);
4306         nw64_mac(RXMAC_MPSZER_CNT, 0);
4307         nw64_mac(RXMAC_CRC_ER_CNT, 0);
4308         nw64_mac(RXMAC_CD_VIO_CNT, 0);
4309         nw64_mac(LINK_FAULT_CNT, 0);
4310 }
4311
4312 static void niu_init_rx_bmac(struct niu *np)
4313 {
4314         struct niu_parent *parent = np->parent;
4315         struct niu_rdc_tables *tp = &parent->rdc_group_cfg[np->port];
4316         int first_rdc_table = tp->first_table_num;
4317         unsigned long i;
4318         u64 val;
4319
4320         nw64_mac(BMAC_ADD_FILT0, 0);
4321         nw64_mac(BMAC_ADD_FILT1, 0);
4322         nw64_mac(BMAC_ADD_FILT2, 0);
4323         nw64_mac(BMAC_ADD_FILT12_MASK, 0);
4324         nw64_mac(BMAC_ADD_FILT00_MASK, 0);
4325         for (i = 0; i < MAC_NUM_HASH; i++)
4326                 nw64_mac(BMAC_HASH_TBL(i), 0);
4327         niu_set_primary_mac_rdc_table(np, first_rdc_table, 1);
4328         niu_set_multicast_mac_rdc_table(np, first_rdc_table, 1);
4329         nw64_mac(BRXMAC_STATUS_MASK, ~(u64)0);
4330
4331         val = nr64_mac(BRXMAC_CONFIG);
4332         val &= ~(BRXMAC_CONFIG_ENABLE |
4333                  BRXMAC_CONFIG_STRIP_PAD |
4334                  BRXMAC_CONFIG_STRIP_FCS |
4335                  BRXMAC_CONFIG_PROMISC |
4336                  BRXMAC_CONFIG_PROMISC_GRP |
4337                  BRXMAC_CONFIG_ADDR_FILT_EN |
4338                  BRXMAC_CONFIG_DISCARD_DIS);
4339         val |= (BRXMAC_CONFIG_HASH_FILT_EN);
4340         nw64_mac(BRXMAC_CONFIG, val);
4341
4342         val = nr64_mac(BMAC_ADDR_CMPEN);
4343         val |= BMAC_ADDR_CMPEN_EN0;
4344         nw64_mac(BMAC_ADDR_CMPEN, val);
4345 }
4346
4347 static void niu_init_rx_mac(struct niu *np)
4348 {
4349         niu_set_primary_mac(np, np->dev->dev_addr);
4350
4351         if (np->flags & NIU_FLAGS_XMAC)
4352                 niu_init_rx_xmac(np);
4353         else
4354                 niu_init_rx_bmac(np);
4355 }
4356
4357 static void niu_enable_tx_xmac(struct niu *np, int on)
4358 {
4359         u64 val = nr64_mac(XMAC_CONFIG);
4360
4361         if (on)
4362                 val |= XMAC_CONFIG_TX_ENABLE;
4363         else
4364                 val &= ~XMAC_CONFIG_TX_ENABLE;
4365         nw64_mac(XMAC_CONFIG, val);
4366 }
4367
4368 static void niu_enable_tx_bmac(struct niu *np, int on)
4369 {
4370         u64 val = nr64_mac(BTXMAC_CONFIG);
4371
4372         if (on)
4373                 val |= BTXMAC_CONFIG_ENABLE;
4374         else
4375                 val &= ~BTXMAC_CONFIG_ENABLE;
4376         nw64_mac(BTXMAC_CONFIG, val);
4377 }
4378
4379 static void niu_enable_tx_mac(struct niu *np, int on)
4380 {
4381         if (np->flags & NIU_FLAGS_XMAC)
4382                 niu_enable_tx_xmac(np, on);
4383         else
4384                 niu_enable_tx_bmac(np, on);
4385 }
4386
4387 static void niu_enable_rx_xmac(struct niu *np, int on)
4388 {
4389         u64 val = nr64_mac(XMAC_CONFIG);
4390
4391         val &= ~(XMAC_CONFIG_HASH_FILTER_EN |
4392                  XMAC_CONFIG_PROMISCUOUS);
4393
4394         if (np->flags & NIU_FLAGS_MCAST)
4395                 val |= XMAC_CONFIG_HASH_FILTER_EN;
4396         if (np->flags & NIU_FLAGS_PROMISC)
4397                 val |= XMAC_CONFIG_PROMISCUOUS;
4398
4399         if (on)
4400                 val |= XMAC_CONFIG_RX_MAC_ENABLE;
4401         else
4402                 val &= ~XMAC_CONFIG_RX_MAC_ENABLE;
4403         nw64_mac(XMAC_CONFIG, val);
4404 }
4405
4406 static void niu_enable_rx_bmac(struct niu *np, int on)
4407 {
4408         u64 val = nr64_mac(BRXMAC_CONFIG);
4409
4410         val &= ~(BRXMAC_CONFIG_HASH_FILT_EN |
4411                  BRXMAC_CONFIG_PROMISC);
4412
4413         if (np->flags & NIU_FLAGS_MCAST)
4414                 val |= BRXMAC_CONFIG_HASH_FILT_EN;
4415         if (np->flags & NIU_FLAGS_PROMISC)
4416                 val |= BRXMAC_CONFIG_PROMISC;
4417
4418         if (on)
4419                 val |= BRXMAC_CONFIG_ENABLE;
4420         else
4421                 val &= ~BRXMAC_CONFIG_ENABLE;
4422         nw64_mac(BRXMAC_CONFIG, val);
4423 }
4424
4425 static void niu_enable_rx_mac(struct niu *np, int on)
4426 {
4427         if (np->flags & NIU_FLAGS_XMAC)
4428                 niu_enable_rx_xmac(np, on);
4429         else
4430                 niu_enable_rx_bmac(np, on);
4431 }
4432
4433 static int niu_init_mac(struct niu *np)
4434 {
4435         int err;
4436
4437         niu_init_xif(np);
4438         err = niu_init_pcs(np);
4439         if (err)
4440                 return err;
4441
4442         err = niu_reset_tx_mac(np);
4443         if (err)
4444                 return err;
4445         niu_init_tx_mac(np);
4446         err = niu_reset_rx_mac(np);
4447         if (err)
4448                 return err;
4449         niu_init_rx_mac(np);
4450
4451         /* This looks hookey but the RX MAC reset we just did will
4452          * undo some of the state we setup in niu_init_tx_mac() so we
4453          * have to call it again.  In particular, the RX MAC reset will
4454          * set the XMAC_MAX register back to it's default value.
4455          */
4456         niu_init_tx_mac(np);
4457         niu_enable_tx_mac(np, 1);
4458
4459         niu_enable_rx_mac(np, 1);
4460
4461         return 0;
4462 }
4463
4464 static void niu_stop_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
4465 {
4466         (void) niu_tx_channel_stop(np, rp->tx_channel);
4467 }
4468
4469 static void niu_stop_tx_channels(struct niu *np)
4470 {
4471         int i;
4472
4473         for (i = 0; i < np->num_tx_rings; i++) {
4474                 struct tx_ring_info *rp = &np->tx_rings[i];
4475
4476                 niu_stop_one_tx_channel(np, rp);
4477         }
4478 }
4479
4480 static void niu_reset_one_tx_channel(struct niu *np, struct tx_ring_info *rp)
4481 {
4482         (void) niu_tx_channel_reset(np, rp->tx_channel);
4483 }
4484
4485 static void niu_reset_tx_channels(struct niu *np)
4486 {
4487         int i;
4488
4489         for (i = 0; i < np->num_tx_rings; i++) {
4490                 struct tx_ring_info *rp = &np->tx_rings[i];
4491
4492                 niu_reset_one_tx_channel(np, rp);
4493         }
4494 }
4495
4496 static void niu_stop_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
4497 {
4498         (void) niu_enable_rx_channel(np, rp->rx_channel, 0);
4499 }
4500
4501 static void niu_stop_rx_channels(struct niu *np)
4502 {
4503         int i;
4504
4505         for (i = 0; i < np->num_rx_rings; i++) {
4506                 struct rx_ring_info *rp = &np->rx_rings[i];
4507
4508                 niu_stop_one_rx_channel(np, rp);
4509         }
4510 }
4511
4512 static void niu_reset_one_rx_channel(struct niu *np, struct rx_ring_info *rp)
4513 {
4514         int channel = rp->rx_channel;
4515
4516         (void) niu_rx_channel_reset(np, channel);
4517         nw64(RX_DMA_ENT_MSK(channel), RX_DMA_ENT_MSK_ALL);
4518         nw64(RX_DMA_CTL_STAT(channel), 0);
4519         (void) niu_enable_rx_channel(np, channel, 0);
4520 }
4521
4522 static void niu_reset_rx_channels(struct niu *np)
4523 {
4524         int i;
4525
4526         for (i = 0; i < np->num_rx_rings; i++) {
4527                 struct rx_ring_info *rp = &np->rx_rings[i];
4528
4529                 niu_reset_one_rx_channel(np, rp);
4530         }
4531 }
4532
4533 static void niu_disable_ipp(struct niu *np)
4534 {
4535         u64 rd, wr, val;
4536         int limit;
4537
4538         rd = nr64_ipp(IPP_DFIFO_RD_PTR);
4539         wr = nr64_ipp(IPP_DFIFO_WR_PTR);
4540         limit = 100;
4541         while (--limit >= 0 && (rd != wr)) {
4542                 rd = nr64_ipp(IPP_DFIFO_RD_PTR);
4543                 wr = nr64_ipp(IPP_DFIFO_WR_PTR);
4544         }
4545         if (limit < 0 &&
4546             (rd != 0 && wr != 1)) {
4547                 dev_err(np->device, PFX "%s: IPP would not quiesce, "
4548                         "rd_ptr[%llx] wr_ptr[%llx]\n",
4549                         np->dev->name,
4550                         (unsigned long long) nr64_ipp(IPP_DFIFO_RD_PTR),
4551                         (unsigned long long) nr64_ipp(IPP_DFIFO_WR_PTR));
4552         }
4553
4554         val = nr64_ipp(IPP_CFIG);
4555         val &= ~(IPP_CFIG_IPP_ENABLE |
4556                  IPP_CFIG_DFIFO_ECC_EN |
4557                  IPP_CFIG_DROP_BAD_CRC |
4558                  IPP_CFIG_CKSUM_EN);
4559         nw64_ipp(IPP_CFIG, val);
4560
4561         (void) niu_ipp_reset(np);
4562 }
4563
4564 static int niu_init_hw(struct niu *np)
4565 {
4566         int i, err;
4567
4568         niudbg(IFUP, "%s: Initialize TXC\n", np->dev->name);
4569         niu_txc_enable_port(np, 1);
4570         niu_txc_port_dma_enable(np, 1);
4571         niu_txc_set_imask(np, 0);
4572
4573         niudbg(IFUP, "%s: Initialize TX channels\n", np->dev->name);
4574         for (i = 0; i < np->num_tx_rings; i++) {
4575                 struct tx_ring_info *rp = &np->tx_rings[i];
4576
4577                 err = niu_init_one_tx_channel(np, rp);
4578                 if (err)
4579                         return err;
4580         }
4581
4582         niudbg(IFUP, "%s: Initialize RX channels\n", np->dev->name);
4583         err = niu_init_rx_channels(np);
4584         if (err)
4585                 goto out_uninit_tx_channels;
4586
4587         niudbg(IFUP, "%s: Initialize classifier\n", np->dev->name);
4588         err = niu_init_classifier_hw(np);
4589         if (err)
4590                 goto out_uninit_rx_channels;
4591
4592         niudbg(IFUP, "%s: Initialize ZCP\n", np->dev->name);
4593         err = niu_init_zcp(np);
4594         if (err)
4595                 goto out_uninit_rx_channels;
4596
4597         niudbg(IFUP, "%s: Initialize IPP\n", np->dev->name);
4598         err = niu_init_ipp(np);
4599         if (err)
4600                 goto out_uninit_rx_channels;
4601
4602         niudbg(IFUP, "%s: Initialize MAC\n", np->dev->name);
4603         err = niu_init_mac(np);
4604         if (err)
4605                 goto out_uninit_ipp;
4606
4607         return 0;
4608
4609 out_uninit_ipp:
4610         niudbg(IFUP, "%s: Uninit IPP\n", np->dev->name);
4611         niu_disable_ipp(np);
4612
4613 out_uninit_rx_channels:
4614         niudbg(IFUP, "%s: Uninit RX channels\n", np->dev->name);
4615         niu_stop_rx_channels(np);
4616         niu_reset_rx_channels(np);
4617
4618 out_uninit_tx_channels:
4619         niudbg(IFUP, "%s: Uninit TX channels\n", np->dev->name);
4620         niu_stop_tx_channels(np);
4621         niu_reset_tx_channels(np);
4622
4623         return err;
4624 }
4625
4626 static void niu_stop_hw(struct niu *np)
4627 {
4628         niudbg(IFDOWN, "%s: Disable interrupts\n", np->dev->name);
4629         niu_enable_interrupts(np, 0);
4630
4631         niudbg(IFDOWN, "%s: Disable RX MAC\n", np->dev->name);
4632         niu_enable_rx_mac(np, 0);
4633
4634         niudbg(IFDOWN, "%s: Disable IPP\n", np->dev->name);
4635         niu_disable_ipp(np);
4636
4637         niudbg(IFDOWN, "%s: Stop TX channels\n", np->dev->name);
4638         niu_stop_tx_channels(np);
4639
4640         niudbg(IFDOWN, "%s: Stop RX channels\n", np->dev->name);
4641         niu_stop_rx_channels(np);
4642
4643         niudbg(IFDOWN, "%s: Reset TX channels\n", np->dev->name);
4644         niu_reset_tx_channels(np);
4645
4646         niudbg(IFDOWN, "%s: Reset RX channels\n", np->dev->name);
4647         niu_reset_rx_channels(np);
4648 }
4649
4650 static int niu_request_irq(struct niu *np)
4651 {
4652         int i, j, err;
4653
4654         err = 0;
4655         for (i = 0; i < np->num_ldg; i++) {
4656                 struct niu_ldg *lp = &np->ldg[i];
4657
4658                 err = request_irq(lp->irq, niu_interrupt,
4659                                   IRQF_SHARED | IRQF_SAMPLE_RANDOM,
4660                                   np->dev->name, lp);
4661                 if (err)
4662                         goto out_free_irqs;
4663
4664         }
4665
4666         return 0;
4667
4668 out_free_irqs:
4669         for (j = 0; j < i; j++) {
4670                 struct niu_ldg *lp = &np->ldg[j];
4671
4672                 free_irq(lp->irq, lp);
4673         }
4674         return err;
4675 }
4676
4677 static void niu_free_irq(struct niu *np)
4678 {
4679         int i;
4680
4681         for (i = 0; i < np->num_ldg; i++) {
4682                 struct niu_ldg *lp = &np->ldg[i];
4683
4684                 free_irq(lp->irq, lp);
4685         }
4686 }
4687
4688 static void niu_enable_napi(struct niu *np)
4689 {
4690         int i;
4691
4692         for (i = 0; i < np->num_ldg; i++)
4693                 napi_enable(&np->ldg[i].napi);
4694 }
4695
4696 static void niu_disable_napi(struct niu *np)
4697 {
4698         int i;
4699
4700         for (i = 0; i < np->num_ldg; i++)
4701                 napi_disable(&np->ldg[i].napi);
4702 }
4703
4704 static int niu_open(struct net_device *dev)
4705 {
4706         struct niu *np = netdev_priv(dev);
4707         int err;
4708
4709         netif_carrier_off(dev);
4710
4711         err = niu_alloc_channels(np);
4712         if (err)
4713                 goto out_err;
4714
4715         err = niu_enable_interrupts(np, 0);
4716         if (err)
4717                 goto out_free_channels;
4718
4719         err = niu_request_irq(np);
4720         if (err)
4721                 goto out_free_channels;
4722
4723         niu_enable_napi(np);
4724
4725         spin_lock_irq(&np->lock);
4726
4727         err = niu_init_hw(np);
4728         if (!err) {
4729                 init_timer(&np->timer);
4730                 np->timer.expires = jiffies + HZ;
4731                 np->timer.data = (unsigned long) np;
4732                 np->timer.function = niu_timer;
4733
4734                 err = niu_enable_interrupts(np, 1);
4735                 if (err)
4736                         niu_stop_hw(np);
4737         }
4738
4739         spin_unlock_irq(&np->lock);
4740
4741         if (err) {
4742                 niu_disable_napi(np);
4743                 goto out_free_irq;
4744         }
4745
4746         netif_start_queue(dev);
4747
4748         if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
4749                 netif_carrier_on(dev);
4750
4751         add_timer(&np->timer);
4752
4753         return 0;
4754
4755 out_free_irq:
4756         niu_free_irq(np);
4757
4758 out_free_channels:
4759         niu_free_channels(np);
4760
4761 out_err:
4762         return err;
4763 }
4764
4765 static void niu_full_shutdown(struct niu *np, struct net_device *dev)
4766 {
4767         cancel_work_sync(&np->reset_task);
4768
4769         niu_disable_napi(np);
4770         netif_stop_queue(dev);
4771
4772         del_timer_sync(&np->timer);
4773
4774         spin_lock_irq(&np->lock);
4775
4776         niu_stop_hw(np);
4777
4778         spin_unlock_irq(&np->lock);
4779 }
4780
4781 static int niu_close(struct net_device *dev)
4782 {
4783         struct niu *np = netdev_priv(dev);
4784
4785         niu_full_shutdown(np, dev);
4786
4787         niu_free_irq(np);
4788
4789         niu_free_channels(np);
4790
4791         niu_handle_led(np, 0);
4792
4793         return 0;
4794 }
4795
4796 static void niu_sync_xmac_stats(struct niu *np)
4797 {
4798         struct niu_xmac_stats *mp = &np->mac_stats.xmac;
4799
4800         mp->tx_frames += nr64_mac(TXMAC_FRM_CNT);
4801         mp->tx_bytes += nr64_mac(TXMAC_BYTE_CNT);
4802
4803         mp->rx_link_faults += nr64_mac(LINK_FAULT_CNT);
4804         mp->rx_align_errors += nr64_mac(RXMAC_ALIGN_ERR_CNT);
4805         mp->rx_frags += nr64_mac(RXMAC_FRAG_CNT);
4806         mp->rx_mcasts += nr64_mac(RXMAC_MC_FRM_CNT);
4807         mp->rx_bcasts += nr64_mac(RXMAC_BC_FRM_CNT);
4808         mp->rx_hist_cnt1 += nr64_mac(RXMAC_HIST_CNT1);
4809         mp->rx_hist_cnt2 += nr64_mac(RXMAC_HIST_CNT2);
4810         mp->rx_hist_cnt3 += nr64_mac(RXMAC_HIST_CNT3);
4811         mp->rx_hist_cnt4 += nr64_mac(RXMAC_HIST_CNT4);
4812         mp->rx_hist_cnt5 += nr64_mac(RXMAC_HIST_CNT5);
4813         mp->rx_hist_cnt6 += nr64_mac(RXMAC_HIST_CNT6);
4814         mp->rx_hist_cnt7 += nr64_mac(RXMAC_HIST_CNT7);
4815         mp->rx_octets += nr64_mac(RXMAC_BT_CNT);
4816         mp->rx_code_violations += nr64_mac(RXMAC_CD_VIO_CNT);
4817         mp->rx_len_errors += nr64_mac(RXMAC_MPSZER_CNT);
4818         mp->rx_crc_errors += nr64_mac(RXMAC_CRC_ER_CNT);
4819 }
4820
4821 static void niu_sync_bmac_stats(struct niu *np)
4822 {
4823         struct niu_bmac_stats *mp = &np->mac_stats.bmac;
4824
4825         mp->tx_bytes += nr64_mac(BTXMAC_BYTE_CNT);
4826         mp->tx_frames += nr64_mac(BTXMAC_FRM_CNT);
4827
4828         mp->rx_frames += nr64_mac(BRXMAC_FRAME_CNT);
4829         mp->rx_align_errors += nr64_mac(BRXMAC_ALIGN_ERR_CNT);
4830         mp->rx_crc_errors += nr64_mac(BRXMAC_ALIGN_ERR_CNT);
4831         mp->rx_len_errors += nr64_mac(BRXMAC_CODE_VIOL_ERR_CNT);
4832 }
4833
4834 static void niu_sync_mac_stats(struct niu *np)
4835 {
4836         if (np->flags & NIU_FLAGS_XMAC)
4837                 niu_sync_xmac_stats(np);
4838         else
4839                 niu_sync_bmac_stats(np);
4840 }
4841
4842 static void niu_get_rx_stats(struct niu *np)
4843 {
4844         unsigned long pkts, dropped, errors, bytes;
4845         int i;
4846
4847         pkts = dropped = errors = bytes = 0;
4848         for (i = 0; i < np->num_rx_rings; i++) {
4849                 struct rx_ring_info *rp = &np->rx_rings[i];
4850
4851                 pkts += rp->rx_packets;
4852                 bytes += rp->rx_bytes;
4853                 dropped += rp->rx_dropped;
4854                 errors += rp->rx_errors;
4855         }
4856         np->net_stats.rx_packets = pkts;
4857         np->net_stats.rx_bytes = bytes;
4858         np->net_stats.rx_dropped = dropped;
4859         np->net_stats.rx_errors = errors;
4860 }
4861
4862 static void niu_get_tx_stats(struct niu *np)
4863 {
4864         unsigned long pkts, errors, bytes;
4865         int i;
4866
4867         pkts = errors = bytes = 0;
4868         for (i = 0; i < np->num_tx_rings; i++) {
4869                 struct tx_ring_info *rp = &np->tx_rings[i];
4870
4871                 pkts += rp->tx_packets;
4872                 bytes += rp->tx_bytes;
4873                 errors += rp->tx_errors;
4874         }
4875         np->net_stats.tx_packets = pkts;
4876         np->net_stats.tx_bytes = bytes;
4877         np->net_stats.tx_errors = errors;
4878 }
4879
4880 static struct net_device_stats *niu_get_stats(struct net_device *dev)
4881 {
4882         struct niu *np = netdev_priv(dev);
4883
4884         niu_get_rx_stats(np);
4885         niu_get_tx_stats(np);
4886
4887         return &np->net_stats;
4888 }
4889
4890 static void niu_load_hash_xmac(struct niu *np, u16 *hash)
4891 {
4892         int i;
4893
4894         for (i = 0; i < 16; i++)
4895                 nw64_mac(XMAC_HASH_TBL(i), hash[i]);
4896 }
4897
4898 static void niu_load_hash_bmac(struct niu *np, u16 *hash)
4899 {
4900         int i;
4901
4902         for (i = 0; i < 16; i++)
4903                 nw64_mac(BMAC_HASH_TBL(i), hash[i]);
4904 }
4905
4906 static void niu_load_hash(struct niu *np, u16 *hash)
4907 {
4908         if (np->flags & NIU_FLAGS_XMAC)
4909                 niu_load_hash_xmac(np, hash);
4910         else
4911                 niu_load_hash_bmac(np, hash);
4912 }
4913
4914 static void niu_set_rx_mode(struct net_device *dev)
4915 {
4916         struct niu *np = netdev_priv(dev);
4917         int i, alt_cnt, err;
4918         struct dev_addr_list *addr;
4919         unsigned long flags;
4920         u16 hash[16] = { 0, };
4921
4922         spin_lock_irqsave(&np->lock, flags);
4923         niu_enable_rx_mac(np, 0);
4924
4925         np->flags &= ~(NIU_FLAGS_MCAST | NIU_FLAGS_PROMISC);
4926         if (dev->flags & IFF_PROMISC)
4927                 np->flags |= NIU_FLAGS_PROMISC;
4928         if ((dev->flags & IFF_ALLMULTI) || (dev->mc_count > 0))
4929                 np->flags |= NIU_FLAGS_MCAST;
4930
4931         alt_cnt = dev->uc_count;
4932         if (alt_cnt > niu_num_alt_addr(np)) {
4933                 alt_cnt = 0;
4934                 np->flags |= NIU_FLAGS_PROMISC;
4935         }
4936
4937         if (alt_cnt) {
4938                 int index = 0;
4939
4940                 for (addr = dev->uc_list; addr; addr = addr->next) {
4941                         err = niu_set_alt_mac(np, index,
4942                                               addr->da_addr);
4943                         if (err)
4944                                 printk(KERN_WARNING PFX "%s: Error %d "
4945                                        "adding alt mac %d\n",
4946                                        dev->name, err, index);
4947                         err = niu_enable_alt_mac(np, index, 1);
4948                         if (err)
4949                                 printk(KERN_WARNING PFX "%s: Error %d "
4950                                        "enabling alt mac %d\n",
4951                                        dev->name, err, index);
4952
4953                         index++;
4954                 }
4955         } else {
4956                 for (i = 0; i < niu_num_alt_addr(np); i++) {
4957                         err = niu_enable_alt_mac(np, i, 0);
4958                         if (err)
4959                                 printk(KERN_WARNING PFX "%s: Error %d "
4960                                        "disabling alt mac %d\n",
4961                                        dev->name, err, i);
4962                 }
4963         }
4964         if (dev->flags & IFF_ALLMULTI) {
4965                 for (i = 0; i < 16; i++)
4966                         hash[i] = 0xffff;
4967         } else if (dev->mc_count > 0) {
4968                 for (addr = dev->mc_list; addr; addr = addr->next) {
4969                         u32 crc = ether_crc_le(ETH_ALEN, addr->da_addr);
4970
4971                         crc >>= 24;
4972                         hash[crc >> 4] |= (1 << (15 - (crc & 0xf)));
4973                 }
4974         }
4975
4976         if (np->flags & NIU_FLAGS_MCAST)
4977                 niu_load_hash(np, hash);
4978
4979         niu_enable_rx_mac(np, 1);
4980         spin_unlock_irqrestore(&np->lock, flags);
4981 }
4982
4983 static int niu_set_mac_addr(struct net_device *dev, void *p)
4984 {
4985         struct niu *np = netdev_priv(dev);
4986         struct sockaddr *addr = p;
4987         unsigned long flags;
4988
4989         if (!is_valid_ether_addr(addr->sa_data))
4990                 return -EINVAL;
4991
4992         memcpy(dev->dev_addr, addr->sa_data, ETH_ALEN);
4993
4994         if (!netif_running(dev))
4995                 return 0;
4996
4997         spin_lock_irqsave(&np->lock, flags);
4998         niu_enable_rx_mac(np, 0);
4999         niu_set_primary_mac(np, dev->dev_addr);
5000         niu_enable_rx_mac(np, 1);
5001         spin_unlock_irqrestore(&np->lock, flags);
5002
5003         return 0;
5004 }
5005
5006 static int niu_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
5007 {
5008         return -EOPNOTSUPP;
5009 }
5010
5011 static void niu_netif_stop(struct niu *np)
5012 {
5013         np->dev->trans_start = jiffies; /* prevent tx timeout */
5014
5015         niu_disable_napi(np);
5016
5017         netif_tx_disable(np->dev);
5018 }
5019
5020 static void niu_netif_start(struct niu *np)
5021 {
5022         /* NOTE: unconditional netif_wake_queue is only appropriate
5023          * so long as all callers are assured to have free tx slots
5024          * (such as after niu_init_hw).
5025          */
5026         netif_wake_queue(np->dev);
5027
5028         niu_enable_napi(np);
5029
5030         niu_enable_interrupts(np, 1);
5031 }
5032
5033 static void niu_reset_task(struct work_struct *work)
5034 {
5035         struct niu *np = container_of(work, struct niu, reset_task);
5036         unsigned long flags;
5037         int err;
5038
5039         spin_lock_irqsave(&np->lock, flags);
5040         if (!netif_running(np->dev)) {
5041                 spin_unlock_irqrestore(&np->lock, flags);
5042                 return;
5043         }
5044
5045         spin_unlock_irqrestore(&np->lock, flags);
5046
5047         del_timer_sync(&np->timer);
5048
5049         niu_netif_stop(np);
5050
5051         spin_lock_irqsave(&np->lock, flags);
5052
5053         niu_stop_hw(np);
5054
5055         err = niu_init_hw(np);
5056         if (!err) {
5057                 np->timer.expires = jiffies + HZ;
5058                 add_timer(&np->timer);
5059                 niu_netif_start(np);
5060         }
5061
5062         spin_unlock_irqrestore(&np->lock, flags);
5063 }
5064
5065 static void niu_tx_timeout(struct net_device *dev)
5066 {
5067         struct niu *np = netdev_priv(dev);
5068
5069         dev_err(np->device, PFX "%s: Transmit timed out, resetting\n",
5070                 dev->name);
5071
5072         schedule_work(&np->reset_task);
5073 }
5074
5075 static void niu_set_txd(struct tx_ring_info *rp, int index,
5076                         u64 mapping, u64 len, u64 mark,
5077                         u64 n_frags)
5078 {
5079         __le64 *desc = &rp->descr[index];
5080
5081         *desc = cpu_to_le64(mark |
5082                             (n_frags << TX_DESC_NUM_PTR_SHIFT) |
5083                             (len << TX_DESC_TR_LEN_SHIFT) |
5084                             (mapping & TX_DESC_SAD));
5085 }
5086
5087 static u64 niu_compute_tx_flags(struct sk_buff *skb, struct ethhdr *ehdr,
5088                                 u64 pad_bytes, u64 len)
5089 {
5090         u16 eth_proto, eth_proto_inner;
5091         u64 csum_bits, l3off, ihl, ret;
5092         u8 ip_proto;
5093         int ipv6;
5094
5095         eth_proto = be16_to_cpu(ehdr->h_proto);
5096         eth_proto_inner = eth_proto;
5097         if (eth_proto == ETH_P_8021Q) {
5098                 struct vlan_ethhdr *vp = (struct vlan_ethhdr *) ehdr;
5099                 __be16 val = vp->h_vlan_encapsulated_proto;
5100
5101                 eth_proto_inner = be16_to_cpu(val);
5102         }
5103
5104         ipv6 = ihl = 0;
5105         switch (skb->protocol) {
5106         case __constant_htons(ETH_P_IP):
5107                 ip_proto = ip_hdr(skb)->protocol;
5108                 ihl = ip_hdr(skb)->ihl;
5109                 break;
5110         case __constant_htons(ETH_P_IPV6):
5111                 ip_proto = ipv6_hdr(skb)->nexthdr;
5112                 ihl = (40 >> 2);
5113                 ipv6 = 1;
5114                 break;
5115         default:
5116                 ip_proto = ihl = 0;
5117                 break;
5118         }
5119
5120         csum_bits = TXHDR_CSUM_NONE;
5121         if (skb->ip_summed == CHECKSUM_PARTIAL) {
5122                 u64 start, stuff;
5123
5124                 csum_bits = (ip_proto == IPPROTO_TCP ?
5125                              TXHDR_CSUM_TCP :
5126                              (ip_proto == IPPROTO_UDP ?
5127                               TXHDR_CSUM_UDP : TXHDR_CSUM_SCTP));
5128
5129                 start = skb_transport_offset(skb) -
5130                         (pad_bytes + sizeof(struct tx_pkt_hdr));
5131                 stuff = start + skb->csum_offset;
5132
5133                 csum_bits |= (start / 2) << TXHDR_L4START_SHIFT;
5134                 csum_bits |= (stuff / 2) << TXHDR_L4STUFF_SHIFT;
5135         }
5136
5137         l3off = skb_network_offset(skb) -
5138                 (pad_bytes + sizeof(struct tx_pkt_hdr));
5139
5140         ret = (((pad_bytes / 2) << TXHDR_PAD_SHIFT) |
5141                (len << TXHDR_LEN_SHIFT) |
5142                ((l3off / 2) << TXHDR_L3START_SHIFT) |
5143                (ihl << TXHDR_IHL_SHIFT) |
5144                ((eth_proto_inner < 1536) ? TXHDR_LLC : 0) |
5145                ((eth_proto == ETH_P_8021Q) ? TXHDR_VLAN : 0) |
5146                (ipv6 ? TXHDR_IP_VER : 0) |
5147                csum_bits);
5148
5149         return ret;
5150 }
5151
5152 static struct tx_ring_info *tx_ring_select(struct niu *np, struct sk_buff *skb)
5153 {
5154         return &np->tx_rings[0];
5155 }
5156
5157 static int niu_start_xmit(struct sk_buff *skb, struct net_device *dev)
5158 {
5159         struct niu *np = netdev_priv(dev);
5160         unsigned long align, headroom;
5161         struct tx_ring_info *rp;
5162         struct tx_pkt_hdr *tp;
5163         unsigned int len, nfg;
5164         struct ethhdr *ehdr;
5165         int prod, i, tlen;
5166         u64 mapping, mrk;
5167
5168         rp = tx_ring_select(np, skb);
5169
5170         if (niu_tx_avail(rp) <= (skb_shinfo(skb)->nr_frags + 1)) {
5171                 netif_stop_queue(dev);
5172                 dev_err(np->device, PFX "%s: BUG! Tx ring full when "
5173                         "queue awake!\n", dev->name);
5174                 rp->tx_errors++;
5175                 return NETDEV_TX_BUSY;
5176         }
5177
5178         if (skb->len < ETH_ZLEN) {
5179                 unsigned int pad_bytes = ETH_ZLEN - skb->len;
5180
5181                 if (skb_pad(skb, pad_bytes))
5182                         goto out;
5183                 skb_put(skb, pad_bytes);
5184         }
5185
5186         len = sizeof(struct tx_pkt_hdr) + 15;
5187         if (skb_headroom(skb) < len) {
5188                 struct sk_buff *skb_new;
5189
5190                 skb_new = skb_realloc_headroom(skb, len);
5191                 if (!skb_new) {
5192                         rp->tx_errors++;
5193                         goto out_drop;
5194                 }
5195                 kfree_skb(skb);
5196                 skb = skb_new;
5197         }
5198
5199         align = ((unsigned long) skb->data & (16 - 1));
5200         headroom = align + sizeof(struct tx_pkt_hdr);
5201
5202         ehdr = (struct ethhdr *) skb->data;
5203         tp = (struct tx_pkt_hdr *) skb_push(skb, headroom);
5204
5205         len = skb->len - sizeof(struct tx_pkt_hdr);
5206         tp->flags = cpu_to_le64(niu_compute_tx_flags(skb, ehdr, align, len));
5207         tp->resv = 0;
5208
5209         len = skb_headlen(skb);
5210         mapping = np->ops->map_single(np->device, skb->data,
5211                                       len, DMA_TO_DEVICE);
5212
5213         prod = rp->prod;
5214
5215         rp->tx_buffs[prod].skb = skb;
5216         rp->tx_buffs[prod].mapping = mapping;
5217
5218         mrk = TX_DESC_SOP;
5219         if (++rp->mark_counter == rp->mark_freq) {
5220                 rp->mark_counter = 0;
5221                 mrk |= TX_DESC_MARK;
5222                 rp->mark_pending++;
5223         }
5224
5225         tlen = len;
5226         nfg = skb_shinfo(skb)->nr_frags;
5227         while (tlen > 0) {
5228                 tlen -= MAX_TX_DESC_LEN;
5229                 nfg++;
5230         }
5231
5232         while (len > 0) {
5233                 unsigned int this_len = len;
5234
5235                 if (this_len > MAX_TX_DESC_LEN)
5236                         this_len = MAX_TX_DESC_LEN;
5237
5238                 niu_set_txd(rp, prod, mapping, this_len, mrk, nfg);
5239                 mrk = nfg = 0;
5240
5241                 prod = NEXT_TX(rp, prod);
5242                 mapping += this_len;
5243                 len -= this_len;
5244         }
5245
5246         for (i = 0; i <  skb_shinfo(skb)->nr_frags; i++) {
5247                 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
5248
5249                 len = frag->size;
5250                 mapping = np->ops->map_page(np->device, frag->page,
5251                                             frag->page_offset, len,
5252                                             DMA_TO_DEVICE);
5253
5254                 rp->tx_buffs[prod].skb = NULL;
5255                 rp->tx_buffs[prod].mapping = mapping;
5256
5257                 niu_set_txd(rp, prod, mapping, len, 0, 0);
5258
5259                 prod = NEXT_TX(rp, prod);
5260         }
5261
5262         if (prod < rp->prod)
5263                 rp->wrap_bit ^= TX_RING_KICK_WRAP;
5264         rp->prod = prod;
5265
5266         nw64(TX_RING_KICK(rp->tx_channel), rp->wrap_bit | (prod << 3));
5267
5268         if (unlikely(niu_tx_avail(rp) <= (MAX_SKB_FRAGS + 1))) {
5269                 netif_stop_queue(dev);
5270                 if (niu_tx_avail(rp) > NIU_TX_WAKEUP_THRESH(rp))
5271                         netif_wake_queue(dev);
5272         }
5273
5274         dev->trans_start = jiffies;
5275
5276 out:
5277         return NETDEV_TX_OK;
5278
5279 out_drop:
5280         rp->tx_errors++;
5281         kfree_skb(skb);
5282         goto out;
5283 }
5284
5285 static int niu_change_mtu(struct net_device *dev, int new_mtu)
5286 {
5287         struct niu *np = netdev_priv(dev);
5288         int err, orig_jumbo, new_jumbo;
5289
5290         if (new_mtu < 68 || new_mtu > NIU_MAX_MTU)
5291                 return -EINVAL;
5292
5293         orig_jumbo = (dev->mtu > ETH_DATA_LEN);
5294         new_jumbo = (new_mtu > ETH_DATA_LEN);
5295
5296         dev->mtu = new_mtu;
5297
5298         if (!netif_running(dev) ||
5299             (orig_jumbo == new_jumbo))
5300                 return 0;
5301
5302         niu_full_shutdown(np, dev);
5303
5304         niu_free_channels(np);
5305
5306         niu_enable_napi(np);
5307
5308         err = niu_alloc_channels(np);
5309         if (err)
5310                 return err;
5311
5312         spin_lock_irq(&np->lock);
5313
5314         err = niu_init_hw(np);
5315         if (!err) {
5316                 init_timer(&np->timer);
5317                 np->timer.expires = jiffies + HZ;
5318                 np->timer.data = (unsigned long) np;
5319                 np->timer.function = niu_timer;
5320
5321                 err = niu_enable_interrupts(np, 1);
5322                 if (err)
5323                         niu_stop_hw(np);
5324         }
5325
5326         spin_unlock_irq(&np->lock);
5327
5328         if (!err) {
5329                 netif_start_queue(dev);
5330                 if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
5331                         netif_carrier_on(dev);
5332
5333                 add_timer(&np->timer);
5334         }
5335
5336         return err;
5337 }
5338
5339 static void niu_get_drvinfo(struct net_device *dev,
5340                             struct ethtool_drvinfo *info)
5341 {
5342         struct niu *np = netdev_priv(dev);
5343         struct niu_vpd *vpd = &np->vpd;
5344
5345         strcpy(info->driver, DRV_MODULE_NAME);
5346         strcpy(info->version, DRV_MODULE_VERSION);
5347         sprintf(info->fw_version, "%d.%d",
5348                 vpd->fcode_major, vpd->fcode_minor);
5349         if (np->parent->plat_type != PLAT_TYPE_NIU)
5350                 strcpy(info->bus_info, pci_name(np->pdev));
5351 }
5352
5353 static int niu_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
5354 {
5355         struct niu *np = netdev_priv(dev);
5356         struct niu_link_config *lp;
5357
5358         lp = &np->link_config;
5359
5360         memset(cmd, 0, sizeof(*cmd));
5361         cmd->phy_address = np->phy_addr;
5362         cmd->supported = lp->supported;
5363         cmd->advertising = lp->advertising;
5364         cmd->autoneg = lp->autoneg;
5365         cmd->speed = lp->active_speed;
5366         cmd->duplex = lp->active_duplex;
5367
5368         return 0;
5369 }
5370
5371 static int niu_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
5372 {
5373         return -EINVAL;
5374 }
5375
5376 static u32 niu_get_msglevel(struct net_device *dev)
5377 {
5378         struct niu *np = netdev_priv(dev);
5379         return np->msg_enable;
5380 }
5381
5382 static void niu_set_msglevel(struct net_device *dev, u32 value)
5383 {
5384         struct niu *np = netdev_priv(dev);
5385         np->msg_enable = value;
5386 }
5387
5388 static int niu_get_eeprom_len(struct net_device *dev)
5389 {
5390         struct niu *np = netdev_priv(dev);
5391
5392         return np->eeprom_len;
5393 }
5394
5395 static int niu_get_eeprom(struct net_device *dev,
5396                           struct ethtool_eeprom *eeprom, u8 *data)
5397 {
5398         struct niu *np = netdev_priv(dev);
5399         u32 offset, len, val;
5400
5401         offset = eeprom->offset;
5402         len = eeprom->len;
5403
5404         if (offset + len < offset)
5405                 return -EINVAL;
5406         if (offset >= np->eeprom_len)
5407                 return -EINVAL;
5408         if (offset + len > np->eeprom_len)
5409                 len = eeprom->len = np->eeprom_len - offset;
5410
5411         if (offset & 3) {
5412                 u32 b_offset, b_count;
5413
5414                 b_offset = offset & 3;
5415                 b_count = 4 - b_offset;
5416                 if (b_count > len)
5417                         b_count = len;
5418
5419                 val = nr64(ESPC_NCR((offset - b_offset) / 4));
5420                 memcpy(data, ((char *)&val) + b_offset, b_count);
5421                 data += b_count;
5422                 len -= b_count;
5423                 offset += b_count;
5424         }
5425         while (len >= 4) {
5426                 val = nr64(ESPC_NCR(offset / 4));
5427                 memcpy(data, &val, 4);
5428                 data += 4;
5429                 len -= 4;
5430                 offset += 4;
5431         }
5432         if (len) {
5433                 val = nr64(ESPC_NCR(offset / 4));
5434                 memcpy(data, &val, len);
5435         }
5436         return 0;
5437 }
5438
5439 static const struct {
5440         const char string[ETH_GSTRING_LEN];
5441 } niu_xmac_stat_keys[] = {
5442         { "tx_frames" },
5443         { "tx_bytes" },
5444         { "tx_fifo_errors" },
5445         { "tx_overflow_errors" },
5446         { "tx_max_pkt_size_errors" },
5447         { "tx_underflow_errors" },
5448         { "rx_local_faults" },
5449         { "rx_remote_faults" },
5450         { "rx_link_faults" },
5451         { "rx_align_errors" },
5452         { "rx_frags" },
5453         { "rx_mcasts" },
5454         { "rx_bcasts" },
5455         { "rx_hist_cnt1" },
5456         { "rx_hist_cnt2" },
5457         { "rx_hist_cnt3" },
5458         { "rx_hist_cnt4" },
5459         { "rx_hist_cnt5" },
5460         { "rx_hist_cnt6" },
5461         { "rx_hist_cnt7" },
5462         { "rx_octets" },
5463         { "rx_code_violations" },
5464         { "rx_len_errors" },
5465         { "rx_crc_errors" },
5466         { "rx_underflows" },
5467         { "rx_overflows" },
5468         { "pause_off_state" },
5469         { "pause_on_state" },
5470         { "pause_received" },
5471 };
5472
5473 #define NUM_XMAC_STAT_KEYS      ARRAY_SIZE(niu_xmac_stat_keys)
5474
5475 static const struct {
5476         const char string[ETH_GSTRING_LEN];
5477 } niu_bmac_stat_keys[] = {
5478         { "tx_underflow_errors" },
5479         { "tx_max_pkt_size_errors" },
5480         { "tx_bytes" },
5481         { "tx_frames" },
5482         { "rx_overflows" },
5483         { "rx_frames" },
5484         { "rx_align_errors" },
5485         { "rx_crc_errors" },
5486         { "rx_len_errors" },
5487         { "pause_off_state" },
5488         { "pause_on_state" },
5489         { "pause_received" },
5490 };
5491
5492 #define NUM_BMAC_STAT_KEYS      ARRAY_SIZE(niu_bmac_stat_keys)
5493
5494 static const struct {
5495         const char string[ETH_GSTRING_LEN];
5496 } niu_rxchan_stat_keys[] = {
5497         { "rx_channel" },
5498         { "rx_packets" },
5499         { "rx_bytes" },
5500         { "rx_dropped" },
5501         { "rx_errors" },
5502 };
5503
5504 #define NUM_RXCHAN_STAT_KEYS    ARRAY_SIZE(niu_rxchan_stat_keys)
5505
5506 static const struct {
5507         const char string[ETH_GSTRING_LEN];
5508 } niu_txchan_stat_keys[] = {
5509         { "tx_channel" },
5510         { "tx_packets" },
5511         { "tx_bytes" },
5512         { "tx_errors" },
5513 };
5514
5515 #define NUM_TXCHAN_STAT_KEYS    ARRAY_SIZE(niu_txchan_stat_keys)
5516
5517 static void niu_get_strings(struct net_device *dev, u32 stringset, u8 *data)
5518 {
5519         struct niu *np = netdev_priv(dev);
5520         int i;
5521
5522         if (stringset != ETH_SS_STATS)
5523                 return;
5524
5525         if (np->flags & NIU_FLAGS_XMAC) {
5526                 memcpy(data, niu_xmac_stat_keys,
5527                        sizeof(niu_xmac_stat_keys));
5528                 data += sizeof(niu_xmac_stat_keys);
5529         } else {
5530                 memcpy(data, niu_bmac_stat_keys,
5531                        sizeof(niu_bmac_stat_keys));
5532                 data += sizeof(niu_bmac_stat_keys);
5533         }
5534         for (i = 0; i < np->num_rx_rings; i++) {
5535                 memcpy(data, niu_rxchan_stat_keys,
5536                        sizeof(niu_rxchan_stat_keys));
5537                 data += sizeof(niu_rxchan_stat_keys);
5538         }
5539         for (i = 0; i < np->num_tx_rings; i++) {
5540                 memcpy(data, niu_txchan_stat_keys,
5541                        sizeof(niu_txchan_stat_keys));
5542                 data += sizeof(niu_txchan_stat_keys);
5543         }
5544 }
5545
5546 static int niu_get_stats_count(struct net_device *dev)
5547 {
5548         struct niu *np = netdev_priv(dev);
5549
5550         return ((np->flags & NIU_FLAGS_XMAC ?
5551                  NUM_XMAC_STAT_KEYS :
5552                  NUM_BMAC_STAT_KEYS) +
5553                 (np->num_rx_rings * NUM_RXCHAN_STAT_KEYS) +
5554                 (np->num_tx_rings * NUM_TXCHAN_STAT_KEYS));
5555 }
5556
5557 static void niu_get_ethtool_stats(struct net_device *dev,
5558                                   struct ethtool_stats *stats, u64 *data)
5559 {
5560         struct niu *np = netdev_priv(dev);
5561         int i;
5562
5563         niu_sync_mac_stats(np);
5564         if (np->flags & NIU_FLAGS_XMAC) {
5565                 memcpy(data, &np->mac_stats.xmac,
5566                        sizeof(struct niu_xmac_stats));
5567                 data += (sizeof(struct niu_xmac_stats) / sizeof(u64));
5568         } else {
5569                 memcpy(data, &np->mac_stats.bmac,
5570                        sizeof(struct niu_bmac_stats));
5571                 data += (sizeof(struct niu_bmac_stats) / sizeof(u64));
5572         }
5573         for (i = 0; i < np->num_rx_rings; i++) {
5574                 struct rx_ring_info *rp = &np->rx_rings[i];
5575
5576                 data[0] = rp->rx_channel;
5577                 data[1] = rp->rx_packets;
5578                 data[2] = rp->rx_bytes;
5579                 data[3] = rp->rx_dropped;
5580                 data[4] = rp->rx_errors;
5581                 data += 5;
5582         }
5583         for (i = 0; i < np->num_tx_rings; i++) {
5584                 struct tx_ring_info *rp = &np->tx_rings[i];
5585
5586                 data[0] = rp->tx_channel;
5587                 data[1] = rp->tx_packets;
5588                 data[2] = rp->tx_bytes;
5589                 data[3] = rp->tx_errors;
5590                 data += 4;
5591         }
5592 }
5593
5594 static u64 niu_led_state_save(struct niu *np)
5595 {
5596         if (np->flags & NIU_FLAGS_XMAC)
5597                 return nr64_mac(XMAC_CONFIG);
5598         else
5599                 return nr64_mac(BMAC_XIF_CONFIG);
5600 }
5601
5602 static void niu_led_state_restore(struct niu *np, u64 val)
5603 {
5604         if (np->flags & NIU_FLAGS_XMAC)
5605                 nw64_mac(XMAC_CONFIG, val);
5606         else
5607                 nw64_mac(BMAC_XIF_CONFIG, val);
5608 }
5609
5610 static void niu_force_led(struct niu *np, int on)
5611 {
5612         u64 val, reg, bit;
5613
5614         if (np->flags & NIU_FLAGS_XMAC) {
5615                 reg = XMAC_CONFIG;
5616                 bit = XMAC_CONFIG_FORCE_LED_ON;
5617         } else {
5618                 reg = BMAC_XIF_CONFIG;
5619                 bit = BMAC_XIF_CONFIG_LINK_LED;
5620         }
5621
5622         val = nr64_mac(reg);
5623         if (on)
5624                 val |= bit;
5625         else
5626                 val &= ~bit;
5627         nw64_mac(reg, val);
5628 }
5629
5630 static int niu_phys_id(struct net_device *dev, u32 data)
5631 {
5632         struct niu *np = netdev_priv(dev);
5633         u64 orig_led_state;
5634         int i;
5635
5636         if (!netif_running(dev))
5637                 return -EAGAIN;
5638
5639         if (data == 0)
5640                 data = 2;
5641
5642         orig_led_state = niu_led_state_save(np);
5643         for (i = 0; i < (data * 2); i++) {
5644                 int on = ((i % 2) == 0);
5645
5646                 niu_force_led(np, on);
5647
5648                 if (msleep_interruptible(500))
5649                         break;
5650         }
5651         niu_led_state_restore(np, orig_led_state);
5652
5653         return 0;
5654 }
5655
5656 static const struct ethtool_ops niu_ethtool_ops = {
5657         .get_drvinfo            = niu_get_drvinfo,
5658         .get_link               = ethtool_op_get_link,
5659         .get_msglevel           = niu_get_msglevel,
5660         .set_msglevel           = niu_set_msglevel,
5661         .get_eeprom_len         = niu_get_eeprom_len,
5662         .get_eeprom             = niu_get_eeprom,
5663         .get_settings           = niu_get_settings,
5664         .set_settings           = niu_set_settings,
5665         .get_strings            = niu_get_strings,
5666         .get_stats_count        = niu_get_stats_count,
5667         .get_ethtool_stats      = niu_get_ethtool_stats,
5668         .phys_id                = niu_phys_id,
5669 };
5670
5671 static int niu_ldg_assign_ldn(struct niu *np, struct niu_parent *parent,
5672                               int ldg, int ldn)
5673 {
5674         if (ldg < NIU_LDG_MIN || ldg > NIU_LDG_MAX)
5675                 return -EINVAL;
5676         if (ldn < 0 || ldn > LDN_MAX)
5677                 return -EINVAL;
5678
5679         parent->ldg_map[ldn] = ldg;
5680
5681         if (np->parent->plat_type == PLAT_TYPE_NIU) {
5682                 /* On N2 NIU, the ldn-->ldg assignments are setup and fixed by
5683                  * the firmware, and we're not supposed to change them.
5684                  * Validate the mapping, because if it's wrong we probably
5685                  * won't get any interrupts and that's painful to debug.
5686                  */
5687                 if (nr64(LDG_NUM(ldn)) != ldg) {
5688                         dev_err(np->device, PFX "Port %u, mis-matched "
5689                                 "LDG assignment "
5690                                 "for ldn %d, should be %d is %llu\n",
5691                                 np->port, ldn, ldg,
5692                                 (unsigned long long) nr64(LDG_NUM(ldn)));
5693                         return -EINVAL;
5694                 }
5695         } else
5696                 nw64(LDG_NUM(ldn), ldg);
5697
5698         return 0;
5699 }
5700
5701 static int niu_set_ldg_timer_res(struct niu *np, int res)
5702 {
5703         if (res < 0 || res > LDG_TIMER_RES_VAL)
5704                 return -EINVAL;
5705
5706
5707         nw64(LDG_TIMER_RES, res);
5708
5709         return 0;
5710 }
5711
5712 static int niu_set_ldg_sid(struct niu *np, int ldg, int func, int vector)
5713 {
5714         if ((ldg < NIU_LDG_MIN || ldg > NIU_LDG_MAX) ||
5715             (func < 0 || func > 3) ||
5716             (vector < 0 || vector > 0x1f))
5717                 return -EINVAL;
5718
5719         nw64(SID(ldg), (func << SID_FUNC_SHIFT) | vector);
5720
5721         return 0;
5722 }
5723
5724 static int __devinit niu_pci_eeprom_read(struct niu *np, u32 addr)
5725 {
5726         u64 frame, frame_base = (ESPC_PIO_STAT_READ_START |
5727                                  (addr << ESPC_PIO_STAT_ADDR_SHIFT));
5728         int limit;
5729
5730         if (addr > (ESPC_PIO_STAT_ADDR >> ESPC_PIO_STAT_ADDR_SHIFT))
5731                 return -EINVAL;
5732
5733         frame = frame_base;
5734         nw64(ESPC_PIO_STAT, frame);
5735         limit = 64;
5736         do {
5737                 udelay(5);
5738                 frame = nr64(ESPC_PIO_STAT);
5739                 if (frame & ESPC_PIO_STAT_READ_END)
5740                         break;
5741         } while (limit--);
5742         if (!(frame & ESPC_PIO_STAT_READ_END)) {
5743                 dev_err(np->device, PFX "EEPROM read timeout frame[%llx]\n",
5744                         (unsigned long long) frame);
5745                 return -ENODEV;
5746         }
5747
5748         frame = frame_base;
5749         nw64(ESPC_PIO_STAT, frame);
5750         limit = 64;
5751         do {
5752                 udelay(5);
5753                 frame = nr64(ESPC_PIO_STAT);
5754                 if (frame & ESPC_PIO_STAT_READ_END)
5755                         break;
5756         } while (limit--);
5757         if (!(frame & ESPC_PIO_STAT_READ_END)) {
5758                 dev_err(np->device, PFX "EEPROM read timeout frame[%llx]\n",
5759                         (unsigned long long) frame);
5760                 return -ENODEV;
5761         }
5762
5763         frame = nr64(ESPC_PIO_STAT);
5764         return (frame & ESPC_PIO_STAT_DATA) >> ESPC_PIO_STAT_DATA_SHIFT;
5765 }
5766
5767 static int __devinit niu_pci_eeprom_read16(struct niu *np, u32 off)
5768 {
5769         int err = niu_pci_eeprom_read(np, off);
5770         u16 val;
5771
5772         if (err < 0)
5773                 return err;
5774         val = (err << 8);
5775         err = niu_pci_eeprom_read(np, off + 1);
5776         if (err < 0)
5777                 return err;
5778         val |= (err & 0xff);
5779
5780         return val;
5781 }
5782
5783 static int __devinit niu_pci_eeprom_read16_swp(struct niu *np, u32 off)
5784 {
5785         int err = niu_pci_eeprom_read(np, off);
5786         u16 val;
5787
5788         if (err < 0)
5789                 return err;
5790
5791         val = (err & 0xff);
5792         err = niu_pci_eeprom_read(np, off + 1);
5793         if (err < 0)
5794                 return err;
5795
5796         val |= (err & 0xff) << 8;
5797
5798         return val;
5799 }
5800
5801 static int __devinit niu_pci_vpd_get_propname(struct niu *np,
5802                                               u32 off,
5803                                               char *namebuf,
5804                                               int namebuf_len)
5805 {
5806         int i;
5807
5808         for (i = 0; i < namebuf_len; i++) {
5809                 int err = niu_pci_eeprom_read(np, off + i);
5810                 if (err < 0)
5811                         return err;
5812                 *namebuf++ = err;
5813                 if (!err)
5814                         break;
5815         }
5816         if (i >= namebuf_len)
5817                 return -EINVAL;
5818
5819         return i + 1;
5820 }
5821
5822 static void __devinit niu_vpd_parse_version(struct niu *np)
5823 {
5824         struct niu_vpd *vpd = &np->vpd;
5825         int len = strlen(vpd->version) + 1;
5826         const char *s = vpd->version;
5827         int i;
5828
5829         for (i = 0; i < len - 5; i++) {
5830                 if (!strncmp(s + i, "FCode ", 5))
5831                         break;
5832         }
5833         if (i >= len - 5)
5834                 return;
5835
5836         s += i + 5;
5837         sscanf(s, "%d.%d", &vpd->fcode_major, &vpd->fcode_minor);
5838
5839         niudbg(PROBE, "VPD_SCAN: FCODE major(%d) minor(%d)\n",
5840                vpd->fcode_major, vpd->fcode_minor);
5841         if (vpd->fcode_major > NIU_VPD_MIN_MAJOR ||
5842             (vpd->fcode_major == NIU_VPD_MIN_MAJOR &&
5843              vpd->fcode_minor >= NIU_VPD_MIN_MINOR))
5844                 np->flags |= NIU_FLAGS_VPD_VALID;
5845 }
5846
5847 /* ESPC_PIO_EN_ENABLE must be set */
5848 static int __devinit niu_pci_vpd_scan_props(struct niu *np,
5849                                             u32 start, u32 end)
5850 {
5851         unsigned int found_mask = 0;
5852 #define FOUND_MASK_MODEL        0x00000001
5853 #define FOUND_MASK_BMODEL       0x00000002
5854 #define FOUND_MASK_VERS         0x00000004
5855 #define FOUND_MASK_MAC          0x00000008
5856 #define FOUND_MASK_NMAC         0x00000010
5857 #define FOUND_MASK_PHY          0x00000020
5858 #define FOUND_MASK_ALL          0x0000003f
5859
5860         niudbg(PROBE, "VPD_SCAN: start[%x] end[%x]\n",
5861                start, end);
5862         while (start < end) {
5863                 int len, err, instance, type, prop_len;
5864                 char namebuf[64];
5865                 u8 *prop_buf;
5866                 int max_len;
5867
5868                 if (found_mask == FOUND_MASK_ALL) {
5869                         niu_vpd_parse_version(np);
5870                         return 1;
5871                 }
5872
5873                 err = niu_pci_eeprom_read(np, start + 2);
5874                 if (err < 0)
5875                         return err;
5876                 len = err;
5877                 start += 3;
5878
5879                 instance = niu_pci_eeprom_read(np, start);
5880                 type = niu_pci_eeprom_read(np, start + 3);
5881                 prop_len = niu_pci_eeprom_read(np, start + 4);
5882                 err = niu_pci_vpd_get_propname(np, start + 5, namebuf, 64);
5883                 if (err < 0)
5884                         return err;
5885
5886                 prop_buf = NULL;
5887                 max_len = 0;
5888                 if (!strcmp(namebuf, "model")) {
5889                         prop_buf = np->vpd.model;
5890                         max_len = NIU_VPD_MODEL_MAX;
5891                         found_mask |= FOUND_MASK_MODEL;
5892                 } else if (!strcmp(namebuf, "board-model")) {
5893                         prop_buf = np->vpd.board_model;
5894                         max_len = NIU_VPD_BD_MODEL_MAX;
5895                         found_mask |= FOUND_MASK_BMODEL;
5896                 } else if (!strcmp(namebuf, "version")) {
5897                         prop_buf = np->vpd.version;
5898                         max_len = NIU_VPD_VERSION_MAX;
5899                         found_mask |= FOUND_MASK_VERS;
5900                 } else if (!strcmp(namebuf, "local-mac-address")) {
5901                         prop_buf = np->vpd.local_mac;
5902                         max_len = ETH_ALEN;
5903                         found_mask |= FOUND_MASK_MAC;
5904                 } else if (!strcmp(namebuf, "num-mac-addresses")) {
5905                         prop_buf = &np->vpd.mac_num;
5906                         max_len = 1;
5907                         found_mask |= FOUND_MASK_NMAC;
5908                 } else if (!strcmp(namebuf, "phy-type")) {
5909                         prop_buf = np->vpd.phy_type;
5910                         max_len = NIU_VPD_PHY_TYPE_MAX;
5911                         found_mask |= FOUND_MASK_PHY;
5912                 }
5913
5914                 if (max_len && prop_len > max_len) {
5915                         dev_err(np->device, PFX "Property '%s' length (%d) is "
5916                                 "too long.\n", namebuf, prop_len);
5917                         return -EINVAL;
5918                 }
5919
5920                 if (prop_buf) {
5921                         u32 off = start + 5 + err;
5922                         int i;
5923
5924                         niudbg(PROBE, "VPD_SCAN: Reading in property [%s] "
5925                                "len[%d]\n", namebuf, prop_len);
5926                         for (i = 0; i < prop_len; i++)
5927                                 *prop_buf++ = niu_pci_eeprom_read(np, off + i);
5928                 }
5929
5930                 start += len;
5931         }
5932
5933         return 0;
5934 }
5935
5936 /* ESPC_PIO_EN_ENABLE must be set */
5937 static void __devinit niu_pci_vpd_fetch(struct niu *np, u32 start)
5938 {
5939         u32 offset;
5940         int err;
5941
5942         err = niu_pci_eeprom_read16_swp(np, start + 1);
5943         if (err < 0)
5944                 return;
5945
5946         offset = err + 3;
5947
5948         while (start + offset < ESPC_EEPROM_SIZE) {
5949                 u32 here = start + offset;
5950                 u32 end;
5951
5952                 err = niu_pci_eeprom_read(np, here);
5953                 if (err != 0x90)
5954                         return;
5955
5956                 err = niu_pci_eeprom_read16_swp(np, here + 1);
5957                 if (err < 0)
5958                         return;
5959
5960                 here = start + offset + 3;
5961                 end = start + offset + err;
5962
5963                 offset += err;
5964
5965                 err = niu_pci_vpd_scan_props(np, here, end);
5966                 if (err < 0 || err == 1)
5967                         return;
5968         }
5969 }
5970
5971 /* ESPC_PIO_EN_ENABLE must be set */
5972 static u32 __devinit niu_pci_vpd_offset(struct niu *np)
5973 {
5974         u32 start = 0, end = ESPC_EEPROM_SIZE, ret;
5975         int err;
5976
5977         while (start < end) {
5978                 ret = start;
5979
5980                 /* ROM header signature?  */
5981                 err = niu_pci_eeprom_read16(np, start +  0);
5982                 if (err != 0x55aa)
5983                         return 0;
5984
5985                 /* Apply offset to PCI data structure.  */
5986                 err = niu_pci_eeprom_read16(np, start + 23);
5987                 if (err < 0)
5988                         return 0;
5989                 start += err;
5990
5991                 /* Check for "PCIR" signature.  */
5992                 err = niu_pci_eeprom_read16(np, start +  0);
5993                 if (err != 0x5043)
5994                         return 0;
5995                 err = niu_pci_eeprom_read16(np, start +  2);
5996                 if (err != 0x4952)
5997                         return 0;
5998
5999                 /* Check for OBP image type.  */
6000                 err = niu_pci_eeprom_read(np, start + 20);
6001                 if (err < 0)
6002                         return 0;
6003                 if (err != 0x01) {
6004                         err = niu_pci_eeprom_read(np, ret + 2);
6005                         if (err < 0)
6006                                 return 0;
6007
6008                         start = ret + (err * 512);
6009                         continue;
6010                 }
6011
6012                 err = niu_pci_eeprom_read16_swp(np, start + 8);
6013                 if (err < 0)
6014                         return err;
6015                 ret += err;
6016
6017                 err = niu_pci_eeprom_read(np, ret + 0);
6018                 if (err != 0x82)
6019                         return 0;
6020
6021                 return ret;
6022         }
6023
6024         return 0;
6025 }
6026
6027 static int __devinit niu_phy_type_prop_decode(struct niu *np,
6028                                               const char *phy_prop)
6029 {
6030         if (!strcmp(phy_prop, "mif")) {
6031                 /* 1G copper, MII */
6032                 np->flags &= ~(NIU_FLAGS_FIBER |
6033                                NIU_FLAGS_10G);
6034                 np->mac_xcvr = MAC_XCVR_MII;
6035         } else if (!strcmp(phy_prop, "xgf")) {
6036                 /* 10G fiber, XPCS */
6037                 np->flags |= (NIU_FLAGS_10G |
6038                               NIU_FLAGS_FIBER);
6039                 np->mac_xcvr = MAC_XCVR_XPCS;
6040         } else if (!strcmp(phy_prop, "pcs")) {
6041                 /* 1G fiber, PCS */
6042                 np->flags &= ~NIU_FLAGS_10G;
6043                 np->flags |= NIU_FLAGS_FIBER;
6044                 np->mac_xcvr = MAC_XCVR_PCS;
6045         } else if (!strcmp(phy_prop, "xgc")) {
6046                 /* 10G copper, XPCS */
6047                 np->flags |= NIU_FLAGS_10G;
6048                 np->flags &= ~NIU_FLAGS_FIBER;
6049                 np->mac_xcvr = MAC_XCVR_XPCS;
6050         } else {
6051                 return -EINVAL;
6052         }
6053         return 0;
6054 }
6055
6056 static void __devinit niu_pci_vpd_validate(struct niu *np)
6057 {
6058         struct net_device *dev = np->dev;
6059         struct niu_vpd *vpd = &np->vpd;
6060         u8 val8;
6061
6062         if (!is_valid_ether_addr(&vpd->local_mac[0])) {
6063                 dev_err(np->device, PFX "VPD MAC invalid, "
6064                         "falling back to SPROM.\n");
6065
6066                 np->flags &= ~NIU_FLAGS_VPD_VALID;
6067                 return;
6068         }
6069
6070         if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {
6071                 dev_err(np->device, PFX "Illegal phy string [%s].\n",
6072                         np->vpd.phy_type);
6073                 dev_err(np->device, PFX "Falling back to SPROM.\n");
6074                 np->flags &= ~NIU_FLAGS_VPD_VALID;
6075                 return;
6076         }
6077
6078         memcpy(dev->perm_addr, vpd->local_mac, ETH_ALEN);
6079
6080         val8 = dev->perm_addr[5];
6081         dev->perm_addr[5] += np->port;
6082         if (dev->perm_addr[5] < val8)
6083                 dev->perm_addr[4]++;
6084
6085         memcpy(dev->dev_addr, dev->perm_addr, dev->addr_len);
6086 }
6087
6088 static int __devinit niu_pci_probe_sprom(struct niu *np)
6089 {
6090         struct net_device *dev = np->dev;
6091         int len, i;
6092         u64 val, sum;
6093         u8 val8;
6094
6095         val = (nr64(ESPC_VER_IMGSZ) & ESPC_VER_IMGSZ_IMGSZ);
6096         val >>= ESPC_VER_IMGSZ_IMGSZ_SHIFT;
6097         len = val / 4;
6098
6099         np->eeprom_len = len;
6100
6101         niudbg(PROBE, "SPROM: Image size %llu\n", (unsigned long long) val);
6102
6103         sum = 0;
6104         for (i = 0; i < len; i++) {
6105                 val = nr64(ESPC_NCR(i));
6106                 sum += (val >>  0) & 0xff;
6107                 sum += (val >>  8) & 0xff;
6108                 sum += (val >> 16) & 0xff;
6109                 sum += (val >> 24) & 0xff;
6110         }
6111         niudbg(PROBE, "SPROM: Checksum %x\n", (int)(sum & 0xff));
6112         if ((sum & 0xff) != 0xab) {
6113                 dev_err(np->device, PFX "Bad SPROM checksum "
6114                         "(%x, should be 0xab)\n", (int) (sum & 0xff));
6115                 return -EINVAL;
6116         }
6117
6118         val = nr64(ESPC_PHY_TYPE);
6119         switch (np->port) {
6120         case 0:
6121                 val8 = (val & ESPC_PHY_TYPE_PORT0) >>
6122                         ESPC_PHY_TYPE_PORT0_SHIFT;
6123                 break;
6124         case 1:
6125                 val8 = (val & ESPC_PHY_TYPE_PORT1) >>
6126                         ESPC_PHY_TYPE_PORT1_SHIFT;
6127                 break;
6128         case 2:
6129                 val8 = (val & ESPC_PHY_TYPE_PORT2) >>
6130                         ESPC_PHY_TYPE_PORT2_SHIFT;
6131                 break;
6132         case 3:
6133                 val8 = (val & ESPC_PHY_TYPE_PORT3) >>
6134                         ESPC_PHY_TYPE_PORT3_SHIFT;
6135                 break;
6136         default:
6137                 dev_err(np->device, PFX "Bogus port number %u\n",
6138                         np->port);
6139                 return -EINVAL;
6140         }
6141         niudbg(PROBE, "SPROM: PHY type %x\n", val8);
6142
6143         switch (val8) {
6144         case ESPC_PHY_TYPE_1G_COPPER:
6145                 /* 1G copper, MII */
6146                 np->flags &= ~(NIU_FLAGS_FIBER |
6147                                NIU_FLAGS_10G);
6148                 np->mac_xcvr = MAC_XCVR_MII;
6149                 break;
6150
6151         case ESPC_PHY_TYPE_1G_FIBER:
6152                 /* 1G fiber, PCS */
6153                 np->flags &= ~NIU_FLAGS_10G;
6154                 np->flags |= NIU_FLAGS_FIBER;
6155                 np->mac_xcvr = MAC_XCVR_PCS;
6156                 break;
6157
6158         case ESPC_PHY_TYPE_10G_COPPER:
6159                 /* 10G copper, XPCS */
6160                 np->flags |= NIU_FLAGS_10G;
6161                 np->flags &= ~NIU_FLAGS_FIBER;
6162                 np->mac_xcvr = MAC_XCVR_XPCS;
6163                 break;
6164
6165         case ESPC_PHY_TYPE_10G_FIBER:
6166                 /* 10G fiber, XPCS */
6167                 np->flags |= (NIU_FLAGS_10G |
6168                               NIU_FLAGS_FIBER);
6169                 np->mac_xcvr = MAC_XCVR_XPCS;
6170                 break;
6171
6172         default:
6173                 dev_err(np->device, PFX "Bogus SPROM phy type %u\n", val8);
6174                 return -EINVAL;
6175         }
6176
6177         val = nr64(ESPC_MAC_ADDR0);
6178         niudbg(PROBE, "SPROM: MAC_ADDR0[%08llx]\n",
6179                (unsigned long long) val);
6180         dev->perm_addr[0] = (val >>  0) & 0xff;
6181         dev->perm_addr[1] = (val >>  8) & 0xff;
6182         dev->perm_addr[2] = (val >> 16) & 0xff;
6183         dev->perm_addr[3] = (val >> 24) & 0xff;
6184
6185         val = nr64(ESPC_MAC_ADDR1);
6186         niudbg(PROBE, "SPROM: MAC_ADDR1[%08llx]\n",
6187                (unsigned long long) val);
6188         dev->perm_addr[4] = (val >>  0) & 0xff;
6189         dev->perm_addr[5] = (val >>  8) & 0xff;
6190
6191         if (!is_valid_ether_addr(&dev->perm_addr[0])) {
6192                 dev_err(np->device, PFX "SPROM MAC address invalid\n");
6193                 dev_err(np->device, PFX "[ \n");
6194                 for (i = 0; i < 6; i++)
6195                         printk("%02x ", dev->perm_addr[i]);
6196                 printk("]\n");
6197                 return -EINVAL;
6198         }
6199
6200         val8 = dev->perm_addr[5];
6201         dev->perm_addr[5] += np->port;
6202         if (dev->perm_addr[5] < val8)
6203                 dev->perm_addr[4]++;
6204
6205         memcpy(dev->dev_addr, dev->perm_addr, dev->addr_len);
6206
6207         val = nr64(ESPC_MOD_STR_LEN);
6208         niudbg(PROBE, "SPROM: MOD_STR_LEN[%llu]\n",
6209                (unsigned long long) val);
6210         if (val >= 8 * 4)
6211                 return -EINVAL;
6212
6213         for (i = 0; i < val; i += 4) {
6214                 u64 tmp = nr64(ESPC_NCR(5 + (i / 4)));
6215
6216                 np->vpd.model[i + 3] = (tmp >>  0) & 0xff;
6217                 np->vpd.model[i + 2] = (tmp >>  8) & 0xff;
6218                 np->vpd.model[i + 1] = (tmp >> 16) & 0xff;
6219                 np->vpd.model[i + 0] = (tmp >> 24) & 0xff;
6220         }
6221         np->vpd.model[val] = '\0';
6222
6223         val = nr64(ESPC_BD_MOD_STR_LEN);
6224         niudbg(PROBE, "SPROM: BD_MOD_STR_LEN[%llu]\n",
6225                (unsigned long long) val);
6226         if (val >= 4 * 4)
6227                 return -EINVAL;
6228
6229         for (i = 0; i < val; i += 4) {
6230                 u64 tmp = nr64(ESPC_NCR(14 + (i / 4)));
6231
6232                 np->vpd.board_model[i + 3] = (tmp >>  0) & 0xff;
6233                 np->vpd.board_model[i + 2] = (tmp >>  8) & 0xff;
6234                 np->vpd.board_model[i + 1] = (tmp >> 16) & 0xff;
6235                 np->vpd.board_model[i + 0] = (tmp >> 24) & 0xff;
6236         }
6237         np->vpd.board_model[val] = '\0';
6238
6239         np->vpd.mac_num =
6240                 nr64(ESPC_NUM_PORTS_MACS) & ESPC_NUM_PORTS_MACS_VAL;
6241         niudbg(PROBE, "SPROM: NUM_PORTS_MACS[%d]\n",
6242                np->vpd.mac_num);
6243
6244         return 0;
6245 }
6246
6247 static int __devinit niu_get_and_validate_port(struct niu *np)
6248 {
6249         struct niu_parent *parent = np->parent;
6250
6251         if (np->port <= 1)
6252                 np->flags |= NIU_FLAGS_XMAC;
6253
6254         if (!parent->num_ports) {
6255                 if (parent->plat_type == PLAT_TYPE_NIU) {
6256                         parent->num_ports = 2;
6257                 } else {
6258                         parent->num_ports = nr64(ESPC_NUM_PORTS_MACS) &
6259                                 ESPC_NUM_PORTS_MACS_VAL;
6260
6261                         if (!parent->num_ports)
6262                                 parent->num_ports = 4;
6263                 }
6264         }
6265
6266         niudbg(PROBE, "niu_get_and_validate_port: port[%d] num_ports[%d]\n",
6267                np->port, parent->num_ports);
6268         if (np->port >= parent->num_ports)
6269                 return -ENODEV;
6270
6271         return 0;
6272 }
6273
6274 static int __devinit phy_record(struct niu_parent *parent,
6275                                 struct phy_probe_info *p,
6276                                 int dev_id_1, int dev_id_2, u8 phy_port,
6277                                 int type)
6278 {
6279         u32 id = (dev_id_1 << 16) | dev_id_2;
6280         u8 idx;
6281
6282         if (dev_id_1 < 0 || dev_id_2 < 0)
6283                 return 0;
6284         if (type == PHY_TYPE_PMA_PMD || type == PHY_TYPE_PCS) {
6285                 if ((id & NIU_PHY_ID_MASK) != NIU_PHY_ID_BCM8704)
6286                         return 0;
6287         } else {
6288                 if ((id & NIU_PHY_ID_MASK) != NIU_PHY_ID_BCM5464R)
6289                         return 0;
6290         }
6291
6292         pr_info("niu%d: Found PHY %08x type %s at phy_port %u\n",
6293                 parent->index, id,
6294                 (type == PHY_TYPE_PMA_PMD ?
6295                  "PMA/PMD" :
6296                  (type == PHY_TYPE_PCS ?
6297                   "PCS" : "MII")),
6298                 phy_port);
6299
6300         if (p->cur[type] >= NIU_MAX_PORTS) {
6301                 printk(KERN_ERR PFX "Too many PHY ports.\n");
6302                 return -EINVAL;
6303         }
6304         idx = p->cur[type];
6305         p->phy_id[type][idx] = id;
6306         p->phy_port[type][idx] = phy_port;
6307         p->cur[type] = idx + 1;
6308         return 0;
6309 }
6310
6311 static int __devinit port_has_10g(struct phy_probe_info *p, int port)
6312 {
6313         int i;
6314
6315         for (i = 0; i < p->cur[PHY_TYPE_PMA_PMD]; i++) {
6316                 if (p->phy_port[PHY_TYPE_PMA_PMD][i] == port)
6317                         return 1;
6318         }
6319         for (i = 0; i < p->cur[PHY_TYPE_PCS]; i++) {
6320                 if (p->phy_port[PHY_TYPE_PCS][i] == port)
6321                         return 1;
6322         }
6323
6324         return 0;
6325 }
6326
6327 static int __devinit count_10g_ports(struct phy_probe_info *p, int *lowest)
6328 {
6329         int port, cnt;
6330
6331         cnt = 0;
6332         *lowest = 32;
6333         for (port = 8; port < 32; port++) {
6334                 if (port_has_10g(p, port)) {
6335                         if (!cnt)
6336                                 *lowest = port;
6337                         cnt++;
6338                 }
6339         }
6340
6341         return cnt;
6342 }
6343
6344 static int __devinit count_1g_ports(struct phy_probe_info *p, int *lowest)
6345 {
6346         *lowest = 32;
6347         if (p->cur[PHY_TYPE_MII])
6348                 *lowest = p->phy_port[PHY_TYPE_MII][0];
6349
6350         return p->cur[PHY_TYPE_MII];
6351 }
6352
6353 static void __devinit niu_n2_divide_channels(struct niu_parent *parent)
6354 {
6355         int num_ports = parent->num_ports;
6356         int i;
6357
6358         for (i = 0; i < num_ports; i++) {
6359                 parent->rxchan_per_port[i] = (16 / num_ports);
6360                 parent->txchan_per_port[i] = (16 / num_ports);
6361
6362                 pr_info(PFX "niu%d: Port %u [%u RX chans] "
6363                         "[%u TX chans]\n",
6364                         parent->index, i,
6365                         parent->rxchan_per_port[i],
6366                         parent->txchan_per_port[i]);
6367         }
6368 }
6369
6370 static void __devinit niu_divide_channels(struct niu_parent *parent,
6371                                           int num_10g, int num_1g)
6372 {
6373         int num_ports = parent->num_ports;
6374         int rx_chans_per_10g, rx_chans_per_1g;
6375         int tx_chans_per_10g, tx_chans_per_1g;
6376         int i, tot_rx, tot_tx;
6377
6378         if (!num_10g || !num_1g) {
6379                 rx_chans_per_10g = rx_chans_per_1g =
6380                         (NIU_NUM_RXCHAN / num_ports);
6381                 tx_chans_per_10g = tx_chans_per_1g =
6382                         (NIU_NUM_TXCHAN / num_ports);
6383         } else {
6384                 rx_chans_per_1g = NIU_NUM_RXCHAN / 8;
6385                 rx_chans_per_10g = (NIU_NUM_RXCHAN -
6386                                     (rx_chans_per_1g * num_1g)) /
6387                         num_10g;
6388
6389                 tx_chans_per_1g = NIU_NUM_TXCHAN / 6;
6390                 tx_chans_per_10g = (NIU_NUM_TXCHAN -
6391                                     (tx_chans_per_1g * num_1g)) /
6392                         num_10g;
6393         }
6394
6395         tot_rx = tot_tx = 0;
6396         for (i = 0; i < num_ports; i++) {
6397                 int type = phy_decode(parent->port_phy, i);
6398
6399                 if (type == PORT_TYPE_10G) {
6400                         parent->rxchan_per_port[i] = rx_chans_per_10g;
6401                         parent->txchan_per_port[i] = tx_chans_per_10g;
6402                 } else {
6403                         parent->rxchan_per_port[i] = rx_chans_per_1g;
6404                         parent->txchan_per_port[i] = tx_chans_per_1g;
6405                 }
6406                 pr_info(PFX "niu%d: Port %u [%u RX chans] "
6407                         "[%u TX chans]\n",
6408                         parent->index, i,
6409                         parent->rxchan_per_port[i],
6410                         parent->txchan_per_port[i]);
6411                 tot_rx += parent->rxchan_per_port[i];
6412                 tot_tx += parent->txchan_per_port[i];
6413         }
6414
6415         if (tot_rx > NIU_NUM_RXCHAN) {
6416                 printk(KERN_ERR PFX "niu%d: Too many RX channels (%d), "
6417                        "resetting to one per port.\n",
6418                        parent->index, tot_rx);
6419                 for (i = 0; i < num_ports; i++)
6420                         parent->rxchan_per_port[i] = 1;
6421         }
6422         if (tot_tx > NIU_NUM_TXCHAN) {
6423                 printk(KERN_ERR PFX "niu%d: Too many TX channels (%d), "
6424                        "resetting to one per port.\n",
6425                        parent->index, tot_tx);
6426                 for (i = 0; i < num_ports; i++)
6427                         parent->txchan_per_port[i] = 1;
6428         }
6429         if (tot_rx < NIU_NUM_RXCHAN || tot_tx < NIU_NUM_TXCHAN) {
6430                 printk(KERN_WARNING PFX "niu%d: Driver bug, wasted channels, "
6431                        "RX[%d] TX[%d]\n",
6432                        parent->index, tot_rx, tot_tx);
6433         }
6434 }
6435
6436 static void __devinit niu_divide_rdc_groups(struct niu_parent *parent,
6437                                             int num_10g, int num_1g)
6438 {
6439         int i, num_ports = parent->num_ports;
6440         int rdc_group, rdc_groups_per_port;
6441         int rdc_channel_base;
6442
6443         rdc_group = 0;
6444         rdc_groups_per_port = NIU_NUM_RDC_TABLES / num_ports;
6445
6446         rdc_channel_base = 0;
6447
6448         for (i = 0; i < num_ports; i++) {
6449                 struct niu_rdc_tables *tp = &parent->rdc_group_cfg[i];
6450                 int grp, num_channels = parent->rxchan_per_port[i];
6451                 int this_channel_offset;
6452
6453                 tp->first_table_num = rdc_group;
6454                 tp->num_tables = rdc_groups_per_port;
6455                 this_channel_offset = 0;
6456                 for (grp = 0; grp < tp->num_tables; grp++) {
6457                         struct rdc_table *rt = &tp->tables[grp];
6458                         int slot;
6459
6460                         pr_info(PFX "niu%d: Port %d RDC tbl(%d) [ ",
6461                                 parent->index, i, tp->first_table_num + grp);
6462                         for (slot = 0; slot < NIU_RDC_TABLE_SLOTS; slot++) {
6463                                 rt->rxdma_channel[slot] =
6464                                         rdc_channel_base + this_channel_offset;
6465
6466                                 printk("%d ", rt->rxdma_channel[slot]);
6467
6468                                 if (++this_channel_offset == num_channels)
6469                                         this_channel_offset = 0;
6470                         }
6471                         printk("]\n");
6472                 }
6473
6474                 parent->rdc_default[i] = rdc_channel_base;
6475
6476                 rdc_channel_base += num_channels;
6477                 rdc_group += rdc_groups_per_port;
6478         }
6479 }
6480
6481 static int __devinit fill_phy_probe_info(struct niu *np,
6482                                          struct niu_parent *parent,
6483                                          struct phy_probe_info *info)
6484 {
6485         unsigned long flags;
6486         int port, err;
6487
6488         memset(info, 0, sizeof(*info));
6489
6490         /* Port 0 to 7 are reserved for onboard Serdes, probe the rest.  */
6491         niu_lock_parent(np, flags);
6492         err = 0;
6493         for (port = 8; port < 32; port++) {
6494                 int dev_id_1, dev_id_2;
6495
6496                 dev_id_1 = mdio_read(np, port,
6497                                      NIU_PMA_PMD_DEV_ADDR, MII_PHYSID1);
6498                 dev_id_2 = mdio_read(np, port,
6499                                      NIU_PMA_PMD_DEV_ADDR, MII_PHYSID2);
6500                 err = phy_record(parent, info, dev_id_1, dev_id_2, port,
6501                                  PHY_TYPE_PMA_PMD);
6502                 if (err)
6503                         break;
6504                 dev_id_1 = mdio_read(np, port,
6505                                      NIU_PCS_DEV_ADDR, MII_PHYSID1);
6506                 dev_id_2 = mdio_read(np, port,
6507                                      NIU_PCS_DEV_ADDR, MII_PHYSID2);
6508                 err = phy_record(parent, info, dev_id_1, dev_id_2, port,
6509                                  PHY_TYPE_PCS);
6510                 if (err)
6511                         break;
6512                 dev_id_1 = mii_read(np, port, MII_PHYSID1);
6513                 dev_id_2 = mii_read(np, port, MII_PHYSID2);
6514                 err = phy_record(parent, info, dev_id_1, dev_id_2, port,
6515                                  PHY_TYPE_MII);
6516                 if (err)
6517                         break;
6518         }
6519         niu_unlock_parent(np, flags);
6520
6521         return err;
6522 }
6523
6524 static int __devinit walk_phys(struct niu *np, struct niu_parent *parent)
6525 {
6526         struct phy_probe_info *info = &parent->phy_probe_info;
6527         int lowest_10g, lowest_1g;
6528         int num_10g, num_1g;
6529         u32 val;
6530         int err;
6531
6532         err = fill_phy_probe_info(np, parent, info);
6533         if (err)
6534                 return err;
6535
6536         num_10g = count_10g_ports(info, &lowest_10g);
6537         num_1g = count_1g_ports(info, &lowest_1g);
6538
6539         switch ((num_10g << 4) | num_1g) {
6540         case 0x24:
6541                 if (lowest_1g == 10)
6542                         parent->plat_type = PLAT_TYPE_VF_P0;
6543                 else if (lowest_1g == 26)
6544                         parent->plat_type = PLAT_TYPE_VF_P1;
6545                 else
6546                         goto unknown_vg_1g_port;
6547
6548                 /* fallthru */
6549         case 0x22:
6550                 val = (phy_encode(PORT_TYPE_10G, 0) |
6551                        phy_encode(PORT_TYPE_10G, 1) |
6552                        phy_encode(PORT_TYPE_1G, 2) |
6553                        phy_encode(PORT_TYPE_1G, 3));
6554                 break;
6555
6556         case 0x20:
6557                 val = (phy_encode(PORT_TYPE_10G, 0) |
6558                        phy_encode(PORT_TYPE_10G, 1));
6559                 break;
6560
6561         case 0x10:
6562                 val = phy_encode(PORT_TYPE_10G, np->port);
6563                 break;
6564
6565         case 0x14:
6566                 if (lowest_1g == 10)
6567                         parent->plat_type = PLAT_TYPE_VF_P0;
6568                 else if (lowest_1g == 26)
6569                         parent->plat_type = PLAT_TYPE_VF_P1;
6570                 else
6571                         goto unknown_vg_1g_port;
6572
6573                 /* fallthru */
6574         case 0x13:
6575                 if ((lowest_10g & 0x7) == 0)
6576                         val = (phy_encode(PORT_TYPE_10G, 0) |
6577                                phy_encode(PORT_TYPE_1G, 1) |
6578                                phy_encode(PORT_TYPE_1G, 2) |
6579                                phy_encode(PORT_TYPE_1G, 3));
6580                 else
6581                         val = (phy_encode(PORT_TYPE_1G, 0) |
6582                                phy_encode(PORT_TYPE_10G, 1) |
6583                                phy_encode(PORT_TYPE_1G, 2) |
6584                                phy_encode(PORT_TYPE_1G, 3));
6585                 break;
6586
6587         case 0x04:
6588                 if (lowest_1g == 10)
6589                         parent->plat_type = PLAT_TYPE_VF_P0;
6590                 else if (lowest_1g == 26)
6591                         parent->plat_type = PLAT_TYPE_VF_P1;
6592                 else
6593                         goto unknown_vg_1g_port;
6594
6595                 val = (phy_encode(PORT_TYPE_1G, 0) |
6596                        phy_encode(PORT_TYPE_1G, 1) |
6597                        phy_encode(PORT_TYPE_1G, 2) |
6598                        phy_encode(PORT_TYPE_1G, 3));
6599                 break;
6600
6601         default:
6602                 printk(KERN_ERR PFX "Unsupported port config "
6603                        "10G[%d] 1G[%d]\n",
6604                        num_10g, num_1g);
6605                 return -EINVAL;
6606         }
6607
6608         parent->port_phy = val;
6609
6610         if (parent->plat_type == PLAT_TYPE_NIU)
6611                 niu_n2_divide_channels(parent);
6612         else
6613                 niu_divide_channels(parent, num_10g, num_1g);
6614
6615         niu_divide_rdc_groups(parent, num_10g, num_1g);
6616
6617         return 0;
6618
6619 unknown_vg_1g_port:
6620         printk(KERN_ERR PFX "Cannot identify platform type, 1gport=%d\n",
6621                lowest_1g);
6622         return -EINVAL;
6623 }
6624
6625 static int __devinit niu_probe_ports(struct niu *np)
6626 {
6627         struct niu_parent *parent = np->parent;
6628         int err, i;
6629
6630         niudbg(PROBE, "niu_probe_ports(): port_phy[%08x]\n",
6631                parent->port_phy);
6632
6633         if (parent->port_phy == PORT_PHY_UNKNOWN) {
6634                 err = walk_phys(np, parent);
6635                 if (err)
6636                         return err;
6637
6638                 niu_set_ldg_timer_res(np, 2);
6639                 for (i = 0; i <= LDN_MAX; i++)
6640                         niu_ldn_irq_enable(np, i, 0);
6641         }
6642
6643         if (parent->port_phy == PORT_PHY_INVALID)
6644                 return -EINVAL;
6645
6646         return 0;
6647 }
6648
6649 static int __devinit niu_classifier_swstate_init(struct niu *np)
6650 {
6651         struct niu_classifier *cp = &np->clas;
6652
6653         niudbg(PROBE, "niu_classifier_swstate_init: num_tcam(%d)\n",
6654                np->parent->tcam_num_entries);
6655
6656         cp->tcam_index = (u16) np->port;
6657         cp->h1_init = 0xffffffff;
6658         cp->h2_init = 0xffff;
6659
6660         return fflp_early_init(np);
6661 }
6662
6663 static void __devinit niu_link_config_init(struct niu *np)
6664 {
6665         struct niu_link_config *lp = &np->link_config;
6666
6667         lp->advertising = (ADVERTISED_10baseT_Half |
6668                            ADVERTISED_10baseT_Full |
6669                            ADVERTISED_100baseT_Half |
6670                            ADVERTISED_100baseT_Full |
6671                            ADVERTISED_1000baseT_Half |
6672                            ADVERTISED_1000baseT_Full |
6673                            ADVERTISED_10000baseT_Full |
6674                            ADVERTISED_Autoneg);
6675         lp->speed = lp->active_speed = SPEED_INVALID;
6676         lp->duplex = lp->active_duplex = DUPLEX_INVALID;
6677 #if 0
6678         lp->loopback_mode = LOOPBACK_MAC;
6679         lp->active_speed = SPEED_10000;
6680         lp->active_duplex = DUPLEX_FULL;
6681 #else
6682         lp->loopback_mode = LOOPBACK_DISABLED;
6683 #endif
6684 }
6685
6686 static int __devinit niu_init_mac_ipp_pcs_base(struct niu *np)
6687 {
6688         switch (np->port) {
6689         case 0:
6690                 np->mac_regs = np->regs + XMAC_PORT0_OFF;
6691                 np->ipp_off  = 0x00000;
6692                 np->pcs_off  = 0x04000;
6693                 np->xpcs_off = 0x02000;
6694                 break;
6695
6696         case 1:
6697                 np->mac_regs = np->regs + XMAC_PORT1_OFF;
6698                 np->ipp_off  = 0x08000;
6699                 np->pcs_off  = 0x0a000;
6700                 np->xpcs_off = 0x08000;
6701                 break;
6702
6703         case 2:
6704                 np->mac_regs = np->regs + BMAC_PORT2_OFF;
6705                 np->ipp_off  = 0x04000;
6706                 np->pcs_off  = 0x0e000;
6707                 np->xpcs_off = ~0UL;
6708                 break;
6709
6710         case 3:
6711                 np->mac_regs = np->regs + BMAC_PORT3_OFF;
6712                 np->ipp_off  = 0x0c000;
6713                 np->pcs_off  = 0x12000;
6714                 np->xpcs_off = ~0UL;
6715                 break;
6716
6717         default:
6718                 dev_err(np->device, PFX "Port %u is invalid, cannot "
6719                         "compute MAC block offset.\n", np->port);
6720                 return -EINVAL;
6721         }
6722
6723         return 0;
6724 }
6725
6726 static void __devinit niu_try_msix(struct niu *np, u8 *ldg_num_map)
6727 {
6728         struct msix_entry msi_vec[NIU_NUM_LDG];
6729         struct niu_parent *parent = np->parent;
6730         struct pci_dev *pdev = np->pdev;
6731         int i, num_irqs, err;
6732         u8 first_ldg;
6733
6734         first_ldg = (NIU_NUM_LDG / parent->num_ports) * np->port;
6735         for (i = 0; i < (NIU_NUM_LDG / parent->num_ports); i++)
6736                 ldg_num_map[i] = first_ldg + i;
6737
6738         num_irqs = (parent->rxchan_per_port[np->port] +
6739                     parent->txchan_per_port[np->port] +
6740                     (np->port == 0 ? 3 : 1));
6741         BUG_ON(num_irqs > (NIU_NUM_LDG / parent->num_ports));
6742
6743 retry:
6744         for (i = 0; i < num_irqs; i++) {
6745                 msi_vec[i].vector = 0;
6746                 msi_vec[i].entry = i;
6747         }
6748
6749         err = pci_enable_msix(pdev, msi_vec, num_irqs);
6750         if (err < 0) {
6751                 np->flags &= ~NIU_FLAGS_MSIX;
6752                 return;
6753         }
6754         if (err > 0) {
6755                 num_irqs = err;
6756                 goto retry;
6757         }
6758
6759         np->flags |= NIU_FLAGS_MSIX;
6760         for (i = 0; i < num_irqs; i++)
6761                 np->ldg[i].irq = msi_vec[i].vector;
6762         np->num_ldg = num_irqs;
6763 }
6764
6765 static int __devinit niu_n2_irq_init(struct niu *np, u8 *ldg_num_map)
6766 {
6767 #ifdef CONFIG_SPARC64
6768         struct of_device *op = np->op;
6769         const u32 *int_prop;
6770         int i;
6771
6772         int_prop = of_get_property(op->node, "interrupts", NULL);
6773         if (!int_prop)
6774                 return -ENODEV;
6775
6776         for (i = 0; i < op->num_irqs; i++) {
6777                 ldg_num_map[i] = int_prop[i];
6778                 np->ldg[i].irq = op->irqs[i];
6779         }
6780
6781         np->num_ldg = op->num_irqs;
6782
6783         return 0;
6784 #else
6785         return -EINVAL;
6786 #endif
6787 }
6788
6789 static int __devinit niu_ldg_init(struct niu *np)
6790 {
6791         struct niu_parent *parent = np->parent;
6792         u8 ldg_num_map[NIU_NUM_LDG];
6793         int first_chan, num_chan;
6794         int i, err, ldg_rotor;
6795         u8 port;
6796
6797         np->num_ldg = 1;
6798         np->ldg[0].irq = np->dev->irq;
6799         if (parent->plat_type == PLAT_TYPE_NIU) {
6800                 err = niu_n2_irq_init(np, ldg_num_map);
6801                 if (err)
6802                         return err;
6803         } else
6804                 niu_try_msix(np, ldg_num_map);
6805
6806         port = np->port;
6807         for (i = 0; i < np->num_ldg; i++) {
6808                 struct niu_ldg *lp = &np->ldg[i];
6809
6810                 netif_napi_add(np->dev, &lp->napi, niu_poll, 64);
6811
6812                 lp->np = np;
6813                 lp->ldg_num = ldg_num_map[i];
6814                 lp->timer = 2; /* XXX */
6815
6816                 /* On N2 NIU the firmware has setup the SID mappings so they go
6817                  * to the correct values that will route the LDG to the proper
6818                  * interrupt in the NCU interrupt table.
6819                  */
6820                 if (np->parent->plat_type != PLAT_TYPE_NIU) {
6821                         err = niu_set_ldg_sid(np, lp->ldg_num, port, i);
6822                         if (err)
6823                                 return err;
6824                 }
6825         }
6826
6827         /* We adopt the LDG assignment ordering used by the N2 NIU
6828          * 'interrupt' properties because that simplifies a lot of
6829          * things.  This ordering is:
6830          *
6831          *      MAC
6832          *      MIF     (if port zero)
6833          *      SYSERR  (if port zero)
6834          *      RX channels
6835          *      TX channels
6836          */
6837
6838         ldg_rotor = 0;
6839
6840         err = niu_ldg_assign_ldn(np, parent, ldg_num_map[ldg_rotor],
6841                                   LDN_MAC(port));
6842         if (err)
6843                 return err;
6844
6845         ldg_rotor++;
6846         if (ldg_rotor == np->num_ldg)
6847                 ldg_rotor = 0;
6848
6849         if (port == 0) {
6850                 err = niu_ldg_assign_ldn(np, parent,
6851                                          ldg_num_map[ldg_rotor],
6852                                          LDN_MIF);
6853                 if (err)
6854                         return err;
6855
6856                 ldg_rotor++;
6857                 if (ldg_rotor == np->num_ldg)
6858                         ldg_rotor = 0;
6859
6860                 err = niu_ldg_assign_ldn(np, parent,
6861                                          ldg_num_map[ldg_rotor],
6862                                          LDN_DEVICE_ERROR);
6863                 if (err)
6864                         return err;
6865
6866                 ldg_rotor++;
6867                 if (ldg_rotor == np->num_ldg)
6868                         ldg_rotor = 0;
6869
6870         }
6871
6872         first_chan = 0;
6873         for (i = 0; i < port; i++)
6874                 first_chan += parent->rxchan_per_port[port];
6875         num_chan = parent->rxchan_per_port[port];
6876
6877         for (i = first_chan; i < (first_chan + num_chan); i++) {
6878                 err = niu_ldg_assign_ldn(np, parent,
6879                                          ldg_num_map[ldg_rotor],
6880                                          LDN_RXDMA(i));
6881                 if (err)
6882                         return err;
6883                 ldg_rotor++;
6884                 if (ldg_rotor == np->num_ldg)
6885                         ldg_rotor = 0;
6886         }
6887
6888         first_chan = 0;
6889         for (i = 0; i < port; i++)
6890                 first_chan += parent->txchan_per_port[port];
6891         num_chan = parent->txchan_per_port[port];
6892         for (i = first_chan; i < (first_chan + num_chan); i++) {
6893                 err = niu_ldg_assign_ldn(np, parent,
6894                                          ldg_num_map[ldg_rotor],
6895                                          LDN_TXDMA(i));
6896                 if (err)
6897                         return err;
6898                 ldg_rotor++;
6899                 if (ldg_rotor == np->num_ldg)
6900                         ldg_rotor = 0;
6901         }
6902
6903         return 0;
6904 }
6905
6906 static void __devexit niu_ldg_free(struct niu *np)
6907 {
6908         if (np->flags & NIU_FLAGS_MSIX)
6909                 pci_disable_msix(np->pdev);
6910 }
6911
6912 static int __devinit niu_get_of_props(struct niu *np)
6913 {
6914 #ifdef CONFIG_SPARC64
6915         struct net_device *dev = np->dev;
6916         struct device_node *dp;
6917         const char *phy_type;
6918         const u8 *mac_addr;
6919         int prop_len;
6920
6921         if (np->parent->plat_type == PLAT_TYPE_NIU)
6922                 dp = np->op->node;
6923         else
6924                 dp = pci_device_to_OF_node(np->pdev);
6925
6926         phy_type = of_get_property(dp, "phy-type", &prop_len);
6927         if (!phy_type) {
6928                 dev_err(np->device, PFX "%s: OF node lacks "
6929                         "phy-type property\n",
6930                         dp->full_name);
6931                 return -EINVAL;
6932         }
6933
6934         if (!strcmp(phy_type, "none"))
6935                 return -ENODEV;
6936
6937         strcpy(np->vpd.phy_type, phy_type);
6938
6939         if (niu_phy_type_prop_decode(np, np->vpd.phy_type)) {
6940                 dev_err(np->device, PFX "%s: Illegal phy string [%s].\n",
6941                         dp->full_name, np->vpd.phy_type);
6942                 return -EINVAL;
6943         }
6944
6945         mac_addr = of_get_property(dp, "local-mac-address", &prop_len);
6946         if (!mac_addr) {
6947                 dev_err(np->device, PFX "%s: OF node lacks "
6948                         "local-mac-address property\n",
6949                         dp->full_name);
6950                 return -EINVAL;
6951         }
6952         if (prop_len != dev->addr_len) {
6953                 dev_err(np->device, PFX "%s: OF MAC address prop len (%d) "
6954                         "is wrong.\n",
6955                         dp->full_name, prop_len);
6956         }
6957         memcpy(dev->perm_addr, mac_addr, dev->addr_len);
6958         if (!is_valid_ether_addr(&dev->perm_addr[0])) {
6959                 int i;
6960
6961                 dev_err(np->device, PFX "%s: OF MAC address is invalid\n",
6962                         dp->full_name);
6963                 dev_err(np->device, PFX "%s: [ \n",
6964                         dp->full_name);
6965                 for (i = 0; i < 6; i++)
6966                         printk("%02x ", dev->perm_addr[i]);
6967                 printk("]\n");
6968                 return -EINVAL;
6969         }
6970
6971         memcpy(dev->dev_addr, dev->perm_addr, dev->addr_len);
6972
6973         return 0;
6974 #else
6975         return -EINVAL;
6976 #endif
6977 }
6978
6979 static int __devinit niu_get_invariants(struct niu *np)
6980 {
6981         int err, have_props;
6982         u32 offset;
6983
6984         err = niu_get_of_props(np);
6985         if (err == -ENODEV)
6986                 return err;
6987
6988         have_props = !err;
6989
6990         err = niu_get_and_validate_port(np);
6991         if (err)
6992                 return err;
6993
6994         err = niu_init_mac_ipp_pcs_base(np);
6995         if (err)
6996                 return err;
6997
6998         if (!have_props) {
6999                 if (np->parent->plat_type == PLAT_TYPE_NIU)
7000                         return -EINVAL;
7001
7002                 nw64(ESPC_PIO_EN, ESPC_PIO_EN_ENABLE);
7003                 offset = niu_pci_vpd_offset(np);
7004                 niudbg(PROBE, "niu_get_invariants: VPD offset [%08x]\n",
7005                        offset);
7006                 if (offset)
7007                         niu_pci_vpd_fetch(np, offset);
7008                 nw64(ESPC_PIO_EN, 0);
7009
7010                 if (np->flags & NIU_FLAGS_VPD_VALID)
7011                         niu_pci_vpd_validate(np);
7012
7013                 if (!(np->flags & NIU_FLAGS_VPD_VALID)) {
7014                         err = niu_pci_probe_sprom(np);
7015                         if (err)
7016                                 return err;
7017                 }
7018         }
7019
7020         err = niu_probe_ports(np);
7021         if (err)
7022                 return err;
7023
7024         niu_ldg_init(np);
7025
7026         niu_classifier_swstate_init(np);
7027         niu_link_config_init(np);
7028
7029         err = niu_determine_phy_disposition(np);
7030         if (!err)
7031                 err = niu_init_link(np);
7032
7033         return err;
7034 }
7035
7036 static LIST_HEAD(niu_parent_list);
7037 static DEFINE_MUTEX(niu_parent_lock);
7038 static int niu_parent_index;
7039
7040 static ssize_t show_port_phy(struct device *dev,
7041                              struct device_attribute *attr, char *buf)
7042 {
7043         struct platform_device *plat_dev = to_platform_device(dev);
7044         struct niu_parent *p = plat_dev->dev.platform_data;
7045         u32 port_phy = p->port_phy;
7046         char *orig_buf = buf;
7047         int i;
7048
7049         if (port_phy == PORT_PHY_UNKNOWN ||
7050             port_phy == PORT_PHY_INVALID)
7051                 return 0;
7052
7053         for (i = 0; i < p->num_ports; i++) {
7054                 const char *type_str;
7055                 int type;
7056
7057                 type = phy_decode(port_phy, i);
7058                 if (type == PORT_TYPE_10G)
7059                         type_str = "10G";
7060                 else
7061                         type_str = "1G";
7062                 buf += sprintf(buf,
7063                                (i == 0) ? "%s" : " %s",
7064                                type_str);
7065         }
7066         buf += sprintf(buf, "\n");
7067         return buf - orig_buf;
7068 }
7069
7070 static ssize_t show_plat_type(struct device *dev,
7071                               struct device_attribute *attr, char *buf)
7072 {
7073         struct platform_device *plat_dev = to_platform_device(dev);
7074         struct niu_parent *p = plat_dev->dev.platform_data;
7075         const char *type_str;
7076
7077         switch (p->plat_type) {
7078         case PLAT_TYPE_ATLAS:
7079                 type_str = "atlas";
7080                 break;
7081         case PLAT_TYPE_NIU:
7082                 type_str = "niu";
7083                 break;
7084         case PLAT_TYPE_VF_P0:
7085                 type_str = "vf_p0";
7086                 break;
7087         case PLAT_TYPE_VF_P1:
7088                 type_str = "vf_p1";
7089                 break;
7090         default:
7091                 type_str = "unknown";
7092                 break;
7093         }
7094
7095         return sprintf(buf, "%s\n", type_str);
7096 }
7097
7098 static ssize_t __show_chan_per_port(struct device *dev,
7099                                     struct device_attribute *attr, char *buf,
7100                                     int rx)
7101 {
7102         struct platform_device *plat_dev = to_platform_device(dev);
7103         struct niu_parent *p = plat_dev->dev.platform_data;
7104         char *orig_buf = buf;
7105         u8 *arr;
7106         int i;
7107
7108         arr = (rx ? p->rxchan_per_port : p->txchan_per_port);
7109
7110         for (i = 0; i < p->num_ports; i++) {
7111                 buf += sprintf(buf,
7112                                (i == 0) ? "%d" : " %d",
7113                                arr[i]);
7114         }
7115         buf += sprintf(buf, "\n");
7116
7117         return buf - orig_buf;
7118 }
7119
7120 static ssize_t show_rxchan_per_port(struct device *dev,
7121                                     struct device_attribute *attr, char *buf)
7122 {
7123         return __show_chan_per_port(dev, attr, buf, 1);
7124 }
7125
7126 static ssize_t show_txchan_per_port(struct device *dev,
7127                                     struct device_attribute *attr, char *buf)
7128 {
7129         return __show_chan_per_port(dev, attr, buf, 1);
7130 }
7131
7132 static ssize_t show_num_ports(struct device *dev,
7133                               struct device_attribute *attr, char *buf)
7134 {
7135         struct platform_device *plat_dev = to_platform_device(dev);
7136         struct niu_parent *p = plat_dev->dev.platform_data;
7137
7138         return sprintf(buf, "%d\n", p->num_ports);
7139 }
7140
7141 static struct device_attribute niu_parent_attributes[] = {
7142         __ATTR(port_phy, S_IRUGO, show_port_phy, NULL),
7143         __ATTR(plat_type, S_IRUGO, show_plat_type, NULL),
7144         __ATTR(rxchan_per_port, S_IRUGO, show_rxchan_per_port, NULL),
7145         __ATTR(txchan_per_port, S_IRUGO, show_txchan_per_port, NULL),
7146         __ATTR(num_ports, S_IRUGO, show_num_ports, NULL),
7147         {}
7148 };
7149
7150 static struct niu_parent * __devinit niu_new_parent(struct niu *np,
7151                                                     union niu_parent_id *id,
7152                                                     u8 ptype)
7153 {
7154         struct platform_device *plat_dev;
7155         struct niu_parent *p;
7156         int i;
7157
7158         niudbg(PROBE, "niu_new_parent: Creating new parent.\n");
7159
7160         plat_dev = platform_device_register_simple("niu", niu_parent_index,
7161                                                    NULL, 0);
7162         if (!plat_dev)
7163                 return NULL;
7164
7165         for (i = 0; attr_name(niu_parent_attributes[i]); i++) {
7166                 int err = device_create_file(&plat_dev->dev,
7167                                              &niu_parent_attributes[i]);
7168                 if (err)
7169                         goto fail_unregister;
7170         }
7171
7172         p = kzalloc(sizeof(*p), GFP_KERNEL);
7173         if (!p)
7174                 goto fail_unregister;
7175
7176         p->index = niu_parent_index++;
7177
7178         plat_dev->dev.platform_data = p;
7179         p->plat_dev = plat_dev;
7180
7181         memcpy(&p->id, id, sizeof(*id));
7182         p->plat_type = ptype;
7183         INIT_LIST_HEAD(&p->list);
7184         atomic_set(&p->refcnt, 0);
7185         list_add(&p->list, &niu_parent_list);
7186         spin_lock_init(&p->lock);
7187
7188         p->rxdma_clock_divider = 7500;
7189
7190         p->tcam_num_entries = NIU_PCI_TCAM_ENTRIES;
7191         if (p->plat_type == PLAT_TYPE_NIU)
7192                 p->tcam_num_entries = NIU_NONPCI_TCAM_ENTRIES;
7193
7194         for (i = CLASS_CODE_USER_PROG1; i <= CLASS_CODE_SCTP_IPV6; i++) {
7195                 int index = i - CLASS_CODE_USER_PROG1;
7196
7197                 p->tcam_key[index] = TCAM_KEY_TSEL;
7198                 p->flow_key[index] = (FLOW_KEY_IPSA |
7199                                       FLOW_KEY_IPDA |
7200                                       FLOW_KEY_PROTO |
7201                                       (FLOW_KEY_L4_BYTE12 <<
7202                                        FLOW_KEY_L4_0_SHIFT) |
7203                                       (FLOW_KEY_L4_BYTE12 <<
7204                                        FLOW_KEY_L4_1_SHIFT));
7205         }
7206
7207         for (i = 0; i < LDN_MAX + 1; i++)
7208                 p->ldg_map[i] = LDG_INVALID;
7209
7210         return p;
7211
7212 fail_unregister:
7213         platform_device_unregister(plat_dev);
7214         return NULL;
7215 }
7216
7217 static struct niu_parent * __devinit niu_get_parent(struct niu *np,
7218                                                     union niu_parent_id *id,
7219                                                     u8 ptype)
7220 {
7221         struct niu_parent *p, *tmp;
7222         int port = np->port;
7223
7224         niudbg(PROBE, "niu_get_parent: platform_type[%u] port[%u]\n",
7225                ptype, port);
7226
7227         mutex_lock(&niu_parent_lock);
7228         p = NULL;
7229         list_for_each_entry(tmp, &niu_parent_list, list) {
7230                 if (!memcmp(id, &tmp->id, sizeof(*id))) {
7231                         p = tmp;
7232                         break;
7233                 }
7234         }
7235         if (!p)
7236                 p = niu_new_parent(np, id, ptype);
7237
7238         if (p) {
7239                 char port_name[6];
7240                 int err;
7241
7242                 sprintf(port_name, "port%d", port);
7243                 err = sysfs_create_link(&p->plat_dev->dev.kobj,
7244                                         &np->device->kobj,
7245                                         port_name);
7246                 if (!err) {
7247                         p->ports[port] = np;
7248                         atomic_inc(&p->refcnt);
7249                 }
7250         }
7251         mutex_unlock(&niu_parent_lock);
7252
7253         return p;
7254 }
7255
7256 static void niu_put_parent(struct niu *np)
7257 {
7258         struct niu_parent *p = np->parent;
7259         u8 port = np->port;
7260         char port_name[6];
7261
7262         BUG_ON(!p || p->ports[port] != np);
7263
7264         niudbg(PROBE, "niu_put_parent: port[%u]\n", port);
7265
7266         sprintf(port_name, "port%d", port);
7267
7268         mutex_lock(&niu_parent_lock);
7269
7270         sysfs_remove_link(&p->plat_dev->dev.kobj, port_name);
7271
7272         p->ports[port] = NULL;
7273         np->parent = NULL;
7274
7275         if (atomic_dec_and_test(&p->refcnt)) {
7276                 list_del(&p->list);
7277                 platform_device_unregister(p->plat_dev);
7278         }
7279
7280         mutex_unlock(&niu_parent_lock);
7281 }
7282
7283 static void *niu_pci_alloc_coherent(struct device *dev, size_t size,
7284                                     u64 *handle, gfp_t flag)
7285 {
7286         dma_addr_t dh;
7287         void *ret;
7288
7289         ret = dma_alloc_coherent(dev, size, &dh, flag);
7290         if (ret)
7291                 *handle = dh;
7292         return ret;
7293 }
7294
7295 static void niu_pci_free_coherent(struct device *dev, size_t size,
7296                                   void *cpu_addr, u64 handle)
7297 {
7298         dma_free_coherent(dev, size, cpu_addr, handle);
7299 }
7300
7301 static u64 niu_pci_map_page(struct device *dev, struct page *page,
7302                             unsigned long offset, size_t size,
7303                             enum dma_data_direction direction)
7304 {
7305         return dma_map_page(dev, page, offset, size, direction);
7306 }
7307
7308 static void niu_pci_unmap_page(struct device *dev, u64 dma_address,
7309                                size_t size, enum dma_data_direction direction)
7310 {
7311         return dma_unmap_page(dev, dma_address, size, direction);
7312 }
7313
7314 static u64 niu_pci_map_single(struct device *dev, void *cpu_addr,
7315                               size_t size,
7316                               enum dma_data_direction direction)
7317 {
7318         return dma_map_single(dev, cpu_addr, size, direction);
7319 }
7320
7321 static void niu_pci_unmap_single(struct device *dev, u64 dma_address,
7322                                  size_t size,
7323                                  enum dma_data_direction direction)
7324 {
7325         dma_unmap_single(dev, dma_address, size, direction);
7326 }
7327
7328 static const struct niu_ops niu_pci_ops = {
7329         .alloc_coherent = niu_pci_alloc_coherent,
7330         .free_coherent  = niu_pci_free_coherent,
7331         .map_page       = niu_pci_map_page,
7332         .unmap_page     = niu_pci_unmap_page,
7333         .map_single     = niu_pci_map_single,
7334         .unmap_single   = niu_pci_unmap_single,
7335 };
7336
7337 static void __devinit niu_driver_version(void)
7338 {
7339         static int niu_version_printed;
7340
7341         if (niu_version_printed++ == 0)
7342                 pr_info("%s", version);
7343 }
7344
7345 static struct net_device * __devinit niu_alloc_and_init(
7346         struct device *gen_dev, struct pci_dev *pdev,
7347         struct of_device *op, const struct niu_ops *ops,
7348         u8 port)
7349 {
7350         struct net_device *dev = alloc_etherdev(sizeof(struct niu));
7351         struct niu *np;
7352
7353         if (!dev) {
7354                 dev_err(gen_dev, PFX "Etherdev alloc failed, aborting.\n");
7355                 return NULL;
7356         }
7357
7358         SET_NETDEV_DEV(dev, gen_dev);
7359
7360         np = netdev_priv(dev);
7361         np->dev = dev;
7362         np->pdev = pdev;
7363         np->op = op;
7364         np->device = gen_dev;
7365         np->ops = ops;
7366
7367         np->msg_enable = niu_debug;
7368
7369         spin_lock_init(&np->lock);
7370         INIT_WORK(&np->reset_task, niu_reset_task);
7371
7372         np->port = port;
7373
7374         return dev;
7375 }
7376
7377 static void __devinit niu_assign_netdev_ops(struct net_device *dev)
7378 {
7379         dev->open = niu_open;
7380         dev->stop = niu_close;
7381         dev->get_stats = niu_get_stats;
7382         dev->set_multicast_list = niu_set_rx_mode;
7383         dev->set_mac_address = niu_set_mac_addr;
7384         dev->do_ioctl = niu_ioctl;
7385         dev->tx_timeout = niu_tx_timeout;
7386         dev->hard_start_xmit = niu_start_xmit;
7387         dev->ethtool_ops = &niu_ethtool_ops;
7388         dev->watchdog_timeo = NIU_TX_TIMEOUT;
7389         dev->change_mtu = niu_change_mtu;
7390 }
7391
7392 static void __devinit niu_device_announce(struct niu *np)
7393 {
7394         struct net_device *dev = np->dev;
7395         int i;
7396
7397         pr_info("%s: NIU Ethernet ", dev->name);
7398         for (i = 0; i < 6; i++)
7399                 printk("%2.2x%c", dev->dev_addr[i],
7400                        i == 5 ? '\n' : ':');
7401
7402         pr_info("%s: Port type[%s] mode[%s:%s] XCVR[%s] phy[%s]\n",
7403                 dev->name,
7404                 (np->flags & NIU_FLAGS_XMAC ? "XMAC" : "BMAC"),
7405                 (np->flags & NIU_FLAGS_10G ? "10G" : "1G"),
7406                 (np->flags & NIU_FLAGS_FIBER ? "FIBER" : "COPPER"),
7407                 (np->mac_xcvr == MAC_XCVR_MII ? "MII" :
7408                  (np->mac_xcvr == MAC_XCVR_PCS ? "PCS" : "XPCS")),
7409                 np->vpd.phy_type);
7410 }
7411
7412 static int __devinit niu_pci_init_one(struct pci_dev *pdev,
7413                                       const struct pci_device_id *ent)
7414 {
7415         unsigned long niureg_base, niureg_len;
7416         union niu_parent_id parent_id;
7417         struct net_device *dev;
7418         struct niu *np;
7419         int err, pos;
7420         u64 dma_mask;
7421         u16 val16;
7422
7423         niu_driver_version();
7424
7425         err = pci_enable_device(pdev);
7426         if (err) {
7427                 dev_err(&pdev->dev, PFX "Cannot enable PCI device, "
7428                         "aborting.\n");
7429                 return err;
7430         }
7431
7432         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
7433             !(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
7434                 dev_err(&pdev->dev, PFX "Cannot find proper PCI device "
7435                         "base addresses, aborting.\n");
7436                 err = -ENODEV;
7437                 goto err_out_disable_pdev;
7438         }
7439
7440         err = pci_request_regions(pdev, DRV_MODULE_NAME);
7441         if (err) {
7442                 dev_err(&pdev->dev, PFX "Cannot obtain PCI resources, "
7443                         "aborting.\n");
7444                 goto err_out_disable_pdev;
7445         }
7446
7447         pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
7448         if (pos <= 0) {
7449                 dev_err(&pdev->dev, PFX "Cannot find PCI Express capability, "
7450                         "aborting.\n");
7451                 goto err_out_free_res;
7452         }
7453
7454         dev = niu_alloc_and_init(&pdev->dev, pdev, NULL,
7455                                  &niu_pci_ops, PCI_FUNC(pdev->devfn));
7456         if (!dev) {
7457                 err = -ENOMEM;
7458                 goto err_out_free_res;
7459         }
7460         np = netdev_priv(dev);
7461
7462         memset(&parent_id, 0, sizeof(parent_id));
7463         parent_id.pci.domain = pci_domain_nr(pdev->bus);
7464         parent_id.pci.bus = pdev->bus->number;
7465         parent_id.pci.device = PCI_SLOT(pdev->devfn);
7466
7467         np->parent = niu_get_parent(np, &parent_id,
7468                                     PLAT_TYPE_ATLAS);
7469         if (!np->parent) {
7470                 err = -ENOMEM;
7471                 goto err_out_free_dev;
7472         }
7473
7474         pci_read_config_word(pdev, pos + PCI_EXP_DEVCTL, &val16);
7475         val16 &= ~PCI_EXP_DEVCTL_NOSNOOP_EN;
7476         val16 |= (PCI_EXP_DEVCTL_CERE |
7477                   PCI_EXP_DEVCTL_NFERE |
7478                   PCI_EXP_DEVCTL_FERE |
7479                   PCI_EXP_DEVCTL_URRE |
7480                   PCI_EXP_DEVCTL_RELAX_EN);
7481         pci_write_config_word(pdev, pos + PCI_EXP_DEVCTL, val16);
7482
7483         dma_mask = DMA_44BIT_MASK;
7484         err = pci_set_dma_mask(pdev, dma_mask);
7485         if (!err) {
7486                 dev->features |= NETIF_F_HIGHDMA;
7487                 err = pci_set_consistent_dma_mask(pdev, dma_mask);
7488                 if (err) {
7489                         dev_err(&pdev->dev, PFX "Unable to obtain 44 bit "
7490                                 "DMA for consistent allocations, "
7491                                 "aborting.\n");
7492                         goto err_out_release_parent;
7493                 }
7494         }
7495         if (err || dma_mask == DMA_32BIT_MASK) {
7496                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
7497                 if (err) {
7498                         dev_err(&pdev->dev, PFX "No usable DMA configuration, "
7499                                 "aborting.\n");
7500                         goto err_out_release_parent;
7501                 }
7502         }
7503
7504         dev->features |= (NETIF_F_SG | NETIF_F_HW_CSUM);
7505
7506         niureg_base = pci_resource_start(pdev, 0);
7507         niureg_len = pci_resource_len(pdev, 0);
7508
7509         np->regs = ioremap_nocache(niureg_base, niureg_len);
7510         if (!np->regs) {
7511                 dev_err(&pdev->dev, PFX "Cannot map device registers, "
7512                         "aborting.\n");
7513                 err = -ENOMEM;
7514                 goto err_out_release_parent;
7515         }
7516
7517         pci_set_master(pdev);
7518         pci_save_state(pdev);
7519
7520         dev->irq = pdev->irq;
7521
7522         niu_assign_netdev_ops(dev);
7523
7524         err = niu_get_invariants(np);
7525         if (err) {
7526                 if (err != -ENODEV)
7527                         dev_err(&pdev->dev, PFX "Problem fetching invariants "
7528                                 "of chip, aborting.\n");
7529                 goto err_out_iounmap;
7530         }
7531
7532         err = register_netdev(dev);
7533         if (err) {
7534                 dev_err(&pdev->dev, PFX "Cannot register net device, "
7535                         "aborting.\n");
7536                 goto err_out_iounmap;
7537         }
7538
7539         pci_set_drvdata(pdev, dev);
7540
7541         niu_device_announce(np);
7542
7543         return 0;
7544
7545 err_out_iounmap:
7546         if (np->regs) {
7547                 iounmap(np->regs);
7548                 np->regs = NULL;
7549         }
7550
7551 err_out_release_parent:
7552         niu_put_parent(np);
7553
7554 err_out_free_dev:
7555         free_netdev(dev);
7556
7557 err_out_free_res:
7558         pci_release_regions(pdev);
7559
7560 err_out_disable_pdev:
7561         pci_disable_device(pdev);
7562         pci_set_drvdata(pdev, NULL);
7563
7564         return err;
7565 }
7566
7567 static void __devexit niu_pci_remove_one(struct pci_dev *pdev)
7568 {
7569         struct net_device *dev = pci_get_drvdata(pdev);
7570
7571         if (dev) {
7572                 struct niu *np = netdev_priv(dev);
7573
7574                 unregister_netdev(dev);
7575                 if (np->regs) {
7576                         iounmap(np->regs);
7577                         np->regs = NULL;
7578                 }
7579
7580                 niu_ldg_free(np);
7581
7582                 niu_put_parent(np);
7583
7584                 free_netdev(dev);
7585                 pci_release_regions(pdev);
7586                 pci_disable_device(pdev);
7587                 pci_set_drvdata(pdev, NULL);
7588         }
7589 }
7590
7591 static int niu_suspend(struct pci_dev *pdev, pm_message_t state)
7592 {
7593         struct net_device *dev = pci_get_drvdata(pdev);
7594         struct niu *np = netdev_priv(dev);
7595         unsigned long flags;
7596
7597         if (!netif_running(dev))
7598                 return 0;
7599
7600         flush_scheduled_work();
7601         niu_netif_stop(np);
7602
7603         del_timer_sync(&np->timer);
7604
7605         spin_lock_irqsave(&np->lock, flags);
7606         niu_enable_interrupts(np, 0);
7607         spin_unlock_irqrestore(&np->lock, flags);
7608
7609         netif_device_detach(dev);
7610
7611         spin_lock_irqsave(&np->lock, flags);
7612         niu_stop_hw(np);
7613         spin_unlock_irqrestore(&np->lock, flags);
7614
7615         pci_save_state(pdev);
7616
7617         return 0;
7618 }
7619
7620 static int niu_resume(struct pci_dev *pdev)
7621 {
7622         struct net_device *dev = pci_get_drvdata(pdev);
7623         struct niu *np = netdev_priv(dev);
7624         unsigned long flags;
7625         int err;
7626
7627         if (!netif_running(dev))
7628                 return 0;
7629
7630         pci_restore_state(pdev);
7631
7632         netif_device_attach(dev);
7633
7634         spin_lock_irqsave(&np->lock, flags);
7635
7636         err = niu_init_hw(np);
7637         if (!err) {
7638                 np->timer.expires = jiffies + HZ;
7639                 add_timer(&np->timer);
7640                 niu_netif_start(np);
7641         }
7642
7643         spin_unlock_irqrestore(&np->lock, flags);
7644
7645         return err;
7646 }
7647
7648 static struct pci_driver niu_pci_driver = {
7649         .name           = DRV_MODULE_NAME,
7650         .id_table       = niu_pci_tbl,
7651         .probe          = niu_pci_init_one,
7652         .remove         = __devexit_p(niu_pci_remove_one),
7653         .suspend        = niu_suspend,
7654         .resume         = niu_resume,
7655 };
7656
7657 #ifdef CONFIG_SPARC64
7658 static void *niu_phys_alloc_coherent(struct device *dev, size_t size,
7659                                      u64 *dma_addr, gfp_t flag)
7660 {
7661         unsigned long order = get_order(size);
7662         unsigned long page = __get_free_pages(flag, order);
7663
7664         if (page == 0UL)
7665                 return NULL;
7666         memset((char *)page, 0, PAGE_SIZE << order);
7667         *dma_addr = __pa(page);
7668
7669         return (void *) page;
7670 }
7671
7672 static void niu_phys_free_coherent(struct device *dev, size_t size,
7673                                    void *cpu_addr, u64 handle)
7674 {
7675         unsigned long order = get_order(size);
7676
7677         free_pages((unsigned long) cpu_addr, order);
7678 }
7679
7680 static u64 niu_phys_map_page(struct device *dev, struct page *page,
7681                              unsigned long offset, size_t size,
7682                              enum dma_data_direction direction)
7683 {
7684         return page_to_phys(page) + offset;
7685 }
7686
7687 static void niu_phys_unmap_page(struct device *dev, u64 dma_address,
7688                                 size_t size, enum dma_data_direction direction)
7689 {
7690         /* Nothing to do.  */
7691 }
7692
7693 static u64 niu_phys_map_single(struct device *dev, void *cpu_addr,
7694                                size_t size,
7695                                enum dma_data_direction direction)
7696 {
7697         return __pa(cpu_addr);
7698 }
7699
7700 static void niu_phys_unmap_single(struct device *dev, u64 dma_address,
7701                                   size_t size,
7702                                   enum dma_data_direction direction)
7703 {
7704         /* Nothing to do.  */
7705 }
7706
7707 static const struct niu_ops niu_phys_ops = {
7708         .alloc_coherent = niu_phys_alloc_coherent,
7709         .free_coherent  = niu_phys_free_coherent,
7710         .map_page       = niu_phys_map_page,
7711         .unmap_page     = niu_phys_unmap_page,
7712         .map_single     = niu_phys_map_single,
7713         .unmap_single   = niu_phys_unmap_single,
7714 };
7715
7716 static unsigned long res_size(struct resource *r)
7717 {
7718         return r->end - r->start + 1UL;
7719 }
7720
7721 static int __devinit niu_of_probe(struct of_device *op,
7722                                   const struct of_device_id *match)
7723 {
7724         union niu_parent_id parent_id;
7725         struct net_device *dev;
7726         struct niu *np;
7727         const u32 *reg;
7728         int err;
7729
7730         niu_driver_version();
7731
7732         reg = of_get_property(op->node, "reg", NULL);
7733         if (!reg) {
7734                 dev_err(&op->dev, PFX "%s: No 'reg' property, aborting.\n",
7735                         op->node->full_name);
7736                 return -ENODEV;
7737         }
7738
7739         dev = niu_alloc_and_init(&op->dev, NULL, op,
7740                                  &niu_phys_ops, reg[0] & 0x1);
7741         if (!dev) {
7742                 err = -ENOMEM;
7743                 goto err_out;
7744         }
7745         np = netdev_priv(dev);
7746
7747         memset(&parent_id, 0, sizeof(parent_id));
7748         parent_id.of = of_get_parent(op->node);
7749
7750         np->parent = niu_get_parent(np, &parent_id,
7751                                     PLAT_TYPE_NIU);
7752         if (!np->parent) {
7753                 err = -ENOMEM;
7754                 goto err_out_free_dev;
7755         }
7756
7757         dev->features |= (NETIF_F_SG | NETIF_F_HW_CSUM);
7758
7759         np->regs = of_ioremap(&op->resource[1], 0,
7760                               res_size(&op->resource[1]),
7761                               "niu regs");
7762         if (!np->regs) {
7763                 dev_err(&op->dev, PFX "Cannot map device registers, "
7764                         "aborting.\n");
7765                 err = -ENOMEM;
7766                 goto err_out_release_parent;
7767         }
7768
7769         np->vir_regs_1 = of_ioremap(&op->resource[2], 0,
7770                                     res_size(&op->resource[2]),
7771                                     "niu vregs-1");
7772         if (!np->vir_regs_1) {
7773                 dev_err(&op->dev, PFX "Cannot map device vir registers 1, "
7774                         "aborting.\n");
7775                 err = -ENOMEM;
7776                 goto err_out_iounmap;
7777         }
7778
7779         np->vir_regs_2 = of_ioremap(&op->resource[3], 0,
7780                                     res_size(&op->resource[3]),
7781                                     "niu vregs-2");
7782         if (!np->vir_regs_2) {
7783                 dev_err(&op->dev, PFX "Cannot map device vir registers 2, "
7784                         "aborting.\n");
7785                 err = -ENOMEM;
7786                 goto err_out_iounmap;
7787         }
7788
7789         niu_assign_netdev_ops(dev);
7790
7791         err = niu_get_invariants(np);
7792         if (err) {
7793                 if (err != -ENODEV)
7794                         dev_err(&op->dev, PFX "Problem fetching invariants "
7795                                 "of chip, aborting.\n");
7796                 goto err_out_iounmap;
7797         }
7798
7799         err = register_netdev(dev);
7800         if (err) {
7801                 dev_err(&op->dev, PFX "Cannot register net device, "
7802                         "aborting.\n");
7803                 goto err_out_iounmap;
7804         }
7805
7806         dev_set_drvdata(&op->dev, dev);
7807
7808         niu_device_announce(np);
7809
7810         return 0;
7811
7812 err_out_iounmap:
7813         if (np->vir_regs_1) {
7814                 of_iounmap(&op->resource[2], np->vir_regs_1,
7815                            res_size(&op->resource[2]));
7816                 np->vir_regs_1 = NULL;
7817         }
7818
7819         if (np->vir_regs_2) {
7820                 of_iounmap(&op->resource[3], np->vir_regs_2,
7821                            res_size(&op->resource[3]));
7822                 np->vir_regs_2 = NULL;
7823         }
7824
7825         if (np->regs) {
7826                 of_iounmap(&op->resource[1], np->regs,
7827                            res_size(&op->resource[1]));
7828                 np->regs = NULL;
7829         }
7830
7831 err_out_release_parent:
7832         niu_put_parent(np);
7833
7834 err_out_free_dev:
7835         free_netdev(dev);
7836
7837 err_out:
7838         return err;
7839 }
7840
7841 static int __devexit niu_of_remove(struct of_device *op)
7842 {
7843         struct net_device *dev = dev_get_drvdata(&op->dev);
7844
7845         if (dev) {
7846                 struct niu *np = netdev_priv(dev);
7847
7848                 unregister_netdev(dev);
7849
7850                 if (np->vir_regs_1) {
7851                         of_iounmap(&op->resource[2], np->vir_regs_1,
7852                                    res_size(&op->resource[2]));
7853                         np->vir_regs_1 = NULL;
7854                 }
7855
7856                 if (np->vir_regs_2) {
7857                         of_iounmap(&op->resource[3], np->vir_regs_2,
7858                                    res_size(&op->resource[3]));
7859                         np->vir_regs_2 = NULL;
7860                 }
7861
7862                 if (np->regs) {
7863                         of_iounmap(&op->resource[1], np->regs,
7864                                    res_size(&op->resource[1]));
7865                         np->regs = NULL;
7866                 }
7867
7868                 niu_ldg_free(np);
7869
7870                 niu_put_parent(np);
7871
7872                 free_netdev(dev);
7873                 dev_set_drvdata(&op->dev, NULL);
7874         }
7875         return 0;
7876 }
7877
7878 static struct of_device_id niu_match[] = {
7879         {
7880                 .name = "network",
7881                 .compatible = "SUNW,niusl",
7882         },
7883         {},
7884 };
7885 MODULE_DEVICE_TABLE(of, niu_match);
7886
7887 static struct of_platform_driver niu_of_driver = {
7888         .name           = "niu",
7889         .match_table    = niu_match,
7890         .probe          = niu_of_probe,
7891         .remove         = __devexit_p(niu_of_remove),
7892 };
7893
7894 #endif /* CONFIG_SPARC64 */
7895
7896 static int __init niu_init(void)
7897 {
7898         int err = 0;
7899
7900         BUILD_BUG_ON(PAGE_SIZE < 4 * 1024);
7901
7902         niu_debug = netif_msg_init(debug, NIU_MSG_DEFAULT);
7903
7904 #ifdef CONFIG_SPARC64
7905         err = of_register_driver(&niu_of_driver, &of_bus_type);
7906 #endif
7907
7908         if (!err) {
7909                 err = pci_register_driver(&niu_pci_driver);
7910 #ifdef CONFIG_SPARC64
7911                 if (err)
7912                         of_unregister_driver(&niu_of_driver);
7913 #endif
7914         }
7915
7916         return err;
7917 }
7918
7919 static void __exit niu_exit(void)
7920 {
7921         pci_unregister_driver(&niu_pci_driver);
7922 #ifdef CONFIG_SPARC64
7923         of_unregister_driver(&niu_of_driver);
7924 #endif
7925 }
7926
7927 module_init(niu_init);
7928 module_exit(niu_exit);